<commit_before>#!/usr/bin/python

"""
This script is what populates the aggregate summary table for crash dump reports.
It lets us efficiently provide up to date reports on the crash dump data.
Examples of these reports are the Top 100 crashers, All Time Crashers, New Crashers
"""
import sys
from operator import itemgetter
import time
import datetime

import psycopg2
import psycopg2.extras

import socorro.lib.util


def calc_tots(crashes, crash_count, configContext, logger):
  """ Calculate the total number of crashes per signature combo, as well as uptime averages. """
  for pvCombo, data in crashes.items():
    for signature, signatureData in crashes[pvCombo].items():
      fullKey = (pvCombo[0], pvCombo[1], pvCombo[2], signature)
      signatureData['total'] = signatureData['win'] + signatureData['mac'] + signatureData['lin']
      signatureData['uptime_average'] = signatureData['uptime'] / crash_count[fullKey]

  if configContext.debug:
    for pvCombo, data in crashes.items():
      for signature, signatureData in crashes[pvCombo].items():
        logger.debug("%s Total: %d" % ((pvCombo, signature), signatureData['total']))
        logger.debug("%s Uptime Average: %d" % ((pvCombo, signature), signatureData['uptime_average']))

def calc_ranks(crashes, configContext, logger):
  """ Calculate the new ranks of the crashes, by total number of crashes """
  ranks = []

  for signature, signatureData in crashes.items():
    ranks.append([signature, signatureData['total']])

  ranks = sorted(ranks, key=itemgetter(1))
  ranks.reverse()

  for rank in range(0, len(ranks)):
    crashes[ranks[rank][0]]['rank'] = rank + 1

  if configContext.debug:
    for signature, data in crashes.items():
      logger.debug("%s is ranked at: %d" % (signature, crashes[signature]['rank']))


def calculateTopCrashes(configContext, logger):
  summary_crashes = {}
  crash_count = {}

  try:
    databaseDSN = "host=%(databaseHost)s dbname=%(databaseName)s user=%(databaseUserName)s password=%(database<PASSWORD>" % configContext
    conn = psycopg2.connect(databaseDSN)
    cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
  except:
    socorro.lib.util.reportExceptionAndAbort(logger)

  columns = ["product", "version", "build", "signature", "uptime", "os_name", "user_id"]

  start_time = datetime.datetime.now()
  end_time = datetime.datetime.now()
  now = datetime.datetime.now()
  p_interval = configContext.processingInterval
  initModeDate = now - datetime.timedelta(days=14)

  if not configContext.initMode:
    startsql = "SELECT last_updated FROM topcrashers ORDER BY last_updated DESC LIMIT 1"
    try:
      cur.execute(startsql)
      row = cur.fetchone()
    except:
      socorro.lib.util.reportExceptionAndAbort(logger)

    if row:
      start_time = row[0]
    else:
      msg = "topcrashers table is empty, use -I to run in initializing mode"
      logger.warn(msg)
      raise Exception(msg)
    end_time = start_time + p_interval
    now = now

  if end_time > now:
    end_time = now

  initLoop = True


  while end_time <= now and initLoop:<commit_msg>Changing default initialization from 14 days to 4 days r=morgamic<commit_after>#!/usr/bin/python

"""
This script is what populates the aggregate summary table for crash dump reports.
It lets us efficiently provide up to date reports on the crash dump data.
Examples of these reports are the Top 100 crashers, All Time Crashers, New Crashers
"""
import sys
from operator import itemgetter
import time
import datetime

import psycopg2
import psycopg2.extras

import socorro.lib.util


def calc_tots(crashes, crash_count, configContext, logger):
  """ Calculate the total number of crashes per signature combo, as well as uptime averages. """
  for pvCombo, data in crashes.items():
    for signature, signatureData in crashes[pvCombo].items():
      fullKey = (pvCombo[0], pvCombo[1], pvCombo[2], signature)
      signatureData['total'] = signatureData['win'] + signatureData['mac'] + signatureData['lin']
      signatureData['uptime_average'] = signatureData['uptime'] / crash_count[fullKey]

  if configContext.debug:
    for pvCombo, data in crashes.items():
      for signature, signatureData in crashes[pvCombo].items():
        logger.debug("%s Total: %d" % ((pvCombo, signature), signatureData['total']))
        logger.debug("%s Uptime Average: %d" % ((pvCombo, signature), signatureData['uptime_average']))

def calc_ranks(crashes, configContext, logger):
  """ Calculate the new ranks of the crashes, by total number of crashes """
  ranks = []

  for signature, signatureData in crashes.items():
    ranks.append([signature, signatureData['total']])

  ranks = sorted(ranks, key=itemgetter(1))
  ranks.reverse()

  for rank in range(0, len(ranks)):
    crashes[ranks[rank][0]]['rank'] = rank + 1

  if configContext.debug:
    for signature, data in crashes.items():
      logger.debug("%s is ranked at: %d" % (signature, crashes[signature]['rank']))


def calculateTopCrashes(configContext, logger):
  summary_crashes = {}
  crash_count = {}

  try:
    databaseDSN = "host=%(databaseHost)s dbname=%(databaseName)s user=%(databaseUserName)s password=%(databasePassword)s" % configContext
    conn = psycopg2.connect(databaseDSN)
    cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
  except:
    socorro.lib.util.reportExceptionAndAbort(logger)

  columns = ["product", "version", "build", "signature", "uptime", "os_name", "user_id"]

  start_time = datetime.datetime.now()
  end_time = datetime.datetime.now()
  now = datetime.datetime.now()
  p_interval = configContext.processingInterval
  initModeDate = now - datetime.timedelta(days=4)

  if not configContext.initMode:
    startsql = "SELECT last_updated FROM topcrashers ORDER BY last_updated DESC LIMIT 1"
    try:
      cur.execute(startsql)
      row = cur.fetchone()
    except:
      socorro.lib.util.reportExceptionAndAbort(logger)

    if row:
      start_time = row[0]
    else:
      msg = "topcrashers table is empty, use -I to run in initializing mode"
      logger.warn(msg)
      raise Exception(msg)
    end_time = start_time + p_interval
    now = now

  if end_time > now:
    end_time = now

  initLoop = True


  while end_time <= now and initLoop:
{'fraction_non_alphanumeric': 0.1166106038763415, 'fraction_numerical': 0.00608681723530354, 'mean_word_length': 4.282571912013537, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '784768', 'n_tokens_mistral': 1957, 'n_tokens_neox': 1904, 'n_words': 544}