<commit_before>#!/usr/bin/python

import simplejson
import sys
import pycurl
import socorro.lib.filesystem

def submitCrashReport (jsonFilePathName, binaryFilePathName, serverURL):
  jsonFile = open(jsonFilePathName)
  try:
    data = simplejson.load(jsonFile)
  finally:
    jsonFile.close()

  c = pycurl.Curl()
  fields = [(str(t[0]), str(t[1])) for t in data.items()]
  fields.append (("upload_file_minidump", (c.FORM_FILE, binaryFilePathName)))
  c.setopt(c.TIMEOUT, 10)
  c.setopt(c.POST, 1)
  c.setopt(c.URL, serverURL)
  c.setopt(c.HTTPPOST, fields)
  c.perform()
  c.close()

def reportErrorToStderr (x):
  print >>sys.stderr, x

def walkFileSystemSubmittingReports (fileSystemRoot, serverURL, errorReporter=reportErrorToStderr):
  for aPath, aFileName, aJsonPathName in socorro.lib.filesystem.findFileGenerator(fileSystemRoot, lambda x: x[2].endswith("json")):
    try:
      dumpfilePathName = os.path.join(aPath, "%s%s" % (aFileName[:-5], ".dump"))
      submitCrashReport(aJsonPathName, dumpfilePathName, serverURL)
    except KeyboardInterrupt:
      break
    except Exception, x:
      errorReporter(x)

if __name__ == '__main__':
  import socorro.lib.ConfigurationManager

  import os.path
  import traceback

  def myErrorReporter (anException):
    print >>sys.stderr, type(anException), anException
    exceptionType, exception, tracebackInfo = sys.exc_info()
    traceback.print_tb(tracebackInfo, None, sys.stderr)

  commandLineOptions = [
    ('c',  'config', True, './config', "the config file"),
    ('u',  'url', True, 'http://crash-reports.stage.mozilla.com/submit', "The url of the server to load test"),
    ('j',  'jsonfile', True, None, 'the pathname of a json file for POST'),
    ('d',  'dumpfile', True, None, 'the pathname of a dumpfile to upload with the POST'),
    ('s',  'searchRoot', True, None, 'a filesystem location to begin a search for json/dump combos'),
    ]

  config = socorro.lib.ConfigurationManager.newConfiguration(configurationOptionsList=commandLineOptions)

  if config.searchRoot:
    walkFileSystemSubmittingReports(config.searchRoot, config.url, myErrorReporter)
  else:
    try:
      submitCrashReport(config.jsonfile, config.dumpfile, config.url)
    except Exception, x:
      myErrorReporter(x)



<commit_msg>added feature to renumber crashes with HangId.  The avoids duplicating HangIds in the database.<commit_after>#!/usr/bin/python

import simplejson
import sys
import pycurl
import socorro.lib.filesystem
import socorro.lib.uuid as uuid

existingHangIdCache = {}

def submitCrashReport (jsonFilePathName, binaryFilePathName, serverURL, uniqueHang):
  jsonFile = open(jsonFilePathName)
  try:
    data = simplejson.load(jsonFile)
  finally:
    jsonFile.close()

  if uniqueHang:
    try:
      if data['HangId'] in existingHangIdCache:
        data['HangId'] = existingHangIdCache
      else:
        data['HangId'] = existingHangIdCache[data['HangId']] = uuid.uuid4()
    except:
      pass

  c = pycurl.Curl()
  fields = [(str(t[0]), str(t[1])) for t in data.items()]
  fields.append (("upload_file_minidump", (c.FORM_FILE, binaryFilePathName)))
  c.setopt(c.TIMEOUT, 10)
  c.setopt(c.POST, 1)
  c.setopt(c.URL, serverURL)
  c.setopt(c.HTTPPOST, fields)
  c.perform()
  c.close()

def reportErrorToStderr (x):
  print >>sys.stderr, x

def walkFileSystemSubmittingReports (fileSystemRoot, serverURL, errorReporter=reportErrorToStderr, uniqueHang=False):
  for aPath, aFileName, aJsonPathName in socorro.lib.filesystem.findFileGenerator(fileSystemRoot, lambda x: x[2].endswith("json")):
    try:
      dumpfilePathName = os.path.join(aPath, "%s%s" % (aFileName[:-5], ".dump"))
      submitCrashReport(aJsonPathName, dumpfilePathName, serverURL, uniqueHang)
    except KeyboardInterrupt:
      break
    except Exception, x:
      errorReporter(x)

if __name__ == '__main__':
  import socorro.lib.ConfigurationManager

  import os.path
  import traceback

  def myErrorReporter (anException):
    print >>sys.stderr, type(anException), anException
    exceptionType, exception, tracebackInfo = sys.exc_info()
    traceback.print_tb(tracebackInfo, None, sys.stderr)

  commandLineOptions = [
    ('c',  'config', True, './config', "the config file"),
    ('u',  'url', True, 'http://crash-reports.stage.mozilla.com/submit', "The url of the server to load test"),
    ('j',  'jsonfile', True, None, 'the pathname of a json file for POST'),
    ('d',  'dumpfile', True, None, 'the pathname of a dumpfile to upload with the POST'),
    ('s',  'searchRoot', True, None, 'a filesystem location to begin a search for json/dump combos'),
    ('i',  'uniqueHangId', False, None, 'coche and uniquify hangids'),
    ]

  config = socorro.lib.ConfigurationManager.newConfiguration(configurationOptionsList=commandLineOptions)

  uniqueHang = 'uniqueHangId' in config

  if config.searchRoot:
    walkFileSystemSubmittingReports(config.searchRoot, config.url, myErrorReporter, uniqueHang)
  else:
    try:
      submitCrashReport(config.jsonfile, config.dumpfile, config.url, uniqueHang)
    except Exception, x:
      myErrorReporter(x)

{'fraction_non_alphanumeric': 0.12441130298273155, 'fraction_numerical': 0.0029434850863422294, 'mean_word_length': 4.410828025477707, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6935375', 'n_tokens_mistral': 1713, 'n_tokens_neox': 1647, 'n_words': 408}