<commit_before>import datetime as dt
import errno
import logging
import os
import time

import psycopg2

import unittest
from nose.tools import *

import socorro.lib.ConfigurationManager as configurationManager
import socorro.lib.util
import socorro.lib.psycopghelper as psy
import socorro.cron.bugzilla as bug
import socorro.database.schema as sch
import socorro.database.database as sdatabase

#from createTables import createCronTables, dropCronTables
import socorro.unittest.testlib.dbtestutil as dbtestutil
from   socorro.unittest.testlib.loggerForTest import TestingLogger
from   socorro.unittest.testlib.testDB import TestDB
import socorro.unittest.testlib.util as tutil

import cronTestconfig as testConfig

def makeBogusReports (connection, cursor, logger):
  # make some bogus data in the reports table
  reportsTable = sch.ReportsTable(logger)
                    # ( uuid,    client_crash_date,   date_processed,             product,   version,   build,   url,              install_age,   last_crash,   uptime,   email,   build_date,   user_id,   user_comments,   app_notes,   distributor,   distributor_version, topmost_filenames, addons_checked, flash_version, hangid, process_type) values
  fakeReportData = [ (( "uuid1", None,                dt.datetime(2009, 05, 04),  "bogus",   "1.0",     "xxx",   "http://cnn.com", 100,           14,           10,       None,    None,         None,      "bogus",         "",          "",            ",",                 None,              None,           None,          None,   None), "BogusClass::bogus_signature (const char**, void *)"),
                     (( "uuid2", None,                dt.datetime(2009, 05, 04),  "bogus",   "1.0",     "xxx",   "http://cnn.com", 100,           14,           10,       None,    None,         None,      "bogus",         "",          "",            ",",                 None,              None,           None,          None,   None), "js3250.dll@0x6cb96"),
                     (( "uuid3", None,                dt.datetime(2009, 05, 04),  "bogus",   "1.0",     "xxx",   "http://cnn.com", 100,           14,           10,       None,    None,         None,      "bogus",         "",          "",            ",",                 None,              None,           None,          None,   None), "libobjc.A.dylib@0x1568c"),
                     (( "uuid4", None,                dt.datetime(2009, 05, 04),  "bogus",   "1.0",     "xxx",   "http://cnn.com", 100,           14,           10,       None,    None,         None,      "bogus",         "",          "",            ",",                 None,              None,           None,          None,   None), "nanojit::LIns::isTramp()"),
                     (( "uuid5", None,                dt.datetime(2009, 05, 04),  "bogus",   "1.0",     "xxx",   "http://cnn.com", 100,           14,           10,       None,    None,         None,      "bogus",         "",          "",            ",",                 None,              None,           None,          None,   None), "libobjc.A.dylib@0x1568c"),
                   ]
  try:
    #altconn = psycopg2.connect(me.dsn)
    altconn = me.database.connection()
    altcur = altconn.cursor()
  except Exception, x:
    print "Exception at line 40:",type(x),x
    raise
  def cursorFunction():
    return altconn, altcur
  for rep, sig in fakeReportData:
    try:
      reportsTable.insert(cursor, rep, cursorFunction, date_processed=rep[2])
      connection.commit()
      cursor.execute("update reports set signature=%s where date_processed = %s and uuid = %s", (sig, rep[2], rep[0]))
      connection.commit()
    except Exception, x:
      print "Exception at line 51", type(x),x
      connection.rollback()
  altconn.close()

class Me: # not quite "self"
  """
  I need stuff to be initialized once per module. Rather than having a bazillion globals, lets just have 'me'
  """
  pass
me = None

def setup_module():
  global me
  if me:
    return
  me = Me()
  me.config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName='Testing bugzilla')
  tutil.nosePrintModule(__file__)
  myDir = os.path.split(__file__)[0]
  if not myDir: myDir = '.'
  replDict = {'testDir':'%s'%myDir}
  for i in me.config:
    try:
      me.config[i] = me.config.get(i)%(replDict)
    except:
      pass
  me.logFilePathname = me.config.logFilePathname
  if not me.logFilePathname:
    me.logFilePathname = 'logs/bugzilla_test.log'
  logFileDir = os.path.split(me.logFilePathname)[0]
  try:
    os.makedirs(logFileDir)
  except OSError,x:
    if errno.EEXIST == x.errno: pass
    else: raise
  f = open(me.logFilePathname,'w')
  f.close()
  fileLog = logging.FileHandler(me.logFilePathname, 'a')
  fileLog.setLevel(logging.DEBUG)
  fileLogFormatter = logging.Formatter('%(asctime)s %(levelname)s - %(message)s')
  fileLog.setFormatter(fileLogFormatter)
  me.fileLogger = logging.getLogger("bugzilla")
  me.fileLogger.addHandler(fileLog)
  # Got trouble?  See what's happening by uncommenting the next three lines
  #stderrLog = logging.StreamHandler()
  #stderrLog.setLevel(10)
  #me.fileLogger.addHandler(stderrLog)

  me.database = sdatabase.Database(me.config)
  #me.dsn = "host=%s dbname=%s user=%s password=%s" % (me.config.databaseHost,me.config.databaseName,
                                                      #me.config.databaseUserName,me.config.databasePassword)
  me.testDB = TestDB()
  me.testDB.removeDB(me.config,me.fileLogger)
  me.testDB.createDB(me.config,me.fileLogger)
  try:
    me.conn = me.database.connection()
    #me.conn = psycopg2.connect(me.dsn)
    #me.cur = me.conn.cursor(cursor_factory=psy.LoggingCursor)
    me.cur = me.conn.cursor()
  except Exception, x:
    print "Exception at line 107",type(x),x
    socorro.lib.util.reportExceptionAndAbort(me.fileLogger)
  makeBogusReports(me.conn, me.cur, me.fileLogger)

def teardown_module():
  global me
  me.testDB.removeDB(me.config,me.fileLogger)
  me.conn.close()
  try:
    os.unlink(me.logFilePathname)
  except:
    pass

class TestBugzilla(unittest.TestCase):
  def setUp(self):
    global me
    self.config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName='Testing bugzilla')

    myDir = os.path.split(__file__)[0]
    if not myDir: myDir = '.'
    replDict = {'testDir':'%s'%myDir}
    for i in self.config:
      try:
        self.config[i] = self.config.get(i)%(replDict)
      except:
        pass
    self.logger = TestingLogger(me.fileLogger)
    self.connection = me.database.connection()
    #self.connection = psycopg2.connect(me.dsn)

    self.testConfig = configurationManager.Config([('t','testPath', True, './TEST-BUGZILLA', ''),
                                                  ('f','testFileName', True, 'lastrun.pickle', ''),
                                                  ('', 'daysIntoPast', True, 0),])
    self.testConfig["persistentDataPathname"] = os.path.join(self.testConfig.testPath, self.testConfig.testFileName)

  def tearDown(self):
    self.logger.clear()

  def test_bugzilla_iterator(self):
    csv = ['bug_id,"bug_status","resolution","short_desc","cf_crash_signature"\n',
           '1,"RESOLVED",,"this is a comment","This sig, while bogus, has a ] bracket"',
           '2,"CLOSED","WONTFIX","comments are not too important","single [@ BogusClass::bogus_sig (const char**) ] signature"',
           '3,"ASSIGNED",,"this is a comment. [@ nanojit::LIns::isTramp()]","[@ js3250.dll@0x6cb96] [@ valid.sig@0x333333]"',
           '4,"CLOSED","RESOLVED","two sigs enter, one sig leaves","[@ layers::Push@0x123456] [@ layers::Push@0x123456]"',
           '5,"ASSIGNED","INCOMPLETE",,"[@ MWSBAR.DLL@0x2589f] and a broken one [@ sadTrombone.DLL@0xb4s455"',
           '6,"ASSIGNED",,"empty crash sigs should not throw errors",""'
           '7,"CLOSED",,"gt 525355 gt","[@gfx::font(nsTArray<nsRefPtr<FontEntry> > const&)]"',
           '8,"CLOSED","RESOLVED","newlines in sigs","[@ legitimate(sig)] \n junk \n [@ another::legitimate(sig) ]"'
          ]
    correct = [ (1, "RESOLVED", "", "this is a comment", set([])),
                (2, "CLOSED", "WONTFIX", "comments are not too important", set(["BogusClass::bogus_sig (const char**)"])),
                (3, "ASSIGNED", "", "this is a comment. [@ nanojit::LIns::isTramp()]", 
                 set(["js3250.dll@0x6cb96", "valid.sig@0x333333"])),
                (4, "CLOSED", "RESOLVED", "two sigs enter, one sig leaves", set(["layers::Push@0x123456"])),
                (5, "ASSIGNED", "INCOMPLETE", "", set(["MWSBAR.DLL@0x2589f"])),
                (6, "ASSIGNED", "", "empty crash sigs should not throw errors", set([])),
                (7, "CLOSED", "", "gt 525355 gt", set(["gfx::font(nsTArray<nsRefPtr<FontEntry> > const&)"])),
                (8, "CLOSED", "RESOLVED", "newlines in sigs", set(['another::legitimate(sig)', 'legitimate(sig)']))
              ]
    for expected, actual in zip(bug.bugzilla_iterator(csv, iter), correct):
      assert expected == actual, "expected %s, got %s" % (str(expected), str(actual))

  def test_signature_is_found(self):
    global me
    assert bug.signature_is_found("js3250.dll@0x6cb96", me.cur)
    assert not bug.signature_is_found("sir_not_appearing_in_this_film", me.cur)
    me.cur.connection.rollback()

  def verify_tables(self, correct):<commit_msg>fixed malformed list in bugzilla cron test<commit_after>import datetime as dt
import errno
import logging
import os
import time

import psycopg2

import unittest
from nose.tools import *

import socorro.lib.ConfigurationManager as configurationManager
import socorro.lib.util
import socorro.lib.psycopghelper as psy
import socorro.cron.bugzilla as bug
import socorro.database.schema as sch
import socorro.database.database as sdatabase

#from createTables import createCronTables, dropCronTables
import socorro.unittest.testlib.dbtestutil as dbtestutil
from   socorro.unittest.testlib.loggerForTest import TestingLogger
from   socorro.unittest.testlib.testDB import TestDB
import socorro.unittest.testlib.util as tutil

import cronTestconfig as testConfig

def makeBogusReports (connection, cursor, logger):
  # make some bogus data in the reports table
  reportsTable = sch.ReportsTable(logger)
                    # ( uuid,    client_crash_date,   date_processed,             product,   version,   build,   url,              install_age,   last_crash,   uptime,   email,   build_date,   user_id,   user_comments,   app_notes,   distributor,   distributor_version, topmost_filenames, addons_checked, flash_version, hangid, process_type) values
  fakeReportData = [ (( "uuid1", None,                dt.datetime(2009, 05, 04),  "bogus",   "1.0",     "xxx",   "http://cnn.com", 100,           14,           10,       None,    None,         None,      "bogus",         "",          "",            ",",                 None,              None,           None,          None,   None), "BogusClass::bogus_signature (const char**, void *)"),
                     (( "uuid2", None,                dt.datetime(2009, 05, 04),  "bogus",   "1.0",     "xxx",   "http://cnn.com", 100,           14,           10,       None,    None,         None,      "bogus",         "",          "",            ",",                 None,              None,           None,          None,   None), "js3250.dll@0x6cb96"),
                     (( "uuid3", None,                dt.datetime(2009, 05, 04),  "bogus",   "1.0",     "xxx",   "http://cnn.com", 100,           14,           10,       None,    None,         None,      "bogus",         "",          "",            ",",                 None,              None,           None,          None,   None), "libobjc.A.dylib@0x1568c"),
                     (( "uuid4", None,                dt.datetime(2009, 05, 04),  "bogus",   "1.0",     "xxx",   "http://cnn.com", 100,           14,           10,       None,    None,         None,      "bogus",         "",          "",            ",",                 None,              None,           None,          None,   None), "nanojit::LIns::isTramp()"),
                     (( "uuid5", None,                dt.datetime(2009, 05, 04),  "bogus",   "1.0",     "xxx",   "http://cnn.com", 100,           14,           10,       None,    None,         None,      "bogus",         "",          "",            ",",                 None,              None,           None,          None,   None), "libobjc.A.dylib@0x1568c"),
                   ]
  try:
    #altconn = psycopg2.connect(me.dsn)
    altconn = me.database.connection()
    altcur = altconn.cursor()
  except Exception, x:
    print "Exception at line 40:",type(x),x
    raise
  def cursorFunction():
    return altconn, altcur
  for rep, sig in fakeReportData:
    try:
      reportsTable.insert(cursor, rep, cursorFunction, date_processed=rep[2])
      connection.commit()
      cursor.execute("update reports set signature=%s where date_processed = %s and uuid = %s", (sig, rep[2], rep[0]))
      connection.commit()
    except Exception, x:
      print "Exception at line 51", type(x),x
      connection.rollback()
  altconn.close()

class Me: # not quite "self"
  """
  I need stuff to be initialized once per module. Rather than having a bazillion globals, lets just have 'me'
  """
  pass
me = None

def setup_module():
  global me
  if me:
    return
  me = Me()
  me.config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName='Testing bugzilla')
  tutil.nosePrintModule(__file__)
  myDir = os.path.split(__file__)[0]
  if not myDir: myDir = '.'
  replDict = {'testDir':'%s'%myDir}
  for i in me.config:
    try:
      me.config[i] = me.config.get(i)%(replDict)
    except:
      pass
  me.logFilePathname = me.config.logFilePathname
  if not me.logFilePathname:
    me.logFilePathname = 'logs/bugzilla_test.log'
  logFileDir = os.path.split(me.logFilePathname)[0]
  try:
    os.makedirs(logFileDir)
  except OSError,x:
    if errno.EEXIST == x.errno: pass
    else: raise
  f = open(me.logFilePathname,'w')
  f.close()
  fileLog = logging.FileHandler(me.logFilePathname, 'a')
  fileLog.setLevel(logging.DEBUG)
  fileLogFormatter = logging.Formatter('%(asctime)s %(levelname)s - %(message)s')
  fileLog.setFormatter(fileLogFormatter)
  me.fileLogger = logging.getLogger("bugzilla")
  me.fileLogger.addHandler(fileLog)
  # Got trouble?  See what's happening by uncommenting the next three lines
  #stderrLog = logging.StreamHandler()
  #stderrLog.setLevel(10)
  #me.fileLogger.addHandler(stderrLog)

  me.database = sdatabase.Database(me.config)
  #me.dsn = "host=%s dbname=%s user=%s password=%s" % (me.config.databaseHost,me.config.databaseName,
                                                      #me.config.databaseUserName,me.config.databasePassword)
  me.testDB = TestDB()
  me.testDB.removeDB(me.config,me.fileLogger)
  me.testDB.createDB(me.config,me.fileLogger)
  try:
    me.conn = me.database.connection()
    #me.conn = psycopg2.connect(me.dsn)
    #me.cur = me.conn.cursor(cursor_factory=psy.LoggingCursor)
    me.cur = me.conn.cursor()
  except Exception, x:
    print "Exception at line 107",type(x),x
    socorro.lib.util.reportExceptionAndAbort(me.fileLogger)
  makeBogusReports(me.conn, me.cur, me.fileLogger)

def teardown_module():
  global me
  me.testDB.removeDB(me.config,me.fileLogger)
  me.conn.close()
  try:
    os.unlink(me.logFilePathname)
  except:
    pass

class TestBugzilla(unittest.TestCase):
  def setUp(self):
    global me
    self.config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName='Testing bugzilla')

    myDir = os.path.split(__file__)[0]
    if not myDir: myDir = '.'
    replDict = {'testDir':'%s'%myDir}
    for i in self.config:
      try:
        self.config[i] = self.config.get(i)%(replDict)
      except:
        pass
    self.logger = TestingLogger(me.fileLogger)
    self.connection = me.database.connection()
    #self.connection = psycopg2.connect(me.dsn)

    self.testConfig = configurationManager.Config([('t','testPath', True, './TEST-BUGZILLA', ''),
                                                  ('f','testFileName', True, 'lastrun.pickle', ''),
                                                  ('', 'daysIntoPast', True, 0),])
    self.testConfig["persistentDataPathname"] = os.path.join(self.testConfig.testPath, self.testConfig.testFileName)

  def tearDown(self):
    self.logger.clear()

  def test_bugzilla_iterator(self):
    csv = ['bug_id,"bug_status","resolution","short_desc","cf_crash_signature"\n',
           '1,"RESOLVED",,"this is a comment","This sig, while bogus, has a ] bracket"',
           '2,"CLOSED","WONTFIX","comments are not too important","single [@ BogusClass::bogus_sig (const char**) ] signature"',
           '3,"ASSIGNED",,"this is a comment. [@ nanojit::LIns::isTramp()]","[@ js3250.dll@0x6cb96] [@ valid.sig@0x333333]"',
           '4,"CLOSED","RESOLVED","two sigs enter, one sig leaves","[@ layers::Push@0x123456] [@ layers::Push@0x123456]"',
           '5,"ASSIGNED","INCOMPLETE",,"[@ MWSBAR.DLL@0x2589f] and a broken one [@ sadTrombone.DLL@0xb4s455"',
           '6,"ASSIGNED",,"empty crash sigs should not throw errors",""',
           '7,"CLOSED",,"gt 525355 gt","[@gfx::font(nsTArray<nsRefPtr<FontEntry> > const&)]"',
           '8,"CLOSED","RESOLVED","newlines in sigs","[@ legitimate(sig)] \n junk \n [@ another::legitimate(sig) ]"'
          ]
    correct = [ (1, "RESOLVED", "", "this is a comment", set([])),
                (2, "CLOSED", "WONTFIX", "comments are not too important", set(["BogusClass::bogus_sig (const char**)"])),
                (3, "ASSIGNED", "", "this is a comment. [@ nanojit::LIns::isTramp()]", 
                 set(["js3250.dll@0x6cb96", "valid.sig@0x333333"])),
                (4, "CLOSED", "RESOLVED", "two sigs enter, one sig leaves", set(["layers::Push@0x123456"])),
                (5, "ASSIGNED", "INCOMPLETE", "", set(["MWSBAR.DLL@0x2589f"])),
                (6, "ASSIGNED", "", "empty crash sigs should not throw errors", set([])),
                (7, "CLOSED", "", "gt 525355 gt", set(["gfx::font(nsTArray<nsRefPtr<FontEntry> > const&)"])),
                (8, "CLOSED", "RESOLVED", "newlines in sigs", set(['another::legitimate(sig)', 'legitimate(sig)']))
              ]
    for expected, actual in zip(bug.bugzilla_iterator(csv, iter), correct):
      assert expected == actual, "expected %s, got %s" % (str(expected), str(actual))

  def test_signature_is_found(self):
    global me
    assert bug.signature_is_found("js3250.dll@0x6cb96", me.cur)
    assert not bug.signature_is_found("sir_not_appearing_in_this_film", me.cur)
    me.cur.connection.rollback()

  def verify_tables(self, correct):
{'fraction_non_alphanumeric': 0.14569288389513108, 'fraction_numerical': 0.024612092027822365, 'mean_word_length': 2.5486994494019366, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4653540', 'n_tokens_mistral': 6355, 'n_tokens_neox': 5983, 'n_words': 1283}