<commit_before>import os.path
import sys
from optparse import OptionParser
from itertools import *
from xml.etree.ElementTree import *
import gzip
import datetime

MISSING_VALUE = "NA"
RUN_REPORT_LIST = "GATK-run-reports"

def main():
    global OPTIONS
    usage = "usage: %prog [options] mode file1 ... fileN"
    parser = OptionParser(usage=usage)
    
    parser.add_option("-v", "--verbose", dest="verbose",
                        action='store_true', default=False,
                        help="If provided, verbose progress will be enabled")         

    parser.add_option("-o", "--o", dest="output",
                        type='string', default=None,
                        help="if provided, output will go here instead of stdout")

    parser.add_option("", "--no-dev", dest="noDev",
                        action='store_true', default=False,
                        help="if provided, only records not coming from a dev version of GATK will be included")

    parser.add_option("", "--max_days", dest="maxDays",
                        type='int', default=None,
                        help="if provided, only records generated within X days of today will be included")
         
    (OPTIONS, args) = parser.parse_args()
    if len(args) == 0:
        parser.error("Requires at least GATKRunReport xml to analyze")

    stage = args[0]
    files = resolveFiles(args[1:])

    # open up the output file
    if OPTIONS.output != None:
        out = openFile(OPTIONS.output,'w')
    else:
        out = sys.stdout

    handler = getHandler(stage)(stage, out)
    handler.initialize(files)

    # parse all of the incoming files
    counter = 0
    for report in readReports(files):
        # todo -- add matching here
        handler.processRecord(report)
        counter += 1

    handler.finalize(files)
    if OPTIONS.output != None: out.close()
    print 'Processed records:', counter 

#
# Stage HANDLERS
#
class StageHandler:
    def __init__(self, name, out):
        self.name = name
        self.out = out
        
    def getName(self): return self.name
        
    def initialize(self, args):
        pass # print 'initialize'
        
    def processRecord(self, record):
        pass # print 'processing record', record
        
    def finalize(self, args):
        pass # print 'Finalize'


# a map from stage strings -> function to handle record
HANDLERS = dict()
def addHandler(name, handler):
    HANDLERS[name] = handler
    
def getHandler(stage):
    return HANDLERS[stage]

def eltIsException(elt):
    return elt.tag == "exception"
    
def parseException(elt):
    return elt.find("message").text, elt.find("stacktrace").find("string").text


class RecordDecoder:
    def __init__(self):
        self.fields = list()
        self.formatters = dict()
    
        def id(elt): return elt.text
        def toString(elt): return '%s' % elt.text
        
        def formatExceptionMsg(elt):
            return '%s' % parseException(elt)[0]

        def formatExceptionAt(elt):
            return '%s' % parseException(elt)[1]
        
        def add(names, func):
            for name in names:
                addComplex(name, [name], [func])

        def addComplex(key, fields, funcs):
            self.fields.extend(fields)
            self.formatters[key] = zip(fields, funcs)
    
        add(["id", "walker-name", "svn-version"], id)
        add(["start-time", "end-time"], toString)      
        add(["run-time", "java-tmp-directory", "working-directory", "user-name", "host-name"], id)
        add(["java", "machine"], toString)
        add(["max-memory", "total-memory", "iterations", "reads"], id)
        addComplex("exception", ["exception-msg", "exception-at"], [formatExceptionMsg, formatExceptionAt])
        # add(["command-line"], toString)          
    
    def decode(self, report):
        bindings = dict()
        for elt in report:
            if elt.tag in self.formatters:
                fieldFormats = self.formatters[elt.tag]
                # we actually care about this tag
                for field, formatter in fieldFormats:
                    bindings[field] = formatter(elt)

        # add missing data
        for field in self.fields:
            if field not in bindings:
                bindings[field] = MISSING_VALUE

        return bindings
    
# def 
class RecordAsTable(StageHandler):
    def __init__(self, name, out):
        StageHandler.__init__(self, name, out)
        
    def initialize(self, args):
        self.decoder = RecordDecoder()
        print >> self.out, "\t".join(self.decoder.fields)

    def processRecord(self, record):
        parsed = self.decoder.decode(record)

        def oneField(field):
            val = MISSING_VALUE
            if field in parsed:
                val = parsed[field]
                if val.find(" ") != -1:
                    val = "\"" + val + "\""
            return val

        print >> self.out, "\t".join([ oneField(field) for field in self.decoder.fields ])

addHandler('table', RecordAsTable)

class RecordAsXML(StageHandler):
    def __init__(self, name, out):
        StageHandler.__init__(self, name, out)

    def initialize(self, args):
        print >> self.out, "<%s>" % RUN_REPORT_LIST
        
    def processRecord(self, record):
        print >> self.out, tostring(record)

    def finalize(self, args):
        print >> self.out, "</%s>" % RUN_REPORT_LIST

addHandler('xml', RecordAsXML)

class Archive(RecordAsXML):
    def __init__(self, name, out):
        RecordAsXML.__init__(self, name, out)

    def finalize(self, args):
        for arg in args:
            print 'Deleting file: ', arg
            os.remove(arg)

addHandler('archive', Archive)

class ExceptionReport(StageHandler):
    #FIELDS = ["Msg", "At", "SVN.versions", "Walkers", 'Occurrences', 'IDs']
    def __init__(self, name, out):
        StageHandler.__init__(self, name, out)
        self.exceptions = []

    def initialize(self, args):<commit_msg>More robust analysis tool<commit_after>import os.path
import sys
from optparse import OptionParser
from itertools import *
from xml.etree.ElementTree import *
import gzip
import datetime

MISSING_VALUE = "NA"
RUN_REPORT_LIST = "GATK-run-reports"

def main():
    global OPTIONS
    usage = "usage: %prog [options] mode file1 ... fileN"
    parser = OptionParser(usage=usage)
    
    parser.add_option("-v", "--verbose", dest="verbose",
                        action='store_true', default=False,
                        help="If provided, verbose progress will be enabled")         

    parser.add_option("-o", "--o", dest="output",
                        type='string', default=None,
                        help="if provided, output will go here instead of stdout")

    parser.add_option("", "--no-dev", dest="noDev",
                        action='store_true', default=False,
                        help="if provided, only records not coming from a dev version of GATK will be included")

    parser.add_option("", "--max_days", dest="maxDays",
                        type='int', default=None,
                        help="if provided, only records generated within X days of today will be included")

    parser.add_option("", "--D", dest="reallyDeleteInArchiveMode",
                        action='store_true', default=False,
                        help="if provided, we'll actually delete records when running in archive mode")
         
    (OPTIONS, args) = parser.parse_args()
    if len(args) == 0:
        parser.error("Requires at least GATKRunReport xml to analyze")

    stage = args[0]
    files = resolveFiles(args[1:])

    # open up the output file
    if OPTIONS.output != None:
        if stage == "archive" and os.path.exists(OPTIONS.output):
            raise "archive output file already exists, aborting!", OPTIONS.output
        out = openFile(OPTIONS.output,'w')
    else:
        out = sys.stdout

    handler = getHandler(stage)(stage, out)
    handler.initialize(files)

    # parse all of the incoming files
    counter = 0
    for report in readReports(files):
        # todo -- add matching here
        handler.processRecord(report)
        counter += 1

    handler.finalize(files)
    if OPTIONS.output != None: out.close()
    print 'Processed records:', counter 

#
# Stage HANDLERS
#
class StageHandler:
    def __init__(self, name, out):
        self.name = name
        self.out = out
        
    def getName(self): return self.name
        
    def initialize(self, args):
        pass # print 'initialize'
        
    def processRecord(self, record):
        pass # print 'processing record', record
        
    def finalize(self, args):
        pass # print 'Finalize'


# a map from stage strings -> function to handle record
HANDLERS = dict()
def addHandler(name, handler):
    HANDLERS[name] = handler
    
def getHandler(stage):
    return HANDLERS[stage]

def eltIsException(elt):
    return elt.tag == "exception"
    
def parseException(elt):
    return elt.find("message").text, elt.find("stacktrace").find("string").text


class RecordDecoder:
    def __init__(self):
        self.fields = list()
        self.formatters = dict()
    
        def id(elt): return elt.text
        def toString(elt): return '%s' % elt.text
        
        def formatExceptionMsg(elt):
            return '%s' % parseException(elt)[0]

        def formatExceptionAt(elt):
            return '%s' % parseException(elt)[1]
        
        def add(names, func):
            for name in names:
                addComplex(name, [name], [func])

        def addComplex(key, fields, funcs):
            self.fields.extend(fields)
            self.formatters[key] = zip(fields, funcs)
    
        add(["id", "walker-name", "svn-version"], id)
        add(["start-time", "end-time"], toString)      
        add(["run-time", "java-tmp-directory", "working-directory", "user-name", "host-name"], id)
        add(["java", "machine"], toString)
        add(["max-memory", "total-memory", "iterations", "reads"], id)
        addComplex("exception", ["exception-msg", "exception-at"], [formatExceptionMsg, formatExceptionAt])
        # add(["command-line"], toString)          
    
    def decode(self, report):
        bindings = dict()
        for elt in report:
            if elt.tag in self.formatters:
                fieldFormats = self.formatters[elt.tag]
                # we actually care about this tag
                for field, formatter in fieldFormats:
                    bindings[field] = formatter(elt)

        # add missing data
        for field in self.fields:
            if field not in bindings:
                bindings[field] = MISSING_VALUE

        return bindings
    
# def 
class RecordAsTable(StageHandler):
    def __init__(self, name, out):
        StageHandler.__init__(self, name, out)
        
    def initialize(self, args):
        self.decoder = RecordDecoder()
        print >> self.out, "\t".join(self.decoder.fields)

    def processRecord(self, record):
        parsed = self.decoder.decode(record)

        def oneField(field):
            val = MISSING_VALUE
            if field in parsed:
                val = parsed[field]
                if val.find(" ") != -1:
                    val = "\"" + val + "\""
            return val

        print >> self.out, "\t".join([ oneField(field) for field in self.decoder.fields ])

addHandler('table', RecordAsTable)

class RecordAsXML(StageHandler):
    def __init__(self, name, out):
        StageHandler.__init__(self, name, out)

    def initialize(self, args):
        print >> self.out, "<%s>" % RUN_REPORT_LIST
        
    def processRecord(self, record):
        print >> self.out, tostring(record)

    def finalize(self, args):
        print >> self.out, "</%s>" % RUN_REPORT_LIST

addHandler('xml', RecordAsXML)

class Archive(RecordAsXML):
    def __init__(self, name, out):
        RecordAsXML.__init__(self, name, out)

    def finalize(self, args):
        RecordAsXML.finalize(self, args)
        for arg in args:
            print 'Deleting file: ', arg
            if OPTIONS.reallyDeleteInArchiveMode:
                os.remove(arg)

addHandler('archive', Archive)

class ExceptionReport(StageHandler):
    #FIELDS = ["Msg", "At", "SVN.versions", "Walkers", 'Occurrences', 'IDs']
    def __init__(self, name, out):
        StageHandler.__init__(self, name, out)
        self.exceptions = []

    def initialize(self, args):
{'fraction_non_alphanumeric': 0.12171818546768366, 'fraction_numerical': 0.0014452027298273785, 'mean_word_length': 2.2505219206680587, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4411278', 'n_tokens_mistral': 3632, 'n_tokens_neox': 3502, 'n_words': 971}