<commit_before>import varnishapi,time,os,sys,syslog,traceback

# LogDataManager responsible for recording of log data
class LogDataManager:
    def __init__(self, logStorage):
        self.logReq = {}
        self.logBereq = {}
        self.logStorage = logStorage

        self.Tags = ['ReqURL', 'BereqURL']
        self.MapTagToZipKinField = {
                'ReqURL': 'span_name',
                'BereqURL': 'span_name'
                }

    def addLogItem(self, vxid, requestType, tag, data):

        #print "type: %s, vxid: %d, tag: %s, data: %s" % (requestType, vxid, tag, data)

        if type == 'c' and tag == 'Begin':
            self.logReq = {}
            self.logBereq = {}
        elif tag == 'End':
            if requestType == 'b':
                #print self.logReq
                #print self.logBereq

                self.logStorage.push(self.logReq)
                self.logStorage.push(self.logBereq)

                self.logReq = {}
                self.logBereq = {}

        if vxid > 0:
            if (requestType == 'c' or requestType == 'b') and tag in self.Tags:
                print 'requestType: %s, tag: %s' % (requestType, tag)
                if requestType == 'c':
                    self.logReq[ self.MapTagToZipKinField[tag] ] = data.rstrip('\x00')
                elif requestType == 'b':
                    self.logBereq[ self.MapTagToZipKinField[tag] ] = data.rstrip('\x00')

    # separate, may be we can do bulk sql inserts later on
    def pushLogForVxId(self, vxid):
        # sql query for insertion
        return<commit_msg>picking up ReqHeader and BereqHeader<commit_after>import varnishapi,time,os,sys,syslog,traceback

# LogDataManager responsible for recording of log data
class LogDataManager:
    def __init__(self, logStorage):
        self.logReq = {}
        self.logBereq = {}
        self.logStorage = logStorage

        self.Tags = ['ReqURL', 'BereqURL', 'ReqHeader', 'BereqHeader']
        self.MapTagToZipKinField = {
                'ReqURL': 'span_name',
                'BereqURL': 'span_name'
                }

    def addLogItem(self, vxid, requestType, tag, data):

        #print "type: %s, vxid: %d, tag: %s, data: %s" % (requestType, vxid, tag, data)

        if type == 'c' and tag == 'Begin':
            self.logReq = {}
            self.logBereq = {}
        elif tag == 'End':
            if requestType == 'b':
                #print self.logReq
                #print self.logBereq

                self.logStorage.push(self.logReq)
                self.logStorage.push(self.logBereq)

                self.logReq = {}
                self.logBereq = {}

        if vxid > 0:
            if (requestType == 'c' or requestType == 'b') and tag in self.Tags:
                if tag == 'ReqHeader' or tag == 'BereqHeader':
                    split = data.split(': ')

                    if split[0] == 'X-Varnish':
                        self.logReq['span_id'] = split[1].rstrip('\x00')
                        self.logBereq['trace_id'] = self.logReq['span_id']
                    #print tag + ' - ' + split[0] + ' -> ' + split[1]
                elif requestType == 'c':
                    self.logReq[ self.MapTagToZipKinField[tag] ] = data.rstrip('\x00')
                elif requestType == 'b':
                    self.logBereq[ self.MapTagToZipKinField[tag] ] = data.rstrip('\x00')

    # separate, may be we can do bulk sql inserts later on
    def pushLogForVxId(self, vxid):
        # sql query for insertion
        return
{'fraction_non_alphanumeric': 0.12599318955732122, 'fraction_numerical': 0.004540295119182747, 'mean_word_length': 1.722007722007722, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '342014', 'n_tokens_mistral': 1148, 'n_tokens_neox': 1060, 'n_words': 253}