<commit_before>#    ____            _           _   
#   |  _ \ _ __ ___ (_) ___  ___| |_ 
#   | |_) | '__/ _ \| |/ _ \/ __| __|
#   |  __/| | | (_) | |  __/ (__| |_ 
#   |_|  _|_|  \___// |\___|\___|\__|
#   | | | | __ _ _|__/ ___| |_ __ _  ___| | __
#   | |_| |/ _` | | | / __| __/ _` |/ __| |/ /
#   |  _  | (_| | |_| \__ \ || (_| | (__|   < 
#   |_| |_|\__,_|\__, |___/\__\__,_|\___|_|\_\
#                |___/                        
#
# ParseEventBolt
#

import logging
import hashlib
import random
from collections import namedtuple
from pyleus.storm import SimpleBolt

log = logging.getLogger('parse_event_bolt')

Record = namedtuple('Record', 'eventid timestamp srcip srcport dstip dstport query qtype ttl')

class ParseEventBolt(SimpleBolt):

    OUTPUT_FIELDS = Record

    def process_tuple(self, tup):
        line, = tup.values
        eventdata = line.split('\t')
        timestamp = eventdata[0].split()[1]
        broid     = eventdata[1]
        ha = hashlib.md5()
        ha.update(timestamp+broid+str(tup.id))
        eventid = ha.hexdigest()

        # ttl does not seem like its very reliable
        ttl = eventdata[20]
        
        if ttl != '-' and not ttl.isdigit():
            ttl = '-'
    
        try: 
            record = Record(
                    eventid,        # eventid md5(timestamp+broid+tup.id)
                    timestamp,      # timestamp
                    eventdata[2],   # srcip
                    eventdata[3],   # srcport
                    eventdata[4],   # dstip
                    eventdata[5],   # dstport
                    eventdata[8],   # query
                    eventdata[12],  # qtype
                    ttl)  # ttl

            log.debug(repr(record))
            self.emit(record, anchors=[tup])
        except IndexError:
            log.debug('[*] IndexError. No emit this tuple.')
            pass

if __name__ == '__main__':
    logging.basicConfig(
        level=logging.DEBUG,
        filename='/var/log/haystack/parse_event_bolt.log',
        format="%(message)s",
        filemode='a',
    )
    ParseEventBolt().run()

<commit_msg>Fix bug in ttl check where sometimes ttl is not present in the input tuple.<commit_after>#    ____            _           _   
#   |  _ \ _ __ ___ (_) ___  ___| |_ 
#   | |_) | '__/ _ \| |/ _ \/ __| __|
#   |  __/| | | (_) | |  __/ (__| |_ 
#   |_|  _|_|  \___// |\___|\___|\__|
#   | | | | __ _ _|__/ ___| |_ __ _  ___| | __
#   | |_| |/ _` | | | / __| __/ _` |/ __| |/ /
#   |  _  | (_| | |_| \__ \ || (_| | (__|   < 
#   |_| |_|\__,_|\__, |___/\__\__,_|\___|_|\_\
#                |___/                        
#
# ParseEventBolt
#

import logging
import hashlib
import random
from collections import namedtuple
from pyleus.storm import SimpleBolt

log = logging.getLogger('parse_event_bolt')

Record = namedtuple('Record', 'eventid timestamp srcip srcport dstip dstport query qtype ttl')

class ParseEventBolt(SimpleBolt):

    OUTPUT_FIELDS = Record

    def process_tuple(self, tup):
        line, = tup.values
        eventdata = line.split('\t')
        timestamp = eventdata[0].split()[1]
        broid     = eventdata[1]
        ha = hashlib.md5()
        ha.update(timestamp+broid+str(tup.id))
        eventid = ha.hexdigest()

        # ttl does not seem like its very reliable
        try:
            ttl = eventdata[20]
            
            if ttl != '-' and not ttl.isdigit():
                ttl = '-'
        except IndexError:
            ttl = '-'
            pass
    
        try: 
            record = Record(
                    eventid,        # eventid md5(timestamp+broid+tup.id)
                    timestamp,      # timestamp
                    eventdata[2],   # srcip
                    eventdata[3],   # srcport
                    eventdata[4],   # dstip
                    eventdata[5],   # dstport
                    eventdata[8],   # query
                    eventdata[12],  # qtype
                    ttl)  # ttl

            log.debug(repr(record))
            self.emit(record, anchors=[tup])
        except IndexError:
            log.debug('[*] IndexError. No emit this tuple.')
            pass

if __name__ == '__main__':
    logging.basicConfig(
        level=logging.DEBUG,
        filename='/var/log/haystack/parse_event_bolt.log',
        format="%(message)s",
        filemode='a',
    )
    ParseEventBolt().run()


{'fraction_non_alphanumeric': 0.14057871952608794, 'fraction_numerical': 0.006379585326953748, 'mean_word_length': 1.4971558589306029, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6944043', 'n_tokens_mistral': 1484, 'n_tokens_neox': 1421, 'n_words': 235}