<commit_before>from csirtg_smrt.utils.ztail import tail
from json import loads as json_loads
import logging
import os
from csirtg_indicator import Indicator
from pprint import pprint
from argparse import ArgumentParser
from argparse import RawDescriptionHelpFormatter
import textwrap
from csirtg_smrt.utils import setup_logging, get_argument_parser
from csirtg_indicator.format import FORMATS
import re
# logging configuration
LOG_FORMAT = '%(asctime)s - %(levelname)s - %(name)s[%(lineno)s] - %(message)s'
logger = logging.getLogger('')

PROVIDER = os.environ.get('CSIRTG_SMRT_PROVIDER')

# https://github.com/JustinAzoff/bro-pdns/blob/master/bro_pdns.py#L27
CORE_FIELDS = set('src ts msg dst'.split())


class BroTailer(object):

    def __init__(self, file, json=False, **kwargs):
        self.headers = {}

        with open(file) as _file:
            line = ''
            it = iter(_file)

            while not line.startswith("#types"):
                line = next(it).rstrip()
                k, v = line[1:].split(None, 1)
                self.headers[k] = v

            self.sep = self.headers['separator'].encode('utf-8').decode("unicode_escape")

            for k, v in self.headers.items():
                if self.sep in v:
                    self.headers[k] = v.split(self.sep)

            self.headers['separator'] = self.sep
            self.fields = self.headers['fields']
            self.types = self.headers['types']
            self.set_sep = self.headers['set_separator']
            self.vectors = [field for field, type in zip(self.fields, self.types) if type.startswith("vector[")]

    def parse_line(self, line):
        if line.startswith("#close"):
            return

        if not len(line):
            return

        parts = line.split(self.sep)

        record = dict(zip(self.fields, parts))

        for k in record:
            if record[k] == '-':
                record[k] = None<commit_msg>problem: bro log parser isn't stripping newline (#249)<commit_after>from csirtg_smrt.utils.ztail import tail
from json import loads as json_loads
import logging
import os
from csirtg_indicator import Indicator
from pprint import pprint
from argparse import ArgumentParser
from argparse import RawDescriptionHelpFormatter
import textwrap
from csirtg_smrt.utils import setup_logging, get_argument_parser
from csirtg_indicator.format import FORMATS
import re
# logging configuration
LOG_FORMAT = '%(asctime)s - %(levelname)s - %(name)s[%(lineno)s] - %(message)s'
logger = logging.getLogger('')

PROVIDER = os.environ.get('CSIRTG_SMRT_PROVIDER')

# https://github.com/JustinAzoff/bro-pdns/blob/master/bro_pdns.py#L27
CORE_FIELDS = set('src ts msg dst'.split())


class BroTailer(object):

    def __init__(self, file, json=False, **kwargs):
        self.headers = {}

        with open(file) as _file:
            line = ''
            it = iter(_file)

            while not line.startswith("#types"):
                line = next(it).rstrip()
                k, v = line[1:].split(None, 1)
                self.headers[k] = v

            self.sep = self.headers['separator'].encode('utf-8').decode("unicode_escape")

            for k, v in self.headers.items():
                if self.sep in v:
                    self.headers[k] = v.split(self.sep)

            self.headers['separator'] = self.sep
            self.fields = self.headers['fields']
            self.types = self.headers['types']
            self.set_sep = self.headers['set_separator']
            self.vectors = [field for field, type in zip(self.fields, self.types) if type.startswith("vector[")]

    def parse_line(self, line):
        if line.startswith("#close"):
            return

        if not len(line):
            return

        parts = line.rstrip().split(self.sep)

        record = dict(zip(self.fields, parts))

        for k in record:
            if record[k] == '-':
                record[k] = None
{'fraction_non_alphanumeric': 0.12067209775967414, 'fraction_numerical': 0.003309572301425662, 'mean_word_length': 2.4986642920747997, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4300222', 'n_tokens_mistral': 1232, 'n_tokens_neox': 1178, 'n_words': 286}