<commit_before>import re
import snmpy

class log_processor(snmpy.plugin):
    def create(self):
        for k, v in sorted(self.conf['objects'].items()):
            extra = {
                'count':  re.compile(v['count']),
                'reset':  re.compile(v['reset']) if 'reset' in v else None,
                'start':  int(v['start']) if 'start' in v else 0,
                'rotate': bool(v['rotate']) if 'rotate' in v else False
            }

            self.data['1.%s' % k] = 'string', v['label']
            self.data['2.%s' % k] = 'integer', extra['start'], extra

        self.tail()

    @snmpy.plugin.task
    def tail(self):
        for line in snmpy.plugin.tail(self.conf['file_name'], True):
            if line is True:
                for item in self.data['2.0':]:
                    if self.data[item:'rotate'] and line is True:
                        self.data[item] = self.data[item:'start']
                continue

            for item in self.data['2.0':]:
                count = self.data[item:'count'].search(line)
                if count:
                    self.data[item] = self.data[item:True] + (int(count.group(1)) if len(count.groups()) > 0 else 1)
                    break
                if self.data[item:'reset'] is not None and self.data[item:'reset'].search(line):
                    self.data[item] = self.data[item:'start']
                    break<commit_msg>Support file globbing for log processor since names could be dynamic (based on hostname, etc.).<commit_after>import glob
import re
import snmpy

class log_processor(snmpy.plugin):
    def create(self):
        for k, v in sorted(self.conf['objects'].items()):
            extra = {
                'count':  re.compile(v['count']),
                'reset':  re.compile(v['reset']) if 'reset' in v else None,
                'start':  int(v['start']) if 'start' in v else 0,
                'rotate': bool(v['rotate']) if 'rotate' in v else False
            }

            self.data['1.%s' % k] = 'string', v['label']
            self.data['2.%s' % k] = 'integer', extra['start'], extra

        self.tail()

    @snmpy.plugin.task
    def tail(self):
        for line in snmpy.plugin.tail(glob.glob(self.conf['file_name'])[0], True):
            if line is True:
                for item in self.data['2.0':]:
                    if self.data[item:'rotate'] and line is True:
                        self.data[item] = self.data[item:'start']
                continue

            for item in self.data['2.0':]:
                count = self.data[item:'count'].search(line)
                if count:
                    self.data[item] = self.data[item:True] + (int(count.group(1)) if len(count.groups()) > 0 else 1)
                    break
                if self.data[item:'reset'] is not None and self.data[item:'reset'].search(line):
                    self.data[item] = self.data[item:'start']
                    break
{'fraction_non_alphanumeric': 0.1514945652173913, 'fraction_numerical': 0.007133152173913043, 'mean_word_length': 1.72181146025878, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '617668', 'n_tokens_mistral': 896, 'n_tokens_neox': 849, 'n_words': 219}