<commit_before>import itertools


class SourceProcessor(object):
    def __init__(self, sources):
        self.sources = sources

    def process(self):
        for source, parser in self.sources.items():
            entries = source.read()
            for entry in parser.parse(entries):
                yield entry


class TriggerProcessor(object):
    def __init__(self, triggers):
        self.triggers = triggers

    def process(self, entries):
        for entry in entries:
            if any(( trigger.is_activated(entry) for trigger in self.triggers )):
                yield entry


class HandlerProcessor(dict):
    def __init__(self, handlers):
        self.handlers = handlers

    def process(self, entries):
        cached_entries = itertools.tee(entries, len(self.handlers))
        joins = zip(
                self.handlers.keys(), 
                self.handlers.values(), 
                cached_entries)

        for handler, formatter, entries in joins:
            entries = formatter.format(entries)
            yield handler.write(entries)
                


class Processor(object):
    def __init__(self, source_processor, trigger_processor, handler_processor):
        self.source_processor = source_processor
        self.trigger_processor = trigger_processor
        self.handler_processor = handler_processor

    def process(self):
        entries = self.source_processor.process()
        entries = self.trigger_processor.process(entries)
        results = self.handler_processor.process(entries)
        return results

    def main(self):
        results = list(self.process())
        return results<commit_msg>define sources, triggers and handlers in "django-style"<commit_after>import itertools


class SourceProcessor(object):
    def __init__(self, sources):
        self.sources = sources

    def process(self):
        for source, parser in self.sources.items():
            entries = source.read()
            for entry in parser.parse(entries):
                yield entry


class TriggerProcessor(object):
    def __init__(self, triggers):
        self.triggers = triggers

    def process(self, entries):
        for entry in entries:
            if any(( trigger.is_activated(entry) for trigger in self.triggers )):
                yield entry


class HandlerProcessor(dict):
    def __init__(self, handlers):
        self.handlers = handlers

    def process(self, entries):
        cached_entries = itertools.tee(entries, len(self.handlers))
        joins = zip(
                self.handlers.keys(), 
                self.handlers.values(), 
                cached_entries)

        for handler, formatter, entries in joins:
            entries = formatter.format(entries)
            yield handler.write(entries)
                


class Processor(object):
    @property
    def source_processor(self):
        try:
            return SourceProcessor(self.sources)
        except AttributeError:
            raise AttributeError('either define sources or source_processor')

    @property
    def trigger_processor(self):
        try:
            return TriggerProcessor(self.triggers)
        except AttributeError:
            raise AttributeError('either define triggers or trigger_processor')

    @property
    def handler_processor(self):
        try:
            return HandlerProcessor(self.handlers)
        except AttributeError:
            raise AttributeError('either define handlers or handler_processor')

    def process(self):
        entries = self.source_processor.process()
        entries = self.trigger_processor.process(entries)
        results = self.handler_processor.process(entries)
        return results

    def main(self):
        results = list(self.process())
        return results
{'fraction_non_alphanumeric': 0.08151885289431758, 'fraction_numerical': 0.0, 'mean_word_length': 2.321869488536155, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '205391', 'n_tokens_mistral': 963, 'n_tokens_neox': 908, 'n_words': 244}