<commit_before>from __future__ import absolute_import

import logging
import traceback

log = logging.getLogger(__name__)

class LoggingMiddleware(object):
    def process_job(self, job):
        log.info("Running job %s", job)

        self.fluent_log(job, state='processing')

    def process_result(self, job, result, duration):
        log.info("Finished job => %r (Time taken: %.2fs)",
            result,
            duration,
        )

        self.fluent_log(job, state='finished', duration=duration)

    def process_exception(self, job, duration, *exc_info):
        exception = ''.join(traceback.format_exception(*exc_info))

        log.error("Exception when processing job (duration: %.2fs): %s",
            duration,
            exception,
        )

        self.fluent_log(
            job,
            state='exception',
            duration=duration,
            exception=exception,
        )

    def fluent_log(self, job, **kwargs):
        data = job.as_dict()

        data.update(kwargs)

        data['fluent_log'] = True

        log.info(data)<commit_msg>Dump data in JSON format<commit_after>from __future__ import absolute_import

import json
import logging
import traceback

log = logging.getLogger(__name__)

class LoggingMiddleware(object):
    def process_job(self, job):
        log.info("Running job %s", job)

        self.fluent_log(job, state='processing')

    def process_result(self, job, result, duration):
        log.info("Finished job => %r (Time taken: %.2fs)",
            result,
            duration,
        )

        self.fluent_log(job, state='finished', duration=duration)

    def process_exception(self, job, duration, *exc_info):
        exception = ''.join(traceback.format_exception(*exc_info))

        log.error("Exception when processing job (duration: %.2fs): %s",
            duration,
            exception,
        )

        self.fluent_log(
            job,
            state='exception',
            duration=duration,
            exception=exception,
        )

    def fluent_log(self, job, **kwargs):
        data = job.as_dict()

        data.update(kwargs)

        data['fluent_log'] = True

        log.info(json.dumps(data))
{'fraction_non_alphanumeric': 0.11484339536995007, 'fraction_numerical': 0.0018157058556513845, 'mean_word_length': 2.3393939393939394, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '125680', 'n_tokens_mistral': 638, 'n_tokens_neox': 618, 'n_words': 141}