<commit_before>import json
import os
import tempfile
import logging

from twisted.internet.protocol import ProcessProtocol
from twisted.internet import defer



class DispatcherProcessProtocol(ProcessProtocol):

    logger = logging.getLogger(__name__ + '.DispatcherProcessProtocol')

    def __init__(self, deferred, log_file, source):
        self.deferred = deferred
        self.log_file = log_file
        self.source = source

    def childDataReceived(self, childFD, data):
        if childFD == 3:
            print data
        self.log_file.write(data)
        self.log_file.flush()

    def processEnded(self, reason):
        # This discards the process exit value.
        self.log_file.close()
        self.deferred.callback(None)


class Job(object):<commit_msg>do something more structure with the output on fd 3<commit_after>import json
import os
import tempfile
import logging

from twisted.internet.protocol import ProcessProtocol
from twisted.internet import defer
from twisted.protocols.basic import LineReceiver


class OOBDataProtocol(LineReceiver):

    delimiter = '\n'

    def lineReceived(self, line):
        print 'lineReceived', repr(line)


class DispatcherProcessProtocol(ProcessProtocol):

    logger = logging.getLogger(__name__ + '.DispatcherProcessProtocol')

    def __init__(self, deferred, log_file, source):
        self.deferred = deferred
        self.log_file = log_file
        self.source = source
        self.oob_data = OOBDataProtocol()

    def childDataReceived(self, childFD, data):
        if childFD == 3:
            self.oob_data.dataReceived(data)
        self.log_file.write(data)
        self.log_file.flush()

    def processEnded(self, reason):
        # This discards the process exit value.
        self.log_file.close()
        self.deferred.callback(None)


class Job(object):
{'fraction_non_alphanumeric': 0.08147745790331341, 'fraction_numerical': 0.0016295491580662683, 'mean_word_length': 3.2054794520547945, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6694313', 'n_tokens_mistral': 531, 'n_tokens_neox': 500, 'n_words': 135}