<commit_before>
from log import logging
LOG = logging.getLogger(__name__)


class Job(object):

    def __init__(self, job, event, config, logger):
        self.job = job
        self.event = event
        self.config = config
        self.logger = logger

        driver = __import__(job["driver"]).Driver
        self.driver = driver(job["name"], *job["driver-args"])
        self.driver.build(logger.stdout(job, "build.txt.gz"))

    def run_script(self, script, event, stdout):
        interpreter = script.get("interpreter")
        if interpreter:
            cmd = interpreter
            for arg in script["args"]:
                value = dict(event)
                for key in arg.split('.'):
                    value = value[key]
                cmd += " %s" % value
            self.driver.run(cmd, stdout, stdin=open(script["path"], "rb"))
        else:
            for cmd in script["commands"]:
                self.driver.run(cmd, stdout)

    def run(self):

        stdout = self.logger.stdout(self.job)
        for build_script in self.job.get("build-scripts", []):
            self.run_script(self.config.scripts[build_script], self.event, stdout)
        for test_cmd in self.job.get("test-commands", []):
            self.driver.run(test_cmd, stdout)
        self.driver.cleanup()


class Project(object):

    def __init__(self, project, event, config):
        self.project = project
        self.event = event
        self.config = config
        logger = self.config.logs["driver"]
        self.logger = __import__(logger).Driver(self.config.logs)
        self.logger.mkdir(self.event)

    def run_jobs(self):
        for job_config in self.project["jobs"]:
            job = Job(job_config, self.event, self.config, self.logger)
            job.run()<commit_msg>Add more details in publshed data<commit_after>
from log import logging
LOG = logging.getLogger(__name__)


class Job(object):

    def __init__(self, job, event, config, logger):
        self.job = job
        self.event = event
        self.config = config
        self.logger = logger
        self.errors = []
        self.name = job["name"]

        print config.drivers
        print job["driver"]
        driver_conf = config.drivers[job["driver"]]
        driver = __import__(driver_conf["driver"]).Driver
        self.driver = driver(driver_conf, job["name"], *job["driver-args"])
        self.driver.build(logger.stdout(job, "build.txt.gz"))

    def run_script(self, script, event, stdout):
        interpreter = script.get("interpreter")
        if interpreter:
            cmd = interpreter
            for arg in script["args"]:
                value = dict(event)
                for key in arg.split('.'):
                    value = value[key]
                cmd += " %s" % value
            return self.driver.run(cmd, stdout, stdin=open(script["path"], "rb"))
        else:
            for cmd in script["commands"]:
                return self.driver.run(cmd, stdout)

    def run(self):

        stdout = self.logger.stdout(self.job)
        for build_script in self.job.get("build-scripts", []):
            self.build_error = self.run_script(
                    self.config.scripts[build_script], self.event, stdout)
        for test_cmd in self.job.get("test-commands", []):
            self.errors.append(self.driver.run(test_cmd, stdout))
        self.driver.cleanup()
        self.error = any(self.errors)


class Project(object):

    def __init__(self, project, event, config):
        self.project = project
        self.event = event
        self.config = config
        self.jobs = []
        logger = self.config.logs["driver"]
        self.logger = __import__(logger).Driver(self.config.logs, self.event)

    def run_jobs(self):
        for job_config in self.project["jobs"]:
            job = Job(job_config, self.event, self.config, self.logger)
            LOG.debug("Running job: %s (Project: %s)" % (job.name, self.project["name"]))
            job.run()
            self.jobs.append(job)
        self.logger.publish_summary(self.jobs)
{'fraction_non_alphanumeric': 0.13031718711581017, 'fraction_numerical': 0.0, 'mean_word_length': 2.1364687740940633, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3245114', 'n_tokens_mistral': 1201, 'n_tokens_neox': 1159, 'n_words': 260}