<commit_before>from __future__ import print_function
import datetime
from functools import wraps
import json
import os
import os.path
import shutil
from subprocess import check_output, STDOUT
from tarfile import TarFile
import tempfile
import traceback
import sys


class Pipeline(object):
    def __init__(self):
        self.debug = False
        self.results = []

    def run(self, name, pipeline, base_dir, *input_files, **kwargs):
        self.results = []
        self.debug = kwargs.get('debug', False)
        self.name = name
        self.input_files = input_files
        self.base_dir = base_dir
        self.launched_dir = os.getcwd()
        self._copy_input_files_to_working_dir()
        self.start_date = datetime.datetime.now()
        os.chdir(self.working_dir)
        pipeline_exception = None
        try:
            pipeline(*input_files)
        except Exception as e:
            pipeline_exception = e
            traceback.print_exc()
        finally:
            os.chdir(self.launched_dir)
            self.save(pipeline_exception)

    def _copy_input_files_to_working_dir(self):
        self.working_dir = tempfile.mkdtemp(prefix='plumbium_{0}_'.format(self.name))
        for i in self.input_files:
            dest_dir = os.path.join(self.working_dir, os.path.dirname(i.filename))
            source = os.path.join(self.base_dir, i.filename)
            if not os.path.exists(dest_dir):
                os.makedirs(dest_dir)
            shutil.copy(source, dest_dir)

    def _store_printed_output(self):
        with open('printed_output.txt', 'w') as printed_output_record:
            for r in self.results:
                printed_output_record.write(r.output)

    def record(self, result):
        self.results.append(result)

    def save(self, exception=None):
        results = {
            'name': self.name,
            'input_files': [repr(f) for f in self.input_files],
            'dir': self.launched_dir,
            'start_date': self.start_date.strftime('%Y%m%d %H:%M'),
        }
        if exception is not None:
            results['pipeline_exception'] = repr(exception)
        results['processes'] = [r.as_dict() for r in self.results]
        basename = '{0}-{1}'.format(
            self.name,
            self.start_date.strftime('%Y%m%d_%H%M')
        )
        with open(basename + '.json', 'w') as f:
            json.dump(results, f, indent=4, separators=(',', ': '))
        archive = TarFile(basename + '.tar', 'w')
        archive.add(self.working_dir, arcname='')
        archive.close()


pipeline = Pipeline()


class OutputRecorder(object):
    def reset(self):
        self.output = ''


_output_recorder = OutputRecorder()


def call(cmd, cwd=None):
    _output_recorder.output += check_output(cmd, stderr=STDOUT, cwd=cwd)
<commit_msg>If an exception occurs in call() immediately print any captured output<commit_after>from __future__ import print_function
import datetime
from functools import wraps
import json
import os
import os.path
import shutil
from subprocess import check_output, STDOUT
from tarfile import TarFile
import tempfile
import traceback
import sys


class Pipeline(object):
    def __init__(self):
        self.debug = False
        self.results = []

    def run(self, name, pipeline, base_dir, *input_files, **kwargs):
        self.results = []
        self.debug = kwargs.get('debug', False)
        self.name = name
        self.input_files = input_files
        self.base_dir = base_dir
        self.launched_dir = os.getcwd()
        self._copy_input_files_to_working_dir()
        self.start_date = datetime.datetime.now()
        os.chdir(self.working_dir)
        pipeline_exception = None
        try:
            pipeline(*input_files)
        except Exception as e:
            pipeline_exception = e
            traceback.print_exc()
        finally:
            os.chdir(self.launched_dir)
            self.save(pipeline_exception)

    def _copy_input_files_to_working_dir(self):
        self.working_dir = tempfile.mkdtemp(prefix='plumbium_{0}_'.format(self.name))
        for i in self.input_files:
            dest_dir = os.path.join(self.working_dir, os.path.dirname(i.filename))
            source = os.path.join(self.base_dir, i.filename)
            if not os.path.exists(dest_dir):
                os.makedirs(dest_dir)
            shutil.copy(source, dest_dir)

    def _store_printed_output(self):
        with open('printed_output.txt', 'w') as printed_output_record:
            for r in self.results:
                printed_output_record.write(r.output)

    def record(self, result):
        self.results.append(result)

    def save(self, exception=None):
        results = {
            'name': self.name,
            'input_files': [repr(f) for f in self.input_files],
            'dir': self.launched_dir,
            'start_date': self.start_date.strftime('%Y%m%d %H:%M'),
        }
        if exception is not None:
            results['pipeline_exception'] = repr(exception)
        results['processes'] = [r.as_dict() for r in self.results]
        basename = '{0}-{1}'.format(
            self.name,
            self.start_date.strftime('%Y%m%d_%H%M')
        )
        with open(basename + '.json', 'w') as f:
            json.dump(results, f, indent=4, separators=(',', ': '))
        archive = TarFile(basename + '.tar', 'w')
        archive.add(self.working_dir, arcname='')
        archive.close()


pipeline = Pipeline()


class OutputRecorder(object):
    def reset(self):
        self.output = ''


_output_recorder = OutputRecorder()


def call(cmd, cwd=None):
    try:
        _output_recorder.output += check_output(cmd, stderr=STDOUT, cwd=cwd)
    except:
        print(_output_recorder.output)
        raise

{'fraction_non_alphanumeric': 0.11346655082536924, 'fraction_numerical': 0.0013900955690703736, 'mean_word_length': 2.4425837320574164, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2455572', 'n_tokens_mistral': 1796, 'n_tokens_neox': 1773, 'n_words': 374}