<commit_before>#/usr/bin/env python2

import argparse
import datetime
import glob
import json
import logging
import os
import shutil
import subprocess
import sys
import time

from contextlib import contextmanager

BASE_DIR_NAME = "rolling-perf"

script_args = None
datastore = None

cli_repo = None
xunitperf_repo = None
empty_dir = None
log_file = None
store_path = None
results_dir = None

launch_time = time.time()

@contextmanager
def PushDir(path):
    prev = os.getcwd()
    try:
        logging.getLogger('shell').info('pushd "{}"'.format(path))
        os.chdir(path)
        yield
    finally:
        logging.getLogger('shell').info('popd')
        os.chdir(prev)

def RunCommand(cmdline, valid_exit_codes=[0], get_output=False, silent=False, suffix=None):
    should_pipe = (not silent) or get_output

    quoted_cmdline = subprocess.list2cmdline(cmdline)
    quoted_cmdline += ' > {}'.format(os.devnull) if not should_pipe else ''
    logging.getLogger('shell').info(quoted_cmdline)
    exe_name = os.path.basename(cmdline[0]).replace('.', '_')

    exe_log_file = log_file
    if suffix != None:
        exe_log_file = exe_log_file.replace('.log', '.{}.log'.format(suffix))

    exe_logger = logging.getLogger(exe_name)
    if len(exe_logger.handlers) == 0:
        ch = logging.StreamHandler()
        ch.setLevel(logging.INFO if script_args.verbose else logging.WARNING)
        ch.setFormatter(logging.Formatter(fmt='%(levelname)s:%(name)s:%(message)s'))
        exe_logger.addHandler(ch)
        fh = logging.FileHandler(exe_log_file)
        fh.setLevel(logging.INFO)
        if suffix == None:
            fh.setFormatter(logging.Formatter(fmt='%(levelname)s:%(name)s:%(message)s'))
        exe_logger.addHandler(fh)

    lines = []
    with open(os.devnull) as devnull:
        proc = subprocess.Popen(
            cmdline,
            stdout=subprocess.PIPE if should_pipe else devnull,
            stderr=subprocess.STDOUT,
            universal_newlines=True
        )

        if proc.stdout != None:
            for line in iter(proc.stdout.readline, ''):
                line = line.rstrip()
                if get_output:
                    lines.append(line)
                exe_logger.info(line)
            proc.stdout.close()<commit_msg>Create separate handlers for each invoked process<commit_after>#/usr/bin/env python2

import argparse
import datetime
import glob
import json
import logging
import os
import shutil
import subprocess
import sys
import time

from contextlib import contextmanager

BASE_DIR_NAME = "rolling-perf"

script_args = None
datastore = None

cli_repo = None
xunitperf_repo = None
empty_dir = None
log_file = None
store_path = None
results_dir = None

launch_time = time.time()

@contextmanager
def PushDir(path):
    prev = os.getcwd()
    try:
        logging.getLogger('shell').info('pushd "{}"'.format(path))
        os.chdir(path)
        yield
    finally:
        logging.getLogger('shell').info('popd')
        os.chdir(prev)

def RunCommand(cmdline, valid_exit_codes=[0], get_output=False, silent=False, suffix=None):
    should_pipe = (not silent) or get_output

    quoted_cmdline = subprocess.list2cmdline(cmdline)
    quoted_cmdline += ' > {}'.format(os.devnull) if not should_pipe else ''
    logging.getLogger('shell').info(quoted_cmdline)
    exe_name = os.path.basename(cmdline[0]).replace('.', '_')

    exe_log_file = log_file
    if suffix != None:
        exe_log_file = exe_log_file.replace('.log', '.{}.log'.format(suffix))

    exe_logger = logging.getLogger(exe_name)
    exe_logger.handlers = []
    fh = logging.FileHandler(exe_log_file)
    fh.setLevel(logging.INFO)
    if suffix == None:
        fh.setFormatter(logging.Formatter(fmt='%(levelname)s:%(name)s:%(message)s'))
    exe_logger.addHandler(fh)
    LogStartMessage(exe_name)
    ch = logging.StreamHandler()
    ch.setLevel(logging.INFO if script_args.verbose else logging.WARNING)
    ch.setFormatter(logging.Formatter(fmt='%(levelname)s:%(name)s:%(message)s'))
    exe_logger.addHandler(ch)

    lines = []
    with open(os.devnull) as devnull:
        proc = subprocess.Popen(
            cmdline,
            stdout=subprocess.PIPE if should_pipe else devnull,
            stderr=subprocess.STDOUT,
            universal_newlines=True
        )

        if proc.stdout != None:
            for line in iter(proc.stdout.readline, ''):
                line = line.rstrip()
                if get_output:
                    lines.append(line)
                exe_logger.info(line)
            proc.stdout.close()
{'fraction_non_alphanumeric': 0.11713094976968634, 'fraction_numerical': 0.0019741171309497698, 'mean_word_length': 2.982532751091703, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6787684', 'n_tokens_mistral': 1489, 'n_tokens_neox': 1427, 'n_words': 284}