<commit_before>import asyncio
import io
import logging
import os
import zipfile

from .. import asutil
from .. import exception

logger = logging.getLogger(__name__)

expect_ok = asutil.expect_ok_closure(exception.AdjustCommandError)

stdout_options = asutil.process_stdout_options
stderr_options = asutil.process_stderr_options

# This provider MAY be "extended" by other providers.
def get_process_provider(execution_name, cmd, get_result_data=None, log_context_option=None, send_log=False):
    @asyncio.coroutine
    def get_result_data_default(work_dir):
        return {}

    get_result_data = get_result_data if get_result_data is not None else get_result_data_default

    @asyncio.coroutine
    def adjust(repo_dir, extra_adjust_parameters, adjust_result):
        nonlocal execution_name
        logger.info('Executing "{execution_name}" using (sub)process adjust provider as "{cmd}".'.format(**locals()))
        log_executable_info(cmd)
        filled_cmd = [p.format(repo_dir=repo_dir) if p.startswith("{repo_dir}") else p for p in cmd]

        # log_context = getattr(asyncio.Task.current_task(), "log_context", None)
        # if log_context_option is None:
        #     if log_context is None:
        #         env = None
        #     else:
        #         env = {
        #             "LOG_CONTEXT": asyncio.Task.current_task().log_context,
        #         }
        # else:
        #     env = None
        #     if log_context is not None:
        #         filled_cmd.append(log_context_option)
        #         filled_cmd.append(log_context)
        env = None
        stdout = None
        try:
            stdout = yield from expect_ok(
                cmd=filled_cmd,
                desc="Adjust subprocess failed.",
                cwd=repo_dir,
                env=env,
                stdout=stdout_options["lines"] if send_log else stdout_options["ignore"],
                stderr=stderr_options["stdout"] if send_log else stderr_options["log_on_error"],
            )
        except exception.CommandError as e:
            logger.error('Adjust subprocess failed, exited code "{e.exit_code}"'.format(**locals()))
            raise

        logger.info("Adjust subprocess exited OK!")

        adjust_result_data = {}
        # TODO adjust_result_data["stdout"] = stdout
        logger.info("PME output:")
        logger.info(stdout)
        adjust_result_data["adjustType"] = execution_name
        adjust_result_data["resultData"] = yield from get_result_data(repo_dir)
        return adjust_result_data

    return adjust


#
# Utility
#

def log_executable_info(cmd):
    first = True
    for c in cmd:
        if c.endswith(".jar"):
            manifest = {}
            try:
                with zipfile.ZipFile(c) as z:
                    with z.open("META-INF/MANIFEST.MF") as bf:<commit_msg>Make log format more readable<commit_after>import asyncio
import io
import logging
import os
import zipfile

from .. import asutil
from .. import exception

logger = logging.getLogger(__name__)

expect_ok = asutil.expect_ok_closure(exception.AdjustCommandError)

stdout_options = asutil.process_stdout_options
stderr_options = asutil.process_stderr_options

# This provider MAY be "extended" by other providers.
def get_process_provider(execution_name, cmd, get_result_data=None, log_context_option=None, send_log=False):
    @asyncio.coroutine
    def get_result_data_default(work_dir):
        return {}

    get_result_data = get_result_data if get_result_data is not None else get_result_data_default

    @asyncio.coroutine
    def adjust(repo_dir, extra_adjust_parameters, adjust_result):
        nonlocal execution_name
        logger.info('Executing "{execution_name}" using (sub)process adjust provider as "{cmd}".'.format(**locals()))
        log_executable_info(cmd)
        filled_cmd = [p.format(repo_dir=repo_dir) if p.startswith("{repo_dir}") else p for p in cmd]

        # log_context = getattr(asyncio.Task.current_task(), "log_context", None)
        # if log_context_option is None:
        #     if log_context is None:
        #         env = None
        #     else:
        #         env = {
        #             "LOG_CONTEXT": asyncio.Task.current_task().log_context,
        #         }
        # else:
        #     env = None
        #     if log_context is not None:
        #         filled_cmd.append(log_context_option)
        #         filled_cmd.append(log_context)
        env = None
        stdout = None
        try:
            stdout = yield from expect_ok(
                cmd=filled_cmd,
                desc="Adjust subprocess failed.",
                cwd=repo_dir,
                env=env,
                stdout=stdout_options["lines"] if send_log else stdout_options["ignore"],
                stderr=stderr_options["stdout"] if send_log else stderr_options["log_on_error"],
            )
        except exception.CommandError as e:
            logger.error('Adjust subprocess failed, exited code "{e.exit_code}"'.format(**locals()))
            raise

        logger.info("Adjust subprocess exited OK!")

        adjust_result_data = {}
        # TODO adjust_result_data["stdout"] = stdout
        if stdout:
            logger.info("Output:")
            logger.info('\n'.join(stdout))
        else:
            logger.info("No Output")

        adjust_result_data["adjustType"] = execution_name
        adjust_result_data["resultData"] = yield from get_result_data(repo_dir)
        return adjust_result_data

    return adjust


#
# Utility
#

def log_executable_info(cmd):
    first = True
    for c in cmd:
        if c.endswith(".jar"):
            manifest = {}
            try:
                with zipfile.ZipFile(c) as z:
                    with z.open("META-INF/MANIFEST.MF") as bf:
{'fraction_non_alphanumeric': 0.09284232365145229, 'fraction_numerical': 0.0, 'mean_word_length': 2.3771161704611794, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2249647', 'n_tokens_mistral': 1702, 'n_tokens_neox': 1646, 'n_words': 373}