<commit_before>from __future__ import print_function
from __future__ import division
import os
import logging
import subprocess
from . import core

APPSDIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'apps')

def dump_config(config, descs):
    """Given a relaxation configuration and an accompanying description
    map, returning a human-readable string describing it.
    """
    optimizations = [r for r in config if r[1]]
    if not optimizations:
        return u'no optimizations'

    out = []
    for ident, param in optimizations:
        out.append(u'{} @ {}'.format(descs[ident], param))
    return u', '.join(out)

def dump_results_human(exp, verbose):
    """Generate human-readable text (as a sequence of lines) for
    the results.
    """
    optimal, suboptimal, bad = core.triage_results(exp.results)

    if verbose and isinstance(exp.pout, str):
        yield 'precise output: {}'.format(exp.pout)
        yield ''

    yield '{} optimal, {} suboptimal, {} bad'.format(
        len(optimal), len(suboptimal), len(bad)
    )
    for res in optimal:
        yield dump_config(res.config, exp.descs)
        yield '{:.1%} error'.format(res.error)
        yield '{} speedup'.format(res.speedup)
        if verbose and isinstance(res.output, str):
            yield 'output: {}'.format(res.output)

    if verbose:
        yield '\nsuboptimal configs:'
        for res in suboptimal:
            yield dump_config(res.config, exp.descs)
            yield '{:.1%} error'.format(res.error)
            yield '{} speedup'.format(res.speedup)

        yield '\nbad configs:'
        for res in bad:
            yield dump_config(res.config, exp.descs)
            yield res.desc

def dump_results_json(exp):
    """Return a JSON-like representation of the results.
    """
    results, _, _ = core.triage_results(exp.results)
    out = []
    for res in results:
        out.append({
            'config': dump_config(res.config, exp.descs),
            'error': res.error,
            'speedup_mu': res.speedup.value,
            'speedup_sigma': res.speedup.error,
        })
    return out

def evaluate(client, appname, verbose=False, reps=1, as_json=False):
    appdir = os.path.join(APPSDIR, appname)
    exp = core.Evaluation(appdir, client, reps)
    
    setup_script = os.path.join(appdir, 'setup.sh')
    if os.path.exists(setup_script):
        logging.info('running setup script')
        with core.chdir(appdir):
            subprocess.check_call(['sh', 'setup.sh'])

    logging.info('starting experiments')
    with client:
        exp.run()
    logging.info('all experiments finished')
    results, descs = exp.results, exp.descs

    if as_json:
        return dump_results_json(exp)
    else:
        return '\n'.join(dump_results_human(exp, verbose))<commit_msg>silence setup.sh output<commit_after>from __future__ import print_function
from __future__ import division
import os
import logging
from . import core

APPSDIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'apps')

def dump_config(config, descs):
    """Given a relaxation configuration and an accompanying description
    map, returning a human-readable string describing it.
    """
    optimizations = [r for r in config if r[1]]
    if not optimizations:
        return u'no optimizations'

    out = []
    for ident, param in optimizations:
        out.append(u'{} @ {}'.format(descs[ident], param))
    return u', '.join(out)

def dump_results_human(exp, verbose):
    """Generate human-readable text (as a sequence of lines) for
    the results.
    """
    optimal, suboptimal, bad = core.triage_results(exp.results)

    if verbose and isinstance(exp.pout, str):
        yield 'precise output: {}'.format(exp.pout)
        yield ''

    yield '{} optimal, {} suboptimal, {} bad'.format(
        len(optimal), len(suboptimal), len(bad)
    )
    for res in optimal:
        yield dump_config(res.config, exp.descs)
        yield '{:.1%} error'.format(res.error)
        yield '{} speedup'.format(res.speedup)
        if verbose and isinstance(res.output, str):
            yield 'output: {}'.format(res.output)

    if verbose:
        yield '\nsuboptimal configs:'
        for res in suboptimal:
            yield dump_config(res.config, exp.descs)
            yield '{:.1%} error'.format(res.error)
            yield '{} speedup'.format(res.speedup)

        yield '\nbad configs:'
        for res in bad:
            yield dump_config(res.config, exp.descs)
            yield res.desc

def dump_results_json(exp):
    """Return a JSON-like representation of the results.
    """
    results, _, _ = core.triage_results(exp.results)
    out = []
    for res in results:
        out.append({
            'config': dump_config(res.config, exp.descs),
            'error': res.error,
            'speedup_mu': res.speedup.value,
            'speedup_sigma': res.speedup.error,
        })
    return out

def evaluate(client, appname, verbose=False, reps=1, as_json=False):
    appdir = os.path.join(APPSDIR, appname)
    exp = core.Evaluation(appdir, client, reps)
    
    setup_script = os.path.join(appdir, 'setup.sh')
    if os.path.exists(setup_script):
        logging.info('running setup script')
        with core.chdir(appdir):
            core.run_cmd(['sh', 'setup.sh'])

    logging.info('starting experiments')
    with client:
        exp.run()
    logging.info('all experiments finished')
    results, descs = exp.results, exp.descs

    if as_json:
        return dump_results_json(exp)
    else:
        return '\n'.join(dump_results_human(exp, verbose))
{'fraction_non_alphanumeric': 0.12408498482413854, 'fraction_numerical': 0.0014283163720764148, 'mean_word_length': 2.6758530183727034, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5247152', 'n_tokens_mistral': 1739, 'n_tokens_neox': 1686, 'n_words': 441}