<commit_before>from __future__ import print_function
import argparse
import json
import sys

import perf


def create_parser():
    parser = argparse.ArgumentParser(description='Display benchmark results.',
                                     prog='-m perf')
    parser.add_argument('-v', '--verbose', action='count', default=0,
                        help='an integer for the accumulator')
    parser.add_argument('-M', '--no-metadata', dest='metadata', action="store_false",
                        default=True,
                        help="Don't show metadata.")
    subparsers = parser.add_subparsers(dest='action')

    show = subparsers.add_parser('show')
    show.add_argument('filename', type=str,
                      help='Result JSON file')

    compare = subparsers.add_parser('compare')
    compare.add_argument('ref_filename', type=str,
                         help='Reference JSON file')
    compare.add_argument('changed_filename', type=str,
                         help='Changed JSON file')

    return parser


def parse_results(filename, default_name=None):
    if filename != '-':
        fp = open(filename)
    else:
        fp = sys.stdin
    with fp:
        result = perf.Results.json_load_from(fp)

    if not result.name and filename != "-":
        name = filename
        if name.lower().endswith('.json'):
            name = name[:-5]
        if name:
            result.name = name
    if not result.name and default_name:
        result.name = default_name

    return result


def display_result(args, results):
    if args.metadata:
        perf._display_metadata(result.get_metadata())

    if args.verbose:
        nrun = len(result.runs)
        for index, run in enumerate(result.runs, 1):
            text = perf._very_verbose_run(run)
            print("Run %s/%s: %s" % (index, nrun, text))
        print()

    print("Average: %s" % result.format(verbose=args.verbose))


def compare_results(args, ref_result, changed_result):
    print("Reference: %s" % ref_result.name)
    print("Changed: %s" % changed_result.name)
    print()

    if args.metadata:
        ref_metadata = ref_result.get_metadata()
        changed_metadata = changed_result.get_metadata()

        common_metadata = perf._common_metadata([ref_metadata, changed_metadata])
        perf._display_metadata(common_metadata,
                               header='Common metadata:')

        for key in common_metadata:
            ref_metadata.pop(key, None)
            changed_metadata.pop(key, None)

        perf._display_metadata(ref_metadata,
                               header='%s metadata:' % ref_result.name)
        perf._display_metadata(changed_metadata,
                               header='%s metadata:' % changed_result.name)

    # Compute means
    ref_samples = ref_result.get_samples()
    changed_samples = changed_result.get_samples()
    ref_avg = perf.mean(ref_samples)
    changed_avg = perf.mean(changed_samples)
    text = ("Average: [%s] %s -> [%s] %s"
            % (ref_result.name,
               ref_result.format(verbose=args.verbose),
               changed_result.name,
               changed_result.format(verbose=args.verbose)))

    # avoid division by zero
    if ref_avg and changed_avg:
        if changed_avg < ref_avg:
            text = "%s: %.1fx faster" % (text, ref_avg /  changed_avg)
        else:
            text= "%s: %.1fx slower" % (text, changed_avg / ref_avg)
    print(text)

    # significant?
    significant, t_score = perf.is_significant(ref_samples, changed_samples)
    if significant:
        print("Significant (t=%.2f)" % t_score)
    else:
        print("Not significant!")


parser = create_parser()
args = parser.parse_args()
action = args.action
if action == 'show':
    result = parse_results(args.filename)
    display_result(args, result)
elif action == 'compare':
    ref_result = parse_results(args.ref_filename, '<ref>')
    changed_result = parse_results(args.changed_filename, '<changed>')
    compare_results(args, ref_result, changed_result)
else:
    parser.print_usage()
    sys.exit(1)<commit_msg>perf compare now accepts multiple changed filenames<commit_after>from __future__ import print_function
import argparse
import json
import sys

import perf


def create_parser():
    parser = argparse.ArgumentParser(description='Display benchmark results.',
                                     prog='-m perf')
    parser.add_argument('-v', '--verbose', action='count', default=0,
                        help='an integer for the accumulator')
    parser.add_argument('-M', '--no-metadata', dest='metadata', action="store_false",
                        default=True,
                        help="Don't show metadata.")
    subparsers = parser.add_subparsers(dest='action')

    show = subparsers.add_parser('show')
    show.add_argument('filename', type=str,
                      help='Result JSON file')

    compare = subparsers.add_parser('compare')
    compare.add_argument('ref_filename', type=str,
                         help='Reference JSON file')
    compare.add_argument('changed_filenames', metavar="changed_filename",
                         type=str, nargs='+',
                         help='Changed JSON file')

    return parser


def parse_results(filename, default_name=None):
    if filename != '-':
        fp = open(filename)
    else:
        fp = sys.stdin
    with fp:
        result = perf.Results.json_load_from(fp)

    if not result.name and filename != "-":
        name = filename
        if name.lower().endswith('.json'):
            name = name[:-5]
        if name:
            result.name = name
    if not result.name and default_name:
        result.name = default_name

    return result


def display_result(args, results):
    if args.metadata:
        perf._display_metadata(result.get_metadata())

    if args.verbose:
        nrun = len(result.runs)
        for index, run in enumerate(result.runs, 1):
            text = perf._very_verbose_run(run)
            print("Run %s/%s: %s" % (index, nrun, text))
        print()

    print("Average: %s" % result.format(verbose=args.verbose))


def compare_results(args, results):
    ref_result = results[0]

    print("Reference: %s" % ref_result.name)
    for index, result in enumerate(results[1:], 1):
        if index > 1:
            prefix = 'Changed #%s' % index
        else:
            prefix = 'Changed'
        print("%s: %s" % (prefix, result.name))
    print()

    if args.metadata:
        metadatas = [result.get_metadata() for result in results]

        common_metadata = perf._common_metadata(metadatas)
        perf._display_metadata(common_metadata,
                               header='Common metadata:')

        for key in common_metadata:
            for metadata in metadatas:
                metadata.pop(key, None)

        for result, metadata in zip(results, metadatas):
            perf._display_metadata(metadata,
                                   header='%s metadata:' % result.name)

    # Compute means
    ref_samples = ref_result.get_samples()
    ref_avg = perf.mean(ref_samples)
    last_index = len(results) - 1
    for index, changed_result in enumerate(results[1:], 1):
        changed_samples = changed_result.get_samples()
        changed_avg = perf.mean(changed_samples)
        text = ("Average: [%s] %s -> [%s] %s"
                % (ref_result.name,
                   ref_result.format(verbose=args.verbose),
                   changed_result.name,
                   changed_result.format(verbose=args.verbose)))

        # avoid division by zero
        if ref_avg and changed_avg:
            if changed_avg < ref_avg:
                text = "%s: %.1fx faster" % (text, ref_avg /  changed_avg)
            else:
                text= "%s: %.1fx slower" % (text, changed_avg / ref_avg)
        print(text)

        # significant?
        significant, t_score = perf.is_significant(ref_samples, changed_samples)
        if significant:
            print("Significant (t=%.2f)" % t_score)
        else:
            print("Not significant!")

        if index != last_index:
            print()


parser = create_parser()
args = parser.parse_args()
action = args.action
if action == 'show':
    result = parse_results(args.filename)
    display_result(args, result)
elif action == 'compare':
    ref_result = parse_results(args.ref_filename, '<ref>')
    results = [ref_result]
    for index, filename in enumerate(args.changed_filenames, 1):
        if index > 1:
            default_name = '<changed #%s>' % index
        else:
            default_name = '<changed>'
        result = parse_results(filename, default_name)
        results.append(result)
    compare_results(args, results)
else:
    parser.print_usage()
    sys.exit(1)
{'fraction_non_alphanumeric': 0.11388224575788634, 'fraction_numerical': 0.002619291652431386, 'mean_word_length': 2.3015037593984964, 'pattern_counts': {'":': 2, '<': 10, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '300407', 'n_tokens_mistral': 2548, 'n_tokens_neox': 2446, 'n_words': 567}