<commit_before># -*- coding: utf-8 -*-

import platform
import time
from math import sqrt

try:
    import texttable as tt
except ImportError:
    print ("please install texttable to show results (sudo apt-get install python-texttable?")

try:
    import psutil
except ImportError:
    print ("please install psutil to show results (sudo apt-get install python-psutil?")

try:
    from cpuinfo import cpuinfo
except ImportError:
    print ("please install cpuinfo to show results (pip install py-cpuinfo")

__author__ = '<NAME>'


def report(results):

    tab = tt.Texttable()
    tab.header(['Benchmark', 'Input', 'Average', 'std dev %', 'std dev', 'runs', 'discarded'])

    tab.set_cols_align(['r','r', 'r','r', 'r', 'r', 'r'])
    tab.set_deco(tab.HEADER | tab.VLINES)

    for benchmark, measures in sorted(results.items()):
        report_benchmark(tab, benchmark, measures)

    table = tab.draw()
    print (table)

    context = '\n%s-%s-%s on %s' % (platform.system(), platform.release(), platform.machine(),
                                  time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime())) + '.'

    print (context)
    print ("%d logical cores (%d physical)" % (psutil.cpu_count(), psutil.cpu_count(False)))
    print (str(psutil.virtual_memory()))

    print ("cpuinfo: %s" % (str(cpuinfo.get_cpu_info())))


def report_benchmark(tab, benchmark, measures):

    trimmed = trim_ends(sorted(measures), 0.1)
    runs = len(trimmed)
    average = sum(trimmed) / runs
    stddev  = sqrt(sum([(measure - average)**2 for measure in trimmed]))
    stddev_relative = stddev / average * 100
    tab.add_row([benchmark.name(), benchmark.input, average, "%2.2f %%" % stddev_relative, stddev, len(measures), len(measures) - runs])

    tab.set_cols_width([35, 10, 10, 10, 10, 10, 10])


def trim_ends(sorted_list, proportion_to_cut):

    size = len(sorted_list)
    left = int(proportion_to_cut * size)
    right = size - left

    return sorted_list[left:right]
<commit_msg>group results by benchmark<commit_after># -*- coding: utf-8 -*-

import platform
import time
from math import sqrt

try:
    import texttable as tt
except ImportError:
    print ("please install texttable to show results (sudo apt-get install python-texttable?")

try:
    import psutil
except ImportError:
    print ("please install psutil to show results (sudo apt-get install python-psutil?")

try:
    from cpuinfo import cpuinfo
except ImportError:
    print ("please install cpuinfo to show results (pip install py-cpuinfo")

__author__ = '<NAME>'


def report(results):
    tab = tt.Texttable()
    tab.header(['Benchmark', 'Input', 'Average', 'std dev %', 'std dev', 'runs', 'discarded'])

    tab.set_cols_align(['r', 'r', 'r', 'r', 'r', 'r', 'r'])
    tab.set_deco(tab.HEADER | tab.VLINES)

    grouped = {}

    for benchmark in results.keys():
        if not grouped.has_key(benchmark.name()):
            grouped[benchmark.name()] = []

        grouped[benchmark.name()].append(benchmark)

    #for benchmark, measures in sorted(results.items()):
    for group in grouped.values():
        for benchmark in sorted(group):
            measures = results[benchmark]
            report_benchmark(tab, benchmark, measures)

        tab.add_row(("··········", "······", "······", "·····", "·······", "·····", "······"))

    table = tab.draw()
    print (table)

    context = '\n%s-%s-%s on %s' % (platform.system(), platform.release(), platform.machine(),
                                    time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime())) + '.'

    print (context)
    print ("%d logical cores (%d physical)" % (psutil.cpu_count(), psutil.cpu_count(False)))
    print (str(psutil.virtual_memory()))

    print ("cpuinfo: %s" % (str(cpuinfo.get_cpu_info())))


def report_benchmark(tab, benchmark, measures):
    trimmed = trim_ends(sorted(measures), 0.1)
    runs = len(trimmed)
    average = sum(trimmed) / runs
    stddev = sqrt(sum([(measure - average) ** 2 for measure in trimmed]))
    stddev_relative = stddev / average * 100
    tab.add_row([benchmark.variation, benchmark.input, average, "%2.2f %%" % stddev_relative, stddev, len(measures),
                 len(measures) - runs])

    tab.set_cols_width([35, 10, 10, 10, 10, 10, 10])


def trim_ends(sorted_list, proportion_to_cut):
    size = len(sorted_list)
    left = int(proportion_to_cut * size)
    right = size - left

    return sorted_list[left:right]

{'fraction_non_alphanumeric': 0.16187457550373557, 'fraction_numerical': 0.010414308354086485, 'mean_word_length': 3.431293881644935, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5068497', 'n_tokens_mistral': 1552, 'n_tokens_neox': 1411, 'n_words': 381}