<commit_before>import numpy as np
from simplejson import load as load_json

from pprint import pprint

data_file = './public/data/results.json'

with open(data_file) as f:
    data = load_json(f)

tests = {}

for d in data:
    device = tests.get(d['device'], {})
    browser = device.get(d['browser']['browser']['name'], {})
    version = browser.get(d['jquery'], {})
    data_points = version.get('data', [])
    data_points.append(d['parse_time'])

    version['data'] = data_points
    browser[d['jquery']] = version
    device[d['browser']['browser']['name']] = browser
    tests[d['device']] = device

#pprint(tests)

results = {}

for device in tests.itervalues():
    for browser in device.itervalues():
        for version, results in browser.items():
            #pprint(results)
            data_points = results['data']
            a = np.array(data_points)
            results['mean'] = np.mean(a, dtype=np.float64)
            results['sigma'] = np.std(a, dtype=np.float64)

pprint(tests)<commit_msg>Added number of data points to analysis.<commit_after>import numpy as np
from simplejson import load as load_json

from pprint import pprint

data_file = './public/data/results.json'

with open(data_file) as f:
    data = load_json(f)

tests = {}

for d in data:
    device = tests.get(d['device'], {})
    browser = device.get(d['browser']['browser']['name'], {})
    version = browser.get(d['jquery'], {})
    data_points = version.get('data', [])
    data_points.append(d['parse_time'])

    version['data'] = data_points
    browser[d['jquery']] = version
    device[d['browser']['browser']['name']] = browser
    tests[d['device']] = device

#pprint(tests)

results = {}

for device in tests.itervalues():
    for browser in device.itervalues():
        for version, results in browser.items():
            #pprint(results)
            data_points = results['data']
            a = np.array(data_points)
            results['mean'] = np.mean(a, dtype=np.float64)
            results['sigma'] = np.std(a, dtype=np.float64)
            results['number'] = len(data_points)
            del results['data']

pprint(tests)
{'fraction_non_alphanumeric': 0.16198501872659177, 'fraction_numerical': 0.003745318352059925, 'mean_word_length': 3.2654690618762476, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7165109', 'n_tokens_mistral': 683, 'n_tokens_neox': 681, 'n_words': 151}