<commit_before>#!/usr/bin/env python
# This Source Code Form is subject to the terms of the Mozilla Public License,
# v. 2.0. If a copy of the MPL was not distributed with this file, You can
# obtain one at http://mozilla.org/MPL/2.0/.

import copy
import glob
import json
import os
import shutil

import stoneridge

class StoneRidgeCollator(object):
    """Takes the data we've collected from our tests and puts it into formats
    the graph server can handle. This is saved into json files for the uploader
    to do its thing with
    """
    def run(self):
        outfiles = glob.glob(os.path.join(stoneridge.xpcoutdir, '*.out'))
        with file(os.path.join(stoneridge.outdir, 'info.json'), 'rb') as f:
            info = json.load(f)

        for ofile in outfiles:
            # Make a new copy of the base info
            results = copy.deepcopy(info)
            results['testrun'] = {'date':None, 'suite':None, 'options':{},
                                  'results':{}, 'results_aux':{'totals':[],
                                                               'tstamps':{}}}

            # Figure out the test-specific data
            fname = os.path.basename(ofile)
            suite = fname.split('.')[0]
            finfo = os.stat(ofile)
            results['testrun']['date'] = int(finfo.st_ctime)
            results['testrun']['suite'] = suite

            # Read the raw data
            with file(ofile, 'rb') as f:
                testinfo = json.load(f)

            # Stick the raw data into the json to be uploaded
            for k, vlist in testinfo.items():
                for v in vlist:
                    if k == 'total':
                        # The graph server calculates totals for us, we just keep
                        # our calculations around for verification in case
                        results['testrun']['results_aux']['totals'].append(v['total'])
                    else:
                        if k in results['testrun']['results']:
                            results['testrun']['results'][k].append(v['total'])
                        else:
                            results['testrun']['results'][k] = [v['total']]

                        if k in results['testrun']['results_aux']['tstamps']:
                            results['testrun']['results_aux']['tstamps'][k].append(
                                    {'start':v['start'], 'stop':v['stop']})
                        else:
                            results['testrun']['results_aux']['tstamps'][k] = \
                                    [{'start':v['start'], 'stop':v['stop']}]

            # Copy the raw data into our output directory
            shutil.copyfile(ofile, os.path.join(stoneridge.outdir, fname))

            # Write our json results for uploading
            upload_filename = 'upload_%s.json' % (suite,)<commit_msg>Put output json in the format expected by the graph server<commit_after>#!/usr/bin/env python
# This Source Code Form is subject to the terms of the Mozilla Public License,
# v. 2.0. If a copy of the MPL was not distributed with this file, You can
# obtain one at http://mozilla.org/MPL/2.0/.

import collections
import copy
import glob
import json
import os
import shutil

import stoneridge

class StoneRidgeCollator(object):
    """Takes the data we've collected from our tests and puts it into formats
    the graph server can handle. This is saved into json files for the uploader
    to do its thing with
    """
    def run(self):
        outfiles = glob.glob(os.path.join(stoneridge.xpcoutdir, '*.out'))
        with file(os.path.join(stoneridge.outdir, 'info.json'), 'rb') as f:
            info = json.load(f)

        for ofile in outfiles:
            # Make a new copy of the base info
            results = copy.deepcopy(info)
            results['testrun'] = {'date':None, 'suite':None, 'options':{}}
            results['results'] = collections.defaultdict(list)
            results['results_aux'] = collections.defaultdict(list)

            # Figure out the test-specific data
            fname = os.path.basename(ofile)
            suite = fname.split('.')[0]
            finfo = os.stat(ofile)
            results['testrun']['date'] = int(finfo.st_ctime)
            results['testrun']['suite'] = suite

            # Read the raw data
            with file(ofile, 'rb') as f:
                testinfo = json.load(f)

            # Stick the raw data into the json to be uploaded
            for k, vlist in testinfo.items():
                for v in vlist:
                    if k == 'total':
                        # The graph server calculates totals for us, we just keep
                        # our calculations around for verification in case
                        results['results_aux']['totals'].append(v['total'])
                    else:
                        results['results'][k].append(v['total'])

                        for s in ('start', 'stop'):
                            key = '%s_%s' % (k, s)
                            results['results_aux'][key].append(v[s])

            # Turn our defaultdicts into regular dicts for jsonification
            results['results'] = dict(results['results'])
            results['results_aux'] = dict(results['results_aux'])

            # Copy the raw data into our output directory
            shutil.copyfile(ofile, os.path.join(stoneridge.outdir, fname))

            # Write our json results for uploading
            upload_filename = 'upload_%s.json' % (suite,)
{'fraction_non_alphanumeric': 0.11570547696771853, 'fraction_numerical': 0.0018135654697134566, 'mean_word_length': 1.7867609903991915, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5359057', 'n_tokens_mistral': 1487, 'n_tokens_neox': 1432, 'n_words': 460}