<commit_before># -*- coding: utf-8 -*-
# Licensed under a 3-clause BSD style license - see LICENSE.rst

from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

import os
import itertools

import six
from six.moves import xrange

from . import util


class Graph(object):
    """
    Manages a single "line" in the resulting plots for the front end.

    Unlike "results", which contain the timings for a single commit,
    these contain the timings for a single benchmark.
    """
    def __init__(self, benchmark_name, params, all_params,
                 summary=False):
        """
        Initially the graph contains no data.  It must be added using
        multiple calls to `add_data_point`.

        Parameters
        ----------
        benchmark_name : str
            A unique string to identify the benchmark, and display in
            the frontend.

        params : dict of str -> str
            A dictionary of parameters describing the benchmark.

        all_params : dict of str -> str
            A dictionary of all parameters that were found amongst all
            benchmark results.  This is used to fill in blanks for
            parameters that might not have been recorded for a
            particular result.

        summary : bool, optional
            Whether to generate summary graph, averaged over parameter
            configurations (if test is parametric).

        """
        # Fill in missing parameters
        for key in six.iterkeys(all_params):
            if key not in params:
                params[key] = None

        self.benchmark_name = benchmark_name
        self.params = params
        self.data_points = {}

        # TODO: Make filename safe
        parts = ['graphs']

        l = list(six.iteritems(self.params))
        l.sort()
        for key, val in l:
            if val is None:
                parts.append(key)
            else:
                parts.append('{0}-{1}'.format(key, val))
        parts.append(benchmark_name)

        self.summary = summary
        self.path = os.path.join(*parts)

    def add_data_point(self, date, value):
        """
        Add a data point to the graph.

        Parameters
        ----------
        date : int
            A Javascript timestamp value representing the time a
            particular commit was merged into the repository.

        value : float or list
            The value(s) to plot in the benchmark.

        """
        # Add simple time series
        self.data_points.setdefault(date, [])
        if value is not None:
            if self.summary and hasattr(value, '__len__'):
                value = _mean_with_none(value)
            self.data_points[date].append(value)

    def get_data(self):
        """
        Get the sorted and reduced data.
        """
        def mean(v):
            if not len(v):
                return None
            else:
                if hasattr(v[0], '__len__'):
                    return [_mean_with_none(x[j] for x in v)
                            for j in range(len(v[0]))]
                else:
                    return _mean_with_none(v)

        val = [(k, mean(v)) for (k, v) in
               six.iteritems(self.data_points)]
        val.sort()

        i = 0
        for i in xrange(len(val)):
            if val[i][1] is not None:
                break

        j = i
        for j in xrange(len(val) - 1, i, -1):
            if val[j][1] is not None:
                break

        return val[i:j+1]

    def save(self, html_dir):
        """
        Save the graph to a .json file used by the frontend.

        Parameters
        ----------
        html_dir : str
            The root of the HTML tree.
        """
        filename = os.path.join(html_dir, self.path + ".json")

        val = self.get_data()

        util.write_json(filename, val)


def _mean_with_none(values):<commit_msg>Downsample summary graphs<commit_after># -*- coding: utf-8 -*-
# Licensed under a 3-clause BSD style license - see LICENSE.rst

from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

import os
import itertools

import six
from six.moves import xrange

from . import util


# This is the maximum number of points to include in summary graphs.
# It is based on the number of pixels in the summary graph display on
# a recent Retina MacBook Pro (3840 pixels across the screen, divided
# by 5 summaries across, divided by 2 for good measure and to account
# for width of the line).
RESAMPLED_POINTS = (3840 / 5 / 2)


class Graph(object):
    """
    Manages a single "line" in the resulting plots for the front end.

    Unlike "results", which contain the timings for a single commit,
    these contain the timings for a single benchmark.
    """
    def __init__(self, benchmark_name, params, all_params,
                 summary=False):
        """
        Initially the graph contains no data.  It must be added using
        multiple calls to `add_data_point`.

        Parameters
        ----------
        benchmark_name : str
            A unique string to identify the benchmark, and display in
            the frontend.

        params : dict of str -> str
            A dictionary of parameters describing the benchmark.

        all_params : dict of str -> str
            A dictionary of all parameters that were found amongst all
            benchmark results.  This is used to fill in blanks for
            parameters that might not have been recorded for a
            particular result.

        summary : bool, optional
            Whether to generate summary graph, averaged over parameter
            configurations (if test is parametric).

        """
        # Fill in missing parameters
        for key in six.iterkeys(all_params):
            if key not in params:
                params[key] = None

        self.benchmark_name = benchmark_name
        self.params = params
        self.data_points = {}

        # TODO: Make filename safe
        parts = ['graphs']

        l = list(six.iteritems(self.params))
        l.sort()
        for key, val in l:
            if val is None:
                parts.append(key)
            else:
                parts.append('{0}-{1}'.format(key, val))
        parts.append(benchmark_name)

        self.summary = summary
        self.path = os.path.join(*parts)

    def add_data_point(self, date, value):
        """
        Add a data point to the graph.

        Parameters
        ----------
        date : int
            A Javascript timestamp value representing the time a
            particular commit was merged into the repository.

        value : float or list
            The value(s) to plot in the benchmark.

        """
        # Add simple time series
        self.data_points.setdefault(date, [])
        if value is not None:
            if self.summary and hasattr(value, '__len__'):
                value = _mean_with_none(value)
            self.data_points[date].append(value)

    def resample_data(self, val):
        if len(val) < RESAMPLED_POINTS:
            return val

        min_time = min(x[0] for x in val)
        max_time = max(x[0] for x in val)
        step_size = int((max_time - min_time) / RESAMPLED_POINTS)

        new_val = []
        j = 0
        for i in xrange(min_time + step_size, max_time + step_size, step_size):
            chunk = []
            while j < len(val) and val[j][0] < i:
                chunk.append(val[j][1])
                j += 1
            if len(chunk):
                new_val.append((i, _mean_with_none(chunk)))
        return new_val

    def get_data(self):
        """
        Get the sorted and reduced data.
        """
        def mean(v):
            if not len(v):
                return None
            else:
                if hasattr(v[0], '__len__'):
                    return [_mean_with_none(x[j] for x in v)
                            for j in range(len(v[0]))]
                else:
                    return _mean_with_none(v)

        val = [(k, mean(v)) for (k, v) in
               six.iteritems(self.data_points)]
        val.sort()

        i = 0
        for i in xrange(len(val)):
            if val[i][1] is not None:
                break

        j = i
        for j in xrange(len(val) - 1, i, -1):
            if val[j][1] is not None:
                break

        val = val[i:j+1]

        if self.summary:
            val = self.resample_data(val)

        return val

    def save(self, html_dir):
        """
        Save the graph to a .json file used by the frontend.

        Parameters
        ----------
        html_dir : str
            The root of the HTML tree.
        """
        filename = os.path.join(html_dir, self.path + ".json")

        val = self.get_data()

        util.write_json(filename, val)


def _mean_with_none(values):
{'fraction_non_alphanumeric': 0.0865028609895658, 'fraction_numerical': 0.004712218108381017, 'mean_word_length': 1.6049094097019287, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4571456', 'n_tokens_mistral': 2485, 'n_tokens_neox': 2363, 'n_words': 861}