<commit_before>#!/usr/bin/env python

import os
import random
try:
    import ujson as json
except ImportError:
    import json
try:
    import cPickle as pk
except ImportError:
    import pickle as pk

from time import time
from math import log, ceil
from collections import namedtuple

from randopt.samplers import Uniform
try:  # Try native statistics module
    from statistics import mean, median, pvariance, pstdev
except ImportError:
    from randopt.statistics import mean, median, pvariance, pstdev


"""
This file implements the Experiment, JSONSummary, and SummaryList classes.
"""

ATTACHMENT_DIR = '_attachments'
ATTACHMENT_EXT = '.pk'

leq = lambda x, y: x.result <= y.result
geq = lambda x, y: x.result >= y.result


class SummaryList(list):

    """
    List of JSON Summaries on steroid.

    Parameters:

    * results - (list) list of JSON Summaries.

    Return type: n/a

    Example:
        
        results = exp.top(10)
        results.mean('alpha')
        results.filter(lambda r: r.result > results.mean())
    """

    def __init__(self, results):
        list.__init__(self, results)
        self.__results = results

    def __getitem__(self, key):
        if isinstance(key, slice):
            return SummaryList(self.__results[key])
        return self.__results[key]

    def count(self):
        return len(self)

    def filter(self, fn):
        results = [r for r in self if fn(r)]
        return SummaryList(results)

    def map(self, fn, key='result'):
        if isinstance(self[0][key], list):
            values = [r[key] for r in self]
            return [fn(v) for v in zip(*values)]
        return fn([r[key] for r in self])

    def min(self, key='result'):
        return self.map(min, key)

    def max(self, key='result'):
        return self.map(max, key)

    def mean(self, key='result'):
        return self.map(mean, key)

    def variance(self, key='result'):
        return self.map(pvariance, key)
<commit_msg>SummaryList: compatibility fix for PY2.7<commit_after>#!/usr/bin/env python

import os
import random
try:
    import ujson as json
except ImportError:
    import json
try:
    import cPickle as pk
except ImportError:
    import pickle as pk

from time import time
from math import log, ceil
from collections import namedtuple

from randopt.samplers import Uniform
try:  # Try native statistics module
    from statistics import mean, median, pvariance, pstdev
except ImportError:
    from randopt.statistics import mean, median, pvariance, pstdev


"""
This file implements the Experiment, JSONSummary, and SummaryList classes.
"""

ATTACHMENT_DIR = '_attachments'
ATTACHMENT_EXT = '.pk'

leq = lambda x, y: x.result <= y.result
geq = lambda x, y: x.result >= y.result


class SummaryList(list):

    """
    List of JSON Summaries on steroid.

    Parameters:

    * results - (list) list of JSON Summaries.

    Return type: n/a

    Example:
        
        results = exp.top(10)
        results.mean('alpha')
        results.filter(lambda r: r.result > results.mean())
    """

    def __init__(self, results):
        list.__init__(self, results)
        self.__results = results

    def __getitem__(self, key):
        if isinstance(key, slice):
            return SummaryList(self.__results[key])
        return self.__results[key]

    def __getslice__(self, i, j):
        return self.__getitem__(slice(i, j))

    def __str__(self):
        return 'SummaryList(' + str(len(self)) + ')'

    def count(self):
        return len(self)

    def filter(self, fn):
        results = [r for r in self if fn(r)]
        return SummaryList(results)

    def map(self, fn, key='result'):
        if isinstance(self[0][key], list):
            values = [r[key] for r in self]
            return [fn(v) for v in zip(*values)]
        return fn([r[key] for r in self])

    def min(self, key='result'):
        return self.map(min, key)

    def max(self, key='result'):
        return self.map(max, key)

    def mean(self, key='result'):
        return self.map(mean, key)

    def variance(self, key='result'):
        return self.map(pvariance, key)

{'fraction_non_alphanumeric': 0.10911736178467507, 'fraction_numerical': 0.0019398642095053346, 'mean_word_length': 2.7296564195298374, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '104721', 'n_tokens_mistral': 1285, 'n_tokens_neox': 1234, 'n_words': 412}