<commit_before>from collections import OrderedDict

def range_generator(start, stop):
    assert start < stop, "Wrong values for range generator"
    distance = float(stop - start)
    return lambda x: (start + (distance/(x - 1)) * i if x > 1 else start for i in range(x))

class Parameter(object):
    def __init__(self, values):
        self.values = values

def param(l):
    if isinstance(l, list):
        # Generator from list
        return lambda x: (m for m in l)
    else:
        # Constant generator
        return lambda x: (l for _ in range(x))

def string_generator(pattern, values):
    return lambda x: (pattern % value for value in values(x))

def clusterize(series, clusters):
    len_series = -1
    for k, v in series.items():
        if len_series == -1:
            len_series = len(v)
        else:
            assert len_series == len(v), 'All series must have the same number of values'

    nclusters = 1
    for cluster_level in clusters:
        nclusters = nclusters * len(cluster_level)

    assert nclusters == len_series, 'Series must contain as many values as clusters'

    def leafs(d):
        if len(d.items()) == 0:
            return [ d ]
        else:
            ret = []
            for k, v in d.items():
                ret += leafs(v)

        return ret

    def walk(d, fun):
        if len(d.items()) == 0:
            fun(d)
        else:
            for k, v in d.items():
                walk(v, clusters)

    ret = OrderedDict()
    for cluster_level in clusters:
        def insert_cluster(d):
            for cluster in cluster_level:
                d[cluster] = OrderedDict()

        walk(ret, insert_cluster)

    for i, leaf in enumerate(leafs(ret)):
        for k, series_vals in series.items():
            leaf[k] = series_vals[i]

    return ret
<commit_msg>Fix clusterize function<commit_after>from collections import OrderedDict

def range_generator(start, stop):
    assert start < stop, "Wrong values for range generator"
    distance = float(stop - start)
    return lambda x: (start + (distance/(x - 1)) * i if x > 1 else start for i in range(x))

class Parameter(object):
    def __init__(self, values):
        self.values = values

def param(l):
    if isinstance(l, list):
        # Generator from list
        return lambda x: (m for m in l)
    else:
        # Constant generator
        return lambda x: (l for _ in range(x))

def string_generator(pattern, values):
    return lambda x: (pattern % value for value in values(x))

def clusterize(series, clusters):
    len_series = -1
    for k, v in series.items():
        if len_series == -1:
            len_series = len(v)
        else:
            assert len_series == len(v), 'All series must have the same number of values'

    nclusters = 1
    for cluster_level in clusters:
        nclusters = nclusters * len(cluster_level)

    assert nclusters == len_series, 'Series must contain as many values as clusters'

    def leafs(d):
        if len(d.items()) == 0:
            return [ d ]
        else:
            ret = []
            for k, v in d.items():
                ret += leafs(v)

        return ret

    def walk(d, fun):
        if len(d.items()) == 0:
            fun(d)
        else:
            for k, v in d.items():
                walk(v, fun)

    ret = OrderedDict()
    for cluster_level in clusters:
        def insert_cluster(d):
            for cluster in cluster_level:
                d[cluster] = OrderedDict()

        walk(ret, insert_cluster)

    for i, leaf in enumerate(leafs(ret)):
        for k, series_vals in series.items():
            leaf[k] = series_vals[i]

    return ret

{'fraction_non_alphanumeric': 0.10073911853271283, 'fraction_numerical': 0.0038324664659184233, 'mean_word_length': 2.0273405136702567, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2086709', 'n_tokens_mistral': 1102, 'n_tokens_neox': 1065, 'n_words': 382}