<commit_before># -*- coding: utf-8 -*-

from random import betavariate, uniform


def weighted_choice(weights):
    choices = range(len(weights))
    total = sum(weights)
    r = uniform(0, total)
    upto = 0
    for c, w in zip(choices, weights):
        if upto + w > r:
            return c
        upto += w
    raise Exception("Error in weighted_choice.")


class DirichletProcess():
    def __init__(self, base_measure, alpha):
        self.base_measure = base_measure
        self.alpha = alpha

        self.cache = []
        self.weights = []
        self.total_stick_used = 0.

    def __call__(self):
        remaining = 1.0 - self.total_stick_used
        i = DirichletProcess.roll_die(self.weights + [remaining])
        if i is not None and i < len(self.weights):
            return self.cache[i]
        else:
            stick_piece = betavariate(1, self.alpha) * remaining
            self.total_stick_used += stick_piece
            self.weights.append(stick_piece)
            new_value = self.base_measure()
            self.cache.append(new_value)
            return new_value

    @staticmethod
    def roll_die(weights):
        if weights:
            return weighted_choice(weights)
        else:
            return None
<commit_msg>Remove roll_die method<commit_after># -*- coding: utf-8 -*-

from random import betavariate, uniform


def weighted_choice(weights):
    choices = range(len(weights))
    total = sum(weights)
    r = uniform(0, total)
    upto = 0
    for c, w in zip(choices, weights):
        if upto + w > r:
            return c
        upto += w
    raise Exception("Error in weighted_choice.")


class DirichletProcess():
    def __init__(self, base_measure, alpha):
        self.base_measure = base_measure
        self.alpha = alpha

        self.cache = []
        self.weights = []
        self.total_stick_used = 0.

    def __call__(self):
        remaining = 1.0 - self.total_stick_used
        i = weighted_choice(self.weights + [remaining])
        if i < len(self.weights):
            return self.cache[i]
        else:
            stick_piece = betavariate(1, self.alpha) * remaining
            self.total_stick_used += stick_piece
            self.weights.append(stick_piece)
            new_value = self.base_measure()
            self.cache.append(new_value)
            return new_value

{'fraction_non_alphanumeric': 0.09817254568635785, 'fraction_numerical': 0.005949851253718657, 'mean_word_length': 2.0651041666666665, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4516035', 'n_tokens_mistral': 710, 'n_tokens_neox': 682, 'n_words': 171}