<commit_before>import typing as tp


def exhaust(iterator: tp.Iterator):
    """
    Iterate till the end of the iterator, discarding values as they go

    :param iterator: iterator to exhaust
    """
    try:
        while True:
            next(iterator)
    except (GeneratorExit, StopIteration):
        pass


class SelfClosingGenerator:
    """
    A wrapper to exhaust the generator in response to closing it.

    This will allow generators to complete that don't provide a .close() method.

    This will additionally exhaust the generator upon deallocation of the generator.

    You can feed it with either generators, or generator-functions, it will behave correctly each time.
    """
    __slots__ = ('generator', 'stopped')

    def __init__(self, generator: tp.Generator):
        self.generator = generator
        self.stopped = False

    def __iter__(self):
        return self.generator

    def __call__(self, *args, **kwargs):
        return SelfClosingGenerator(self.generator(*args, **kwargs))

    def send(self, obj: tp.Any):
        self.generator.send(obj)

    def __next__(self):
        try:
            return next(self.generator)
        except (StopIteration, GeneratorExit):
            self.stopped = True
            raise

    def close(self):
        if not self.stopped:
            try:
                exhaust(self.generator)
            except TypeError:
                pass
            self.stopped = True
        raise GeneratorExit()

    def __del__(self):
        try:
            self.close()
        except GeneratorExit:
            pass


# noinspection PyPep8Naming
class hint_with_length:
    """
    Accepting a generator, return it additionally providing
    a specified __length_hint__

    :param generator: generator to decorate
    :param length: length hint to provide
    """

    __slots__ = ('generator', 'length')

    def __init__(self, generator: tp.Generator, length: int):
        self.generator = generator
        self.length = length

    def __call__(self, *args, **kwargs):
        return hint_with_length(self.generator(*args, **kwargs), self.length)

    def close(self):
        return self.generator.close()

    def send(self, obj):
        return self.generator.send(obj)

    def __iter__(self):
        return self.generator

    def __next__(self):
        return next(self.generator)

    def __length_hint__(self) -> int:
        return self.length


<commit_msg>doc update<commit_after>import typing as tp


def exhaust(iterator: tp.Iterator):
    """
    Iterate till the end of the iterator, discarding values as they go

    :param iterator: iterator to exhaust
    """
    try:
        while True:
            next(iterator)
    except (GeneratorExit, StopIteration):
        pass


class SelfClosingGenerator:
    """
    A wrapper to exhaust the generator in response to closing it.

    This will allow generators to complete that don't provide a .close() method.

    This will additionally exhaust the generator upon deallocation of the generator.

    You can feed it with either generators, or generator-functions, it will behave correctly each time.
    """
    __slots__ = ('generator', 'stopped')

    def __init__(self, generator: tp.Generator):
        self.generator = generator
        self.stopped = False

    def __iter__(self):
        return self.generator

    def __call__(self, *args, **kwargs):
        return SelfClosingGenerator(self.generator(*args, **kwargs))

    def send(self, obj: tp.Any):
        self.generator.send(obj)

    def __next__(self):
        try:
            return next(self.generator)
        except (StopIteration, GeneratorExit):
            self.stopped = True
            raise

    def close(self):
        if not self.stopped:
            try:
                exhaust(self.generator)
            except TypeError:
                pass
            self.stopped = True
        raise GeneratorExit()

    def __del__(self):
        try:
            self.close()
        except GeneratorExit:
            pass


# noinspection PyPep8Naming
class hint_with_length:
    """
    Accepting a generator, return it additionally providing
    a specified __length_hint__

    You can provide generator-generating functions as well

    :param generator: generator to decorate
    :param length: length hint to provide
    """

    __slots__ = ('generator', 'length')

    def __init__(self, generator: tp.Generator, length: int):
        self.generator = generator
        self.length = length

    def __call__(self, *args, **kwargs):
        return hint_with_length(self.generator(*args, **kwargs), self.length)

    def close(self):
        return self.generator.close()

    def send(self, obj):
        return self.generator.send(obj)

    def __iter__(self):
        return self.generator

    def __next__(self):
        return next(self.generator)

    def __length_hint__(self) -> int:
        return self.length



{'fraction_non_alphanumeric': 0.08489614841701956, 'fraction_numerical': 0.00040330711837063924, 'mean_word_length': 2.3536173089925625, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3800315', 'n_tokens_mistral': 1349, 'n_tokens_neox': 1300, 'n_words': 428}