<commit_before>import collections
import functools
import inspect
import io


class Buffer:
    def __init__(self):
        self.queue = collections.deque()
        self.position = 0

    def write(self, chunk):
        self.queue.append(chunk)

    def _get_queue(self):
        assert len(self.queue) > 0 or self.position == 0, ("We can't have a positive position "
                                                           "on an empty queue.")

        q = iter(self.queue)
        try:
            data = next(q)
            yield data[self.position:]
        except StopIteration:
            pass
        else:
            yield from q

    def _get_data(self, nbytes):
        if nbytes == 0:
            return len(self.queue), 0, b''.join(self._get_queue())
        else:
            acc = io.BytesIO()
            q = self._get_queue()
            segments_read = 0
            position = self.position
            to_read = nbytes

            while True:
                try:
                    segment_len = acc.write(next(q))
                    to_read -= segment_len
                    segments_read += 1
                except StopIteration:
                    break

                if acc.tell() >= nbytes:
                    assert to_read <= 0
                    position += segment_len + to_read
                    break

                position = 0

            acc.seek(0)
            return segments_read, position, acc.read(nbytes)

    def peek(self, nbytes=0):
        _, _, data = self._get_data(nbytes)
        return data

    def read(self, nbytes=0):
        segment_read, position, data = self._get_data(nbytes)
        if position > 0:
            segment_read -= 1
        for i in range(segment_read):
            self.queue.popleft()
        self.position = position

        assert len(self.queue) > 0 or self.position == 0, ("We can't have a positive position "
                                                           "on an empty queue.")

        return data

    def __len__(self):
        return sum(len(d) for d in self.queue) - self.position

    def __repr__(self):
        return '<{self.__class__} {self.queue!r}>'.format(self=self)


_no_result = object()
_state_ended = object()

<commit_msg>Better repr() for Buffer objects<commit_after>import collections
import functools
import inspect
import io


class Buffer:
    def __init__(self):
        self.queue = collections.deque()
        self.position = 0

    def write(self, chunk):
        self.queue.append(chunk)

    def _get_queue(self):
        assert len(self.queue) > 0 or self.position == 0, ("We can't have a positive position "
                                                           "on an empty queue.")

        q = iter(self.queue)
        try:
            data = next(q)
            yield data[self.position:]
        except StopIteration:
            pass
        else:
            yield from q

    def _get_data(self, nbytes):
        if nbytes == 0:
            return len(self.queue), 0, b''.join(self._get_queue())
        else:
            acc = io.BytesIO()
            q = self._get_queue()
            segments_read = 0
            position = self.position
            to_read = nbytes

            while True:
                try:
                    segment_len = acc.write(next(q))
                    to_read -= segment_len
                    segments_read += 1
                except StopIteration:
                    break

                if acc.tell() >= nbytes:
                    assert to_read <= 0
                    position += segment_len + to_read
                    break

                position = 0

            acc.seek(0)
            return segments_read, position, acc.read(nbytes)

    def peek(self, nbytes=0):
        _, _, data = self._get_data(nbytes)
        return data

    def read(self, nbytes=0):
        segment_read, position, data = self._get_data(nbytes)
        if position > 0:
            segment_read -= 1
        for i in range(segment_read):
            self.queue.popleft()
        self.position = position

        assert len(self.queue) > 0 or self.position == 0, ("We can't have a positive position "
                                                           "on an empty queue.")

        return data

    def __len__(self):
        return sum(len(d) for d in self.queue) - self.position

    def __repr__(self):
        return '<{self.__class__.__name__} {self.queue!r} pos={self.position}>'.format(self=self)


_no_result = object()
_state_ended = object()


{'fraction_non_alphanumeric': 0.09192628345765687, 'fraction_numerical': 0.007020623080298377, 'mean_word_length': 1.396950578338591, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2386105', 'n_tokens_mistral': 1279, 'n_tokens_neox': 1189, 'n_words': 333}