<commit_before>from __future__ import absolute_import

from tempfile import NamedTemporaryFile


class LogBuffer(object):
    def __init__(self, chunk_size=4096):
        self.chunk_size = chunk_size
        self.fp = NamedTemporaryFile()

    def fileno(self):
        return self.fp.fileno()

    def write(self, chunk):
        self.fp.write(chunk)
        self.fp.flush()

    def close(self, force=False):
        self.fp.flush()
        if force:
            self.fp.close()

    def flush(self):
        self.fp.flush()

    def iter_chunks(self):
        chunk_size = self.chunk_size
        result = ''
        offset = 0
        with open(self.fp.name) as fp:
            for chunk in fp:
                result += chunk
                while len(result) >= chunk_size:
                    newline_pos = result.rfind('\n', 0, chunk_size)
                    if newline_pos == -1:
                        newline_pos = chunk_size
                    else:
                        newline_pos += 1
                    yield (offset, result[:newline_pos])
                    offset += newline_pos
                    result = result[newline_pos:]
            if result:
                yield(offset, result)

        self.close(True)<commit_msg>Dont open second fd to logbuffer<commit_after>from __future__ import absolute_import

from tempfile import NamedTemporaryFile


class LogBuffer(object):
    def __init__(self, chunk_size=4096):
        self.chunk_size = chunk_size
        self.fp = NamedTemporaryFile()

    def fileno(self):
        return self.fp.fileno()

    def write(self, chunk):
        self.fp.write(chunk)
        self.fp.flush()

    def close(self, force=False):
        self.fp.flush()
        if force:
            self.fp.close()

    def flush(self):
        self.fp.flush()

    def iter_chunks(self):
        chunk_size = self.chunk_size
        result = ''
        offset = 0
        self.fp.seek(0)
        for chunk in self.fp:
            result += chunk
            while len(result) >= chunk_size:
                newline_pos = result.rfind('\n', 0, chunk_size)
                if newline_pos == -1:
                    newline_pos = chunk_size
                else:
                    newline_pos += 1
                yield (offset, result[:newline_pos])
                offset += newline_pos
                result = result[newline_pos:]
        if result:
            yield(offset, result)
        self.close(True)
{'fraction_non_alphanumeric': 0.0934199837530463, 'fraction_numerical': 0.006904955320877335, 'mean_word_length': 1.573667711598746, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6097004', 'n_tokens_mistral': 730, 'n_tokens_neox': 686, 'n_words': 161}