<commit_before>from threading import RLock
import io

import core

BUFFER_SIZE = io.DEFAULT_BUFFER_SIZE


class DecompressReader(io.RawIOBase):
    def __init__(self, fp, decomp_fn, **decomp_args):
        self._fp = fp
        self._eof = False
        self._pos = 0  # Current offset in decompressed stream

        # Set to size of decompressed stream once it is known
        self._size = -1

        self._decomp_fn = decomp_fn
        self._decomp_args = decomp_args

    def close(self):
        return super(EncodedFile).close()

    def readable(self):
        return True

    def seekable(self):
        return self._fp.seekable()

    def readinto(self, b):
        with memoryview(b) as view, view.cast('B') as byte_view:
            data = self.read(len(byte_view))
            byte_view[:len(data)] = data
        return len(data)

    def read(self, size=-1):
        if size < 0:
            return self.readall()

        if not size or self._eof:
            return b''

        raw_chunk = self._fp.read(size)
        if raw_chunk:
            data = self._decomp_fn(raw_chunk, **self._decomp_args)
        else:
            data = None

        if not data:
            self._eof = True
            self._size = self._pos
            return b''
        self._pos += len(data)
        return data

    # Rewind the file to the beginning of the data stream.
    def _rewind(self):
        self._fp.seek(0)
        self._eof = False
        self._pos = 0

    def seek(self, offset, whence=io.SEEK_SET):
        if whence == io.SEEK_SET:
            pass
        elif whence == io.SEEK_CUR:
            offset += self._pos
        elif whence == io.SEEK_END:
            if self._size < 0:
                # Finish reading the file
                while self.read(io.DEFAULT_BUFFER_SIZE):
                    pass
            offset += self._size
        else:
            raise ValueError('Invalid value for whence: {}'.format(whence))

        # Make it so that offset is the number of bytes to skip forward.
        if offset < self._pos:
            self._rewind()
        else:
            offset -= self._pos

        # Read and discard data until we reach the desired position
        while offset > 0:
            data = self.read(min(io.DEFAULT_BUFFER_SIZE, offset))
            if not data:
                break
            offset -= len(data)

        return self._pos

    def tell(self):
        return self._pos


_MODE_CLOSED = 0
_MODE_READ = 1
_MODE_WRITE = 2


class EncodedFile(io.BufferedIOBase):
    def __init__(self, filename=None, mode=None,
                 _fp=None, **compress_options):
        self._lock = RLock()
        self._mode = _MODE_CLOSED

        if filename:
            self._fp = open(filename, mode)
        elif _fp:
            self._fp = _fp<commit_msg>Added call to __builtins__.open instead of plain open<commit_after>from threading import RLock
import io

import core

BUFFER_SIZE = io.DEFAULT_BUFFER_SIZE


class DecompressReader(io.RawIOBase):
    def __init__(self, fp, decomp_fn, **decomp_args):
        self._fp = fp
        self._eof = False
        self._pos = 0  # Current offset in decompressed stream

        # Set to size of decompressed stream once it is known
        self._size = -1

        self._decomp_fn = decomp_fn
        self._decomp_args = decomp_args

    def close(self):
        return super(EncodedFile).close()

    def readable(self):
        return True

    def seekable(self):
        return self._fp.seekable()

    def readinto(self, b):
        with memoryview(b) as view, view.cast('B') as byte_view:
            data = self.read(len(byte_view))
            byte_view[:len(data)] = data
        return len(data)

    def read(self, size=-1):
        if size < 0:
            return self.readall()

        if not size or self._eof:
            return b''

        raw_chunk = self._fp.read(size)
        if raw_chunk:
            data = self._decomp_fn(raw_chunk, **self._decomp_args)
        else:
            data = None

        if not data:
            self._eof = True
            self._size = self._pos
            return b''
        self._pos += len(data)
        return data

    # Rewind the file to the beginning of the data stream.
    def _rewind(self):
        self._fp.seek(0)
        self._eof = False
        self._pos = 0

    def seek(self, offset, whence=io.SEEK_SET):
        if whence == io.SEEK_SET:
            pass
        elif whence == io.SEEK_CUR:
            offset += self._pos
        elif whence == io.SEEK_END:
            if self._size < 0:
                # Finish reading the file
                while self.read(io.DEFAULT_BUFFER_SIZE):
                    pass
            offset += self._size
        else:
            raise ValueError('Invalid value for whence: {}'.format(whence))

        # Make it so that offset is the number of bytes to skip forward.
        if offset < self._pos:
            self._rewind()
        else:
            offset -= self._pos

        # Read and discard data until we reach the desired position
        while offset > 0:
            data = self.read(min(io.DEFAULT_BUFFER_SIZE, offset))
            if not data:
                break
            offset -= len(data)

        return self._pos

    def tell(self):
        return self._pos


_MODE_CLOSED = 0
_MODE_READ = 1
_MODE_WRITE = 2


class EncodedFile(io.BufferedIOBase):
    def __init__(self, filename=None, mode=None,
                 _fp=None, **compress_options):
        self._lock = RLock()
        self._mode = _MODE_CLOSED

        if filename:
            self._fp = __builtins__.open(filename, mode)
        elif _fp:
            self._fp = _fp
{'fraction_non_alphanumeric': 0.08686477932125901, 'fraction_numerical': 0.0038684719535783366, 'mean_word_length': 1.7111534795042898, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5583703', 'n_tokens_mistral': 1710, 'n_tokens_neox': 1629, 'n_words': 480}