<commit_before>import cython

cdef int64 UNIT_SIZE = 8
cdef int64 UNIT_OFFSET = UNIT_SIZE - 1

cdef class State:
    def __init__(State self, BitStream stream):
        self.offsetRead = stream.offsetRead
        self.offsetWrite = stream.offsetWrite
        self.stream = stream

class ReadError(Exception):
    def __init__(self, message):
        Exception.__init__(self, message)

cdef class BitStream:
    def __init__(self, stream = None):
        self.database = defaultFactory.database

        if stream == None:
            self.stream = []
            self.offsetRead = 0
            self.offsetWrite = 0
        else:
            self.stream = stream
            self.offsetRead = 0
            self.offsetWrite = len(stream) * 8

    def __len__(self):
        return self.offsetWrite - self.offsetRead

    cpdef write(BitStream self, value, type key, int count = -1):
        cdef TypeInfo typeinfo = self.database[key]
        cdef validator = typeinfo.validator
        cdef writer = typeinfo.writer
        cdef int64 byte
        cdef int64 bit

        if count == -1:
            byte = self.offsetWrite / UNIT_SIZE
            bit = self.offsetWrite % UNIT_SIZE

            if validator and not validator(value):
                raise Exception("Input did not validate")

            writer(self, value, byte, bit)

            # Advance pointer
            self.offsetWrite += typeinfo.length
        else:
            for i in range(count):
                byte = self.offsetWrite / UNIT_SIZE
                bit = self.offsetWrite % UNIT_SIZE

                if validator and not validator(value[i]):
                    raise Exception("Input did not validate")

                writer(self, value[i], byte, bit)

                # Advance pointer
                self.offsetWrite += typeinfo.length

    cpdef read(BitStream self, type key, int count = -1):
        cdef TypeInfo typeinfo = self.database[key]
        cdef State state
        cdef int64 byte
        cdef int64 bit

        output = []

        # Allow the reader to read an arbitrary number of bits
        #   from the stream. In order to properly recover we save
        #   our state and restore it in the event of a underrun
        #   exception.
        if typeinfo.length == -1:
            raise NotImplementedError("Types with an arbitrary size aren't properly support yet")

            state = self.save()
            output = []
            try:
                for i in range(count):
                    output.append(typeinfo.reader(self))
            except:
                self.restore(state)
                raise
            return output

        # Otherwise, If the reader knows ahead of time how much data it
        #   will consume, we can take control of the pointers, knowing
        #   if we're going to underrun before we read. So we don't have
        #   to save our state.
        else:
            # If no count was specified, read only a single object from the
            #    stream and directly return it.
            if count == -1:
                # Check for underrun
                if self.offsetRead + typeinfo.length > self.offsetWrite:
                    raise ReadError("end of stream")

                byte = self.offsetRead / UNIT_SIZE
                bit = self.offsetRead % UNIT_SIZE

                output = typeinfo.reader(self, byte, bit)

                # Advance pointer
                self.offsetRead += typeinfo.length

                return output

            # Return an array
            else:
                # Check for underrun
                if self.offsetRead + (typeinfo.length * count) > self.offsetWrite:
                    raise ReadError("end of stream")<commit_msg>Properly raise errors on stream xruns<commit_after>import cython

cdef int64 UNIT_SIZE = 8
cdef int64 UNIT_OFFSET = UNIT_SIZE - 1

cdef class State:
    def __init__(State self, BitStream stream):
        self.offsetRead = stream.offsetRead
        self.offsetWrite = stream.offsetWrite
        self.stream = stream

class ReadError(Exception):
    def __init__(self, message):
        Exception.__init__(self, message)

class WriteError(Exception):
    def __init__(self, message):
        Exception.__init__(self, message)

cdef class BitStream:
    def __init__(self, stream = None):
        self.database = defaultFactory.database

        if stream == None:
            self.stream = []
            self.offsetRead = 0
            self.offsetWrite = 0
        else:
            self.stream = stream
            self.offsetRead = 0
            self.offsetWrite = len(stream) * 8

    def __len__(self):
        return self.offsetWrite - self.offsetRead

    cpdef write(BitStream self, value, type key, int count = -1):
        cdef TypeInfo typeinfo = self.database[key]
        cdef validator = typeinfo.validator
        cdef writer = typeinfo.writer
        cdef int64 byte
        cdef int64 bit

        if count == -1:
            byte = self.offsetWrite / UNIT_SIZE
            bit = self.offsetWrite % UNIT_SIZE

            if validator and not validator(value):
                raise WriteError("Input did not validate")

            writer(self, value, byte, bit)

            # Advance pointer
            self.offsetWrite += typeinfo.length
        else:
            if count < 0:
                raise WriteError("Invalid count specified")

            if count > len(value):
                raise WriteError("Count is greater than the length of the input")

            for i in range(count):
                byte = self.offsetWrite / UNIT_SIZE
                bit = self.offsetWrite % UNIT_SIZE

                if validator and not validator(value[i]):
                    raise WriteError("Input did not validate")

                writer(self, value[i], byte, bit)

                # Advance pointer
                self.offsetWrite += typeinfo.length

    cpdef read(BitStream self, type key, int count = -1):
        cdef TypeInfo typeinfo = self.database[key]
        cdef State state
        cdef int64 byte
        cdef int64 bit

        output = []

        # Allow the reader to read an arbitrary number of bits
        #   from the stream. In order to properly recover we save
        #   our state and restore it in the event of a underrun
        #   exception.
        if typeinfo.length == -1:
            raise NotImplementedError("Types with an arbitrary size aren't properly support yet")

            state = self.save()
            output = []
            try:
                for i in range(count):
                    output.append(typeinfo.reader(self))
            except:
                self.restore(state)
                raise
            return output

        # Otherwise, If the reader knows ahead of time how much data it
        #   will consume, we can take control of the pointers, knowing
        #   if we're going to underrun before we read. So we don't have
        #   to save our state.
        else:
            # If no count was specified, read only a single object from the
            #    stream and directly return it.
            if count == -1:
                # Check for underrun
                if self.offsetRead + typeinfo.length > self.offsetWrite:
                    raise ReadError("end of stream")

                byte = self.offsetRead / UNIT_SIZE
                bit = self.offsetRead % UNIT_SIZE

                output = typeinfo.reader(self, byte, bit)

                # Advance pointer
                self.offsetRead += typeinfo.length

                return output

            # Return an array
            else:
                if count < 0:
                    raise ReadError("Invalid count specified")

                # Check for underrun
                if self.offsetRead + (typeinfo.length * count) > self.offsetWrite:
                    raise ReadError("end of stream")
{'fraction_non_alphanumeric': 0.06835986858731362, 'fraction_numerical': 0.006065200909780136, 'mean_word_length': 1.6036184210526316, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4014879', 'n_tokens_mistral': 2078, 'n_tokens_neox': 1934, 'n_words': 690}