<commit_before>from cbor.CBORStream import CBORStream
from cbor.MajorType import MajorType
from cbor.State import State


class ArrayInfo(State):

    def run(self, stream: CBORStream, handler):
        info = stream.read(1)
        length = ord(info) & 0b00011111
        handler('[')
        if length < 24:
            return [MajorType(), ArrayRead(length)]
        elif length == 24:
            return [ArrayLen(1)]
        elif length == 25:
            return [ArrayLen(2)]
        elif length == 26:
            return [ArrayLen(4)]
        elif length == 27:
            return [ArrayLen(8)]
        elif length == 31:
            return [MajorType(), ArrayInf()]
        return []


class ArrayRead(State):

    def __init__(self, n: int):
        self.n = n

    def run(self, stream: CBORStream, handler):
        if self.n > 0:
            handler(',')
            return [MajorType(), ArrayRead(self.n - 1)]
        handler(']')
        return []


class ArrayLen(State):

    def __init__(self, n: int):
        self.n = n

    def run(self, stream: CBORStream, handler):
        info = stream.read(self.n)
        length = int.from_bytes(info, byteorder='big')
        return [MajorType(), ArrayRead(length)]


class ArrayInf(State):

    def run(self, stream: CBORStream, handler):
        return [MajorType(), ArrayInf()]<commit_msg>You can't handle all these commits<commit_after>from cbor.CBORStream import CBORStream
from cbor.MajorType import MajorType
from cbor.State import State


class ArrayInfo(State):

    def run(self, stream: CBORStream, handler):
        info = stream.read(1)
        length = ord(info) & 0b00011111
        handler('[')
        if length < 24:
            return [MajorType(), ArrayRead(length)]
        elif length == 24:
            return [ArrayLen(1)]
        elif length == 25:
            return [ArrayLen(2)]
        elif length == 26:
            return [ArrayLen(4)]
        elif length == 27:
            return [ArrayLen(8)]
        elif length == 31:
            return [MajorType(), ArrayInf()]
        return []


class ArrayRead(State):

    def __init__(self, n: int):
        self.n = n

    def run(self, stream: CBORStream, handler):
        if self.n > 0:
            handler(',')
            return [MajorType(), ArrayRead(self.n - 1)]
        handler(']')
        return []


class ArrayLen(State):

    def __init__(self, n: int):
        self.n = n

    def run(self, stream: CBORStream, handler):
        info = stream.read(self.n)
        length = int.from_bytes(info, byteorder='big')
        return [MajorType(), ArrayRead(length)]


class ArrayInf(State):

    def run(self, stream: CBORStream, handler):
        handler(',')
        return [MajorType(), ArrayInf()]
{'fraction_non_alphanumeric': 0.12527312454479242, 'fraction_numerical': 0.02039329934450109, 'mean_word_length': 2.1683967704728953, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1731219', 'n_tokens_mistral': 873, 'n_tokens_neox': 799, 'n_words': 229}