<commit_before># BER encoder
import string
from pyasn1.type import base, tag, univ, char, useful
from pyasn1.codec.ber import eoo
from pyasn1 import error

class Error(Exception): pass

class AbstractItemEncoder:
    supportIndefLenMode = 1
    def encodeTag(self, t, isConstructed):
        v = t[0]|t[1]
        if isConstructed:
            v = v|tag.tagFormatConstructed
        if t[2] < 31:
            return chr(v|t[2])
        else:
            longTag = t[2]
            s = chr(longTag&0x7f)
            longTag = longTag >> 7
            while longTag:
                s = chr(0x80|(longTag&0x7f)) + s
                longTag = longTag >> 7
            return chr(v|0x1F) + s

    def encodeLength(self, length, defMode):
        if not defMode and self.supportIndefLenMode:
            return '\x80'
        if length < 0x80:
            return chr(length)
        else:
            substrate = ''
            while length:
                substrate = chr(length&0xff) + substrate
                length = length >> 8
            if len(substrate) > 126:
                raise Error('Length octets overflow (%d)' % len(substrate))
            return chr(0x80 | len(substrate)) + substrate

    def encodeValue(self, encodeFun, value, defMode, maxChunkSize):
        raise Error('Not implemented')

    def _encodeEndOfOctets(self, encodeFun, defMode):
        if defMode or not self.supportIndefLenMode:
            return ''
        else:
            return encodeFun(eoo.endOfOctets, defMode)<commit_msg>cache substrate length at encodeLength() for better performance<commit_after># BER encoder
import string
from pyasn1.type import base, tag, univ, char, useful
from pyasn1.codec.ber import eoo
from pyasn1 import error

class Error(Exception): pass

class AbstractItemEncoder:
    supportIndefLenMode = 1
    def encodeTag(self, t, isConstructed):
        v = t[0]|t[1]
        if isConstructed:
            v = v|tag.tagFormatConstructed
        if t[2] < 31:
            return chr(v|t[2])
        else:
            longTag = t[2]
            s = chr(longTag&0x7f)
            longTag = longTag >> 7
            while longTag:
                s = chr(0x80|(longTag&0x7f)) + s
                longTag = longTag >> 7
            return chr(v|0x1F) + s

    def encodeLength(self, length, defMode):
        if not defMode and self.supportIndefLenMode:
            return '\x80'
        if length < 0x80:
            return chr(length)
        else:
            substrate = ''
            while length:
                substrate = chr(length&0xff) + substrate
                length = length >> 8
            substrateLen = len(substrate)
            if substrateLen > 126:
                raise Error('Length octets overflow (%d)' % substrateLen)
            return chr(0x80 | substrateLen) + substrate

    def encodeValue(self, encodeFun, value, defMode, maxChunkSize):
        raise Error('Not implemented')

    def _encodeEndOfOctets(self, encodeFun, defMode):
        if defMode or not self.supportIndefLenMode:
            return ''
        else:
            return encodeFun(eoo.endOfOctets, defMode)
{'fraction_non_alphanumeric': 0.08866837387964148, 'fraction_numerical': 0.02240717029449424, 'mean_word_length': 1.9342723004694835, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '895947', 'n_tokens_mistral': 968, 'n_tokens_neox': 912, 'n_words': 256}