<commit_before># BER decoder
import types
from pyasn1.type import tag, univ, char, useful
from pyasn1.codec.ber import eoo
from pyasn1 import error

class AbstractDecoder:
    protoComponent = None
    def valueDecoder(self, substrate, asn1Spec, tagSet,
                     length, state, decodeFun):
        raise error.PyAsn1Error('Decoder not implemented for %s' % tagSet)

    def indefLenValueDecoder(self, substrate, asn1Spec, tagSet,
                     length, state, decodeFun):
        raise error.PyAsn1Error('Indefinite length mode decoder not implemented for %s' % tagSet)

class AbstractSimpleDecoder(AbstractDecoder):
    def _createComponent(self, asn1Spec, tagSet, value=None):
        if asn1Spec is None:
            return self.protoComponent.clone(value, tagSet)
        else:
            return asn1Spec.clone(value)
        
class AbstractConstructedDecoder(AbstractDecoder):
    def _createComponent(self, asn1Spec, tagSet, value=None):
        if asn1Spec is None:
            return self.protoComponent.clone(tagSet)
        else:
            return asn1Spec.clone()
                                
class EndOfOctetsDecoder(AbstractSimpleDecoder):
    def valueDecoder(self, substrate, asn1Spec, tagSet,
                     length, state, decodeFun):
        return eoo.endOfOctets, substrate

class IntegerDecoder(AbstractSimpleDecoder):
    protoComponent = univ.Integer(0)
    precomputedValues = {
        '\x00': 0,
        '\x01': 1,
        '\x02': 2,
        '\x03': 3,
        '\x04': 4,
        '\x05': 5,
        '\x06': 6,
        '\x07': 7,
        '\x08': 8,
        '\x09': 9,
        'xff': -1,
        'xfe': -2,
        'xfd': -3,
        'xfc': -4,
        'xfb': -5
        }
    
    def _valueFilter(self, value):
        try:
            return int(value)
        except OverflowError:
            return value
        
    def valueDecoder(self, substrate, asn1Spec, tagSet, length,
                     state, decodeFun):
        if not substrate:
            raise error.PyAsn1Error('Empty substrate')
        if substrate in self.precomputedValues:
            value = self.precomputedValues[substrate]
        else:
            firstOctet = ord(substrate[0])
            if firstOctet & 0x80:
                value = -1L
            else:
                value = 0L
            for octet in substrate:
                value = value << 8 | ord(octet)
            value = self._valueFilter(value)
            if value == -1:
                print repr(substrate)
        return self._createComponent(asn1Spec, tagSet, value), substrate

class BooleanDecoder(IntegerDecoder):
    protoComponent = univ.Boolean(0)
    def _valueFilter(self, value):
        if value:
            return 1
        else:
            return 0

class BitStringDecoder(AbstractSimpleDecoder):
    protoComponent = univ.BitString(())
    def valueDecoder(self, substrate, asn1Spec, tagSet, length,
                     state, decodeFun):
        if tagSet[0][1] == tag.tagFormatSimple:    # XXX what tag to check?
            if not substrate:
                raise error.PyAsn1Error('Missing initial octet')
            trailingBits = ord(substrate[0])
            if trailingBits > 7:
                raise error.PyAsn1Error(
                    'Trailing bits overflow %s' % trailingBits
                    )
            substrate = substrate[1:]
            lsb = p = 0; l = len(substrate)-1; b = []<commit_msg>stray print removed<commit_after># BER decoder
import types
from pyasn1.type import tag, univ, char, useful
from pyasn1.codec.ber import eoo
from pyasn1 import error

class AbstractDecoder:
    protoComponent = None
    def valueDecoder(self, substrate, asn1Spec, tagSet,
                     length, state, decodeFun):
        raise error.PyAsn1Error('Decoder not implemented for %s' % tagSet)

    def indefLenValueDecoder(self, substrate, asn1Spec, tagSet,
                     length, state, decodeFun):
        raise error.PyAsn1Error('Indefinite length mode decoder not implemented for %s' % tagSet)

class AbstractSimpleDecoder(AbstractDecoder):
    def _createComponent(self, asn1Spec, tagSet, value=None):
        if asn1Spec is None:
            return self.protoComponent.clone(value, tagSet)
        else:
            return asn1Spec.clone(value)
        
class AbstractConstructedDecoder(AbstractDecoder):
    def _createComponent(self, asn1Spec, tagSet, value=None):
        if asn1Spec is None:
            return self.protoComponent.clone(tagSet)
        else:
            return asn1Spec.clone()
                                
class EndOfOctetsDecoder(AbstractSimpleDecoder):
    def valueDecoder(self, substrate, asn1Spec, tagSet,
                     length, state, decodeFun):
        return eoo.endOfOctets, substrate

class IntegerDecoder(AbstractSimpleDecoder):
    protoComponent = univ.Integer(0)
    precomputedValues = {
        '\x00': 0,
        '\x01': 1,
        '\x02': 2,
        '\x03': 3,
        '\x04': 4,
        '\x05': 5,
        '\x06': 6,
        '\x07': 7,
        '\x08': 8,
        '\x09': 9,
        'xff': -1,
        'xfe': -2,
        'xfd': -3,
        'xfc': -4,
        'xfb': -5
        }
    
    def _valueFilter(self, value):
        try:
            return int(value)
        except OverflowError:
            return value
        
    def valueDecoder(self, substrate, asn1Spec, tagSet, length,
                     state, decodeFun):
        if not substrate:
            raise error.PyAsn1Error('Empty substrate')
        if substrate in self.precomputedValues:
            value = self.precomputedValues[substrate]
        else:
            firstOctet = ord(substrate[0])
            if firstOctet & 0x80:
                value = -1L
            else:
                value = 0L
            for octet in substrate:
                value = value << 8 | ord(octet)
            value = self._valueFilter(value)
        return self._createComponent(asn1Spec, tagSet, value), substrate

class BooleanDecoder(IntegerDecoder):
    protoComponent = univ.Boolean(0)
    def _valueFilter(self, value):
        if value:
            return 1
        else:
            return 0

class BitStringDecoder(AbstractSimpleDecoder):
    protoComponent = univ.BitString(())
    def valueDecoder(self, substrate, asn1Spec, tagSet, length,
                     state, decodeFun):
        if tagSet[0][1] == tag.tagFormatSimple:    # XXX what tag to check?
            if not substrate:
                raise error.PyAsn1Error('Missing initial octet')
            trailingBits = ord(substrate[0])
            if trailingBits > 7:
                raise error.PyAsn1Error(
                    'Trailing bits overflow %s' % trailingBits
                    )
            substrate = substrate[1:]
            lsb = p = 0; l = len(substrate)-1; b = []
{'fraction_non_alphanumeric': 0.09306409130816505, 'fraction_numerical': 0.02151009657594381, 'mean_word_length': 1.8839662447257384, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '636153', 'n_tokens_mistral': 2040, 'n_tokens_neox': 1923, 'n_words': 517}