<commit_before>from pysnmp.proto import rfc1902
from pyasn1.type import univ
from pyasn1.codec.ber import encoder
from pyasn1.compat.octets import octs2str
from snmpsim.grammar import abstract
from snmpsim import error

class WalkGrammar(abstract.AbstractGrammar):
    # case-insensitive keys as snmpwalk output tend to vary
    tagMap = {
        'OID:': rfc1902.ObjectName,
        'INTEGER:': rfc1902.Integer,
        'STRING:': rfc1902.OctetString,
        'BITS:': rfc1902.Bits,
        'HEX-STRING:': rfc1902.OctetString,
        'GAUGE32:': rfc1902.Gauge32,
        'COUNTER32:': rfc1902.Counter32,
        'COUNTER64:': rfc1902.Counter64,
        'IPADDRESS:': rfc1902.IpAddress,
        'OPAQUE:': rfc1902.Opaque,
        'UNSIGNED32:': rfc1902.Unsigned32,  # this is not needed
        'TIMETICKS:': rfc1902.TimeTicks     # this is made up
    }

    # possible DISPLAY-HINTs parsing should occur here
    def __stringFilter(value):
        if not value:
            return value
        elif value[0] == value[-1] == '"':
            return value[1:-1]
        elif value.find(':') > 0:
            for x in value.split(':'):
                for y in x:
                    if y not in '0123456789ABCDEFabcdef':
                        return value
            return [ int(x, 16) for x in value.split(':') ]
        else:
            return value

    def __opaqueFilter(value):
        if value.upper().startswith('FLOAT: '):
            return encoder.encode(univ.Real(float(value[7:])))
        else:
            return [int(y, 16) for y in value.split(' ')]

    def __bitsFilter(value):
        return ''.join(['%.2x' % int(y, 16) for y in value.split(' ')])

    def __hexStringFilter(value):
        return ''.join([chr(int(y, 16)) for y in value.split(' ')])<commit_msg>BITS and HEX-STRING filters made Py3k compatible<commit_after>from pysnmp.proto import rfc1902
from pyasn1.type import univ
from pyasn1.codec.ber import encoder
from pyasn1.compat.octets import octs2str, ints2octs
from snmpsim.grammar import abstract
from snmpsim import error

class WalkGrammar(abstract.AbstractGrammar):
    # case-insensitive keys as snmpwalk output tend to vary
    tagMap = {
        'OID:': rfc1902.ObjectName,
        'INTEGER:': rfc1902.Integer,
        'STRING:': rfc1902.OctetString,
        'BITS:': rfc1902.Bits,
        'HEX-STRING:': rfc1902.OctetString,
        'GAUGE32:': rfc1902.Gauge32,
        'COUNTER32:': rfc1902.Counter32,
        'COUNTER64:': rfc1902.Counter64,
        'IPADDRESS:': rfc1902.IpAddress,
        'OPAQUE:': rfc1902.Opaque,
        'UNSIGNED32:': rfc1902.Unsigned32,  # this is not needed
        'TIMETICKS:': rfc1902.TimeTicks     # this is made up
    }

    # possible DISPLAY-HINTs parsing should occur here
    def __stringFilter(value):
        if not value:
            return value
        elif value[0] == value[-1] == '"':
            return value[1:-1]
        elif value.find(':') > 0:
            for x in value.split(':'):
                for y in x:
                    if y not in '0123456789ABCDEFabcdef':
                        return value
            return [ int(x, 16) for x in value.split(':') ]
        else:
            return value

    def __opaqueFilter(value):
        if value.upper().startswith('FLOAT: '):
            return encoder.encode(univ.Real(float(value[7:])))
        else:
            return [int(y, 16) for y in value.split(' ')]

    def __bitsFilter(value):
        return ints2octs([int(y, 16) for y in value.split(' ')])

    def __hexStringFilter(value):
        return [int(y, 16) for y in value.split(' ')]
{'fraction_non_alphanumeric': 0.12122052704576976, 'fraction_numerical': 0.05436893203883495, 'mean_word_length': 2.2841530054644807, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1424977', 'n_tokens_mistral': 1324, 'n_tokens_neox': 1197, 'n_words': 298}