<commit_before>import collections
import struct

varint_types = [
  'int32',
  'int64',
  'uint32',
  'uint64',
  'sint32',
  'sint64',
  'bool',
  'enum',
]
_64bit_types = ['fixed64', 'sfixed64', 'double']
delimited_types = ['string', 'bytes', 'embedded', 'packed']
_32bit_types = ['fixed32', 'sfixed32', 'float']
struct_formats = dict([
    ('fixed64', '<Q'),
    ('sfixed64', '<q'),
    ('double', '<d'),
    ('fixed32', '<I'),
    ('sfixed32', '<i'),
    ('float', '<f'),
])
joinstr = ''.join


def decode(proto, data):
    data = [ord(d) for d in data]
    return _decode(proto, data)


def _decode(proto, data):
    index = 0
    length = len(data)
    msg = collections.defaultdict(list)
    while index < length:
        # optimization: avoid function calls to decode_varint, decode_key
        item = 128
        key = 0
        left = 0
        while item & 128:
            item = data[index]
            index += 1
            value = (item & 127) << left
            key += value
            left += 7
        field = key >> 3
        wire = key & 7
        # end optimization
        if field not in proto:
            continue
        info = proto[field]
        _type = info['type']
        repeated = info.get('scope') == 'repeated'
        name = info['name']
        if wire == 0:
            # TODO support int32, int64, uint32, sint32, sint64
            # optimization: avoid function call to decode_varint
            item = 128
            num = 0
            left = 0
            while item & 128:
                item = data[index]
                index += 1
                value = (item & 127) << left
                num += value
                left += 7
            # end optimization
            if _type == 'bool':
                num = bool(num)
            msg[name].append(num)
            if not repeated:
                msg[name] = num
            continue
        if wire == 1:
            fmt = struct_formats[_type]
            # optimization: avoid function call to decode_struct
            last = index
            index += 8
            values = [chr(i) for i in data[last:index]]
            values = joinstr(values)
            num = struct.unpack(fmt, values)
            # end optimization
            msg[name].append(num[0])
            if not repeated:
                msg[name] = num[0]
            continue
        if wire == 2:
            # TODO support bytes and packed repeated fields
            # optimization: avoid function call to decode_delimited
            item = 128
            _length = 0
            left = 0
            while item & 128:
                item = data[index]
                index += 1
                value = (item & 127) << left
                _length += value
                left += 7
            last = index
            index += _length
            item = data[last:index]
            # end optimization
            if _type == 'embedded':
                item = _decode(info['message'], item)
            if _type == 'string':
                item = joinstr([chr(i) for i in item])
            msg[name].append(item)
            if not repeated:
                msg[name] = item
            continue
        if wire == 5:
            fmt = struct_formats[_type]
            # optimization: avoid function call to decode_struct
            last = index
            index += 4
            values = [chr(i) for i in data[last:index]]
            values = ''.join(values)
            num = struct.unpack(fmt, values)
            # end optimization
            msg[name].append(num[0])
            if not repeated:
                msg[name] = num[0]
            continue
        raise ValueError(
          'invalid wire type: {wire}'.format(wire=wire)
        )
    return msg


def encode(proto, msg):
    data = _encode(proto, msg)
    data = [chr(d) for d in data]
    return  ''.join(data)


def _encode(proto, msg):
    data =  []
    for k, values in msg.items():
        info = proto[k]
        field = info['field']
        _type = info['type']
        repeated = info.get('scope') == 'repeated'
        if _type in varint_types:
            if repeated:<commit_msg>Do not use defaultdict in encoder._decode<commit_after>import struct

varint_types = [
  'int32',
  'int64',
  'uint32',
  'uint64',
  'sint32',
  'sint64',
  'bool',
  'enum',
]
_64bit_types = ['fixed64', 'sfixed64', 'double']
delimited_types = ['string', 'bytes', 'embedded', 'packed']
_32bit_types = ['fixed32', 'sfixed32', 'float']
struct_formats = dict([
    ('fixed64', '<Q'),
    ('sfixed64', '<q'),
    ('double', '<d'),
    ('fixed32', '<I'),
    ('sfixed32', '<i'),
    ('float', '<f'),
])
joinstr = ''.join


def decode(proto, data):
    data = [ord(d) for d in data]
    return _decode(proto, data)


def _decode(proto, data):
    index = 0
    length = len(data)
    msg = dict()
    while index < length:
        # optimization: avoid function calls to decode_varint, decode_key
        item = 128
        key = 0
        left = 0
        while item & 128:
            item = data[index]
            index += 1
            value = (item & 127) << left
            key += value
            left += 7
        field = key >> 3
        wire = key & 7
        # end optimization
        if field not in proto:
            continue
        info = proto[field]
        _type = info['type']
        repeated = info.get('scope') == 'repeated'
        name = info['name']
        if repeated and name not in msg:
            msg[name] = list()
        if wire == 0:
            # TODO support int32, int64, uint32, sint32, sint64
            # optimization: avoid function call to decode_varint
            item = 128
            num = 0
            left = 0
            while item & 128:
                item = data[index]
                index += 1
                value = (item & 127) << left
                num += value
                left += 7
            # end optimization
            if _type == 'bool':
                num = bool(num)
            if repeated:
                msg[name].append(num)
                continue
            msg[name] = num
            continue
        if wire == 1:
            fmt = struct_formats[_type]
            # optimization: avoid function call to decode_struct
            last = index
            index += 8
            values = [chr(i) for i in data[last:index]]
            values = joinstr(values)
            num = struct.unpack(fmt, values)
            # end optimization
            if repeated:
                msg[name].append(num[0])
                continue
            msg[name] = num[0]
            continue
        if wire == 2:
            # TODO support bytes and packed repeated fields
            # optimization: avoid function call to decode_delimited
            item = 128
            _length = 0
            left = 0
            while item & 128:
                item = data[index]
                index += 1
                value = (item & 127) << left
                _length += value
                left += 7
            last = index
            index += _length
            item = data[last:index]
            # end optimization
            if _type == 'embedded':
                item = _decode(info['message'], item)
                if repeated:
                    msg[name].append(item)
                    continue
                msg[name] = item
            if _type == 'string':
                item = joinstr([chr(i) for i in item])
                if repeated:
                    msg[name].append(item)
                    continue
                msg[name] = item
            continue
        if wire == 5:
            fmt = struct_formats[_type]
            # optimization: avoid function call to decode_struct
            last = index
            index += 4
            values = [chr(i) for i in data[last:index]]
            values = ''.join(values)
            num = struct.unpack(fmt, values)
            # end optimization
            if repeated:
                msg[name].append(num[0])
                continue
            msg[name] = num[0]
            continue
        raise ValueError(
          'invalid wire type: {wire}'.format(wire=wire)
        )
    return msg


def encode(proto, msg):
    data = _encode(proto, msg)
    data = [chr(d) for d in data]
    return  ''.join(data)


def _encode(proto, msg):
    data =  []
    for k, values in msg.items():
        info = proto[k]
        field = info['field']
        _type = info['type']
        repeated = info.get('scope') == 'repeated'
        if _type in varint_types:
            if repeated:
{'fraction_non_alphanumeric': 0.11194895591647332, 'fraction_numerical': 0.021809744779582366, 'mean_word_length': 1.3933925596890617, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7517179', 'n_tokens_mistral': 2533, 'n_tokens_neox': 2335, 'n_words': 719}