<commit_before>#!/bin/python
import sys
import binascii
import struct
import pprint
import opcodes

def addLineNums(codeatt, linenumatt):
    pc2ln = {}
    for lnt in linenumatt['04_line_number_table']:
        pc2ln[lnt['01_start_pc']] = '%4d' % lnt['02_line_number']
    
    codes = []
    for line in codeatt['06_code']:
        pc, ln, code = line.split(':')
        ln = pc2ln.get(int(pc), ln)
        codes.append('%s:%s:%s' % (pc, ln, code)) # replace '?:' with linenum
    codeatt['06_code'] = codes

def parseAttribute(att, info, cf):
    if att['00_attribute_name'] == 'Code': #Code
        att['03_max_stack'] = struct.unpack('>h', info[:2])[0]
        att['04_max_locals'] = struct.unpack('>h', info[2:4])[0]
        att['05_code_length'] = struct.unpack('>i', info[4:8])[0]
        att['06_code'] = []
        i = 8
        while i < 8 + att['05_code_length']:
            opsize = 0
            mnem, fmt, plen, add_comment = opcodes.code2mnem[ord(info[i])]
            opsize += 1
            if mnem == 'lookupswitch':
                while (i + opsize) % 4 != 0: opsize += 1 # skip to 4-byte boundary
                args = struct.unpack('>2i', info[i+opsize:i+opsize+8])
                # now get pairs
                args += struct.unpack('>%di' % (args[1]*2), info[i+opsize+8:i+opsize+8+(args[1]*8)])
                opsize += 8 + (args[1] * 8)
            elif mnem == 'tableswitch':
                while (i + opsize) % 4 != 0: opsize += 1 # skip to 4-byte boundary
                args = struct.unpack('>3i', info[i+opsize:i+opsize+12])
                offsets = args[2] - args[1] + 1
                args += struct.unpack('>%di' % offsets, info[i+opsize+12:i+opsize+12+(offsets*4)])
                opsize += 12 + (offsets*4)
            elif mnem == 'wide':
                args = struct.unpack('>Bh', info[i+1:i+4])
                opsize += 3
                if opcodes.code2mnem[args[0]][0] == 'iinc':
                    args += struct.unpack('>h', info[i+4:i+6])
                    opsize += 2
            else:
                args = struct.unpack(fmt, info[i+1:i+1+plen])
                opsize += plen
            code = '%4d:     ?:%s' % (i-8, mnem)
            for arg in args:
                code += ' ' + str(arg)
            att['06_code'].append(add_comment(cf, code, args))
            i += opsize
        
        att['07_exception_table_length'] = struct.unpack('>h', info[i:i+2])[0]
        i += 2
        att['08_exception_table'] = []
        for x in range(att['07_exception_table_length']):
            et = {}
            et['01_start_pc'] = struct.unpack('>h', info[i:i+2])[0]
            et['02_end_pc'] = struct.unpack('>h', info[i+2:i+4])[0]
            et['03_handler_pc'] = struct.unpack('>h', info[i+4:i+6])[0]
            et['04_catch_type'] = struct.unpack('>h', info[i+6:i+8])[0]
            i += 8
            att['08_exception_table'].append(et)
        att['09_attributes_count'] = struct.unpack('>h', info[i:i+2])[0]
        i += 2
        att['10_attribute_info'] = []
        
        for x in range(att['09_attributes_count']):
            attai = {}
            attai['01_attribute_name_index'] = struct.unpack('>h', info[i:i+2])[0]
            attai['00_attribute_name'] = cf['05_cp_info'][attai['01_attribute_name_index']]['03_bytes']
            attai['02_attribute_length'] = struct.unpack('>i', info[i+2:i+6])[0]
            parseAttribute(attai, struct.unpack('%ds' % attai['02_attribute_length'], info[i+6:i+6+attai['02_attribute_length']])[0], cf)
            i += 6+attai['02_attribute_length']
            att['10_attribute_info'].append(attai)
            if attai['00_attribute_name'] == 'LineNumberTable':
                addLineNums(att, attai)
    elif att['00_attribute_name'] == 'LineNumberTable': # LineNumberTable
        att['03_line_number_table_length'] = struct.unpack('>h', info[:2])[0]
        i = 2
        att['04_line_number_table'] = []
        for x in range(att['03_line_number_table_length']):
            lnt = {}
            lnt['01_start_pc'] = struct.unpack('>h', info[i:i+2])[0]
            lnt['02_line_number'] = struct.unpack('>h', info[i+2:i+4])[0]
            i += 4
            att['04_line_number_table'].append(lnt)
    elif att['00_attribute_name'] == 'LocalVariableTable': # LocalVariableTable
        att['03_local_variable_table_length'] = struct.unpack('>h', info[:2])[0]
        i = 2
        att['04_local_variable_table'] = []
        for x in range(att['03_local_variable_table_length']):
            lvt = {}
            lvt['01_start_pc'] = struct.unpack('>h', info[i:i+2])[0]
            lvt['02_length'] = struct.unpack('>h', info[i+2:i+4])[0]
            lvt['03_name_index'] = struct.unpack('>h', info[i+4:i+6])[0]
            lvt['04_descriptor_index'] = struct.unpack('>h', info[i+6:i+8])[0]
            lvt['05_index'] = struct.unpack('>h', info[i+8:i+10])[0]
            lvt['00_name'] = cf['05_cp_info'][lvt['04_descriptor_index']]['03_bytes'] + ' ' + cf['05_cp_info'][lvt['03_name_index']]['03_bytes']
            i += 10
            att['04_local_variable_table'].append(lvt)
    elif att['00_attribute_name'] == 'Exceptions': # Exceptions
        att['03_number_of_exceptions'] = struct.unpack('>h', info[:2])[0]
        i = 2
        att['04_exception_index_table'] = []
        for x in range(att['03_number_of_exceptions']):
            eit = {}
            eit['01_exception_index'] = struct.unpack('>h', info[i:i+2])[0]
            eit['00_exception_name'] = cf['05_cp_info'][cf['05_cp_info'][eit['01_exception_index']]['02_name_index']]['03_bytes']
            i += 2
            att['04_exception_index_table'].append(eit)
    elif att['00_attribute_name'] == 'ConstantValue': # ConstantValue
        att['01_constantvalue_index'] = struct.unpack('>h', info[:2])[0]
        att['00_attribute_value'] = cf['05_cp_info'][att['01_constantvalue_index']]['00_value']
    elif att['00_attribute_name'] == 'SourceFile': # SourceFile
        att['03_sourcefile_index'] = struct.unpack('>h', info[:2])[0]
        att['00_sourcefile'] = cf['05_cp_info'][att['03_sourcefile_index']]['03_bytes']
    elif att['00_attribute_name'] == 'Deprecated': # Deprecated - nothing to do
        pass
    elif att['00_attribute_name'] == 'RuntimeVisibleAnnotations': # RuntimeVisibleAnnotations
        att['03_info'] = info
        att['03_runtimevisibleannotations_num'] = struct.unpack('>h', info[:2])[0]
        i = 2
        att['04_runtimevisibleannotations_table'] = []
        for x in range(att['03_runtimevisibleannotations_num']):
            an = {}
            i += parseAnnotation(an, info[i:], cf)
            att['04_runtimevisibleannotations_table'].append(an)
    else:
        print '#unhandled attribute', att['00_attribute_name']
        att['03_info'] = info

def parseAnnotation(an, info, cf):
    an['01_type_index'] = struct.unpack('>h', info[:2])[0]
    an['00_type'] = cf['05_cp_info'][an['01_type_index']]['03_bytes']
    an['02_num_element_value_pairs'] = struct.unpack('>h', info[2:4])[0]
    an['03_element_value_pairs'] = []
    i = 4
    for x in range(an['02_num_element_value_pairs']):
        el = {}
        el['01_element_name_index'] = struct.unpack('>h', info[i:i+2])[0]
        el['00_name'] = cf['05_cp_info'][el['01_element_name_index']]['03_bytes']
        el['02_tag'] = info[i+2]
        el['03_element_value'] = {}
        i += 2 + parseElementValue(el['03_element_value'], info[i+2:], cf)
        an['03_element_value_pairs'].append(el)
    return i

def parseElementValue(ev, info, cf):
        ev['01_tag'] = info[0]
        if ev['01_tag'] in 'BCDFIJSZs':
            ev['02_element_value_index'] = struct.unpack('>h', info[1:3])[0]
            ev['02_element_value'] = cf['05_cp_info'][ev['02_element_value_index']]['03_bytes']
            return 3
        elif ev['01_tag'] == 'e':
            ev['02_type_name_index'] = struct.unpack('>h', info[:2])[0]
            ev['02_type_name'] = cf['05_cp_info'][ev['02_type_name_index']]['03_bytes']
            ev['02_const_name_index'] = struct.unpack('>h', info[2:4])[0]
            ev['02_const_name'] = cf['05_cp_info'][ev['02_const_name_index']]['03_bytes']
            return 5
        elif ev['01_tag'] == 'c':
            ev['02_class_info_index'] = struct.unpack('>h', info[:2])[0]
            ev['02_class_info'] = cf['05_cp_info'][ev['02_class_info_index']]['03_bytes']
            return 3
        elif ev['01_tag'] == '@':
            ev['02_annotation'] = {}
            i += parseAnnotation(el['02_annotation'], info[1:], cf)
        elif ev['02_tag'] == '[':
            ev['02_num_values'] = struct.unpack('>h', info[1:3])[0]
            ev['03_values'] = []
            i += 3
            for x in range():
                ev2 = {}
                i += parseElementValue(ev2, info[3:], cf)
                ev['03_values'].append(ev2)
            return i
        else:
            raise '#unknown element and value tag %s' % el['02_tag']
    
def writeAttribute(att, linenum_cp_index=None):
    result = ''
    if att['00_attribute_name'] == 'Code': # Code
        result += struct.pack('>h', att['03_max_stack'])
        result += struct.pack('>h', att['04_max_locals'])
        code = writeCode(att, linenum_cp_index)
        result += struct.pack('>i', len(code))
        result += code
        result += struct.pack('>h', att['07_exception_table_length'])
        for et in att['08_exception_table']:
            result += struct.pack('>h', et['01_start_pc'])
            result += struct.pack('>h', et['02_end_pc'])
            result += struct.pack('>h', et['03_handler_pc'])
            result += struct.pack('>h', et['04_catch_type'])
        result += struct.pack('>h', len(att['10_attribute_info']))
        
        for attai in att['10_attribute_info']:
            result += writeAttribute(attai, linenum_cp_index)
    elif att['00_attribute_name'] == 'LineNumberTable': # LineNumberTable
        result += struct.pack('>h', len(att['04_line_number_table']))
        for lnt in att['04_line_number_table']:
            result += struct.pack('>h', lnt['01_start_pc'])<commit_msg>better exception table length read<commit_after>#!/bin/python
import sys
import binascii
import struct
import pprint
import opcodes

def addLineNums(codeatt, linenumatt):
    pc2ln = {}
    for lnt in linenumatt['04_line_number_table']:
        pc2ln[lnt['01_start_pc']] = '%4d' % lnt['02_line_number']
    
    codes = []
    for line in codeatt['06_code']:
        pc, ln, code = line.split(':')
        ln = pc2ln.get(int(pc), ln)
        codes.append('%s:%s:%s' % (pc, ln, code)) # replace '?:' with linenum
    codeatt['06_code'] = codes

def parseAttribute(att, info, cf):
    if att['00_attribute_name'] == 'Code': #Code
        att['03_max_stack'] = struct.unpack('>h', info[:2])[0]
        att['04_max_locals'] = struct.unpack('>h', info[2:4])[0]
        att['05_code_length'] = struct.unpack('>i', info[4:8])[0]
        att['06_code'] = []
        i = 8
        while i < 8 + att['05_code_length']:
            opsize = 0
            mnem, fmt, plen, add_comment = opcodes.code2mnem[ord(info[i])]
            opsize += 1
            if mnem == 'lookupswitch':
                while (i + opsize) % 4 != 0: opsize += 1 # skip to 4-byte boundary
                args = struct.unpack('>2i', info[i+opsize:i+opsize+8])
                # now get pairs
                args += struct.unpack('>%di' % (args[1]*2), info[i+opsize+8:i+opsize+8+(args[1]*8)])
                opsize += 8 + (args[1] * 8)
            elif mnem == 'tableswitch':
                while (i + opsize) % 4 != 0: opsize += 1 # skip to 4-byte boundary
                args = struct.unpack('>3i', info[i+opsize:i+opsize+12])
                offsets = args[2] - args[1] + 1
                args += struct.unpack('>%di' % offsets, info[i+opsize+12:i+opsize+12+(offsets*4)])
                opsize += 12 + (offsets*4)
            elif mnem == 'wide':
                args = struct.unpack('>Bh', info[i+1:i+4])
                opsize += 3
                if opcodes.code2mnem[args[0]][0] == 'iinc':
                    args += struct.unpack('>h', info[i+4:i+6])
                    opsize += 2
            else:
                args = struct.unpack(fmt, info[i+1:i+1+plen])
                opsize += plen
            code = '%4d:     ?:%s' % (i-8, mnem)
            for arg in args:
                code += ' ' + str(arg)
            att['06_code'].append(add_comment(cf, code, args))
            i += opsize
        
        att['07_exception_table_length'] = struct.unpack('>h', info[i:i+2])[0]
        i += 2
        att['08_exception_table'] = []
        for x in range(att['07_exception_table_length']):
            et = {}
            et['01_start_pc'] = struct.unpack('>h', info[i:i+2])[0]
            et['02_end_pc'] = struct.unpack('>h', info[i+2:i+4])[0]
            et['03_handler_pc'] = struct.unpack('>h', info[i+4:i+6])[0]
            et['04_catch_type'] = struct.unpack('>h', info[i+6:i+8])[0]
            i += 8
            att['08_exception_table'].append(et)
        att['09_attributes_count'] = struct.unpack('>h', info[i:i+2])[0]
        i += 2
        att['10_attribute_info'] = []
        
        for x in range(att['09_attributes_count']):
            attai = {}
            attai['01_attribute_name_index'] = struct.unpack('>h', info[i:i+2])[0]
            attai['00_attribute_name'] = cf['05_cp_info'][attai['01_attribute_name_index']]['03_bytes']
            attai['02_attribute_length'] = struct.unpack('>i', info[i+2:i+6])[0]
            parseAttribute(attai, struct.unpack('%ds' % attai['02_attribute_length'], info[i+6:i+6+attai['02_attribute_length']])[0], cf)
            i += 6+attai['02_attribute_length']
            att['10_attribute_info'].append(attai)
            if attai['00_attribute_name'] == 'LineNumberTable':
                addLineNums(att, attai)
    elif att['00_attribute_name'] == 'LineNumberTable': # LineNumberTable
        att['03_line_number_table_length'] = struct.unpack('>h', info[:2])[0]
        i = 2
        att['04_line_number_table'] = []
        for x in range(att['03_line_number_table_length']):
            lnt = {}
            lnt['01_start_pc'] = struct.unpack('>h', info[i:i+2])[0]
            lnt['02_line_number'] = struct.unpack('>h', info[i+2:i+4])[0]
            i += 4
            att['04_line_number_table'].append(lnt)
    elif att['00_attribute_name'] == 'LocalVariableTable': # LocalVariableTable
        att['03_local_variable_table_length'] = struct.unpack('>h', info[:2])[0]
        i = 2
        att['04_local_variable_table'] = []
        for x in range(att['03_local_variable_table_length']):
            lvt = {}
            lvt['01_start_pc'] = struct.unpack('>h', info[i:i+2])[0]
            lvt['02_length'] = struct.unpack('>h', info[i+2:i+4])[0]
            lvt['03_name_index'] = struct.unpack('>h', info[i+4:i+6])[0]
            lvt['04_descriptor_index'] = struct.unpack('>h', info[i+6:i+8])[0]
            lvt['05_index'] = struct.unpack('>h', info[i+8:i+10])[0]
            lvt['00_name'] = cf['05_cp_info'][lvt['04_descriptor_index']]['03_bytes'] + ' ' + cf['05_cp_info'][lvt['03_name_index']]['03_bytes']
            i += 10
            att['04_local_variable_table'].append(lvt)
    elif att['00_attribute_name'] == 'Exceptions': # Exceptions
        att['03_number_of_exceptions'] = struct.unpack('>h', info[:2])[0]
        i = 2
        att['04_exception_index_table'] = []
        for x in range(att['03_number_of_exceptions']):
            eit = {}
            eit['01_exception_index'] = struct.unpack('>h', info[i:i+2])[0]
            eit['00_exception_name'] = cf['05_cp_info'][cf['05_cp_info'][eit['01_exception_index']]['02_name_index']]['03_bytes']
            i += 2
            att['04_exception_index_table'].append(eit)
    elif att['00_attribute_name'] == 'ConstantValue': # ConstantValue
        att['01_constantvalue_index'] = struct.unpack('>h', info[:2])[0]
        att['00_attribute_value'] = cf['05_cp_info'][att['01_constantvalue_index']]['00_value']
    elif att['00_attribute_name'] == 'SourceFile': # SourceFile
        att['03_sourcefile_index'] = struct.unpack('>h', info[:2])[0]
        att['00_sourcefile'] = cf['05_cp_info'][att['03_sourcefile_index']]['03_bytes']
    elif att['00_attribute_name'] == 'Deprecated': # Deprecated - nothing to do
        pass
    elif att['00_attribute_name'] == 'RuntimeVisibleAnnotations': # RuntimeVisibleAnnotations
        att['03_info'] = info
        att['03_runtimevisibleannotations_num'] = struct.unpack('>h', info[:2])[0]
        i = 2
        att['04_runtimevisibleannotations_table'] = []
        for x in range(att['03_runtimevisibleannotations_num']):
            an = {}
            i += parseAnnotation(an, info[i:], cf)
            att['04_runtimevisibleannotations_table'].append(an)
    else:
        print '#unhandled attribute', att['00_attribute_name']
        att['03_info'] = info

def parseAnnotation(an, info, cf):
    an['01_type_index'] = struct.unpack('>h', info[:2])[0]
    an['00_type'] = cf['05_cp_info'][an['01_type_index']]['03_bytes']
    an['02_num_element_value_pairs'] = struct.unpack('>h', info[2:4])[0]
    an['03_element_value_pairs'] = []
    i = 4
    for x in range(an['02_num_element_value_pairs']):
        el = {}
        el['01_element_name_index'] = struct.unpack('>h', info[i:i+2])[0]
        el['00_name'] = cf['05_cp_info'][el['01_element_name_index']]['03_bytes']
        el['02_tag'] = info[i+2]
        el['03_element_value'] = {}
        i += 2 + parseElementValue(el['03_element_value'], info[i+2:], cf)
        an['03_element_value_pairs'].append(el)
    return i

def parseElementValue(ev, info, cf):
        ev['01_tag'] = info[0]
        if ev['01_tag'] in 'BCDFIJSZs':
            ev['02_element_value_index'] = struct.unpack('>h', info[1:3])[0]
            ev['02_element_value'] = cf['05_cp_info'][ev['02_element_value_index']]['03_bytes']
            return 3
        elif ev['01_tag'] == 'e':
            ev['02_type_name_index'] = struct.unpack('>h', info[:2])[0]
            ev['02_type_name'] = cf['05_cp_info'][ev['02_type_name_index']]['03_bytes']
            ev['02_const_name_index'] = struct.unpack('>h', info[2:4])[0]
            ev['02_const_name'] = cf['05_cp_info'][ev['02_const_name_index']]['03_bytes']
            return 5
        elif ev['01_tag'] == 'c':
            ev['02_class_info_index'] = struct.unpack('>h', info[:2])[0]
            ev['02_class_info'] = cf['05_cp_info'][ev['02_class_info_index']]['03_bytes']
            return 3
        elif ev['01_tag'] == '@':
            ev['02_annotation'] = {}
            i += parseAnnotation(el['02_annotation'], info[1:], cf)
        elif ev['02_tag'] == '[':
            ev['02_num_values'] = struct.unpack('>h', info[1:3])[0]
            ev['03_values'] = []
            i += 3
            for x in range():
                ev2 = {}
                i += parseElementValue(ev2, info[3:], cf)
                ev['03_values'].append(ev2)
            return i
        else:
            raise '#unknown element and value tag %s' % el['02_tag']
    
def writeAttribute(att, linenum_cp_index=None):
    result = ''
    if att['00_attribute_name'] == 'Code': # Code
        result += struct.pack('>h', att['03_max_stack'])
        result += struct.pack('>h', att['04_max_locals'])
        code = writeCode(att, linenum_cp_index)
        result += struct.pack('>i', len(code))
        result += code
        result += struct.pack('>h', len(att['08_exception_table']))
        for et in att['08_exception_table']:
            result += struct.pack('>h', et['01_start_pc'])
            result += struct.pack('>h', et['02_end_pc'])
            result += struct.pack('>h', et['03_handler_pc'])
            result += struct.pack('>h', et['04_catch_type'])
        result += struct.pack('>h', len(att['10_attribute_info']))
        
        for attai in att['10_attribute_info']:
            result += writeAttribute(attai, linenum_cp_index)
    elif att['00_attribute_name'] == 'LineNumberTable': # LineNumberTable
        result += struct.pack('>h', len(att['04_line_number_table']))
        for lnt in att['04_line_number_table']:
            result += struct.pack('>h', lnt['01_start_pc'])
{'fraction_non_alphanumeric': 0.18388990282641937, 'fraction_numerical': 0.0469590095200513, 'mean_word_length': 2.6922236386814786, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 103, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '856540', 'n_tokens_mistral': 7540, 'n_tokens_neox': 7104, 'n_words': 1093}