<commit_before>#!/usr/bin/env python

# Generates code for encoding assembly instructions (and potentially decoding
# in the future for a disassembler). The format that is parsed matches the
# format used in the Intel architecture manual, read it for further details.

import pprint
import re
import sys
from collections import namedtuple

pp = pprint.PrettyPrinter(indent=4)

Instr = namedtuple('Instr', ['opcode', 'encodings'])
Encoding = namedtuple('Encoding',
        ['args', 'arg_order', 'rex_prefix', 'opcode_num', 'reg_and_rm', 'opcode_extension',
         'immediate_size'])

def generate_encoder(input_filename, output_filename):
    instrs = {}

    with open(input_filename, 'r') as f:
        for line in f.readlines():
            instr, encoding = line.rstrip('\n').split('=')

            instr_components = instr.replace(',', ' ').split()
            assert 1 <= len(instr_components) <= 3
            opcode = instr_components[0]
            args = instr_components[1:]

            if args == ['r64', 'r/m64']:
                arg_order = 'RM'
            elif args == ['r/m64', 'r64']:
                arg_order = 'MR'
            else:
                arg_order = 'INVALID'

            match = re.match(
                    r' *(?:REX\.([WRXB]+) *\+ *)?([0-9a-fA-F]+) *(/.)? *(ib)?',
                    encoding)
            groups = match.groups()
            rex_prefix = -1
            if groups[0]:
                rex_prefix = 0b01000000
                for c in groups[0]:
                    rex_prefix |= {
                        'W': 1 << 3,
                        'R': 1 << 2,
                        'X': 1 << 1,
                        'B': 1 << 0
                    }[c]

            opcode_num = int(groups[1], 16)

            slash = groups[2]
            if slash:
                slash = slash[1]

            if slash and ('0' <= slash <= '7'):
                opcode_extension = ord(slash) - ord('0')
            else:
                opcode_extension = -1

            reg_and_rm = slash == 'r'

            if groups[3]:
                immediate_size = 8
            else:
                immediate_size = -1

            encoding = Encoding(args, arg_order, rex_prefix, opcode_num,
                    reg_and_rm, opcode_extension, immediate_size)
            
            # @TODO: We should sort encodings by immediate size (ascending) so
            # that the smallest encoding gets selected automatically.
            if opcode not in instrs:
                instrs[opcode] = Instr(opcode, [])
            instrs[opcode].encodings.append(encoding)

    output = []
    output.append("""
// @NOTE: This is an automatically generated file! Do not edit it!
//        It was generated from '%s', edit that instead

static u32 assemble_instr(FILE *output_file, AsmInstr *instr)
{
\tswitch (instr->op) {
""" % input_filename)

    #pp.pprint(instrs)

    for opcode in instrs:
        output.append("\tcase %s:\n" % opcode)
        for encoding in instrs[opcode].encodings:
            if len(encoding.args) == 0:
                indent = '\t\t'
            else:
                output.append("\t\tif (%s)" % ' && '.join(
                    arg_condition(arg, i) for i, arg in enumerate(encoding.args)))
                indent = '\n\t\t\t'
            output.append("%sreturn encode_instr(output_file, instr, %s);\n"
                    % (indent,
                        ', '.join(map(to_c_val,
                            [encoding.arg_order, encoding.rex_prefix,
                            encoding.opcode_num, encoding.reg_and_rm,
                            encoding.opcode_extension, encoding.immediate_size]))))

        output.append("\t\tbreak;\n")

    output.append("""
\t}
\t
\tfputs("Unimplemented instruction:\\n", stderr);
\tdump_asm_instr(instr);
\t
\tUNIMPLEMENTED;
}
""")

    with open(output_filename, 'w') as f:
        f.writelines(output)

def arg_condition(arg, i):
    if arg == 'r/m64':
        return '((instr->args[%d].type == REGISTER) || (instr->args[%d].type == OFFSET_REGISTER))' % (i, i)
    if arg == 'r64':
        return '(instr->args[%d].type == REGISTER) && !instr->args[%d].is_deref' % (i, i)
    if arg == 'imm8':
        return '(is_const_and_fits(instr->args[%d], 8))' % i
<commit_msg>Avoid "unhandled case in switch" in generated code.<commit_after>#!/usr/bin/env python

# Generates code for encoding assembly instructions (and potentially decoding
# in the future for a disassembler). The format that is parsed matches the
# format used in the Intel architecture manual, read it for further details.

import pprint
import re
import sys
from collections import namedtuple

pp = pprint.PrettyPrinter(indent=4)

Instr = namedtuple('Instr', ['opcode', 'encodings'])
Encoding = namedtuple('Encoding',
        ['args', 'arg_order', 'rex_prefix', 'opcode_num', 'reg_and_rm', 'opcode_extension',
         'immediate_size'])

def generate_encoder(input_filename, output_filename):
    instrs = {}

    with open(input_filename, 'r') as f:
        for line in f.readlines():
            instr, encoding = line.rstrip('\n').split('=')

            instr_components = instr.replace(',', ' ').split()
            assert 1 <= len(instr_components) <= 3
            opcode = instr_components[0]
            args = instr_components[1:]

            if args == ['r64', 'r/m64']:
                arg_order = 'RM'
            elif args == ['r/m64', 'r64']:
                arg_order = 'MR'
            else:
                arg_order = 'INVALID'

            match = re.match(
                    r' *(?:REX\.([WRXB]+) *\+ *)?([0-9a-fA-F]+) *(/.)? *(ib)?',
                    encoding)
            groups = match.groups()
            rex_prefix = -1
            if groups[0]:
                rex_prefix = 0b01000000
                for c in groups[0]:
                    rex_prefix |= {
                        'W': 1 << 3,
                        'R': 1 << 2,
                        'X': 1 << 1,
                        'B': 1 << 0
                    }[c]

            opcode_num = int(groups[1], 16)

            slash = groups[2]
            if slash:
                slash = slash[1]

            if slash and ('0' <= slash <= '7'):
                opcode_extension = ord(slash) - ord('0')
            else:
                opcode_extension = -1

            reg_and_rm = slash == 'r'

            if groups[3]:
                immediate_size = 8
            else:
                immediate_size = -1

            encoding = Encoding(args, arg_order, rex_prefix, opcode_num,
                    reg_and_rm, opcode_extension, immediate_size)
            
            # @TODO: We should sort encodings by immediate size (ascending) so
            # that the smallest encoding gets selected automatically.
            if opcode not in instrs:
                instrs[opcode] = Instr(opcode, [])
            instrs[opcode].encodings.append(encoding)

    output = []
    output.append("""
// @NOTE: This is an automatically generated file! Do not edit it!
//        It was generated from '%s', edit that instead

static u32 assemble_instr(FILE *output_file, AsmInstr *instr)
{
\tswitch (instr->op) {
""" % input_filename)

    #pp.pprint(instrs)

    for opcode in instrs:
        output.append("\tcase %s:\n" % opcode)
        for encoding in instrs[opcode].encodings:
            if len(encoding.args) == 0:
                indent = '\t\t'
            else:
                output.append("\t\tif (%s)" % ' && '.join(
                    arg_condition(arg, i) for i, arg in enumerate(encoding.args)))
                indent = '\n\t\t\t'
            output.append("%sreturn encode_instr(output_file, instr, %s);\n"
                    % (indent,
                        ', '.join(map(to_c_val,
                            [encoding.arg_order, encoding.rex_prefix,
                            encoding.opcode_num, encoding.reg_and_rm,
                            encoding.opcode_extension, encoding.immediate_size]))))

        output.append("\t\tbreak;\n")

    output.append("""
\tdefault: break;
\t}
\t
\tfputs("Unimplemented instruction:\\n", stderr);
\tdump_asm_instr(instr);
\t
\tUNIMPLEMENTED;
}
""")

    with open(output_filename, 'w') as f:
        f.writelines(output)

def arg_condition(arg, i):
    if arg == 'r/m64':
        return '((instr->args[%d].type == REGISTER) || (instr->args[%d].type == OFFSET_REGISTER))' % (i, i)
    if arg == 'r64':
        return '(instr->args[%d].type == REGISTER) && !instr->args[%d].is_deref' % (i, i)
    if arg == 'imm8':
        return '(is_const_and_fits(instr->args[%d], 8))' % i

{'fraction_non_alphanumeric': 0.13725034835113795, 'fraction_numerical': 0.013005109150023224, 'mean_word_length': 1.7570422535211268, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6288041', 'n_tokens_mistral': 2726, 'n_tokens_neox': 2553, 'n_words': 643}