<commit_before>import struct as _struct
import platform as _platform
import re
from archinfo.archerror import ArchError

try:
    import pyvex as _pyvex
except ImportError:
    _pyvex = None

try:
    import unicorn as _unicorn
except ImportError:
    _unicorn = None

try:
    import capstone as _capstone
except ImportError:
    _capstone = None

import logging
l = logging.getLogger('archinfo.arch')
l.addHandler(logging.NullHandler())

class Arch(object):
    """
    A collection of information about a given architecture. This class should be subclasses for each different
    architecture, and then that subclass should be registered with the ``register_arch`` method.

    A good number of assumptions are made that code is being processed under the VEX IR - for instance, it is expected
    the register file offsets are expected to match code generated by PyVEX.

    Arches maybe compared with == and !=.

    :ivar str name: The name of the arch
    :ivar int bits: The number of bits in a word
    :ivar str vex_arch: The VEX enum name used to identify this arch
    :ivar str qemu_name: The name used by QEMU to identify this arch
    :ivar str ida_processor: The processor string used by IDA to identify this arch
    :ivar str triplet: The triplet used to identify a linux system on this arch
    :ivar int max_inst_bytes: The maximum number of bytes in a single instruction
    :ivar int ip_offset: The offset of the instruction pointer in the register file
    :ivar int sp_offset: The offset of the stack pointer in the register file
    :ivar int bp_offset: The offset of the base pointer in the register file
    :ivar int lr_offset: The offset of the link register (return address) in the register file
    :ivar int ret_offset: The offset of the return value register in the register file
    :ivar bool vex_conditional_helpers: Whether libVEX will generate code to process the conditional flags for this
            arch using ccalls
    :ivar int syscall_num_offset: The offset in the register file where the syscall number is stored
    :ivar bool call_pushes_ret: Whether this arch's call instruction causes a stack push
    :ivar int stack_change: The change to the stack pointer caused by a push instruction
    :ivar str memory_endness: The endness of memory, as a VEX enum
    :ivar str register_endness: The endness of registers, as a VEX enum. Should usually be same as above
    :ivar dict sizeof: A mapping from C type to variable size in bits
    :ivar cs_arch: The capstone arch value for this arch
    :ivar cs_mode: The capstone mode value for this arch
    :ivar uc_arch: The unicorn engine arch value for this arch
    :ivar uc_mode: The unicorn engine mode value for this arch
    :ivar uc_const: The unicorn engine constants module for this arch
    :ivar uc_prefix: The prefix used for variables in the unicorn engine constants module
    :ivar list function_prologs: A list of regular expressions matching the bytes for common function prologues
    :ivar list function_epilogs: A list of regular expressions matching the bytes for common function epilogues
    :ivar str ret_instruction: The bytes for a return instruction
    :ivar str nop_instruction: The bytes for a nop instruction
    :ivar int instruction_alignment: The instruction alignment requirement
    :ivar list default_register_values: A weird listing describing how registers should be initialized for purposes of
            sanity
    :ivar dict entry_register_values: A mapping from register name to a description of the value that should be in it
            at program entry on linux
    :ivar list default_symbolic_register: Honestly, who knows what this is supposed to do. Fill it with the names of
            the general purpose registers.
    :ivar dict register_names: A mapping from register file offset to register name
    :ivar dict registers: A mapping from register name to a tuple of (register file offset, size in bytes)
    :ivar list lib_paths: A listing of common locations where shared libraries for this architecture may be found
    :ivar str got_section_name: The name of the GOT section in ELFs
    :ivar str ld_linux_name: The name of the linux dynamic loader program
    :ivar TLSArchInfo elf_tls: A description of how thread-local storage works
    """
    def __init__(self, endness):
        if endness not in ('Iend_LE', 'Iend_BE'):
            raise ArchError('Must pass a valid VEX endness: "Iend_LE" or "Iend_BE"')

        if _pyvex:
            self.vex_archinfo = _pyvex.default_vex_archinfo()
        if endness == 'Iend_BE':
            if self.vex_archinfo:
                self.vex_archinfo['endness'] = _pyvex.vex_endness_from_string('VexEndnessBE')
            self.memory_endness = 'Iend_BE'
            self.register_endness = 'Iend_BE'
            if _capstone:
                self.cs_mode -= _capstone.CS_MODE_LITTLE_ENDIAN
                self.cs_mode += _capstone.CS_MODE_BIG_ENDIAN
            self.ret_instruction = reverse_ends(self.ret_instruction)
            self.nop_instruction = reverse_ends(self.nop_instruction)

        # generate regitster mapping (offset, size): name
        self.register_size_names = {}
        for k, v in self.registers.iteritems():
            if v in self.register_size_names and k not in self.register_names:
                continue
            self.register_size_names[v] = k

        # unicorn specific stuff
        if self.uc_mode is not None:
            if endness == 'Iend_BE':
                self.uc_mode -= _unicorn.UC_MODE_LITTLE_ENDIAN
                self.uc_mode += _unicorn.UC_MODE_BIG_ENDIAN
            self.uc_regs = { }
            # map register names to unicorn const
            for r in self.register_names.itervalues():
                reg_name = self.uc_prefix + 'REG_' + r.upper()
                if hasattr(self.uc_const, reg_name):
                    self.uc_regs[r] = getattr(self.uc_const, reg_name)

    def copy(self):
        """
        Produce a copy of this instance of this arch.
        """<commit_msg>Add a special hack for X86 and AMD64 for better mapping from register offsets and sizes to register names.<commit_after>import struct as _struct
import platform as _platform
import re
from archinfo.archerror import ArchError

try:
    import pyvex as _pyvex
except ImportError:
    _pyvex = None

try:
    import unicorn as _unicorn
except ImportError:
    _unicorn = None

try:
    import capstone as _capstone
except ImportError:
    _capstone = None

import logging
l = logging.getLogger('archinfo.arch')
l.addHandler(logging.NullHandler())

class Arch(object):
    """
    A collection of information about a given architecture. This class should be subclasses for each different
    architecture, and then that subclass should be registered with the ``register_arch`` method.

    A good number of assumptions are made that code is being processed under the VEX IR - for instance, it is expected
    the register file offsets are expected to match code generated by PyVEX.

    Arches maybe compared with == and !=.

    :ivar str name: The name of the arch
    :ivar int bits: The number of bits in a word
    :ivar str vex_arch: The VEX enum name used to identify this arch
    :ivar str qemu_name: The name used by QEMU to identify this arch
    :ivar str ida_processor: The processor string used by IDA to identify this arch
    :ivar str triplet: The triplet used to identify a linux system on this arch
    :ivar int max_inst_bytes: The maximum number of bytes in a single instruction
    :ivar int ip_offset: The offset of the instruction pointer in the register file
    :ivar int sp_offset: The offset of the stack pointer in the register file
    :ivar int bp_offset: The offset of the base pointer in the register file
    :ivar int lr_offset: The offset of the link register (return address) in the register file
    :ivar int ret_offset: The offset of the return value register in the register file
    :ivar bool vex_conditional_helpers: Whether libVEX will generate code to process the conditional flags for this
            arch using ccalls
    :ivar int syscall_num_offset: The offset in the register file where the syscall number is stored
    :ivar bool call_pushes_ret: Whether this arch's call instruction causes a stack push
    :ivar int stack_change: The change to the stack pointer caused by a push instruction
    :ivar str memory_endness: The endness of memory, as a VEX enum
    :ivar str register_endness: The endness of registers, as a VEX enum. Should usually be same as above
    :ivar dict sizeof: A mapping from C type to variable size in bits
    :ivar cs_arch: The capstone arch value for this arch
    :ivar cs_mode: The capstone mode value for this arch
    :ivar uc_arch: The unicorn engine arch value for this arch
    :ivar uc_mode: The unicorn engine mode value for this arch
    :ivar uc_const: The unicorn engine constants module for this arch
    :ivar uc_prefix: The prefix used for variables in the unicorn engine constants module
    :ivar list function_prologs: A list of regular expressions matching the bytes for common function prologues
    :ivar list function_epilogs: A list of regular expressions matching the bytes for common function epilogues
    :ivar str ret_instruction: The bytes for a return instruction
    :ivar str nop_instruction: The bytes for a nop instruction
    :ivar int instruction_alignment: The instruction alignment requirement
    :ivar list default_register_values: A weird listing describing how registers should be initialized for purposes of
            sanity
    :ivar dict entry_register_values: A mapping from register name to a description of the value that should be in it
            at program entry on linux
    :ivar list default_symbolic_register: Honestly, who knows what this is supposed to do. Fill it with the names of
            the general purpose registers.
    :ivar dict register_names: A mapping from register file offset to register name
    :ivar dict registers: A mapping from register name to a tuple of (register file offset, size in bytes)
    :ivar list lib_paths: A listing of common locations where shared libraries for this architecture may be found
    :ivar str got_section_name: The name of the GOT section in ELFs
    :ivar str ld_linux_name: The name of the linux dynamic loader program
    :ivar TLSArchInfo elf_tls: A description of how thread-local storage works
    """
    def __init__(self, endness):
        if endness not in ('Iend_LE', 'Iend_BE'):
            raise ArchError('Must pass a valid VEX endness: "Iend_LE" or "Iend_BE"')

        if _pyvex:
            self.vex_archinfo = _pyvex.default_vex_archinfo()
        if endness == 'Iend_BE':
            if self.vex_archinfo:
                self.vex_archinfo['endness'] = _pyvex.vex_endness_from_string('VexEndnessBE')
            self.memory_endness = 'Iend_BE'
            self.register_endness = 'Iend_BE'
            if _capstone:
                self.cs_mode -= _capstone.CS_MODE_LITTLE_ENDIAN
                self.cs_mode += _capstone.CS_MODE_BIG_ENDIAN
            self.ret_instruction = reverse_ends(self.ret_instruction)
            self.nop_instruction = reverse_ends(self.nop_instruction)

        # generate regitster mapping (offset, size): name
        self.register_size_names = {}
        for k, v in self.registers.iteritems():

            # special hacks for X86 and AMD64 - don't translate register names to bp, sp, etc.
            if self.name in {'X86', 'AMD64'} and k in {'bp', 'sp', 'ip'}:
                continue

            if v in self.register_size_names and k not in self.register_names:
                continue
            self.register_size_names[v] = k

        # unicorn specific stuff
        if self.uc_mode is not None:
            if endness == 'Iend_BE':
                self.uc_mode -= _unicorn.UC_MODE_LITTLE_ENDIAN
                self.uc_mode += _unicorn.UC_MODE_BIG_ENDIAN
            self.uc_regs = { }
            # map register names to unicorn const
            for r in self.register_names.itervalues():
                reg_name = self.uc_prefix + 'REG_' + r.upper()
                if hasattr(self.uc_const, reg_name):
                    self.uc_regs[r] = getattr(self.uc_const, reg_name)

    def copy(self):
        """
        Produce a copy of this instance of this arch.
        """
{'fraction_non_alphanumeric': 0.04944655409226792, 'fraction_numerical': 0.0009695402763189787, 'mean_word_length': 2.9993537964458805, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1150865', 'n_tokens_mistral': 3480, 'n_tokens_neox': 3332, 'n_words': 1514}