<commit_before>import sys
import types

import ana
import pyvex
import simuvex
import logging
l = logging.getLogger("angr.vexer")

class SerializableIRSB(ana.Storable):
    __slots__ = [ '_state', '_irsb', '_addr' ]

    def __init__(self, *args, **kwargs):
        self._state = args, kwargs
        self._irsb = pyvex.IRSB(*args, **kwargs)
        self._addr = next(a.addr for a in self._irsb.statements if isinstance(a, pyvex.IRStmt.IMark))

    def __dir__(self):
        return dir(self._irsb) + self._all_slots()

    def __getattr__(self, a):
        try:
            return object.__getattribute__(self, a)
        except AttributeError:
            return getattr(self._irsb, a)

    def __setattr__(self, a, v):
        try:
            return object.__setattr__(self, a, v)
        except AttributeError:
            return setattr(self._irsb, a, v)

    def _ana_getstate(self):
        return self._state

    def _ana_setstate(self, s):
        self.__init__(*(s[0]), **(s[1]))

    def _ana_getliteral(self):
        return self._crawl_vex(self._irsb)

    def instruction_addrs(self):
        return [ s.addr for s in self._irsb.statements if isinstance(s, pyvex.IRStmt.IMark) ]

    @property
    def json(self):
        return self._ana_getliteral()

    def _crawl_vex(self, p):
        if isinstance(p, (int, str, float, long, bool)): return p
        elif isinstance(p, (tuple, list, set)): return [ self._crawl_vex(e) for e in p ]
        elif isinstance(p, dict): return { k:self._crawl_vex(p[k]) for k in p }

        attr_keys = set()
        for k in dir(p):
            if k in [ 'wrapped' ] or k.startswith('_'):
                continue

            if isinstance(getattr(p, k), (types.BuiltinFunctionType, types.BuiltinMethodType, types.FunctionType, types.ClassType, type, types.UnboundMethodType)):
                continue

            attr_keys.add(k)

        vdict = { }
        for k in attr_keys:
            vdict[k] = self._crawl_vex(getattr(p, k))

        if isinstance(p, pyvex.IRSB):
            vdict['statements'] = self._crawl_vex(p.statements)
            vdict['instructions'] = self._crawl_vex(p.instructions)
            vdict['addr'] = self._addr
        elif isinstance(p, pyvex.IRTypeEnv):
            vdict['types'] = self._crawl_vex(p.types)

        return vdict

class VEXer:
    def __init__(self, mem, arch, max_size=None, num_inst=None, traceflags=None, use_cache=None, opt_level=None):
        self.mem = mem
        self.arch = arch
        self.max_size = 400 if max_size is None else max_size
        self.num_inst = 99 if num_inst is None else num_inst
        self.traceflags = 0 if traceflags is None else traceflags
        self.use_cache = True if use_cache is None else use_cache
        self.opt_level = 1 if opt_level is None else opt_level
        self.irsb_cache = { }


    def block(self, addr, max_size=None, num_inst=None, traceflags=0, thumb=False, backup_state=None, opt_level=None):
        """
        Returns a pyvex block starting at address addr

        Optional params:

        @param max_size: the maximum size of the block, in bytes
        @param num_inst: the maximum number of instructions
        @param traceflags: traceflags to be passed to VEX. Default: 0
        """
        max_size = self.max_size if max_size is None else max_size
        num_inst = self.num_inst if num_inst is None else num_inst
        opt_level = self.opt_level if opt_level is None else opt_level

        if thumb:
            addr &= ~1

        # TODO: FIXME: figure out what to do if we're about to exhaust the memory
        # (we can probably figure out how many instructions we have left by talking to IDA)

        # Try to find the actual size of the block, stop at the first keyerror
        arr = []
        for i in range(addr, addr+max_size):
            try:
                arr.append(self.mem[i])
            except KeyError:
                if backup_state:
                    if i in backup_state.memory:
                        val = backup_state.mem_expr(backup_state.BVV(i), backup_state.BVV(1))
                        try:
                            val = backup_state.se.exactly_n_int(val, 1)[0]
                            val = chr(val)
                        except simuvex.SimValueError:
                            break

                        arr.append(val)
                    else:
                        break
                else:
                    break

        buff = "".join(arr)

        if not buff:
            raise AngrMemoryError("No bytes in memory for block starting at 0x%x." % addr)

        # deal with thumb mode in ARM, sending an odd address and an offset
        # into the string
        byte_offset = 0
        if thumb:
            byte_offset = 1
            addr += 1

        l.debug("Creating pyvex.IRSB of arch %s at 0x%x", self.arch.name, addr)

        if self.use_cache:
            cache_key = (buff, addr, num_inst, self.arch.vex_arch, byte_offset, thumb, opt_level)
            if cache_key in self.irsb_cache:
                return self.irsb_cache[cache_key]

        pyvex.set_iropt_level(opt_level)
        try:
            if num_inst:
                block = SerializableIRSB(bytes=buff, mem_addr=addr, num_inst=num_inst, arch=self.arch.vex_arch,
                                   endness=self.arch.vex_endness, bytes_offset=byte_offset, traceflags=traceflags)
            else:
                block = SerializableIRSB(bytes=buff, mem_addr=addr, arch=self.arch.vex_arch,
                                   endness=self.arch.vex_endness, bytes_offset=byte_offset, traceflags=traceflags)
        except pyvex.PyVEXError:
            l.debug("VEX translation error at 0x%x", addr)
            l.debug("Using bytes: " + buff.encode('hex'))
            e_type, value, traceback = sys.exc_info()
            raise AngrTranslationError, ("Translation error", e_type, value), traceback

        if self.use_cache:
            self.irsb_cache[cache_key] = block

        block = self._post_process(block)

        return block

    def _post_process(self, block):
        '''
        Do some post-processing work here.<commit_msg>put max_size into consideration for irsb cache key in vexer.<commit_after>import sys
import types

import ana
import pyvex
import simuvex
import logging
l = logging.getLogger("angr.vexer")

class SerializableIRSB(ana.Storable):
    __slots__ = [ '_state', '_irsb', '_addr' ]

    def __init__(self, *args, **kwargs):
        self._state = args, kwargs
        self._irsb = pyvex.IRSB(*args, **kwargs)
        self._addr = next(a.addr for a in self._irsb.statements if isinstance(a, pyvex.IRStmt.IMark))

    def __dir__(self):
        return dir(self._irsb) + self._all_slots()

    def __getattr__(self, a):
        try:
            return object.__getattribute__(self, a)
        except AttributeError:
            return getattr(self._irsb, a)

    def __setattr__(self, a, v):
        try:
            return object.__setattr__(self, a, v)
        except AttributeError:
            return setattr(self._irsb, a, v)

    def _ana_getstate(self):
        return self._state

    def _ana_setstate(self, s):
        self.__init__(*(s[0]), **(s[1]))

    def _ana_getliteral(self):
        return self._crawl_vex(self._irsb)

    def instruction_addrs(self):
        return [ s.addr for s in self._irsb.statements if isinstance(s, pyvex.IRStmt.IMark) ]

    @property
    def json(self):
        return self._ana_getliteral()

    def _crawl_vex(self, p):
        if isinstance(p, (int, str, float, long, bool)): return p
        elif isinstance(p, (tuple, list, set)): return [ self._crawl_vex(e) for e in p ]
        elif isinstance(p, dict): return { k:self._crawl_vex(p[k]) for k in p }

        attr_keys = set()
        for k in dir(p):
            if k in [ 'wrapped' ] or k.startswith('_'):
                continue

            if isinstance(getattr(p, k), (types.BuiltinFunctionType, types.BuiltinMethodType, types.FunctionType, types.ClassType, type, types.UnboundMethodType)):
                continue

            attr_keys.add(k)

        vdict = { }
        for k in attr_keys:
            vdict[k] = self._crawl_vex(getattr(p, k))

        if isinstance(p, pyvex.IRSB):
            vdict['statements'] = self._crawl_vex(p.statements)
            vdict['instructions'] = self._crawl_vex(p.instructions)
            vdict['addr'] = self._addr
        elif isinstance(p, pyvex.IRTypeEnv):
            vdict['types'] = self._crawl_vex(p.types)

        return vdict

class VEXer:
    def __init__(self, mem, arch, max_size=None, num_inst=None, traceflags=None, use_cache=None, opt_level=None):
        self.mem = mem
        self.arch = arch
        self.max_size = 400 if max_size is None else max_size
        self.num_inst = 99 if num_inst is None else num_inst
        self.traceflags = 0 if traceflags is None else traceflags
        self.use_cache = True if use_cache is None else use_cache
        self.opt_level = 1 if opt_level is None else opt_level
        self.irsb_cache = { }


    def block(self, addr, max_size=None, num_inst=None, traceflags=0, thumb=False, backup_state=None, opt_level=None):
        """
        Returns a pyvex block starting at address addr

        Optional params:

        @param max_size: the maximum size of the block, in bytes
        @param num_inst: the maximum number of instructions
        @param traceflags: traceflags to be passed to VEX. Default: 0
        """
        max_size = self.max_size if max_size is None else max_size
        num_inst = self.num_inst if num_inst is None else num_inst
        opt_level = self.opt_level if opt_level is None else opt_level

        if thumb:
            addr &= ~1

        # TODO: FIXME: figure out what to do if we're about to exhaust the memory
        # (we can probably figure out how many instructions we have left by talking to IDA)

        # Try to find the actual size of the block, stop at the first keyerror
        arr = []
        for i in range(addr, addr+max_size):
            try:
                arr.append(self.mem[i])
            except KeyError:
                if backup_state:
                    if i in backup_state.memory:
                        val = backup_state.mem_expr(backup_state.BVV(i), backup_state.BVV(1))
                        try:
                            val = backup_state.se.exactly_n_int(val, 1)[0]
                            val = chr(val)
                        except simuvex.SimValueError:
                            break

                        arr.append(val)
                    else:
                        break
                else:
                    break

        buff = "".join(arr)

        if not buff:
            raise AngrMemoryError("No bytes in memory for block starting at 0x%x." % addr)

        # deal with thumb mode in ARM, sending an odd address and an offset
        # into the string
        byte_offset = 0
        if thumb:
            byte_offset = 1
            addr += 1

        l.debug("Creating pyvex.IRSB of arch %s at 0x%x", self.arch.name, addr)

        if self.use_cache:
            cache_key = (buff, addr, max_size, num_inst, self.arch.vex_arch, byte_offset, thumb, opt_level)
            if cache_key in self.irsb_cache:
                return self.irsb_cache[cache_key]

        pyvex.set_iropt_level(opt_level)
        try:
            if num_inst:
                block = SerializableIRSB(bytes=buff, mem_addr=addr, num_inst=num_inst, arch=self.arch.vex_arch,
                                   endness=self.arch.vex_endness, bytes_offset=byte_offset, traceflags=traceflags)
            else:
                block = SerializableIRSB(bytes=buff, mem_addr=addr, arch=self.arch.vex_arch,
                                   endness=self.arch.vex_endness, bytes_offset=byte_offset, traceflags=traceflags)
        except pyvex.PyVEXError:
            l.debug("VEX translation error at 0x%x", addr)
            l.debug("Using bytes: " + buff.encode('hex'))
            e_type, value, traceback = sys.exc_info()
            raise AngrTranslationError, ("Translation error", e_type, value), traceback

        if self.use_cache:
            self.irsb_cache[cache_key] = block

        block = self._post_process(block)

        return block

    def _post_process(self, block):
        '''
        Do some post-processing work here.
{'fraction_non_alphanumeric': 0.09118002416431736, 'fraction_numerical': 0.003383004430124849, 'mean_word_length': 2.0062953995157384, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '598336', 'n_tokens_mistral': 3846, 'n_tokens_neox': 3634, 'n_words': 1027}