<commit_before>#!/usr/bin/env python

import functools
import itertools
#import weakref

import logging
l = logging.getLogger("simuvex.s_state")

import ana
from archinfo import arch_from_id

def arch_overrideable(f):
    @functools.wraps(f)
    def wrapped_f(self, *args, **kwargs):
        if hasattr(self.arch, f.__name__):
            arch_f = getattr(self.arch, f.__name__)
            return arch_f(self, *args, **kwargs)
        else:
            return f(self, *args, **kwargs)
    return wrapped_f

from .plugins import default_plugins

# This is a counter for the state-merging symbolic variables
merge_counter = itertools.count()

class SimState(ana.Storable): # pylint: disable=R0904
    '''The SimState represents the state of a program, including its memory, registers, and so forth.'''

    def __init__(self, arch="AMD64", plugins=None, memory_backer=None, mode=None, options=None, add_options=None, remove_options=None, special_memory_filler=None):
        # the architecture is used for function simulations (autorets) and the bitness
        if isinstance(arch, str):
            self.arch = arch_from_id(arch)
        else:
            self.arch = arch

        # the options
        if options is None:
            if mode is None:
                l.warning("SimState defaulting to symbolic mode.")
                mode = "symbolic"
            options = o.modes[mode]

        options = set(options)
        if add_options is not None:
            options |= add_options
        if remove_options is not None:
            options -= remove_options
        self.options = options
        self.mode = mode

        # plugins
        self.plugins = { }
        if plugins is not None:
            for n,p in plugins.iteritems():
                self.register_plugin(n, p)

        if not self.has_plugin('memory'):
            # we don't set the memory endness because, unlike registers, it's hard to understand
            # which endness the data should be read

            if o.ABSTRACT_MEMORY in self.options:
                # We use SimAbstractMemory in static mode
                # Convert memory_backer into 'global' region
                if memory_backer is not None:
                    memory_backer = {'global': memory_backer}

                self.register_plugin('memory', SimAbstractMemory(memory_backer, memory_id="mem"))
            else:
                self.register_plugin('memory', SimSymbolicMemory(memory_backer, memory_id="mem"))
            self.register_plugin('mem', SimMemView())
        if not self.has_plugin('registers'):
            self.register_plugin('registers', SimSymbolicMemory(memory_id="reg", endness=self.arch.register_endness))
            self.register_plugin('regs', SimRegNameView())

        # This is used in static mode as we don't have any constraints there
        self._satisfiable = True

        # states are big, so let's give them UUIDs for ANA right away to avoid
        # extra pickling
        self.make_uuid()

        self.uninitialized_access_handler = None
        self._special_memory_filler = special_memory_filler

    def _ana_getstate(self):
        s = dict(ana.Storable._ana_getstate(self))
        s['plugins'] = { k:v for k,v in s['plugins'].iteritems() if k != 'inspector' }
        return s

    def _ana_setstate(self, s):
        ana.Storable._ana_setstate(self, s)
        for p in self.plugins.values():
            p.set_state(self)

    # easier access to some properties

    @property
    def ip(self):
        '''
        Get the instruction pointer expression.
        :return: an expression
        '''
        return self.regs.ip

    @ip.setter
    def ip(self, val):
        self.regs.ip = val

    # accessors for memory and registers and such
    @property
    def memory(self):
        return self.get_plugin('memory')

    @property
    def registers(self):
        return self.get_plugin('registers')

    @property
    def se(self):
        return self.get_plugin('solver_engine')

    @property
    def inspect(self):
        return self.get_plugin('inspector')

    @property
    def log(self):
        return self.get_plugin('log')

    @property
    def scratch(self):
        return self.get_plugin('scratch')

    @property
    def posix(self):
        return self.get_plugin('posix')

    @property
    def libc(self):
        return self.get_plugin('libc')

    @property
    def cgc(self):
        return self.get_plugin('cgc')

    @property
    def regs(self):
        return self.get_plugin('regs')

    @property
    def mem(self):
        return self.get_plugin('mem')

    @property
    def gdb(self):
        return self.get_plugin('gdb')

    @property
    def procedure_data(self):
        return self.get_plugin('procedure_data')

    def _inspect(self, *args, **kwargs):
        if self.has_plugin('inspector'):
            self.inspect.action(*args, **kwargs)

    #
    # Plugins
    #

    def has_plugin(self, name):
        return name in self.plugins

    def get_plugin(self, name):
        if name not in self.plugins:
            p = default_plugins[name]()
            self.register_plugin(name, p)
            return p
        return self.plugins[name]

    def register_plugin(self, name, plugin):
        #l.debug("Adding plugin %s of type %s", name, plugin.__class__.__name__)
        plugin.set_state(self)
        self.plugins[name] = plugin
        return plugin

    def release_plugin(self, name):
        if name in self.plugins:
            del self.plugins[name]

    #
    # Constraint pass-throughs
    #

    def simplify(self, *args): return self.se.simplify(*args)

    def add_constraints(self, *args, **kwargs):
        if len(args) > 0 and isinstance(args[0], (list, tuple)):
            raise Exception("Tuple or list passed to add_constraints!")

        if o.TRACK_CONSTRAINTS in self.options and len(args) > 0:
            if o.SIMPLIFY_CONSTRAINTS in self.options:
                constraints = [ self.simplify(a) for a in args ]
            else:
                constraints = args

            self._inspect('constraints', BP_BEFORE, added_constraints=constraints)
            self.se.add(*constraints)
            self._inspect('constraints', BP_AFTER)

        if 'action' in kwargs and kwargs['action'] and o.TRACK_CONSTRAINT_ACTIONS in self.options and len(args) > 0:
            for arg in args:
                if self.se.symbolic(arg):
                    sac = SimActionConstraint(self, arg)
                    self.log.add_action(sac)

        if o.ABSTRACT_SOLVER in self.options and len(args) > 0:
            for arg in args:
                if self.se.is_false(arg):
                    self._satisfiable = False
                    return
                if self.se.is_true(arg):
                    continue
                else:
                    # We take the argument, extract a list of constrained SIs out of it (if we could, of course), and
                    # then replace each original SI the intersection of original SI and the constrained one.

                    _, converted = self.se.constraint_to_si(arg)

                    for original_expr, constrained_si in converted:
                        if not original_expr.variables:
                            l.error('Incorrect original_expression to replace in add_constraints(). ' +
                                    'This is due to defects in VSA logics inside claripy. Please report ' +
                                    'to Fish and he will fix it if he\'s free.')
                            continue

                        new_expr = original_expr.intersection(constrained_si)
                        self.registers.replace_all(original_expr, new_expr)
                        for _, region in self.memory.regions.items():
                            region.memory.replace_all(original_expr, new_expr)

                        l.debug("SimState.add_constraints: Applied to final state.")
        elif o.SYMBOLIC not in self.options and len(args) > 0:
            for arg in args:
                if self.se.is_false(arg):
                    self._satisfiable = False
                    return

    def BV(self, name, size, explicit_name=None):
        size = self.arch.bits if size is None else size

        self._inspect('symbolic_variable', BP_BEFORE, symbolic_name=name, symbolic_size=size)
        v = self.se.BitVec(name, size, explicit_name=explicit_name)
        self._inspect('symbolic_variable', BP_AFTER, symbolic_expr=v)
        return v

    def BVV(self, value, size=None):
        if isinstance(value, str):
            v = 0
            for c in value:
                v = v << 8
                v += ord(c)
            size = len(value)*8
            value = v
        size = self.arch.bits if size is None else size
        return self.se.BitVecVal(value, size)

    def StridedInterval(self, name=None, bits=0, stride=None, lower_bound=None, upper_bound=None, to_conv=None):
        return self.se.StridedInterval(name=name,
                                       bits=bits,
                                       stride=stride,
                                       lower_bound=lower_bound,
                                       upper_bound=upper_bound,
                                       to_conv=to_conv)

    def satisfiable(self, **kwargs):
        if o.ABSTRACT_SOLVER in self.options or o.SYMBOLIC not in self.options:
            extra_constraints = kwargs.pop('extra_constraints', ())
            for e in extra_constraints:
                if self.se.is_false(e):
                    return False

            return self._satisfiable
        else:
            return self.se.satisfiable(**kwargs)

    def downsize(self):
        if 'solver_engine' in self.plugins:
            self.se.downsize()

    #
    # State branching operations
    #

    # Returns a dict that is a copy of all the state's plugins
    def copy_plugins(self):
        return { n: p.copy() for n,p in self.plugins.iteritems() }

    def copy(self):
        '''
        Returns a copy of the state.
        '''

        c_arch = self.arch
        c_plugins = self.copy_plugins()
        state = SimState(arch=c_arch, plugins=c_plugins, options=self.options, mode=self.mode)

        state.uninitialized_access_handler = self.uninitialized_access_handler
        state._special_memory_filler = self._special_memory_filler

        return state
<commit_msg>s_state: copy the arch when we copy the state<commit_after>#!/usr/bin/env python

import functools
import itertools
#import weakref

import logging
l = logging.getLogger("simuvex.s_state")

import ana
from archinfo import arch_from_id

def arch_overrideable(f):
    @functools.wraps(f)
    def wrapped_f(self, *args, **kwargs):
        if hasattr(self.arch, f.__name__):
            arch_f = getattr(self.arch, f.__name__)
            return arch_f(self, *args, **kwargs)
        else:
            return f(self, *args, **kwargs)
    return wrapped_f

from .plugins import default_plugins

# This is a counter for the state-merging symbolic variables
merge_counter = itertools.count()

class SimState(ana.Storable): # pylint: disable=R0904
    '''The SimState represents the state of a program, including its memory, registers, and so forth.'''

    def __init__(self, arch="AMD64", plugins=None, memory_backer=None, mode=None, options=None, add_options=None, remove_options=None, special_memory_filler=None):
        # the architecture is used for function simulations (autorets) and the bitness
        if isinstance(arch, str):
            self.arch = arch_from_id(arch)
        else:
            self.arch = arch

        # the options
        if options is None:
            if mode is None:
                l.warning("SimState defaulting to symbolic mode.")
                mode = "symbolic"
            options = o.modes[mode]

        options = set(options)
        if add_options is not None:
            options |= add_options
        if remove_options is not None:
            options -= remove_options
        self.options = options
        self.mode = mode

        # plugins
        self.plugins = { }
        if plugins is not None:
            for n,p in plugins.iteritems():
                self.register_plugin(n, p)

        if not self.has_plugin('memory'):
            # we don't set the memory endness because, unlike registers, it's hard to understand
            # which endness the data should be read

            if o.ABSTRACT_MEMORY in self.options:
                # We use SimAbstractMemory in static mode
                # Convert memory_backer into 'global' region
                if memory_backer is not None:
                    memory_backer = {'global': memory_backer}

                self.register_plugin('memory', SimAbstractMemory(memory_backer, memory_id="mem"))
            else:
                self.register_plugin('memory', SimSymbolicMemory(memory_backer, memory_id="mem"))
            self.register_plugin('mem', SimMemView())
        if not self.has_plugin('registers'):
            self.register_plugin('registers', SimSymbolicMemory(memory_id="reg", endness=self.arch.register_endness))
            self.register_plugin('regs', SimRegNameView())

        # This is used in static mode as we don't have any constraints there
        self._satisfiable = True

        # states are big, so let's give them UUIDs for ANA right away to avoid
        # extra pickling
        self.make_uuid()

        self.uninitialized_access_handler = None
        self._special_memory_filler = special_memory_filler

    def _ana_getstate(self):
        s = dict(ana.Storable._ana_getstate(self))
        s['plugins'] = { k:v for k,v in s['plugins'].iteritems() if k != 'inspector' }
        return s

    def _ana_setstate(self, s):
        ana.Storable._ana_setstate(self, s)
        for p in self.plugins.values():
            p.set_state(self)

    # easier access to some properties

    @property
    def ip(self):
        '''
        Get the instruction pointer expression.
        :return: an expression
        '''
        return self.regs.ip

    @ip.setter
    def ip(self, val):
        self.regs.ip = val

    # accessors for memory and registers and such
    @property
    def memory(self):
        return self.get_plugin('memory')

    @property
    def registers(self):
        return self.get_plugin('registers')

    @property
    def se(self):
        return self.get_plugin('solver_engine')

    @property
    def inspect(self):
        return self.get_plugin('inspector')

    @property
    def log(self):
        return self.get_plugin('log')

    @property
    def scratch(self):
        return self.get_plugin('scratch')

    @property
    def posix(self):
        return self.get_plugin('posix')

    @property
    def libc(self):
        return self.get_plugin('libc')

    @property
    def cgc(self):
        return self.get_plugin('cgc')

    @property
    def regs(self):
        return self.get_plugin('regs')

    @property
    def mem(self):
        return self.get_plugin('mem')

    @property
    def gdb(self):
        return self.get_plugin('gdb')

    @property
    def procedure_data(self):
        return self.get_plugin('procedure_data')

    def _inspect(self, *args, **kwargs):
        if self.has_plugin('inspector'):
            self.inspect.action(*args, **kwargs)

    #
    # Plugins
    #

    def has_plugin(self, name):
        return name in self.plugins

    def get_plugin(self, name):
        if name not in self.plugins:
            p = default_plugins[name]()
            self.register_plugin(name, p)
            return p
        return self.plugins[name]

    def register_plugin(self, name, plugin):
        #l.debug("Adding plugin %s of type %s", name, plugin.__class__.__name__)
        plugin.set_state(self)
        self.plugins[name] = plugin
        return plugin

    def release_plugin(self, name):
        if name in self.plugins:
            del self.plugins[name]

    #
    # Constraint pass-throughs
    #

    def simplify(self, *args): return self.se.simplify(*args)

    def add_constraints(self, *args, **kwargs):
        if len(args) > 0 and isinstance(args[0], (list, tuple)):
            raise Exception("Tuple or list passed to add_constraints!")

        if o.TRACK_CONSTRAINTS in self.options and len(args) > 0:
            if o.SIMPLIFY_CONSTRAINTS in self.options:
                constraints = [ self.simplify(a) for a in args ]
            else:
                constraints = args

            self._inspect('constraints', BP_BEFORE, added_constraints=constraints)
            self.se.add(*constraints)
            self._inspect('constraints', BP_AFTER)

        if 'action' in kwargs and kwargs['action'] and o.TRACK_CONSTRAINT_ACTIONS in self.options and len(args) > 0:
            for arg in args:
                if self.se.symbolic(arg):
                    sac = SimActionConstraint(self, arg)
                    self.log.add_action(sac)

        if o.ABSTRACT_SOLVER in self.options and len(args) > 0:
            for arg in args:
                if self.se.is_false(arg):
                    self._satisfiable = False
                    return
                if self.se.is_true(arg):
                    continue
                else:
                    # We take the argument, extract a list of constrained SIs out of it (if we could, of course), and
                    # then replace each original SI the intersection of original SI and the constrained one.

                    _, converted = self.se.constraint_to_si(arg)

                    for original_expr, constrained_si in converted:
                        if not original_expr.variables:
                            l.error('Incorrect original_expression to replace in add_constraints(). ' +
                                    'This is due to defects in VSA logics inside claripy. Please report ' +
                                    'to Fish and he will fix it if he\'s free.')
                            continue

                        new_expr = original_expr.intersection(constrained_si)
                        self.registers.replace_all(original_expr, new_expr)
                        for _, region in self.memory.regions.items():
                            region.memory.replace_all(original_expr, new_expr)

                        l.debug("SimState.add_constraints: Applied to final state.")
        elif o.SYMBOLIC not in self.options and len(args) > 0:
            for arg in args:
                if self.se.is_false(arg):
                    self._satisfiable = False
                    return

    def BV(self, name, size, explicit_name=None):
        size = self.arch.bits if size is None else size

        self._inspect('symbolic_variable', BP_BEFORE, symbolic_name=name, symbolic_size=size)
        v = self.se.BitVec(name, size, explicit_name=explicit_name)
        self._inspect('symbolic_variable', BP_AFTER, symbolic_expr=v)
        return v

    def BVV(self, value, size=None):
        if isinstance(value, str):
            v = 0
            for c in value:
                v = v << 8
                v += ord(c)
            size = len(value)*8
            value = v
        size = self.arch.bits if size is None else size
        return self.se.BitVecVal(value, size)

    def StridedInterval(self, name=None, bits=0, stride=None, lower_bound=None, upper_bound=None, to_conv=None):
        return self.se.StridedInterval(name=name,
                                       bits=bits,
                                       stride=stride,
                                       lower_bound=lower_bound,
                                       upper_bound=upper_bound,
                                       to_conv=to_conv)

    def satisfiable(self, **kwargs):
        if o.ABSTRACT_SOLVER in self.options or o.SYMBOLIC not in self.options:
            extra_constraints = kwargs.pop('extra_constraints', ())
            for e in extra_constraints:
                if self.se.is_false(e):
                    return False

            return self._satisfiable
        else:
            return self.se.satisfiable(**kwargs)

    def downsize(self):
        if 'solver_engine' in self.plugins:
            self.se.downsize()

    #
    # State branching operations
    #

    # Returns a dict that is a copy of all the state's plugins
    def copy_plugins(self):
        return { n: p.copy() for n,p in self.plugins.iteritems() }

    def copy(self):
        '''
        Returns a copy of the state.
        '''

        c_arch = self.arch.copy()
        c_plugins = self.copy_plugins()
        state = SimState(arch=c_arch, plugins=c_plugins, options=self.options, mode=self.mode)

        state.uninitialized_access_handler = self.uninitialized_access_handler
        state._special_memory_filler = self._special_memory_filler

        return state

{'fraction_non_alphanumeric': 0.08924386627767947, 'fraction_numerical': 0.0015304414367018986, 'mean_word_length': 1.9392746696654484, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7417308', 'n_tokens_mistral': 6001, 'n_tokens_neox': 5714, 'n_words': 1584}