<commit_before>import collections
import claripy

class SimVariable:

    __slots__ = ['ident', 'name', 'region', 'category']

    def __init__(self, ident=None, name=None, region=None, category=None):
        """
        :param ident: A unique identifier provided by user or the program. Usually a string.
        :param str name: Name of this variable.
        """
        self.ident = ident
        self.name = name
        self.region = region if region is not None else ""
        self.category = category


class SimConstantVariable(SimVariable):

    __slots__ = ['value', '_hash']

    def __init__(self, ident=None, value=None, region=None):
        super(SimConstantVariable, self).__init__(ident=ident, region=region)
        self.value = value
        self._hash = None

    def __repr__(self):
        s = "<%s|const %s>" % (self.region, self.value)

        return s

    def __eq__(self, other):
        if not isinstance(other, SimConstantVariable):
            return False

        if self.value is None or other.value is None:
            # they may or may not represent the same constant. return not equal to be safe
            return False

        return self.ident == other.ident and self.value == other.value and self.region == other.region

    def __hash__(self):
        if self._hash is None:
            self._hash = hash(('const', self.value, self.ident, self.region, self.ident))
        return self._hash


class SimTemporaryVariable(SimVariable):

    __slots__ = ['tmp_id', '_hash']

    def __init__(self, tmp_id):
        SimVariable.__init__(self)

        self.tmp_id = tmp_id
        self._hash = None

    def __repr__(self):
        s = "<tmp %d>" % (self.tmp_id)

        return s

    def __hash__(self):
        if self._hash is None:
            self._hash = hash('tmp_%d' % (self.tmp_id))
        return self._hash

    def __eq__(self, other):
        if isinstance(other, SimTemporaryVariable):
            return hash(self) == hash(other)

        return False


class SimRegisterVariable(SimVariable):

    __slots__ = ['reg', 'size', '_hash']

    def __init__(self, reg_offset, size, ident=None, name=None, region=None, category=None):
        SimVariable.__init__(self, ident=ident, name=name, region=region, category=category)

        self.reg = reg_offset
        self.size = size
        self._hash = None

    def __repr__(self):

        ident_str = "[%s]" % self.ident if self.ident else ""
        region_str = hex(self.region) if isinstance(self.region, int) else self.region

        s = "<%s%s|Reg %s, %sB>" % (region_str, ident_str, self.reg, self.size)

        return s

    def __hash__(self):
        if self._hash is None:
            self._hash = hash(('reg', self.region, self.reg, self.size, self.ident))
        return self._hash

    def __eq__(self, other):
        if isinstance(other, SimRegisterVariable):
            return self.ident == other.ident and \
                   self.reg == other.reg and \
                   self.size == other.size and \
                   self.region == other.region

        return False


class SimMemoryVariable(SimVariable):

    __slots__ = ['addr', 'size', '_hash']

    def __init__(self, addr, size, ident=None, name=None, region=None, category=None):
        SimVariable.__init__(self, ident=ident, name=name, region=region, category=category)

        self.addr = addr

        if isinstance(size, claripy.ast.BV) and not size.symbolic:
            # Convert it to a concrete number
            size = size._model_concrete.value

        self.size = size
        self._hash = None

    def __repr__(self):
        if type(self.size) is int:
            size = '%d' % self.size
        else:
            size = '%s' % self.size

        if type(self.addr) is int:
            s = "<%s|Mem %#x %s>" % (self.region, self.addr, size)
        else:
            s = "<%s|Mem %s %s>" % (self.region, self.addr, size)

        return s

    def __hash__(self):
        if self._hash is not None:
            return self._hash

        if isinstance(self.addr, AddressWrapper):
            addr_hash = hash(self.addr)
        elif type(self.addr) is int:
            addr_hash = self.addr
        elif self.addr._model_concrete is not self.addr:
            addr_hash = hash(self.addr._model_concrete)
        elif self.addr._model_vsa is not self.addr:
            addr_hash = hash(self.addr._model_vsa)
        elif self.addr._model_z3 is not self.addr:
            addr_hash = hash(self.addr._model_z3)
        else:
            addr_hash = hash(self.addr)
        self._hash = hash((addr_hash, hash(self.size), self.ident))

        return self._hash

    def __eq__(self, other):
        if isinstance(other, SimMemoryVariable):
            return self.ident == other.ident and \
                   self.addr == other.addr and \
                   self.size == other.size

        return False


class SimStackVariable(SimMemoryVariable):

    __slots__ = ['base', 'offset']

    def __init__(self, offset, size, base='sp', base_addr=None, ident=None, name=None, region=None, category=None):
        if offset > 0x1000000 and isinstance(offset, int):
            # I don't think any positive stack offset will be greater than that...
            # convert it to a negative number
            mask = (1 << offset.bit_length()) - 1
            offset = - ((0 - offset) & mask)

        if base_addr is not None:
            addr = offset + base_addr
        else:
            # TODO: this is not optimal
            addr = offset
<commit_msg>SimMemoryVariable: Add a bits property.<commit_after>import collections
import claripy

class SimVariable:

    __slots__ = ['ident', 'name', 'region', 'category']

    def __init__(self, ident=None, name=None, region=None, category=None):
        """
        :param ident: A unique identifier provided by user or the program. Usually a string.
        :param str name: Name of this variable.
        """
        self.ident = ident
        self.name = name
        self.region = region if region is not None else ""
        self.category = category


class SimConstantVariable(SimVariable):

    __slots__ = ['value', '_hash']

    def __init__(self, ident=None, value=None, region=None):
        super(SimConstantVariable, self).__init__(ident=ident, region=region)
        self.value = value
        self._hash = None

    def __repr__(self):
        s = "<%s|const %s>" % (self.region, self.value)

        return s

    def __eq__(self, other):
        if not isinstance(other, SimConstantVariable):
            return False

        if self.value is None or other.value is None:
            # they may or may not represent the same constant. return not equal to be safe
            return False

        return self.ident == other.ident and self.value == other.value and self.region == other.region

    def __hash__(self):
        if self._hash is None:
            self._hash = hash(('const', self.value, self.ident, self.region, self.ident))
        return self._hash


class SimTemporaryVariable(SimVariable):

    __slots__ = ['tmp_id', '_hash']

    def __init__(self, tmp_id):
        SimVariable.__init__(self)

        self.tmp_id = tmp_id
        self._hash = None

    def __repr__(self):
        s = "<tmp %d>" % (self.tmp_id)

        return s

    def __hash__(self):
        if self._hash is None:
            self._hash = hash('tmp_%d' % (self.tmp_id))
        return self._hash

    def __eq__(self, other):
        if isinstance(other, SimTemporaryVariable):
            return hash(self) == hash(other)

        return False


class SimRegisterVariable(SimVariable):

    __slots__ = ['reg', 'size', '_hash']

    def __init__(self, reg_offset, size, ident=None, name=None, region=None, category=None):
        SimVariable.__init__(self, ident=ident, name=name, region=region, category=category)

        self.reg = reg_offset
        self.size = size
        self._hash = None

    def __repr__(self):

        ident_str = "[%s]" % self.ident if self.ident else ""
        region_str = hex(self.region) if isinstance(self.region, int) else self.region

        s = "<%s%s|Reg %s, %sB>" % (region_str, ident_str, self.reg, self.size)

        return s

    def __hash__(self):
        if self._hash is None:
            self._hash = hash(('reg', self.region, self.reg, self.size, self.ident))
        return self._hash

    def __eq__(self, other):
        if isinstance(other, SimRegisterVariable):
            return self.ident == other.ident and \
                   self.reg == other.reg and \
                   self.size == other.size and \
                   self.region == other.region

        return False


class SimMemoryVariable(SimVariable):

    __slots__ = ['addr', 'size', '_hash']

    def __init__(self, addr, size, ident=None, name=None, region=None, category=None):
        SimVariable.__init__(self, ident=ident, name=name, region=region, category=category)

        self.addr = addr

        if isinstance(size, claripy.ast.BV) and not size.symbolic:
            # Convert it to a concrete number
            size = size._model_concrete.value

        self.size = size
        self._hash = None

    def __repr__(self):
        if type(self.size) is int:
            size = '%d' % self.size
        else:
            size = '%s' % self.size

        if type(self.addr) is int:
            s = "<%s|Mem %#x %s>" % (self.region, self.addr, size)
        else:
            s = "<%s|Mem %s %s>" % (self.region, self.addr, size)

        return s

    def __hash__(self):
        if self._hash is not None:
            return self._hash

        if isinstance(self.addr, AddressWrapper):
            addr_hash = hash(self.addr)
        elif type(self.addr) is int:
            addr_hash = self.addr
        elif self.addr._model_concrete is not self.addr:
            addr_hash = hash(self.addr._model_concrete)
        elif self.addr._model_vsa is not self.addr:
            addr_hash = hash(self.addr._model_vsa)
        elif self.addr._model_z3 is not self.addr:
            addr_hash = hash(self.addr._model_z3)
        else:
            addr_hash = hash(self.addr)
        self._hash = hash((addr_hash, hash(self.size), self.ident))

        return self._hash

    def __eq__(self, other):
        if isinstance(other, SimMemoryVariable):
            return self.ident == other.ident and \
                   self.addr == other.addr and \
                   self.size == other.size

        return False

    @property
    def bits(self):
        return self.size * 8


class SimStackVariable(SimMemoryVariable):

    __slots__ = ['base', 'offset']

    def __init__(self, offset, size, base='sp', base_addr=None, ident=None, name=None, region=None, category=None):
        if offset > 0x1000000 and isinstance(offset, int):
            # I don't think any positive stack offset will be greater than that...
            # convert it to a negative number
            mask = (1 << offset.bit_length()) - 1
            offset = - ((0 - offset) & mask)

        if base_addr is not None:
            addr = offset + base_addr
        else:
            # TODO: this is not optimal
            addr = offset

{'fraction_non_alphanumeric': 0.11063526052819414, 'fraction_numerical': 0.0024089935760171306, 'mean_word_length': 2.137139658550238, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4078243', 'n_tokens_mistral': 3333, 'n_tokens_neox': 3204, 'n_words': 940}