<commit_before>import bintrees
import cooldict
import claripy
import cffi
import cle

from ..s_errors import SimMemoryError, SimSegfaultError
from .. import s_options as options
from .memory_object import SimMemoryObject
from claripy.ast.bv import BV

_ffi = cffi.FFI()

import logging
l = logging.getLogger('simuvex.storage.paged_memory')

class Page(object):
    """
    Page object, allowing for more flexibility than just a raw dict.
    """

    PROT_READ = 1
    PROT_WRITE = 2
    PROT_EXEC = 4

    def __init__(
        self, page_addr, page_size, permissions=None,
        executable=False, storage=None, sinkhole=None
    ):
        """
        Create a new page object. Carries permissions information.
        Permissions default to RW unless `executable` is True,
        in which case permissions default to RWX.

        :param int page_addr: The base address of the page.
        :param int page_size: The size of the page.
        :param bool executable: Whether the page is executable. Typically,
                            this will depend on whether the binary has an
                            executable stack.
        :param claripy.AST permissions: A 3-bit bitvector setting specific permissions
                            for EXEC, READ, and WRITE
        :param bintrees.AVLTree storage: A binary tree with the page data.
        :param claripy.AST sinkhole: A "default" value for the page.
        """

        self._page_addr = page_addr
        self._page_size = page_size
        self._storage = bintrees.AVLTree() if storage is None else storage
        self._sinkhole = sinkhole

        if permissions is None:
            perms = Page.PROT_READ|Page.PROT_WRITE
            if executable:
                perms |= Page.PROT_EXEC
            self.permissions = claripy.BVV(perms, 3) # 3 bits is enough for PROT_EXEC, PROT_WRITE, PROT_READ, PROT_NONE
        else:
            self.permissions = permissions

    @property
    def concrete_permissions(self):
        if self.permissions.symbolic:
            return 7
        else:
            return self.permissions.args[0]

    def sinkhole(self, v, wipe=False):
        if wipe:
            self._storage = bintrees.AVLTree()
        self._sinkhole = v

    def keys(self):
        if self._sinkhole is not None:
            return range(self._page_size)
        else:
            n = 0
            addrs = [ ]
            while n < self._page_size:
                try:
                    page_idx, mo = self._storage.ceiling_item(n)
                    addrs.extend(range(
                        max(0, mo.base - self._page_addr),
                        min(mo.last_addr+1-self._page_addr, self._page_size)
                    ))
                    n = page_idx+mo.length
                except KeyError:
                    break
            return addrs

    def __contains__(self, item):
        return self.load_mo(item, 1)[0] is not None

    def __getitem__(self, idx):
        i = self.load_mo(idx, 1)[0]
        if i is None:
            raise KeyError(idx)
        else:
            return i

    def __setitem__(self, idx, item):
        if idx is self._sinkhole:
            if idx in self._storage:
                del self._storage[idx]
            return
        else:
            self._storage[idx] = item

    def _resolve_range(self, mo):
        length = mo.length
        start =  mo.base - self._page_addr
        if start < 0:
            length = length + start
            start = 0
        length = min(length, self._page_size)
        if length <= 0:
            l.warning("Nothing left of the memory object to store in SimPage.")

        return start, length

    def replace_mo(self, old_mo, new_mo):
        if old_mo is self._sinkhole:
            self._sinkhole = new_mo

        start, length = self._resolve_range(old_mo)
        possible_items = list(self._storage.item_slice(start, start+length))
        for a,v in possible_items:
            if v is old_mo:
                self._storage[a] = new_mo

    def store_mo(self, new_mo, overwrite=True):
        start, length = self._resolve_range(new_mo)
        try:
            _, floor_value = self._storage.floor_item(start)
        except KeyError:
            floor_value = None

        after_items = list(self._storage.item_slice(start, start+length))<commit_msg>remove misleading page.__setitem__<commit_after>import bintrees
import cooldict
import claripy
import cffi
import cle

from ..s_errors import SimMemoryError, SimSegfaultError
from .. import s_options as options
from .memory_object import SimMemoryObject
from claripy.ast.bv import BV

_ffi = cffi.FFI()

import logging
l = logging.getLogger('simuvex.storage.paged_memory')

class Page(object):
    """
    Page object, allowing for more flexibility than just a raw dict.
    """

    PROT_READ = 1
    PROT_WRITE = 2
    PROT_EXEC = 4

    def __init__(
        self, page_addr, page_size, permissions=None,
        executable=False, storage=None, sinkhole=None
    ):
        """
        Create a new page object. Carries permissions information.
        Permissions default to RW unless `executable` is True,
        in which case permissions default to RWX.

        :param int page_addr: The base address of the page.
        :param int page_size: The size of the page.
        :param bool executable: Whether the page is executable. Typically,
                            this will depend on whether the binary has an
                            executable stack.
        :param claripy.AST permissions: A 3-bit bitvector setting specific permissions
                            for EXEC, READ, and WRITE
        :param bintrees.AVLTree storage: A binary tree with the page data.
        :param claripy.AST sinkhole: A "default" value for the page.
        """

        self._page_addr = page_addr
        self._page_size = page_size
        self._storage = bintrees.AVLTree() if storage is None else storage
        self._sinkhole = sinkhole

        if permissions is None:
            perms = Page.PROT_READ|Page.PROT_WRITE
            if executable:
                perms |= Page.PROT_EXEC
            self.permissions = claripy.BVV(perms, 3) # 3 bits is enough for PROT_EXEC, PROT_WRITE, PROT_READ, PROT_NONE
        else:
            self.permissions = permissions

    @property
    def concrete_permissions(self):
        if self.permissions.symbolic:
            return 7
        else:
            return self.permissions.args[0]

    def sinkhole(self, v, wipe=False):
        if wipe:
            self._storage = bintrees.AVLTree()
        self._sinkhole = v

    def keys(self):
        if self._sinkhole is not None:
            return range(self._page_size)
        else:
            n = 0
            addrs = [ ]
            while n < self._page_size:
                try:
                    page_idx, mo = self._storage.ceiling_item(n)
                    addrs.extend(range(
                        max(0, mo.base - self._page_addr),
                        min(mo.last_addr+1-self._page_addr, self._page_size)
                    ))
                    n = page_idx+mo.length
                except KeyError:
                    break
            return addrs

    def __contains__(self, item):
        return self.load_mo(item, 1)[0] is not None

    def __getitem__(self, idx):
        i = self.load_mo(idx, 1)[0]
        if i is None:
            raise KeyError(idx)
        else:
            return i

    def _resolve_range(self, mo):
        length = mo.length
        start =  mo.base - self._page_addr
        if start < 0:
            length = length + start
            start = 0
        length = min(length, self._page_size)
        if length <= 0:
            l.warning("Nothing left of the memory object to store in SimPage.")

        return start, length

    def replace_mo(self, old_mo, new_mo):
        if old_mo is self._sinkhole:
            self._sinkhole = new_mo

        start, length = self._resolve_range(old_mo)
        possible_items = list(self._storage.item_slice(start, start+length))
        for a,v in possible_items:
            if v is old_mo:
                self._storage[a] = new_mo

    def store_mo(self, new_mo, overwrite=True):
        start, length = self._resolve_range(new_mo)
        try:
            _, floor_value = self._storage.floor_item(start)
        except KeyError:
            floor_value = None

        after_items = list(self._storage.item_slice(start, start+length))
{'fraction_non_alphanumeric': 0.07483233321567243, 'fraction_numerical': 0.004235792446170138, 'mean_word_length': 1.8220451527224435, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '645047', 'n_tokens_mistral': 2487, 'n_tokens_neox': 2371, 'n_words': 717}