<commit_before>import claripy
import logging
import itertools

from .memory_object import SimMemoryObject
from ..state_plugins.plugin import SimStatePlugin
from ..state_plugins.sim_action_object import SimActionObject
from ..state_plugins.symbolic_memory import SimSymbolicMemory
from .. import sim_options

l = logging.getLogger("angr.storage.file")

file_counter = itertools.count()
dialogue_counter = itertools.count()

class Flags: # pylint: disable=W0232,
    O_RDONLY = 0
    O_WRONLY = 1
    O_RDWR = 2
    O_ACCMODE = 3 # bitmask for read/write mode
    O_APPEND = 4096
    O_ASYNC = 64
    O_CLOEXEC = 512
    # TODO mode for this flag
    O_CREAT = 256
    O_DIRECT = 262144
    O_DIRECTORY = 2097152
    O_EXCL = 2048
    O_LARGEFILE = 1048576
    O_NOATIME = 16777216
    O_NOCTTY = 1024
    O_NOFOLLOW = 4194304
    O_NONBLOCK = 8192
    O_NODELAY = 8192
    O_SYNC = 67174400
    O_TRUNC = 1024


def _deps_unpack(a):
    if isinstance(a, SimActionObject):
        return a.ast, a.reg_deps, a.tmp_deps
    else:
        return a, None, None

class SimFileBase(SimStatePlugin):
    """
    SimFiles are the storage mechanisms used by SimFileDescriptors.

    Different types of SimFiles can have drastically different interfaces, and as a result there's not much that can be
    specified on this base class. All the read and write methods take a ``pos`` argument, which may have different
    semantics per-class. ``0`` will always be a valid position to use, though, and the next position you should use
    is part of the return tuple.

    Some simfiles are "streams", meaning that the position that reads come from is determined not by the position you
    pass in (it will in fact be ignored), but by an internal variable. This is stored as ``.pos`` if you care to read
    it. Don't write to it. The same lack-of-semantics applies to this field as well.

    :ivar name:     The name of the file. Purely for cosmetic purposes
    :ivar ident:    The identifier of the file, typically autogenerated from the name and a nonce. Purely for cosmetic
                    purposes, but does appear in symbolic values autogenerated in the file.
    :ivar seekable: Bool indicating whether seek operations on this file should succeed. If this is True, then ``pos``
                    must be a number of bytes from the start of the file.
    :ivar writable: Bool indicating whether writing to this file is allowed.
    :ivar pos:      If the file is a stream, this will be the current position. Otherwise, None.
    """

    seekable = False
    pos = None

    def __init__(self, name, writable=True, ident=None, **kwargs):
        self.name = name
        self.ident = ident
        self.writable = writable

        if ident is None:
            nice_name = self.name if all(0x20 <= ord(c) <= 0x7f for c in self.name) else '???'
            self.ident = 'file_%d_%s' % (next(file_counter), nice_name)

        if 'memory_id' in kwargs:
            kwargs['memory_id'] = self.ident
        super(SimFileBase, self).__init__(**kwargs)

    def concretize(self, **kwargs):
        """
        Return a concretization of the contents of the file. The type of the return value of this method will vary
        depending on which kind of SimFile you're using.
        """
        raise NotImplementedError

    def read(self, pos, size, **kwargs):
        """
        Read some data from the file.

        :param pos:     The offset in the file to read from.
        :param size:    The size to read. May be symbolic.
        :return:        A tuple of the data read (a bitvector of the length that is the maximum length of the read), the actual size of the read, and the new file position pointer.
        """
        raise NotImplementedError

    def write(self, pos, data, size=None, **kwargs):
        """
        Write some data to the file.

        :param pos:     The offset in the file to write to. May be ignored if the file is a stream or device.
        :param data:    The data to write as a bitvector
        :param size:    The optional size of the data to write. If not provided will default to the length of the data.
                        Must be constrained to less than or equal to the size of the data.
        :return:        The new file position pointer.
        """
        raise NotImplementedError

    @property
    def size(self):
        """
        The number of data bytes stored by the file at present. May be a symbolic value.
        """
        raise NotImplementedError

class SimFile(SimFileBase, SimSymbolicMemory):
    """
    The normal SimFile is meant to files on disk. It subclasses SimSymbolicMemory so loads and stores to/from
    it are very simple.

    :param name:        The name of the file
    :param content:     Optional initial content for the file as a string or bitvector
    :param size:        Optional size of the file. If content is not specified, it defaults to zero
    :param has_end:     Whether the size boundary is treated as the end of the file or a frontier at which new content
                        will be generated. If unspecified, will pick its value based on options.FILES_HAVE_EOF. Another
                        caveat is that if the size is also unspecified this value will default to False.
    :param seekable:    Optional bool indicating whether seek operations on this file should succeed, default True.
    :param writable:    Whether writing to this file is allowed

    :ivar has_end:      Whether this file has an EOF
    """
    def __init__(self, name, content=None, size=None, has_end=None, seekable=True, writable=True, ident=None, **kwargs):
        kwargs['memory_id'] = kwargs.get('memory_id', 'file')
        super(SimFile, self).__init__(name, writable=writable, ident=ident, **kwargs)
        self._size = size
        self.has_end = has_end
        self.seekable = seekable

        # this is hacky because we need to work around not having a state yet
        content = _deps_unpack(content)[0]
        if type(content) is bytes:
            content = claripy.BVV(content)
        elif type(content) is unicode:
            content = claripy.BVV(content.encode('utf-8'))
        elif content is None:
            pass
        elif isinstance(content, claripy.Bits):
            pass
        else:
            raise TypeError("Can't handle SimFile content of type %s" % type(content))

        if content is not None:
            mo = SimMemoryObject(content, 0, length=len(content)//8)
            self.mem.store_memory_object(mo)

            if self._size is None:
                self._size = len(content) // 8
        else:
            if self._size is None:
                self._size = 0
                if has_end is None:
                    self.has_end = False

    @property
    def category(self): # override trying to determine from self.id to allow arbitrary idents
        return 'file'

    def set_state(self, state):
        super(SimFile, self).set_state(state)
        if self.has_end is None:
            self.has_end = sim_options.FILES_HAVE_EOF in state.options

        if type(self._size) in (int, long):
            self._size = claripy.BVV(self._size, state.arch.bits)

    @property
    def size(self):
        return self._size

    def concretize(self, **kwargs):
        """
        Return a concretization of the contents of the file, as a flat bytestring.
        """
        size = self.state.solver.eval(self._size, **kwargs)
        data = self.load(0, size)

        kwargs['cast_to'] = kwargs.get('cast_to', str)
        kwargs['extra_constraints'] = kwargs.get('extra_constraints', ()) + (self._size == size,)
        return self.state.solver.eval(data, **kwargs)

    def read(self, pos, size, **kwargs):
        # Step 1: figure out a reasonable concrete size to use for the memory load
        # since we don't want to concretize anything
        if self.state.solver.symbolic(size):
            try:<commit_msg>Coerce extra_constraints into a tuple before trying to perform operations on it<commit_after>import claripy
import logging
import itertools

from .memory_object import SimMemoryObject
from ..state_plugins.plugin import SimStatePlugin
from ..state_plugins.sim_action_object import SimActionObject
from ..state_plugins.symbolic_memory import SimSymbolicMemory
from .. import sim_options

l = logging.getLogger("angr.storage.file")

file_counter = itertools.count()
dialogue_counter = itertools.count()

class Flags: # pylint: disable=W0232,
    O_RDONLY = 0
    O_WRONLY = 1
    O_RDWR = 2
    O_ACCMODE = 3 # bitmask for read/write mode
    O_APPEND = 4096
    O_ASYNC = 64
    O_CLOEXEC = 512
    # TODO mode for this flag
    O_CREAT = 256
    O_DIRECT = 262144
    O_DIRECTORY = 2097152
    O_EXCL = 2048
    O_LARGEFILE = 1048576
    O_NOATIME = 16777216
    O_NOCTTY = 1024
    O_NOFOLLOW = 4194304
    O_NONBLOCK = 8192
    O_NODELAY = 8192
    O_SYNC = 67174400
    O_TRUNC = 1024


def _deps_unpack(a):
    if isinstance(a, SimActionObject):
        return a.ast, a.reg_deps, a.tmp_deps
    else:
        return a, None, None

class SimFileBase(SimStatePlugin):
    """
    SimFiles are the storage mechanisms used by SimFileDescriptors.

    Different types of SimFiles can have drastically different interfaces, and as a result there's not much that can be
    specified on this base class. All the read and write methods take a ``pos`` argument, which may have different
    semantics per-class. ``0`` will always be a valid position to use, though, and the next position you should use
    is part of the return tuple.

    Some simfiles are "streams", meaning that the position that reads come from is determined not by the position you
    pass in (it will in fact be ignored), but by an internal variable. This is stored as ``.pos`` if you care to read
    it. Don't write to it. The same lack-of-semantics applies to this field as well.

    :ivar name:     The name of the file. Purely for cosmetic purposes
    :ivar ident:    The identifier of the file, typically autogenerated from the name and a nonce. Purely for cosmetic
                    purposes, but does appear in symbolic values autogenerated in the file.
    :ivar seekable: Bool indicating whether seek operations on this file should succeed. If this is True, then ``pos``
                    must be a number of bytes from the start of the file.
    :ivar writable: Bool indicating whether writing to this file is allowed.
    :ivar pos:      If the file is a stream, this will be the current position. Otherwise, None.
    """

    seekable = False
    pos = None

    def __init__(self, name, writable=True, ident=None, **kwargs):
        self.name = name
        self.ident = ident
        self.writable = writable

        if ident is None:
            nice_name = self.name if all(0x20 <= ord(c) <= 0x7f for c in self.name) else '???'
            self.ident = 'file_%d_%s' % (next(file_counter), nice_name)

        if 'memory_id' in kwargs:
            kwargs['memory_id'] = self.ident
        super(SimFileBase, self).__init__(**kwargs)

    def concretize(self, **kwargs):
        """
        Return a concretization of the contents of the file. The type of the return value of this method will vary
        depending on which kind of SimFile you're using.
        """
        raise NotImplementedError

    def read(self, pos, size, **kwargs):
        """
        Read some data from the file.

        :param pos:     The offset in the file to read from.
        :param size:    The size to read. May be symbolic.
        :return:        A tuple of the data read (a bitvector of the length that is the maximum length of the read), the actual size of the read, and the new file position pointer.
        """
        raise NotImplementedError

    def write(self, pos, data, size=None, **kwargs):
        """
        Write some data to the file.

        :param pos:     The offset in the file to write to. May be ignored if the file is a stream or device.
        :param data:    The data to write as a bitvector
        :param size:    The optional size of the data to write. If not provided will default to the length of the data.
                        Must be constrained to less than or equal to the size of the data.
        :return:        The new file position pointer.
        """
        raise NotImplementedError

    @property
    def size(self):
        """
        The number of data bytes stored by the file at present. May be a symbolic value.
        """
        raise NotImplementedError

class SimFile(SimFileBase, SimSymbolicMemory):
    """
    The normal SimFile is meant to files on disk. It subclasses SimSymbolicMemory so loads and stores to/from
    it are very simple.

    :param name:        The name of the file
    :param content:     Optional initial content for the file as a string or bitvector
    :param size:        Optional size of the file. If content is not specified, it defaults to zero
    :param has_end:     Whether the size boundary is treated as the end of the file or a frontier at which new content
                        will be generated. If unspecified, will pick its value based on options.FILES_HAVE_EOF. Another
                        caveat is that if the size is also unspecified this value will default to False.
    :param seekable:    Optional bool indicating whether seek operations on this file should succeed, default True.
    :param writable:    Whether writing to this file is allowed

    :ivar has_end:      Whether this file has an EOF
    """
    def __init__(self, name, content=None, size=None, has_end=None, seekable=True, writable=True, ident=None, **kwargs):
        kwargs['memory_id'] = kwargs.get('memory_id', 'file')
        super(SimFile, self).__init__(name, writable=writable, ident=ident, **kwargs)
        self._size = size
        self.has_end = has_end
        self.seekable = seekable

        # this is hacky because we need to work around not having a state yet
        content = _deps_unpack(content)[0]
        if type(content) is bytes:
            content = claripy.BVV(content)
        elif type(content) is unicode:
            content = claripy.BVV(content.encode('utf-8'))
        elif content is None:
            pass
        elif isinstance(content, claripy.Bits):
            pass
        else:
            raise TypeError("Can't handle SimFile content of type %s" % type(content))

        if content is not None:
            mo = SimMemoryObject(content, 0, length=len(content)//8)
            self.mem.store_memory_object(mo)

            if self._size is None:
                self._size = len(content) // 8
        else:
            if self._size is None:
                self._size = 0
                if has_end is None:
                    self.has_end = False

    @property
    def category(self): # override trying to determine from self.id to allow arbitrary idents
        return 'file'

    def set_state(self, state):
        super(SimFile, self).set_state(state)
        if self.has_end is None:
            self.has_end = sim_options.FILES_HAVE_EOF in state.options

        if type(self._size) in (int, long):
            self._size = claripy.BVV(self._size, state.arch.bits)

    @property
    def size(self):
        return self._size

    def concretize(self, **kwargs):
        """
        Return a concretization of the contents of the file, as a flat bytestring.
        """
        size = self.state.solver.eval(self._size, **kwargs)
        data = self.load(0, size)

        kwargs['cast_to'] = kwargs.get('cast_to', str)
        kwargs['extra_constraints'] = tuple(kwargs.get('extra_constraints', ())) + (self._size == size,)
        return self.state.solver.eval(data, **kwargs)

    def read(self, pos, size, **kwargs):
        # Step 1: figure out a reasonable concrete size to use for the memory load
        # since we don't want to concretize anything
        if self.state.solver.symbolic(size):
            try:
{'fraction_non_alphanumeric': 0.07369933170944976, 'fraction_numerical': 0.012116669789519705, 'mean_word_length': 2.4228302693458743, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '767831', 'n_tokens_mistral': 4679, 'n_tokens_neox': 4430, 'n_words': 1834}