<commit_before>import os
import sys
import platform
import logging
from collections import OrderedDict

import archinfo
from archinfo.arch_soot import ArchSoot

from .address_translator import AT
from .utils import ALIGN_UP, key_bisect_insort_left, key_bisect_floor_key

try:
    import claripy
except ImportError:
    claripy = None

__all__ = ('Loader',)

l = logging.getLogger(name=__name__)


class Loader:
    """
    The loader loads all the objects and exports an abstraction of the memory of the process. What you see here is an
    address space with loaded and rebased binaries.

    :param main_binary:         The path to the main binary you're loading, or a file-like object with the binary
                                in it.

    The following parameters are optional.

    :param auto_load_libs:      Whether to automatically load shared libraries that loaded objects depend on.
    :param concrete_target:     Whether to instantiate a concrete target for a concrete execution of the process.
                                if this is the case we will need to instantiate a SimConcreteEngine that wraps the
                                ConcreteTarget provided by the user.
    :param force_load_libs:     A list of libraries to load regardless of if they're required by a loaded object.
    :param skip_libs:           A list of libraries to never load, even if they're required by a loaded object.
    :param main_opts:           A dictionary of options to be used loading the main binary.
    :param lib_opts:            A dictionary mapping library names to the dictionaries of options to be used when
                                loading them.
    :param ld_path:      A list of paths in which we can search for shared libraries.
    :param use_system_libs:     Whether or not to search the system load path for requested libraries. Default True.
    :param ignore_import_version_numbers:
                                Whether libraries with different version numbers in the filename will be considered
                                equivalent, for example libc.so.6 and libc.so.0
    :param case_insensitive:    If this is set to True, filesystem loads will be done case-insensitively regardless of
                                the case-sensitivity of the underlying filesystem.
    :param rebase_granularity:  The alignment to use for rebasing shared objects
    :param except_missing_libs: Throw an exception when a shared library can't be found.
    :param aslr:                Load libraries in symbolic address space. Do not use this option.
    :param page_size:           The granularity with which data is mapped into memory. Set to 1 if you are working
                                in a non-paged environment.
    :param preload_libs:        Similar to `force_load_libs` but will provide for symbol resolution, with precedence
                                over any dependencies.
    :ivar memory:               The loaded, rebased, and relocated memory of the program.
    :vartype memory:            cle.memory.Clemory
    :ivar main_object:          The object representing the main binary (i.e., the executable).
    :ivar shared_objects:       A dictionary mapping loaded library names to the objects representing them.
    :ivar all_objects:          A list containing representations of all the different objects loaded.
    :ivar requested_names:      A set containing the names of all the different shared libraries that were marked as a
                                dependency by somebody.
    :ivar initial_load_objects: A list of all the objects that were loaded as a result of the initial load request.

    When reference is made to a dictionary of options, it requires a dictionary with zero or more of the following keys:

    - backend :             "elf", "pe", "mach-o", "blob" : which loader backend to use
    - arch :                The archinfo.Arch object to use for the binary
    - base_addr :           The address to rebase the object at
    - entry_point :         The entry point to use for the object

    More keys are defined on a per-backend basis.
    """

    def __init__(self, main_binary, auto_load_libs=True, concrete_target = None,
                 force_load_libs=(), skip_libs=(),
                 main_opts=None, lib_opts=None, ld_path=(), use_system_libs=True,
                 ignore_import_version_numbers=True, case_insensitive=False, rebase_granularity=0x1000000,
                 except_missing_libs=False, aslr=False, perform_relocations=True,
                 page_size=0x1, preload_libs=(), arch=None):
        if hasattr(main_binary, 'seek') and hasattr(main_binary, 'read'):
            self._main_binary_path = None
            self._main_binary_stream = main_binary
        else:
            self._main_binary_path = os.path.realpath(str(main_binary))
            self._main_binary_stream = None

        # whether we are presently in the middle of a load cycle
        self._juggling = False

        # auto_load_libs doesn't make any sense if we have a concrete target.
        if concrete_target:
            auto_load_libs = False

        self._auto_load_libs = auto_load_libs
        self._satisfied_deps = dict((x, False) for x in skip_libs)
        self._main_opts = {} if main_opts is None else main_opts
        self._lib_opts = {} if lib_opts is None else lib_opts<commit_msg>Reduce rebase granularity by 16x. hopefully this doesn't break anything<commit_after>import os
import sys
import platform
import logging
from collections import OrderedDict

import archinfo
from archinfo.arch_soot import ArchSoot

from .address_translator import AT
from .utils import ALIGN_UP, key_bisect_insort_left, key_bisect_floor_key

try:
    import claripy
except ImportError:
    claripy = None

__all__ = ('Loader',)

l = logging.getLogger(name=__name__)


class Loader:
    """
    The loader loads all the objects and exports an abstraction of the memory of the process. What you see here is an
    address space with loaded and rebased binaries.

    :param main_binary:         The path to the main binary you're loading, or a file-like object with the binary
                                in it.

    The following parameters are optional.

    :param auto_load_libs:      Whether to automatically load shared libraries that loaded objects depend on.
    :param concrete_target:     Whether to instantiate a concrete target for a concrete execution of the process.
                                if this is the case we will need to instantiate a SimConcreteEngine that wraps the
                                ConcreteTarget provided by the user.
    :param force_load_libs:     A list of libraries to load regardless of if they're required by a loaded object.
    :param skip_libs:           A list of libraries to never load, even if they're required by a loaded object.
    :param main_opts:           A dictionary of options to be used loading the main binary.
    :param lib_opts:            A dictionary mapping library names to the dictionaries of options to be used when
                                loading them.
    :param ld_path:      A list of paths in which we can search for shared libraries.
    :param use_system_libs:     Whether or not to search the system load path for requested libraries. Default True.
    :param ignore_import_version_numbers:
                                Whether libraries with different version numbers in the filename will be considered
                                equivalent, for example libc.so.6 and libc.so.0
    :param case_insensitive:    If this is set to True, filesystem loads will be done case-insensitively regardless of
                                the case-sensitivity of the underlying filesystem.
    :param rebase_granularity:  The alignment to use for rebasing shared objects
    :param except_missing_libs: Throw an exception when a shared library can't be found.
    :param aslr:                Load libraries in symbolic address space. Do not use this option.
    :param page_size:           The granularity with which data is mapped into memory. Set to 1 if you are working
                                in a non-paged environment.
    :param preload_libs:        Similar to `force_load_libs` but will provide for symbol resolution, with precedence
                                over any dependencies.
    :ivar memory:               The loaded, rebased, and relocated memory of the program.
    :vartype memory:            cle.memory.Clemory
    :ivar main_object:          The object representing the main binary (i.e., the executable).
    :ivar shared_objects:       A dictionary mapping loaded library names to the objects representing them.
    :ivar all_objects:          A list containing representations of all the different objects loaded.
    :ivar requested_names:      A set containing the names of all the different shared libraries that were marked as a
                                dependency by somebody.
    :ivar initial_load_objects: A list of all the objects that were loaded as a result of the initial load request.

    When reference is made to a dictionary of options, it requires a dictionary with zero or more of the following keys:

    - backend :             "elf", "pe", "mach-o", "blob" : which loader backend to use
    - arch :                The archinfo.Arch object to use for the binary
    - base_addr :           The address to rebase the object at
    - entry_point :         The entry point to use for the object

    More keys are defined on a per-backend basis.
    """

    def __init__(self, main_binary, auto_load_libs=True, concrete_target = None,
                 force_load_libs=(), skip_libs=(),
                 main_opts=None, lib_opts=None, ld_path=(), use_system_libs=True,
                 ignore_import_version_numbers=True, case_insensitive=False, rebase_granularity=0x100000,
                 except_missing_libs=False, aslr=False, perform_relocations=True,
                 page_size=0x1, preload_libs=(), arch=None):
        if hasattr(main_binary, 'seek') and hasattr(main_binary, 'read'):
            self._main_binary_path = None
            self._main_binary_stream = main_binary
        else:
            self._main_binary_path = os.path.realpath(str(main_binary))
            self._main_binary_stream = None

        # whether we are presently in the middle of a load cycle
        self._juggling = False

        # auto_load_libs doesn't make any sense if we have a concrete target.
        if concrete_target:
            auto_load_libs = False

        self._auto_load_libs = auto_load_libs
        self._satisfied_deps = dict((x, False) for x in skip_libs)
        self._main_opts = {} if main_opts is None else main_opts
        self._lib_opts = {} if lib_opts is None else lib_opts
{'fraction_non_alphanumeric': 0.04679871027176417, 'fraction_numerical': 0.002487333026255182, 'mean_word_length': 2.399937362981522, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '291385', 'n_tokens_mistral': 2813, 'n_tokens_neox': 2648, 'n_words': 1196}