<commit_before>import hashlib

from androguard.core.analysis.analysis import *
from androguard.core.bytecodes.dvm import *
from androguard.decompiler.decompiler import *
from androguard.core import androconf

import pickle
import logging

log = logging.getLogger("androguard.session")


def Save(session, filename):
    """
    save your session to use it later.

    example::

        s = session.Session()
        session.Save(s, "msession.p")

    :param session: A Session object to save
    :param filename: output filename to save the session
    :type filename: string

    """
    with open(filename, "wb") as fd:
        pickle.dump(session, fd)


def Load(filename):
    """
      load your session!

      example::

          s = session.Load("mysession.p")

      :param filename: the filename where the session has been saved
      :type filename: string

      :rtype: the elements of your session :)

    """
    with open(filename, "rb") as fd:
        return pickle.load(fd)


class Session(object):
    """
    A Session is able to store multiple APK, DEX or ODEX files and can be pickled
    to disk in order to resume work later.

    The main function used in Sessions is probably :meth:`add`, which adds files
    to the session and performs analysis on them.

    Afterwards, the files can be gathered using methods such as
    :meth:`get_objects_apk`, :meth:`get_objects_dex` or :meth:`get_classes`.

    example::

        s = Session()
        digest = s.add("some.apk")

        print("SHA256 of the file: {}".format(digest))

        a, d, dx = s.get_objects_apk("some.apk", digest)
        print(a.get_package())

        # Reset the Session for a fresh set of files
        s.reset()

        digest2 = s.add("classes.dex")
        print("SHA256 of the file: {}".format(digest2))
        for h, d, dx in s.get_objects_dex():
            print("SHA256 of the DEX file: {}".format(h))


    """
    def __init__(self, export_ipython=False):
        """
        Create a new Session object

        :param export_ipython: set to True in order to create attributes for the
        use in iPython
        """
        self._setup_objects()
        self.export_ipython = export_ipython

    def _setup_objects(self):
        self.analyzed_files = collections.defaultdict(list)
        self.analyzed_digest = dict()
        self.analyzed_apk = dict()

        # Stores Analysis Objects
        # needs to be ordered to return the outermost element when searching for
        # classes
        self.analyzed_vms = collections.OrderedDict()

        # Dict of digest and DalvikVMFormat/DalvikOdexFormat
        # Actually not needed, as we have Analysis objects which store the DEX
        # files as well, but we do not remove it here for legacy reasons
        self.analyzed_dex = dict()

    def reset(self):
        """
        Reset the current session, delete all added files.
        """
        self._setup_objects()

    def isOpen(self):
        """
        Test if any file was analyzed in this session

        :return: `True` if any file was analyzed, `False` otherwise
        """
        return len(self.analyzed_digest) > 0
<commit_msg>Allow higher recursion limit for saving sessions<commit_after>import hashlib
from androguard.core.analysis.analysis import *
from androguard.core.bytecodes.dvm import *
from androguard.decompiler.decompiler import *
from androguard.core import androconf

import pickle
import logging
import datetime

log = logging.getLogger("androguard.session")


def Save(session, filename=None):
    """
    save your session to use it later.

    Returns the filename of the written file.
    If not filename is given, a file named `androguard_session_<DATE>.ag` will
    be created in the current working directory.
    `<DATE>` is a timestamp with the following format: `%Y-%m-%d_%H%M%S`.

    This function will overwrite existing files without asking.

    If the file could not written, None is returned.

    example::

        s = session.Session()
        session.Save(s, "msession.ag")

    :param session: A Session object to save
    :param filename: output filename to save the session
    :type filename: string

    """

    if not filename:
        filename = "androguard_session_{:%Y-%m-%d_%H%M%S}.ag".format(datetime.datetime.now())

    if os.path.isfile(filename):
        log.warning("{} already exists, overwriting!")

    # Setting the recursion limit according to the documentation:
    # https://docs.python.org/3/library/pickle.html#what-can-be-pickled-and-unpickled
    #
    # Some larger APKs require a high recursion limit.
    # Tested to be above 35000 for some files, setting to 50k to be sure.
    # You might want to set this even higher if you encounter problems
    reclimit = sys.getrecursionlimit()
    sys.setrecursionlimit(50000)
    saved = False
    try:
        with open(filename, "wb") as fd:
            pickle.dump(session, fd)
        saved = True
    except RecursionError as e:
        log.exception("Recursion Limit hit while saving. "
                      "Current Recursion limit: {}. "
                      "Please report this error!".format(sys.getrecursionlimit()))
        # Remove partially written file
        os.unlink(filename)

    sys.setrecursionlimit(reclimit)
    return filename if saved else None


def Load(filename):
    """
      load your session!

      example::

          s = session.Load("mysession.ag")

      :param filename: the filename where the session has been saved
      :type filename: string

      :rtype: the elements of your session :)

    """
    with open(filename, "rb") as fd:
        return pickle.load(fd)


class Session:
    """
    A Session is able to store multiple APK, DEX or ODEX files and can be pickled
    to disk in order to resume work later.

    The main function used in Sessions is probably :meth:`add`, which adds files
    to the session and performs analysis on them.

    Afterwards, the files can be gathered using methods such as
    :meth:`get_objects_apk`, :meth:`get_objects_dex` or :meth:`get_classes`.

    example::

        s = Session()
        digest = s.add("some.apk")

        print("SHA256 of the file: {}".format(digest))

        a, d, dx = s.get_objects_apk("some.apk", digest)
        print(a.get_package())

        # Reset the Session for a fresh set of files
        s.reset()

        digest2 = s.add("classes.dex")
        print("SHA256 of the file: {}".format(digest2))
        for h, d, dx in s.get_objects_dex():
            print("SHA256 of the DEX file: {}".format(h))


    """
    def __init__(self, export_ipython=False):
        """
        Create a new Session object

        :param export_ipython: set to True in order to create attributes for the
        use in iPython
        """
        self._setup_objects()
        self.export_ipython = export_ipython

    def save(self, filename=None):
        """
        Save the current session, see also :func:`~androguard.session.Save`.
        """
        return Save(self, filename)

    def _setup_objects(self):
        self.analyzed_files = collections.defaultdict(list)
        self.analyzed_digest = dict()
        self.analyzed_apk = dict()

        # Stores Analysis Objects
        # needs to be ordered to return the outermost element when searching for
        # classes
        self.analyzed_vms = collections.OrderedDict()

        # Dict of digest and DalvikVMFormat/DalvikOdexFormat
        # Actually not needed, as we have Analysis objects which store the DEX
        # files as well, but we do not remove it here for legacy reasons
        self.analyzed_dex = dict()

    def reset(self):
        """
        Reset the current session, delete all added files.
        """
        self._setup_objects()

    def isOpen(self):
        """
        Test if any file was analyzed in this session

        :return: `True` if any file was analyzed, `False` otherwise
        """
        return len(self.analyzed_digest) > 0

{'fraction_non_alphanumeric': 0.09122719320169957, 'fraction_numerical': 0.004623844038990252, 'mean_word_length': 2.517802197802198, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1962012', 'n_tokens_mistral': 2337, 'n_tokens_neox': 2200, 'n_words': 797}