<commit_before>"""
@file
@brief Profiling class.
"""
import sys
import inspect
import numpy
import pandas
from ._event_profiler import CEventProfiler  # pylint: disable=E0611


class EventProfiler:
    """
    This profiler profiles both memory and function calls.
    It stores events and produces a timeline.
    See :ref:`l-example-event-profling` for an example
    of use.

    :param size: size of the buffer to store events

    The profiler stores every event about function calls and returns,
    and memory allocation. It does not give the time spent in every function
    but a timeline with the series of events which occured.
    There are two pieces:

    * A C++ class which logs every event into a bug buffer,
      every event is described by five int64 (id_frame, event, timestamp,
      value1, value2).
    * A python class which wraps the first one.

    Function calls are caught by using :func:`sys.setprofile`.
    Memory calls are caught by using :epkg:`PyMem_SetAllocator`.
    Once the profiled code has ended, results are returned by
    @see meth retrieve_results. This function extends the information
    stored in the C++ class with file names, function names, line numbers,
    memory deallocated.

    .. note::
        The program crashes if the profiled code raises an exception.
        The default memory allocator is not restored and any allocation
        is logged into a buffer which was deleted. The exception must be
        caught or class @see cl WithEventProfiler must be used.
    """

    _event_mapping = {
        'call': 1,
        'return': 2,
        'c_call': 3,
        'c_return': 4,
        'exception': 5,
        'c_exception': 6,
        'profiler_call': 10,
        'profiler_return': 11,
        'profiler_start': 100,
        'profiler_sort': 101,
        'malloc': 1000,
        'calloc': 1001,
        'realloc': 1002,
        'free': 1003,
        'realloc_free': 1004,
    }

    def __init__(self, size=1000000):
        self._started = False
        self._prof = None
        self._frames = {-1: inspect.currentframe()}
        self._args = {-1: None, 0: None, id(None): None}
        self._buffer = CEventProfiler(size)
        self._events = []

    @property
    def n_columns(self):
        """
        Returns the number of stored informations by the profiler
        in memory. This corresponds to the number of columns returned
        by @see meth retrieve_raw_results.
        """
        return self._buffer.n_columns()

    def start(self):
        """
        Starts the profiling.
        """
        if self._started:
            raise RuntimeError(
                "The profiler was already started. It cannot be done again.")
        self._frames[0] = inspect.currentframe()
        self._started = True
        self._setup_profiler()
        self._buffer.log_event(-1, -1, 100, 0, 0)
        self._buffer.start()

    def stop(self):
        """
        Stops the profiling.
        """
        if not self._started:
            raise RuntimeError(
                "The profiler was not started. It must be done first.")
        self._buffer.stop()
        self._buffer.log_event(-1, -1, 101, 0, 0)
        self._restore_profiler()
        self._started = False

    def log_event(self, frame, event, arg):
        """
        Logs an event in the database.

        :param frame: (frame), see :mod:`inspect`
        :param event: (str)
            kind of event
        :param value1: (int)
            additional value to the event
        :param value2: (int)
            additional value to the event
        :param arg: None or...
        """
        idf = id(frame)
        if idf not in self._frames:
            self._frames[idf] = frame
        if hasattr(arg, '__qualname__'):
            ida = id(arg)
        else:
            ida = id(None)
        if ida not in self._args:
            self._args[ida] = arg
        r = self._buffer.log_event(
            idf, ida, EventProfiler._event_mapping[event], 0, 0)
        if not r:
            self._empty_cache()

    def _setup_profiler(self):
        """
        This relies on :func:`sys.setprofile` and :func:`sys.getprofile`.
        """
        self._prof = sys.getprofile()
        sys.setprofile(self.log_event)

    def _restore_profiler(self):
        """
        This relies on :func:`sys.setprofile` and :func:`sys.getprofile`.
        """
        sys.setprofile(self._prof)
        self._prof = None

    def _empty_cache(self):
        """
        Empties the cache.
        """
        self._buffer.log_event(-1, -1, 10, 0, 0)
        self._buffer.lock()
        size = len(self._buffer)
        if size == 0:
            self._buffer.unlock()
            self._buffer.log_event(-1, -1, 11, 0, 0)
            return 0
        store = numpy.empty(
            (size, self._buffer.n_columns()), dtype=numpy.int64)
        self._buffer.dump(store, False)
        self._buffer.clear(False)
        self._buffer.unlock()

        self._events.append(store)
        self._buffer.log_event(-1, -1, 11, store.shape[0], 0)
        return size

    def retrieve_raw_results(self, empty_cache=True):
        """
        Retrieves the raw results. Difficult to interpret.

        :param empty_cache: retrives the data from the C++ container and
            stores it in a numpy array before
        :return: numpy array
        """
        if empty_cache:
            self._empty_cache()
        res = numpy.vstack(self._events)
        # Filling information about the size of freed buffer.
        # id_frame, event, time, value1, value2
        memo = {}
        for i in range(res.shape[0]):
            event = res[i, 2]
            if event < 1000:
                continue
            ptr = res[i, 4]
            if event <= 1002:
                memo[ptr] = res[i, 5]
            else:
                if ptr not in memo:
                    # The system is freeing a non-existing pointer.
                    res[i, 5] = 0
                else:
                    res[i, 5] = -memo[ptr]
                    del memo[ptr]<commit_msg>documentation<commit_after>"""
@file
@brief Profiling class.
"""
import sys
import inspect
import numpy
import pandas
from ._event_profiler import CEventProfiler  # pylint: disable=E0611


class EventProfiler:
    """
    This profiler profiles both memory and function calls.
    It stores events and produces a timeline.
    See :ref:`l-example-event-profling` for an example
    of use.

    :param size: size of the buffer to store events

    The profiler stores every event about function calls and returns,
    and memory allocation. It does not give the time spent in every function
    but a timeline with the series of events which occured.
    There are two pieces:

    * A C++ class which logs every event into a bug buffer,
      every event is described by five int64 (id_frame, id_arg, event,
      timestamp, value1, value2).
    * A python class which wraps the first one.

    Function calls are caught by using :func:`sys.setprofile`.
    Memory calls are caught by using :epkg:`PyMem_SetAllocator`.
    Once the profiled code has ended, results are returned by
    @see meth retrieve_results. This function extends the information
    stored in the C++ class with file names, function names, line numbers,
    memory deallocated.

    .. note::
        The program crashes if the profiled code raises an exception.
        The default memory allocator is not restored and any allocation
        is logged into a buffer which was deleted. The exception must be
        caught or class @see cl WithEventProfiler must be used.
    """

    _event_mapping = {
        'call': 1,
        'return': 2,
        'c_call': 3,
        'c_return': 4,
        'exception': 5,
        'c_exception': 6,
        'profiler_call': 10,
        'profiler_return': 11,
        'profiler_start': 100,
        'profiler_sort': 101,
        'malloc': 1000,
        'calloc': 1001,
        'realloc': 1002,
        'free': 1003,
        'realloc_free': 1004,
    }

    def __init__(self, size=1000000):
        self._started = False
        self._prof = None
        self._frames = {-1: inspect.currentframe()}
        self._args = {-1: None, 0: None, id(None): None}
        self._buffer = CEventProfiler(size)
        self._events = []

    @property
    def n_columns(self):
        """
        Returns the number of stored informations by the profiler
        in memory. This corresponds to the number of columns returned
        by @see meth retrieve_raw_results.
        """
        return self._buffer.n_columns()

    def start(self):
        """
        Starts the profiling.
        """
        if self._started:
            raise RuntimeError(
                "The profiler was already started. It cannot be done again.")
        self._frames[0] = inspect.currentframe()
        self._started = True
        self._setup_profiler()
        self._buffer.log_event(-1, -1, 100, 0, 0)
        self._buffer.start()

    def stop(self):
        """
        Stops the profiling.
        """
        if not self._started:
            raise RuntimeError(
                "The profiler was not started. It must be done first.")
        self._buffer.stop()
        self._buffer.log_event(-1, -1, 101, 0, 0)
        self._restore_profiler()
        self._started = False

    def log_event(self, frame, event, arg):
        """
        Logs an event in the database.

        :param frame: (frame), see :mod:`inspect`
        :param event: (str)
            kind of event
        :param value1: (int)
            additional value to the event
        :param value2: (int)
            additional value to the event
        :param arg: None or...
        """
        idf = id(frame)
        if idf not in self._frames:
            self._frames[idf] = frame
        if type(arg) == type(open):
            ida = id(arg)
        else:
            ida = id(None)
        if ida not in self._args:
            self._args[ida] = arg
        r = self._buffer.log_event(
            idf, ida, EventProfiler._event_mapping[event], 0, 0)
        if not r:
            self._empty_cache()

    def _setup_profiler(self):
        """
        This relies on :func:`sys.setprofile` and :func:`sys.getprofile`.
        """
        self._prof = sys.getprofile()
        sys.setprofile(self.log_event)

    def _restore_profiler(self):
        """
        This relies on :func:`sys.setprofile` and :func:`sys.getprofile`.
        """
        sys.setprofile(self._prof)
        self._prof = None

    def _empty_cache(self):
        """
        Empties the cache.
        """
        self._buffer.log_event(-1, -1, 10, 0, 0)
        self._buffer.lock()
        size = len(self._buffer)
        if size == 0:
            self._buffer.unlock()
            self._buffer.log_event(-1, -1, 11, 0, 0)
            return 0
        store = numpy.empty(
            (size, self._buffer.n_columns()), dtype=numpy.int64)
        self._buffer.dump(store, False)
        self._buffer.clear(False)
        self._buffer.unlock()

        self._events.append(store)
        self._buffer.log_event(-1, -1, 11, store.shape[0], 0)
        return size

    def retrieve_raw_results(self, empty_cache=True):
        """
        Retrieves the raw results. Difficult to interpret.

        :param empty_cache: retrives the data from the C++ container and
            stores it in a numpy array before
        :return: numpy array
        """
        if empty_cache:
            self._empty_cache()
        res = numpy.vstack(self._events)
        # Filling information about the size of freed buffer.
        # id_frame, id_arg, event, time, value1, value2
        memo = {}
        for i in range(res.shape[0]):
            event = res[i, 2]
            if event < 1000:
                continue
            ptr = res[i, 4]
            if event <= 1002:
                memo[ptr] = res[i, 5]
            else:
                if ptr not in memo:
                    # The system is freeing a non-existing pointer.
                    res[i, 5] = 0
                else:
                    res[i, 5] = -memo[ptr]
                    del memo[ptr]
{'fraction_non_alphanumeric': 0.09667896678966789, 'fraction_numerical': 0.01836818368183682, 'mean_word_length': 1.8362790697674418, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6804066', 'n_tokens_mistral': 3667, 'n_tokens_neox': 3439, 'n_words': 1157}