<commit_before>"""
 yappi.py
 Yet Another Python Profiler

 <NAME> 2013
"""
import os
import sys
import _yappi
import pickle
import marshal
import threading
        
class YappiError(Exception): pass

__all__ = ['start', 'stop', 'get_func_stats', 'get_thread_stats', 'clear_stats', 'is_running',
           'get_clock_type', 'set_clock_type',  'get_mem_usage']
           
CRLF = '\n'
COLUMN_GAP = 2
TIME_COLUMN_LEN = 8 # 0.000000, 12345.98, precision is microsecs
YPICKLE_PROTOCOL = 2

SORT_TYPES_FUNCSTATS = {"name":0, "callcount":3, "totaltime":6, "subtime":7, "avgtime":10,
                        "ncall":3, "ttot":6, "tsub":7, "tavg":10}
SORT_TYPES_CHILDFUNCSTATS = {"name":10, "callcount":1, "totaltime":3, "subtime":4, "avgtime":5,
                        "ncall":1, "ttot":3, "tsub":4, "tavg":5}
SORT_TYPES_THREADSTATS = {"name":0, "id":1, "totaltime":2, "schedcount":3,
                          "ttot":2, "scnt":3}
SORT_ORDERS = {"ascending":0, "asc":0, "descending":1, "desc":1}
DEFAULT_SORT_TYPE = "totaltime"
DEFAULT_SORT_ORDER = "desc"

CLOCK_TYPES = {"WALL":0, "CPU":1}

def _validate_sorttype(sort_type, list):
    sort_type = sort_type.lower()
    if sort_type not in list:
        raise YappiError("Invalid SortType parameter.[%s]" % (sort_type))
    return sort_type
       
def _validate_sortorder(sort_order):
    sort_order = sort_order.lower()
    if sort_order not in SORT_ORDERS:
        raise YappiError("Invalid SortOrder parameter.[%s]" % (sort_order))
    return sort_order
        
"""
 _callback will only be called once per-thread. _yappi will detect
 the new thread and changes the profilefunc param of the ThreadState
 structure. This is an internal function please don't mess with it.
"""
def _callback(frame, event, arg):
    _yappi._profile_event(frame, event, arg)
    return _callback
    
"""
function to prettify time columns in stats.
"""
def _fft(x):
    _rprecision = 6
    while(_rprecision > 0):
        _fmt = "%0." + "%d" % (_rprecision) + "f"
        s = _fmt % (x)
        if len(s) <= TIME_COLUMN_LEN:
            break
        _rprecision -= 1
    return s
    
def _func_fullname(builtin, module, lineno, name):
    if builtin: 
        return "%s.%s" % (module, name)
    else:
        return "%s:%d %s" % (module, lineno, name)
    
"""
Converts our internal yappi's YFuncStats (YSTAT type) to PSTAT. So there are 
some differences between the statistics parameters. The PSTAT format is as following:

PSTAT expects a dict. entry as following:

stats[("mod_name", line_no, "func_name")] = \
    ( total_call_count, actual_call_count, total_time, cumulative_time, 
    {
        ("mod_name", line_no, "func_name") : 
        (total_call_count, --> total count caller called the callee
        actual_call_count, --> total count caller called the callee - (recursive calls)
        total_time,        --> total time caller spent _only_ for this function (not further subcalls)
        cumulative_time)   --> total time caller spent for this function
    } --> callers dict
    )
    
Note that in PSTAT the total time spent in the function is called as cumulative_time and 
the time spent _only_ in the function as total_time. From Yappi's perspective, this means:

total_time (inline time) = tsub
cumulative_time (total time) = ttot

Other than that we hold called functions in a profile entry as named 'children'. On the
other hand, PSTAT expects to have a dict of callers of the function. So we also need to 
convert the information to that.       

PSTAT only expects to have the above dict to be saved.
"""
def convert2pstats(stats):
    """
    Converts the internal stat type of yappi(which is returned by a call to YFuncStats.get())
    as pstats object.
    """
    if not isinstance(stats, YFuncStats):
        raise YappiError("Source stats must be derived from YFuncStats.")
    
    import pstats
    class _PStatHolder:
        def __init__(self, d):
            self.stats = d
        def create_stats(self):
            pass                
    def pstat_id(fs):
        return (fs.module, fs.lineno, fs.name)
    
    _pdict = {}
    
    # convert callees to callers
    _callers = {}
    for fs in stats:
        if not fs in _callers:
            _callers[fs] = {}
            
        for ct in fs.children:
            if not ct in _callers:
                _callers[ct] = {}
            _callers[ct][pstat_id(fs)] = (ct.ncall, ct.nactualcall, ct.tsub ,ct.ttot)
    
    # populate the pstat dict.
    for fs in stats:
        _pdict[pstat_id(fs)] = (fs.ncall, fs.nactualcall, fs.tsub, fs.ttot, _callers[fs], )        
     
    return pstats.Stats(_PStatHolder(_pdict))
    
def profile(clock_type="cpu", profile_builtins=False, complete_callback=None):
    def _profile_dec(func):
        def wrapper(*args, **kwargs):
            clear_stats()
            set_clock_type(clock_type)
            start(profile_builtins, profile_threads=False)
            func(*args, **kwargs)
            stop()
            if complete_callback is None:
                get_func_stats().print_all()
            else:
                complete_callback(func, get_func_stats())
            clear_stats()
        return wrapper
    return _profile_dec    
    
class StatString(object):
    """
    Class to prettify/trim a profile result column.
    """
    _s = ""
    _TRAIL_DOT = ".."

    def __init__(self, s):
        self._s = str(s)

    def ltrim(self, length):
        if len(self._s) > length:<commit_msg>return the result of decorated function. Tests ongoing.<commit_after>"""
 yappi.py
 Yet Another Python Profiler

 <NAME> 2013
"""
import os
import sys
import _yappi
import pickle
import marshal
import threading
        
class YappiError(Exception): pass

__all__ = ['start', 'stop', 'get_func_stats', 'get_thread_stats', 'clear_stats', 'is_running',
           'get_clock_type', 'set_clock_type',  'get_mem_usage']
           
CRLF = '\n'
COLUMN_GAP = 2
TIME_COLUMN_LEN = 8 # 0.000000, 12345.98, precision is microsecs
YPICKLE_PROTOCOL = 2

SORT_TYPES_FUNCSTATS = {"name":0, "callcount":3, "totaltime":6, "subtime":7, "avgtime":10,
                        "ncall":3, "ttot":6, "tsub":7, "tavg":10}
SORT_TYPES_CHILDFUNCSTATS = {"name":10, "callcount":1, "totaltime":3, "subtime":4, "avgtime":5,
                        "ncall":1, "ttot":3, "tsub":4, "tavg":5}
SORT_TYPES_THREADSTATS = {"name":0, "id":1, "totaltime":2, "schedcount":3,
                          "ttot":2, "scnt":3}
SORT_ORDERS = {"ascending":0, "asc":0, "descending":1, "desc":1}
DEFAULT_SORT_TYPE = "totaltime"
DEFAULT_SORT_ORDER = "desc"

CLOCK_TYPES = {"WALL":0, "CPU":1}

def _validate_sorttype(sort_type, list):
    sort_type = sort_type.lower()
    if sort_type not in list:
        raise YappiError("Invalid SortType parameter.[%s]" % (sort_type))
    return sort_type
       
def _validate_sortorder(sort_order):
    sort_order = sort_order.lower()
    if sort_order not in SORT_ORDERS:
        raise YappiError("Invalid SortOrder parameter.[%s]" % (sort_order))
    return sort_order
        
"""
 _callback will only be called once per-thread. _yappi will detect
 the new thread and changes the profilefunc param of the ThreadState
 structure. This is an internal function please don't mess with it.
"""
def _callback(frame, event, arg):
    _yappi._profile_event(frame, event, arg)
    return _callback
    
"""
function to prettify time columns in stats.
"""
def _fft(x):
    _rprecision = 6
    while(_rprecision > 0):
        _fmt = "%0." + "%d" % (_rprecision) + "f"
        s = _fmt % (x)
        if len(s) <= TIME_COLUMN_LEN:
            break
        _rprecision -= 1
    return s
    
def _func_fullname(builtin, module, lineno, name):
    if builtin: 
        return "%s.%s" % (module, name)
    else:
        return "%s:%d %s" % (module, lineno, name)
    
"""
Converts our internal yappi's YFuncStats (YSTAT type) to PSTAT. So there are 
some differences between the statistics parameters. The PSTAT format is as following:

PSTAT expects a dict. entry as following:

stats[("mod_name", line_no, "func_name")] = \
    ( total_call_count, actual_call_count, total_time, cumulative_time, 
    {
        ("mod_name", line_no, "func_name") : 
        (total_call_count, --> total count caller called the callee
        actual_call_count, --> total count caller called the callee - (recursive calls)
        total_time,        --> total time caller spent _only_ for this function (not further subcalls)
        cumulative_time)   --> total time caller spent for this function
    } --> callers dict
    )
    
Note that in PSTAT the total time spent in the function is called as cumulative_time and 
the time spent _only_ in the function as total_time. From Yappi's perspective, this means:

total_time (inline time) = tsub
cumulative_time (total time) = ttot

Other than that we hold called functions in a profile entry as named 'children'. On the
other hand, PSTAT expects to have a dict of callers of the function. So we also need to 
convert the information to that.       

PSTAT only expects to have the above dict to be saved.
"""
def convert2pstats(stats):
    """
    Converts the internal stat type of yappi(which is returned by a call to YFuncStats.get())
    as pstats object.
    """
    if not isinstance(stats, YFuncStats):
        raise YappiError("Source stats must be derived from YFuncStats.")
    
    import pstats
    class _PStatHolder:
        def __init__(self, d):
            self.stats = d
        def create_stats(self):
            pass                
    def pstat_id(fs):
        return (fs.module, fs.lineno, fs.name)
    
    _pdict = {}
    
    # convert callees to callers
    _callers = {}
    for fs in stats:
        if not fs in _callers:
            _callers[fs] = {}
            
        for ct in fs.children:
            if not ct in _callers:
                _callers[ct] = {}
            _callers[ct][pstat_id(fs)] = (ct.ncall, ct.nactualcall, ct.tsub ,ct.ttot)
    
    # populate the pstat dict.
    for fs in stats:
        _pdict[pstat_id(fs)] = (fs.ncall, fs.nactualcall, fs.tsub, fs.ttot, _callers[fs], )        
     
    return pstats.Stats(_PStatHolder(_pdict))
    
def profile(clock_type="cpu", profile_builtins=False, complete_callback=None):
    def _profile_dec(func):
        def wrapper(*args, **kwargs):
            clear_stats()
            set_clock_type(clock_type)
            start(profile_builtins, profile_threads=False)
            result = func(*args, **kwargs)
            stop()
            if complete_callback is None:
                get_func_stats().print_all()
            else:
                complete_callback(func, get_func_stats())
            clear_stats()
            return result
        return wrapper
    return _profile_dec    
    
class StatString(object):
    """
    Class to prettify/trim a profile result column.
    """
    _s = ""
    _TRAIL_DOT = ".."

    def __init__(self, s):
        self._s = str(s)

    def ltrim(self, length):
        if len(self._s) > length:
{'fraction_non_alphanumeric': 0.10909585562709713, 'fraction_numerical': 0.010701006620114265, 'mean_word_length': 2.6711051930758987, 'pattern_counts': {'":': 60, '<': 7, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1126874', 'n_tokens_mistral': 3629, 'n_tokens_neox': 3534, 'n_words': 1049}