<commit_before>from __future__ import unicode_literals, division, absolute_import
import logging

from flexget import options, plugin
from flexget.event import event
from flexget.logger import console

try:
    from guppy import hpy
except ImportError:
    # this will leave the plugin unloaded
    raise plugin.DependencyError(issued_by='memusage', missing='ext lib `guppy`', silent=True)

log = logging.getLogger('mem_usage')


"""
http://blog.mfabrik.com/2008/03/07/debugging-django-memory-leak-with-trackrefs-and-guppy/

# Print memory statistics
def update():
    print heapy.heap()

# Print relative memory consumption since last sycle
def update():
    print heapy.heap()
    heapy.setref()

# Print relative memory consumption w/heap traversing
def update()
    print heapy.heap().get_rp(40)
    heapy.setref()
"""

heapy = None


@event('manager.startup')
def on_manager_startup(manager):
    if not manager.options.mem_usage:
        return
    global heapy
    heapy = hpy()


@event('manager.shutdown')
def on_manager_shutdown(manager):
    if not manager.options.mem_usage:
        return
    global heapy
    console('Calculating memory usage:')
    console(heapy.heap())
    console('-' * 79)
    console('Calculating report (this may take a while):')
    console(heapy.heap().get_rp(40))
    heapy = None


@event('options.register')
def register_parser_arguments():
    options.get_parser().add_argument('--mem-usage', action='store_true', dest='mem_usage', default=False,
                                               help='display memory usage debug information')<commit_msg>Added resource module memory usage<commit_after>from __future__ import unicode_literals, division, absolute_import
import logging

from flexget import options, plugin
from flexget.event import event
from flexget.logger import console

try:
    from guppy import hpy
except ImportError:
    # this will leave the plugin unloaded
    raise plugin.DependencyError(issued_by='memusage', missing='ext lib `guppy`', silent=True)

log = logging.getLogger('mem_usage')


"""
http://blog.mfabrik.com/2008/03/07/debugging-django-memory-leak-with-trackrefs-and-guppy/

# Print memory statistics
def update():
    print heapy.heap()

# Print relative memory consumption since last sycle
def update():
    print heapy.heap()
    heapy.setref()

# Print relative memory consumption w/heap traversing
def update()
    print heapy.heap().get_rp(40)
    heapy.setref()
"""

heapy = None


@event('manager.startup')
def on_manager_startup(manager):
    if not manager.options.mem_usage:
        return
    global heapy
    heapy = hpy()


@event('manager.shutdown')
def on_manager_shutdown(manager):
    if not manager.options.mem_usage:
        return

    import resource
    print 'Resource Module memory usage: %s (kb)' % resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
    global heapy
    console('Heapy module calculating memory usage:')
    console(heapy.heap())
    console('-' * 79)
    console('Heapy module calculating report (this may take a while):')
    console(heapy.heap().get_rp(40))
    heapy = None


@event('options.register')
def register_parser_arguments():
    options.get_parser().add_argument('--mem-usage', action='store_true', dest='mem_usage', default=False,
                                               help='display memory usage debug information')
{'fraction_non_alphanumeric': 0.10949122285034216, 'fraction_numerical': 0.008330853912526033, 'mean_word_length': 3.802857142857143, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2117631', 'n_tokens_mistral': 1024, 'n_tokens_neox': 986, 'n_words': 263}