<commit_before>import inspect
import pkg_resources
import requests
import concurrent.futures
import time

from statscache.schedule import Schedule
import statscache.plugins

import logging
log = logging.getLogger("fedmsg")


def find_stats_consumer(hub):
    """ Find the caching StatsConsumer associated with the given hub """
    for cons in hub.consumers:
        if 'StatsConsumer' in str(type(cons)):
            return cons

    raise ValueError('StatsConsumer not found.')


def datagrep(start, stop, quantum=100):
    """ Yield messages generated in the given time interval from datagrepper

    Messages are ordered ascending by age (from oldest to newest), so that
    models may be kept up-to-date through some point, to allow restart in case
    of failure. Messages are generated in collections of the given quantum at a
    time.
    """
    MAX_WORKERS = 8
    endpoint = 'https://apps.fedoraproject.org/datagrepper/raw/'
    session = requests.Session()
    session.params = {
        'start': time.mktime(start.timetuple()),
        'order': 'asc',
        'rows_per_page': quantum,
    }
    if stop is not None:
        session.params['end'] = time.mktime(stop.timetuple())
    query = lambda page: session.get(endpoint, params={ 'page': page })

    # Manually perform the first request in order to get the page count and
    # (hopefully) spawn some persistent connections prior to entering the
    # executor map.
    data = query(1).json()
    yield data['raw_messages']
    pages = int(data['pages'])
    del data

    with concurrent.futures.ThreadPoolExecutor(MAX_WORKERS) as executor:
        for response in executor.map(query, xrange(2, pages+1)):
            yield response.json()['raw_messages']


def init_plugins(config):
    """ Initialize all available plugins using the given configuration.

    Plugin classes and collections of plugin classes are searched for at all
    entry-points registered under statscache.plugin. A plugin class is defined
    to be a class that inherits from statscache.plugin.BasePlugin
    """
    def init_plugin(plugin_class):<commit_msg>Add commented-out datagrepper profiling code<commit_after>import inspect
import pkg_resources
import requests
import concurrent.futures
import time

from statscache.schedule import Schedule
import statscache.plugins

import logging
log = logging.getLogger("fedmsg")


def find_stats_consumer(hub):
    """ Find the caching StatsConsumer associated with the given hub """
    for cons in hub.consumers:
        if 'StatsConsumer' in str(type(cons)):
            return cons

    raise ValueError('StatsConsumer not found.')


def datagrep(start, stop, quantum=100):
    """ Yield messages generated in the given time interval from datagrepper

    Messages are ordered ascending by age (from oldest to newest), so that
    models may be kept up-to-date through some point, to allow restart in case
    of failure. Messages are generated in collections of the given quantum at a
    time.
    """
    MAX_WORKERS = 8
    endpoint = 'https://apps.fedoraproject.org/datagrepper/raw/'
    session = requests.Session()
    session.params = {
        'start': time.mktime(start.timetuple()),
        'order': 'asc',
        'rows_per_page': quantum,
    }
    if stop is not None:
        session.params['end'] = time.mktime(stop.timetuple())
    query = lambda page: session.get(endpoint, params={ 'page': page })

    # Manually perform the first request in order to get the page count and
    # (hopefully) spawn some persistent connections prior to entering the
    # executor map.
    data = query(1).json()
    yield data['raw_messages']
    pages = int(data['pages'])
    del data

    with concurrent.futures.ThreadPoolExecutor(MAX_WORKERS) as executor:
        # Uncomment the lines of code in this block to log profiling data
        #page = 1
        #net_time = time.time()
        for response in executor.map(query, xrange(2, pages+1)):
            #net_time = time.time() - net_time
            #cpu_time = time.time()
            yield response.json()['raw_messages']
            #page += 1
            #cpu_time = time.time() - cpu_time
            #log.info("Processed page {}/{}: {}ms NET {}ms CPU".format(
            #    page,
            #    pages,
            #    int(net_time * 1000),
            #    int(cpu_time * 1000)
            #))
            #net_time = time.time()


def init_plugins(config):
    """ Initialize all available plugins using the given configuration.

    Plugin classes and collections of plugin classes are searched for at all
    entry-points registered under statscache.plugin. A plugin class is defined
    to be a class that inherits from statscache.plugin.BasePlugin
    """
    def init_plugin(plugin_class):
{'fraction_non_alphanumeric': 0.0904701931150294, 'fraction_numerical': 0.005037783375314861, 'mean_word_length': 3.017706576728499, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '876593', 'n_tokens_mistral': 1340, 'n_tokens_neox': 1289, 'n_words': 463}