<commit_before>
"""
    This module defines a set of methods useful in handling series of metrics objects to build more complex results.
"""

__author__ = "<NAME>"
__date__ = "12/28/2012"
__license__ = "GPL (version 2 or later)"

import re
from datetime import datetime, timedelta
from dateutil.parser import parse as date_parse

import user_metric as um
import threshold as th
from blocks import Blocks
from bytes_added import BytesAdded
from survival import Survival
import revert_rate as rr
from time_to_threshold import TimeToThreshold
from edit_rate import EditRate
from namespace_of_edits import NamespaceEdits, namespace_edits_sum

import src.etl.data_loader as dl
import src.etl.aggregator as agg
import src.etl.time_series_process_methods as tspm

from config import logging

INTERVALS_PER_THREAD = 10
MAX_THREADS = 10

# Registered metrics types
metric_dict = {
    'threshold' : th.Threshold,
    'survival' : Survival,
    'revert' : rr.RevertRate,
    'bytes_added' : BytesAdded,
    'blocks' : Blocks,
    'time_to_threshold' : TimeToThreshold,
    'edit_rate' : EditRate,
    'namespace_edits' : NamespaceEdits,
    }

aggregator_dict = {
    'sum+bytes_added' : agg.list_sum_indices,
    'sum+edit_rate' : agg.list_sum_indices,
    'sum+namespace_edits' : namespace_edits_sum,
    'average+threshold' : th.threshold_editors_agg,
    'average+revert' : rr.reverted_revs_agg,
    }

def get_metric_names(): return metric_dict.keys()
def get_param_types(metric_handle): return metric_dict[metric_handle]()._param_types

def get_agg_key(agg_handle, metric_handle):
    """ Compose the metric dependent aggregator handle """
    try:
        return '+'.join([agg_handle, metric_handle])
    except TypeError:
        return ''

def process_data_request(metric_handle, users, agg_handle='', **kwargs):

    # create shorthand method refs
    to_string = dl.DataLoader().cast_elems_to_string
    get_timestamp = um.UserMetric._get_timestamp

    start = get_timestamp(kwargs['date_start']) if 'date_start' in kwargs else get_timestamp(datetime.now() + timedelta(days=-1))
    end = get_timestamp(kwargs['date_end']) if 'date_end' in kwargs else get_timestamp(datetime.now())
    kwargs['date_start'] = start
    kwargs['date_end'] = end

    # Initialize the results
    results = dict()
    metric_class = metric_dict[metric_handle]
    metric_obj = metric_class(**kwargs)
    results['header'] = " ".join(metric_obj.header())
    for key in metric_obj.__dict__:
        if re.search(r'_.*_', key):
            results[str(key[1:-1])] = str(metric_obj.__dict__[key])
    results['metric'] = dict()

    # Get the aggregator if there is one
    aggregator_func = None

    aggregator_key = get_agg_key(agg_handle, metric_handle)
    if aggregator_key in aggregator_dict.keys():
        aggregator_func = aggregator_dict[aggregator_key]

    time_series = True if 'time_series' in kwargs else False

    if aggregator_func:

        if time_series:

            interval = int(kwargs['interval'])      # interval length in hours

            total_intervals = (date_parse(end) - date_parse(start)).total_seconds() / (3600 * interval)
            num_threads = min(MAX_THREADS, int(total_intervals / INTERVALS_PER_THREAD))

            logging.info('Metrics Manager: Initiating time series for %(metric)s with %(agg)s from '
                         '%(start)s to %(end)s.' % {
                'metric' : metric_class.__name__,
                'agg' : aggregator_func.__name__,<commit_msg>Mod. interface to process_data_request() and handling of aggregator handle.<commit_after>
"""
    This module defines a set of methods useful in handling series of metrics objects to build more complex results.
"""

__author__ = "<NAME>"
__date__ = "12/28/2012"
__license__ = "GPL (version 2 or later)"

import re
from datetime import datetime, timedelta
from dateutil.parser import parse as date_parse

import user_metric as um
import threshold as th
from blocks import Blocks
from bytes_added import BytesAdded
from survival import Survival
import revert_rate as rr
from time_to_threshold import TimeToThreshold
from edit_rate import EditRate
from namespace_of_edits import NamespaceEdits, namespace_edits_sum

import src.etl.data_loader as dl
import src.etl.aggregator as agg
import src.etl.time_series_process_methods as tspm

from config import logging

INTERVALS_PER_THREAD = 10
MAX_THREADS = 10

# Registered metrics types
metric_dict = {
    'threshold' : th.Threshold,
    'survival' : Survival,
    'revert' : rr.RevertRate,
    'bytes_added' : BytesAdded,
    'blocks' : Blocks,
    'time_to_threshold' : TimeToThreshold,
    'edit_rate' : EditRate,
    'namespace_edits' : NamespaceEdits,
    }

aggregator_dict = {
    'sum+bytes_added' : agg.list_sum_indices,
    'sum+edit_rate' : agg.list_sum_indices,
    'sum+namespace_edits' : namespace_edits_sum,
    'average+threshold' : th.threshold_editors_agg,
    'average+revert' : rr.reverted_revs_agg,
    }

def get_metric_names(): return metric_dict.keys()
def get_param_types(metric_handle): return metric_dict[metric_handle]()._param_types

def get_agg_key(agg_handle, metric_handle):
    """ Compose the metric dependent aggregator handle """
    try:
        return '+'.join([agg_handle, metric_handle])
    except TypeError:
        return ''

def process_data_request(metric_handle, users, **kwargs):

    # create shorthand method refs
    to_string = dl.DataLoader().cast_elems_to_string
    get_timestamp = um.UserMetric._get_timestamp

    aggregator = kwargs['aggregator'] if 'aggrgator' in kwargs else None
    start = get_timestamp(kwargs['date_start']) if 'date_start' in kwargs else get_timestamp(datetime.now() + timedelta(days=-1))
    end = get_timestamp(kwargs['date_end']) if 'date_end' in kwargs else get_timestamp(datetime.now())
    kwargs['date_start'] = start
    kwargs['date_end'] = end

    # Initialize the results
    results = dict()
    metric_class = metric_dict[metric_handle]
    metric_obj = metric_class(**kwargs)
    results['header'] = " ".join(metric_obj.header())
    for key in metric_obj.__dict__:
        if re.search(r'_.*_', key):
            results[str(key[1:-1])] = str(metric_obj.__dict__[key])
    results['metric'] = dict()

    # Get the aggregator if there is one
    aggregator_func = None

    aggregator_key = get_agg_key(aggregator, metric_handle)
    if aggregator_key in aggregator_dict.keys():
        aggregator_func = aggregator_dict[aggregator_key]

    time_series = True if 'time_series' in kwargs else False

    if aggregator_func:

        if time_series:

            interval = int(kwargs['interval'])      # interval length in hours

            total_intervals = (date_parse(end) - date_parse(start)).total_seconds() / (3600 * interval)
            num_threads = min(MAX_THREADS, int(total_intervals / INTERVALS_PER_THREAD))

            logging.info('Metrics Manager: Initiating time series for %(metric)s with %(agg)s from '
                         '%(start)s to %(end)s.' % {
                'metric' : metric_class.__name__,
                'agg' : aggregator_func.__name__,
{'fraction_non_alphanumeric': 0.09744094488188976, 'fraction_numerical': 0.00562429696287964, 'mean_word_length': 3.8752570253598355, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6790909', 'n_tokens_mistral': 2264, 'n_tokens_neox': 2174, 'n_words': 560}