<commit_before># Copyright (C) [2015-] [Thomson Reuters LLC]
# Copyright (C) [2015-] [<NAME>]

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""InfluxGraph utility functions"""

from __future__ import absolute_import, print_function
import datetime
import sys
import re
import hashlib

from .constants import INFLUXDB_AGGREGATIONS
try:
    from .ext.classes.tree import NodeTreeIndex
    from .ext.templates import get_series_with_tags
except ImportError:
    from .classes.tree import NodeTreeIndex
    from .templates import get_series_with_tags

def calculate_interval(start_time, end_time, deltas=None):
    """Calculates wanted data series interval according to start and end times

    Returns interval in seconds
    :param start_time: Start time in seconds from epoch
    :param end_time: End time in seconds from epoch
    :type start_time: int
    :type end_time: int
    :param deltas: Delta configuration to use. Defaults hardcoded if no
    configuration is provided
    :type deltas: dict(max time range of query in seconds: interval to use in seconds)

    :rtype: int - *Interval in seconds*
    """
    time_delta = end_time - start_time
    deltas = deltas if deltas else {
        # 15 min -> 10s
        900: 10,
        # 30 min -> 30s
        1800: 30,
        # # 1 hour -> 1s
        # 3600 : 1,
        # # 1 day -> 30s
        # 86400 : 30,
        # 3 days -> 1min
        259200: 60,
        # 7 days -> 5min
        604800: 300,
        # 14 days -> 10min
        1209600: 600,
        # 28 days -> 15min
        2419200: 900,
        # 2 months -> 30min
        4838400: 1800,
        # 4 months -> 1hour
        9676800: 3600,
        # 12 months -> 3hours
        31536000: 7200,
        # 4 years -> 12hours
        126144000: 43200,
        }
    for delta in sorted(deltas.keys()):
        if time_delta <= delta:
            return deltas[delta]
    # 1 day default, or if time range > max configured (4 years default max)
    return 86400

def get_retention_policy(interval, retention_policies):
    """Get appropriate retention policy for interval provided

    :param interval: Interval of query in seconds
    :type interval: int
    :param retention_policies: Retention policy configuration
    :type retention_policies: dict(max time range of interval in seconds: retention policy name)

    :rtype: ``str`` or ``None``
    """
    if not retention_policies:
        return
    for retention_interval in sorted(retention_policies.keys()):
        if interval <= retention_interval:
            return retention_policies[retention_interval]
    # In the case that desired interval is beyond configured interval range,
    # return policy for max interval
    return retention_policies[max(sorted(retention_policies.keys()))]

class Query(object):
    """Graphite-API compatible query class"""

    def __init__(self, pattern):
        self.pattern = pattern


class NullStatsd(object):
    """Fake StatsClient compatible class to use when statsd is not configured"""

    def __enter__(self):
        return self

    def __exit__(self, _type, value, traceback):
        pass

    def timer(self, key, val=None):
        return self

    def timing(self, key, val):
        pass

    def start(self):
        pass

    def stop(self):
        pass

def _compile_aggregation_patterns(aggregation_functions):
    """Compile aggregation function patterns to compiled regex objects"""
    if not aggregation_functions:
        return
    compiled_aggregations = {}
    for pattern in aggregation_functions.keys():
        if not aggregation_functions[pattern] in INFLUXDB_AGGREGATIONS:
            sys.stderr.write("Requested aggregation function '%s' is not a valid InfluxDB "
                             "aggregation function - ignoring..\n" % (
                                 aggregation_functions[pattern],))
            continue
        try:
            compiled_aggregations[re.compile(r'%s' % (pattern,))] = aggregation_functions[pattern]
        except re.error:
            sys.stderr.write("Error compiling regex pattern '%s' - ignoring..\n" % (
                pattern,))
    return compiled_aggregations

def get_aggregation_func(path, aggregation_functions):
    """Lookup aggregation function for path, if any.
    Defaults to 'mean'.

    :param path: Path to lookup
    :type path: str
    :param aggregation_functions: Aggregation function configuration
    :type aggregation_functions: dict(<pattern>: <compiled regex>)
    """
    if not aggregation_functions:
        return 'mean'
    for pattern in aggregation_functions:
        if pattern.search(path):
            return aggregation_functions[pattern]
    return 'mean'

def _retrieve_named_field_data(infl_data, path_measurements, measurement, _data):
    measurement_paths = path_measurements[measurement]['paths'][:]
    points = list(infl_data.get_points(measurement))
    for columns in points:
        point_fields = columns.keys()
        for field in point_fields:
            try:
                metric = [p for p in measurement_paths
                          if field in path_measurements[measurement]['fields']
                          and field in p][0]
                del measurement_paths[measurement_paths.index(metric)]
            except IndexError:
                continue
            _data[metric] = [d[field]
                             for d in points]

def _retrieve_field_data(infl_data, path_measurements, measurement,
                         metric, _data):<commit_msg>Updated values reading to correctly handle identical fields and measurement from multiple tags<commit_after># Copyright (C) [2015-] [Thomson Reuters LLC]
# Copyright (C) [2015-] [<NAME>]

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""InfluxGraph utility functions"""

from __future__ import absolute_import, print_function
import datetime
import sys
import re
import hashlib

from .constants import INFLUXDB_AGGREGATIONS
try:
    from .ext.classes.tree import NodeTreeIndex
    from .ext.templates import get_series_with_tags
except ImportError:
    from .classes.tree import NodeTreeIndex
    from .templates import get_series_with_tags

def calculate_interval(start_time, end_time, deltas=None):
    """Calculates wanted data series interval according to start and end times

    Returns interval in seconds
    :param start_time: Start time in seconds from epoch
    :param end_time: End time in seconds from epoch
    :type start_time: int
    :type end_time: int
    :param deltas: Delta configuration to use. Defaults hardcoded if no
    configuration is provided
    :type deltas: dict(max time range of query in seconds: interval to use in seconds)

    :rtype: int - *Interval in seconds*
    """
    time_delta = end_time - start_time
    deltas = deltas if deltas else {
        # 15 min -> 10s
        900: 10,
        # 30 min -> 30s
        1800: 30,
        # # 1 hour -> 1s
        # 3600 : 1,
        # # 1 day -> 30s
        # 86400 : 30,
        # 3 days -> 1min
        259200: 60,
        # 7 days -> 5min
        604800: 300,
        # 14 days -> 10min
        1209600: 600,
        # 28 days -> 15min
        2419200: 900,
        # 2 months -> 30min
        4838400: 1800,
        # 4 months -> 1hour
        9676800: 3600,
        # 12 months -> 3hours
        31536000: 7200,
        # 4 years -> 12hours
        126144000: 43200,
        }
    for delta in sorted(deltas.keys()):
        if time_delta <= delta:
            return deltas[delta]
    # 1 day default, or if time range > max configured (4 years default max)
    return 86400

def get_retention_policy(interval, retention_policies):
    """Get appropriate retention policy for interval provided

    :param interval: Interval of query in seconds
    :type interval: int
    :param retention_policies: Retention policy configuration
    :type retention_policies: dict(max time range of interval in seconds: retention policy name)

    :rtype: ``str`` or ``None``
    """
    if not retention_policies:
        return
    for retention_interval in sorted(retention_policies.keys()):
        if interval <= retention_interval:
            return retention_policies[retention_interval]
    # In the case that desired interval is beyond configured interval range,
    # return policy for max interval
    return retention_policies[max(sorted(retention_policies.keys()))]

class Query(object):
    """Graphite-API compatible query class"""

    def __init__(self, pattern):
        self.pattern = pattern


class NullStatsd(object):
    """Fake StatsClient compatible class to use when statsd is not configured"""

    def __enter__(self):
        return self

    def __exit__(self, _type, value, traceback):
        pass

    def timer(self, key, val=None):
        return self

    def timing(self, key, val):
        pass

    def start(self):
        pass

    def stop(self):
        pass

def _compile_aggregation_patterns(aggregation_functions):
    """Compile aggregation function patterns to compiled regex objects"""
    if not aggregation_functions:
        return
    compiled_aggregations = {}
    for pattern in aggregation_functions.keys():
        if not aggregation_functions[pattern] in INFLUXDB_AGGREGATIONS:
            sys.stderr.write("Requested aggregation function '%s' is not a valid InfluxDB "
                             "aggregation function - ignoring..\n" % (
                                 aggregation_functions[pattern],))
            continue
        try:
            compiled_aggregations[re.compile(r'%s' % (pattern,))] = aggregation_functions[pattern]
        except re.error:
            sys.stderr.write("Error compiling regex pattern '%s' - ignoring..\n" % (
                pattern,))
    return compiled_aggregations

def get_aggregation_func(path, aggregation_functions):
    """Lookup aggregation function for path, if any.
    Defaults to 'mean'.

    :param path: Path to lookup
    :type path: str
    :param aggregation_functions: Aggregation function configuration
    :type aggregation_functions: dict(<pattern>: <compiled regex>)
    """
    if not aggregation_functions:
        return 'mean'
    for pattern in aggregation_functions:
        if pattern.search(path):
            return aggregation_functions[pattern]
    return 'mean'

def _retrieve_named_field_data(infl_data, path_measurements, measurement, _data):
    measurement_paths = path_measurements[measurement]['paths'][:]
    field_keys = infl_data.get_points(measurement).next().keys()
    point_fields = sorted([k for k in field_keys if k != 'time'])
    for field in point_fields:
        try:
            metric = [p for p in measurement_paths
                        if field in path_measurements[measurement]['fields']
                        and field in p][0]
            del measurement_paths[measurement_paths.index(metric)]
        except IndexError:
            continue
        _data[metric] = [d[field]
                        for d in infl_data.get_points(measurement)]
    path_measurements[measurement]['paths'] = measurement_paths

def _retrieve_field_data(infl_data, path_measurements, measurement,
                         metric, _data):
{'fraction_non_alphanumeric': 0.07934943321833415, 'fraction_numerical': 0.026942664695252177, 'mean_word_length': 2.6793593230583257, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4512583', 'n_tokens_mistral': 3595, 'n_tokens_neox': 3236, 'n_words': 1148}