<commit_before>import os
import sys
import imp
import re
from inspect import isclass
import sre_constants
import logging
import json

from elasticsearch import Elasticsearch
from elasticsearch.exceptions import TransportError


query_all = {
    "query_string": {
        "query": "*"
    }
}


def es_query(query, k, v):
    return {
        'query': {
            query: {
                k: v
            }
        }
    }


def es_regexp(k, v):
    return {
        'regexp': {
            k: v
        }
    }


def es_prefix(k, v):
    return {
        'prefix': {
            k: v
        }
    }


def hit_to_metric(hit):
    tags = {}
    for tag in hit['_source']['tags']:
        (k, v) = tag.split('=', 1)
        tags[str(k)] = str(v)
    return {
        'id': hit['_id'],
        'tags': tags
    }


class PluginError(Exception):

    def __init__(self, plugin, msg, underlying_error):
        Exception.__init__(self, msg)
        self.plugin = plugin
        self.msg = msg
        self.underlying_error = underlying_error

    def __str__(self):
        return "%s -> %s (%s)" % (self.plugin, self.msg, self.underlying_error)


def regex_unanchor(regexp):
    """
    ES regexps are anchored, so adding .* is necessary at the beginning and
    end to get a substring match. But we want to skip that if front/end
    anchors are explicitly used.

    Also avoid doubling up wildcards, in case the regexp engine is not smart
    about backtracking.
    """

    if regexp.startswith('^'):
        regexp = regexp.lstrip('^')
    elif not regexp.startswith('.*'):
        regexp = '.*' + regexp
    if regexp.endswith('$'):
        regexp = regexp.rstrip('$')
    elif not regexp.endswith('.*'):
        regexp += '.*'
    return regexp


def build_es_match_tag_equality(key, term):
    return es_query('match', 'tags', '%s=%s' % (key, term))


def build_es_match_tag_exists(key):
    return es_prefix('tags', key + '=')


def build_es_match_any_tag_value(term):
    return es_regexp('tags', ".*=" + re.escape(term))


def build_es_match_tag_regex(key, term):
    return es_regexp('tags', '%s=%s' % (re.escape(key), regex_unanchor(term)))


def build_es_match_tag_name_regex(key):
    return es_regexp('tags', '%s=.*' % regex_unanchor(key))


def build_es_match_tag_value_regex(term):
    return es_regexp('tags', '.*=%s' % regex_unanchor(term))


def build_es_match_id_regex(key):
    # here 'id' is to be interpreted loosely, as in the old
    # (python-native datastructures) approach where we used
    # Plugin.get_target_id to have an id that contains the graphite
    # metric, but also the tags etc. so if the user types just a
    # word, we want the metrics to be returned where the id or tags
    # are matched
    return {'or': [es_regexp('_id', regex_unanchor(key)),
                   es_regexp('tags', regex_unanchor(key))]}


def build_es_match_negate(ast):
    return {'not': build_es_expr(ast)}


def build_es_match_or(*asts):
    return {'or': map(build_es_expr, asts)}


def build_es_match_and(*asts):
    return {'and': map(build_es_expr, asts)}


def build_es_expr(ast):
    return globals()['build_es_' + ast[0]](*ast[1:])


def build_es_query(ast):
    return {
        "filtered": {
            "query": {"match_all": {}},
            "filter": build_es_expr(ast)
        }
    }


class StructuredMetrics(object):

    def __init__(self, config=None, logger=logging):
        self.plugins = []
        if hasattr(config, 'es_host') and hasattr(config, 'es_port') and hasattr(config, 'es_index'):
            es_host = config.es_host.replace('http://', '').replace('https://', '')
            self.es = Elasticsearch([{"host": es_host, "port": config.es_port}])
            self.es_index = config.es_index
        self.logger = logger
        self.config = config

    def load_plugins(self):
        '''
        loads all the plugins sub-modules
        returns encountered errors, doesn't raise them because
        whoever calls this function defines how any errors are
        handled. meanwhile, loading must continue
        '''
        from . import plugins
        errors = []
        plugin_dirs = getattr(self.config, "metric_plugin_dirs", ('**builtins**',))
        for plugin_dir in plugin_dirs:
            if plugin_dir == '**builtins**':
                plugin_dir = os.path.dirname(plugins.__file__)
            newplugins, newerrors = self.load_plugins_from(plugin_dir, plugins, self.config)
            self.plugins.extend(newplugins)
            errors.extend(newerrors)
        # sort plugins by their matching priority
        self.plugins = sorted(self.plugins, key=lambda t: t[1].priority, reverse=True)
        return errors

    def load_plugins_from(self, plugin_dir, package, config):
        # import in sorted order to let it be predictable; lets user plugins import
        # pieces of other plugins imported earlier
        plugins = []
        errors = []
        Plugin = package.Plugin
        for f in sorted(os.listdir(plugin_dir)):
            if f == '__init__.py' or not f.endswith(".py"):
                continue
            mname = f[:-3]
            qualifiedname = package.__name__ + '.' + mname
            imp.acquire_lock()
            try:
                self.logger.debug("loading plugin %s" % qualifiedname)
                module = imp.load_source(qualifiedname, os.path.join(plugin_dir, f))
                sys.modules[qualifiedname] = module<commit_msg>bugfix: properly use custom plugin dirs<commit_after>import os
import sys
import imp
import re
from inspect import isclass
import sre_constants
import logging
import json

from elasticsearch import Elasticsearch
from elasticsearch.exceptions import TransportError


query_all = {
    "query_string": {
        "query": "*"
    }
}


def es_query(query, k, v):
    return {
        'query': {
            query: {
                k: v
            }
        }
    }


def es_regexp(k, v):
    return {
        'regexp': {
            k: v
        }
    }


def es_prefix(k, v):
    return {
        'prefix': {
            k: v
        }
    }


def hit_to_metric(hit):
    tags = {}
    for tag in hit['_source']['tags']:
        (k, v) = tag.split('=', 1)
        tags[str(k)] = str(v)
    return {
        'id': hit['_id'],
        'tags': tags
    }


class PluginError(Exception):

    def __init__(self, plugin, msg, underlying_error):
        Exception.__init__(self, msg)
        self.plugin = plugin
        self.msg = msg
        self.underlying_error = underlying_error

    def __str__(self):
        return "%s -> %s (%s)" % (self.plugin, self.msg, self.underlying_error)


def regex_unanchor(regexp):
    """
    ES regexps are anchored, so adding .* is necessary at the beginning and
    end to get a substring match. But we want to skip that if front/end
    anchors are explicitly used.

    Also avoid doubling up wildcards, in case the regexp engine is not smart
    about backtracking.
    """

    if regexp.startswith('^'):
        regexp = regexp.lstrip('^')
    elif not regexp.startswith('.*'):
        regexp = '.*' + regexp
    if regexp.endswith('$'):
        regexp = regexp.rstrip('$')
    elif not regexp.endswith('.*'):
        regexp += '.*'
    return regexp


def build_es_match_tag_equality(key, term):
    return es_query('match', 'tags', '%s=%s' % (key, term))


def build_es_match_tag_exists(key):
    return es_prefix('tags', key + '=')


def build_es_match_any_tag_value(term):
    return es_regexp('tags', ".*=" + re.escape(term))


def build_es_match_tag_regex(key, term):
    return es_regexp('tags', '%s=%s' % (re.escape(key), regex_unanchor(term)))


def build_es_match_tag_name_regex(key):
    return es_regexp('tags', '%s=.*' % regex_unanchor(key))


def build_es_match_tag_value_regex(term):
    return es_regexp('tags', '.*=%s' % regex_unanchor(term))


def build_es_match_id_regex(key):
    # here 'id' is to be interpreted loosely, as in the old
    # (python-native datastructures) approach where we used
    # Plugin.get_target_id to have an id that contains the graphite
    # metric, but also the tags etc. so if the user types just a
    # word, we want the metrics to be returned where the id or tags
    # are matched
    return {'or': [es_regexp('_id', regex_unanchor(key)),
                   es_regexp('tags', regex_unanchor(key))]}


def build_es_match_negate(ast):
    return {'not': build_es_expr(ast)}


def build_es_match_or(*asts):
    return {'or': map(build_es_expr, asts)}


def build_es_match_and(*asts):
    return {'and': map(build_es_expr, asts)}


def build_es_expr(ast):
    return globals()['build_es_' + ast[0]](*ast[1:])


def build_es_query(ast):
    return {
        "filtered": {
            "query": {"match_all": {}},
            "filter": build_es_expr(ast)
        }
    }


class StructuredMetrics(object):

    def __init__(self, config=None, logger=logging):
        self.plugins = []
        if hasattr(config, 'es_host') and hasattr(config, 'es_port') and hasattr(config, 'es_index'):
            es_host = config.es_host.replace('http://', '').replace('https://', '')
            self.es = Elasticsearch([{"host": es_host, "port": config.es_port}])
            self.es_index = config.es_index
        self.logger = logger
        self.config = config

    def load_plugins(self):
        '''
        loads all the plugins sub-modules
        returns encountered errors, doesn't raise them because
        whoever calls this function defines how any errors are
        handled. meanwhile, loading must continue
        '''
        from . import plugins
        errors = []
        plugin_dirs = self.config.locations_plugins_structured_metrics
        for plugin_dir in plugin_dirs:
            if plugin_dir == '**builtins**':
                plugin_dir = os.path.dirname(plugins.__file__)
            newplugins, newerrors = self.load_plugins_from(plugin_dir, plugins, self.config)
            self.plugins.extend(newplugins)
            errors.extend(newerrors)
        # sort plugins by their matching priority
        self.plugins = sorted(self.plugins, key=lambda t: t[1].priority, reverse=True)
        return errors

    def load_plugins_from(self, plugin_dir, package, config):
        # import in sorted order to let it be predictable; lets user plugins import
        # pieces of other plugins imported earlier
        plugins = []
        errors = []
        Plugin = package.Plugin
        for f in sorted(os.listdir(plugin_dir)):
            if f == '__init__.py' or not f.endswith(".py"):
                continue
            mname = f[:-3]
            qualifiedname = package.__name__ + '.' + mname
            imp.acquire_lock()
            try:
                self.logger.debug("loading plugin %s" % qualifiedname)
                module = imp.load_source(qualifiedname, os.path.join(plugin_dir, f))
                sys.modules[qualifiedname] = module
{'fraction_non_alphanumeric': 0.12203172274685982, 'fraction_numerical': 0.0009168423947923352, 'mean_word_length': 2.5221181788827898, 'pattern_counts': {'":': 16, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6977782', 'n_tokens_mistral': 3434, 'n_tokens_neox': 3339, 'n_words': 888}