<commit_before># -*- coding: utf-8 -*-
import re
import logging
from collections import namedtuple

import six


logger = logging.getLogger(__name__)


RuleInput = namedtuple('RuleInput', ['result_log', 'node'])
RuleMatch = namedtuple('RuleMatch', ['rule', 'result', 'node'])


class Network(object):
    """
    A grouping of condition nodes.  Contains the base logic for running the
    conditions on some input.
    """
    def __init__(self):
        self.conditions = {}
        self.alert_conditions = []

    def make_condition(self, klass, *args, **kwargs):
        """
        Memoizing constructor for conditions.  Uses the input config as the cache key.
        """
        # Calculate the memoization key.  We do this by creating a 3-tuple of
        # (condition class name, args, kwargs).  There is some nuance to this,
        # though: we need to put args/kwargs in the right format.  We
        # recursively iterate through lists/dicts and convert them to tuples,
        # and extract the memoization key from instances of BaseCondition.
        def tupleify(obj):
            if isinstance(obj, BaseCondition):
                return obj.__network_memo_key
            elif isinstance(obj, tuple):
                return tuple(tupleify(x) for x in obj)
            elif isinstance(obj, list):
                return tuple(tupleify(x) for x in obj)
            elif isinstance(obj, dict):
                items = ((tupleify(k), tupleify(v)) for k, v in obj.items())
                return tuple(sorted(items))
            else:
                return obj

        args_tuple = tupleify(args)
        kwargs_tuple = tupleify(kwargs)

        key = (klass.__name__, args_tuple, kwargs_tuple)
        if key in self.conditions:
            return self.conditions[key]

        # Instantiate the condition class.  Also, save the memoization key on
        # the class, so it can be retrieved (above).
        inst = klass(*args, **kwargs)
        inst.__network_memo_key = key

        # Save the condition
        self.conditions[key] = inst
        return inst

    def make_alert_condition(self, alert, dependent, rule_id=None):
        self.alert_conditions.append((alert, dependent, rule_id))

    def process(self, entry, node):
        input = RuleInput(result_log=entry, node=node)

        # Step 1: Mark all conditions as 'not evaluated'.
        for condition in self.conditions.values():
            condition.evaluated = False

        # Step 2: For each alerter condition, we tell it to 'process' a new
        # input.  This will propagate "upstream" to each condition node and
        # evaluate the dependent chain of conditions.  We then check if the
        # condition has triggered.
        alerts = set()
        for (alert, upstream, rule_id) in self.alert_conditions:
            if upstream.run(input):
                alerts.add((alert, rule_id))

        # Step 3: Return all alerts to the caller.
        return alerts

    def parse_query(self, query, alerters=None, rule_id=None):
        """
        Parse a query output from jQuery.QueryBuilder.
        """
        def parse_condition(d):
            op = d['operator']
            value = d['value']
            
            # If this is a "column operator" - i.e. operating on a particular
            # value in a column - then we need to give a custom extraction
            # function that knows how to get this value from a query.
            column_name = None
            if d['field'] == 'column':
                # Strip 'column_' prefix to get the 'real' operator.
                op = op[7:]

                if isinstance(value, basestring):
                    column_name = value
                else:
                    # The 'value' array will look like ['column_name', 'actual value']
                    column_name, value = value

            klass = OPERATOR_MAP.get(op)
            if not klass:
                raise ValueError("Unsupported operator: {0}".format(op))
            
            inst = self.make_condition(klass, d['field'], value, column_name=column_name)<commit_msg>python3 compatibility<commit_after># -*- coding: utf-8 -*-
import re
import logging
from collections import namedtuple

import six


logger = logging.getLogger(__name__)


RuleInput = namedtuple('RuleInput', ['result_log', 'node'])
RuleMatch = namedtuple('RuleMatch', ['rule', 'result', 'node'])


class Network(object):
    """
    A grouping of condition nodes.  Contains the base logic for running the
    conditions on some input.
    """
    def __init__(self):
        self.conditions = {}
        self.alert_conditions = []

    def make_condition(self, klass, *args, **kwargs):
        """
        Memoizing constructor for conditions.  Uses the input config as the cache key.
        """
        # Calculate the memoization key.  We do this by creating a 3-tuple of
        # (condition class name, args, kwargs).  There is some nuance to this,
        # though: we need to put args/kwargs in the right format.  We
        # recursively iterate through lists/dicts and convert them to tuples,
        # and extract the memoization key from instances of BaseCondition.
        def tupleify(obj):
            if isinstance(obj, BaseCondition):
                return obj.__network_memo_key
            elif isinstance(obj, tuple):
                return tuple(tupleify(x) for x in obj)
            elif isinstance(obj, list):
                return tuple(tupleify(x) for x in obj)
            elif isinstance(obj, dict):
                items = ((tupleify(k), tupleify(v)) for k, v in obj.items())
                return tuple(sorted(items))
            else:
                return obj

        args_tuple = tupleify(args)
        kwargs_tuple = tupleify(kwargs)

        key = (klass.__name__, args_tuple, kwargs_tuple)
        if key in self.conditions:
            return self.conditions[key]

        # Instantiate the condition class.  Also, save the memoization key on
        # the class, so it can be retrieved (above).
        inst = klass(*args, **kwargs)
        inst.__network_memo_key = key

        # Save the condition
        self.conditions[key] = inst
        return inst

    def make_alert_condition(self, alert, dependent, rule_id=None):
        self.alert_conditions.append((alert, dependent, rule_id))

    def process(self, entry, node):
        input = RuleInput(result_log=entry, node=node)

        # Step 1: Mark all conditions as 'not evaluated'.
        for condition in self.conditions.values():
            condition.evaluated = False

        # Step 2: For each alerter condition, we tell it to 'process' a new
        # input.  This will propagate "upstream" to each condition node and
        # evaluate the dependent chain of conditions.  We then check if the
        # condition has triggered.
        alerts = set()
        for (alert, upstream, rule_id) in self.alert_conditions:
            if upstream.run(input):
                alerts.add((alert, rule_id))

        # Step 3: Return all alerts to the caller.
        return alerts

    def parse_query(self, query, alerters=None, rule_id=None):
        """
        Parse a query output from jQuery.QueryBuilder.
        """
        def parse_condition(d):
            op = d['operator']
            value = d['value']
            
            # If this is a "column operator" - i.e. operating on a particular
            # value in a column - then we need to give a custom extraction
            # function that knows how to get this value from a query.
            column_name = None
            if d['field'] == 'column':
                # Strip 'column_' prefix to get the 'real' operator.
                op = op[7:]

                if isinstance(value, six.string_types):
                    column_name = value
                else:
                    # The 'value' array will look like ['column_name', 'actual value']
                    column_name, value = value

            klass = OPERATOR_MAP.get(op)
            if not klass:
                raise ValueError("Unsupported operator: {0}".format(op))
            
            inst = self.make_condition(klass, d['field'], value, column_name=column_name)
{'fraction_non_alphanumeric': 0.0891510581367064, 'fraction_numerical': 0.0018243736317197761, 'mean_word_length': 2.033198081888602, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3343896', 'n_tokens_mistral': 2191, 'n_tokens_neox': 2109, 'n_words': 772}