<commit_before># -*- coding: utf-8 -*-
import copy
from datetime import timedelta
import json
import os.path
import time

from elastalert.util import dt_to_ts
from elastalert.util import EAException
from elastalert.util import elastalert_logger
from elastalert.util import hashable
from elastalert.util import lookup_es_key
from elastalert.util import new_get_event_ts
from elastalert.util import pretty_ts
from elastalert.util import ts_to_dt
from elastalert.ruletypes import BlacklistRule, CompareRule
from elastalert.ruletypes import EventWindow
from elastalert.ruletypes import FrequencyRule


UPDATE_INTERVAL = 60.0


class BlacklistDurationRule(CompareRule):
    required_options = frozenset(['query_key', 'compound_compare_key',
                                  'ignore_null', 'blacklist', 'timeframe'])
    change_map = {}
    occurrence_time = {}

    def __init__(self, rules, args=None):
        super(BlacklistDurationRule, self).__init__(rules, args=None)
        self.expand_entries('blacklist')

    def compare(self, event):
        key = hashable(lookup_es_key(event, self.rules['query_key']))
        values = []
        elastalert_logger.debug(" Previous Values of compare keys  " + str(self.occurrences))
        for val in self.rules['compound_compare_key']:
            lookup_value = lookup_es_key(event, val)
            values.append(lookup_value)
        elastalert_logger.debug(" Current Values of compare keys   " + str(values))

        start = changed = False
        for val in values:
            if not isinstance(val, bool) and not val and self.rules['ignore_null']:
                return False
        # If we have seen this key before, compare it to the new value
        if key in self.occurrences:
            for idx, previous_values in enumerate(self.occurrences[key]):
                elastalert_logger.debug(" " + str(previous_values) + " " + str(values[idx]))
                start = (previous_values != values[idx] and
                         values[idx] in self.rules['blacklist'])
                changed = (previous_values != values[idx] and
                           previous_values in self.rules['blacklist'])
                if start or changed:
                    break
            if changed:
                entry = [self.occurrences[key], None, None]
                # If using timeframe, only return true if the time delta is < timeframe
                if key in self.occurrence_time:
                    changed = (event[self.rules['timestamp_field']] - self.occurrence_time[key] <=
                               self.rules['timeframe'])
                    old_time = self.occurrence_time[key]
                    new_time = event[self.rules['timestamp_field']]
                    duration = new_time - old_time
                    entry[1], entry[2] = [old_time, duration.total_seconds()]
                self.change_map[key] = tuple(entry)

        if key not in self.occurrences or start or changed:
            # Update the current value and time
            elastalert_logger.debug(" Setting current value of compare keys values " + str(values))
            self.occurrences[key] = values
            self.occurrence_time[key] = event[self.rules['timestamp_field']]
        elastalert_logger.debug("Final result of comparision between previous and current values " +
                                str(changed))
        return changed

    def add_match(self, match):
        # TODO this is not technically correct
        # if the term changes multiple times before an alert is sent
        # this data will be overwritten with the most recent change
        change = self.change_map.get(hashable(lookup_es_key(match, self.rules['query_key'])))
        extra = {}
        if change:
            extra = {'value': change[0],
                     'start_time': change[1],
                     'duration': change[2]}
            elastalert_logger.debug("Description of the changed records  " +
                                    str(dict(match.items() + extra.items())))
        super(BlacklistDurationRule, self).add_match(dict(match.items() + extra.items()))


class CompoundBlacklistRule(BlacklistRule):
    """ A CompareRule where the compare function checks a given key against a blacklist """
    def compare(self, event):
        terms = lookup_es_key(event, self.rules['compare_key'])
        if not isinstance(terms, list):
            terms = [terms]
        for term in terms:
            if term in self.rules['blacklist']:
                return True
        return False


class ProfiledFrequencyRule(FrequencyRule):
    """ A rule that matches if num_events number of events occur within a timeframe """
    required_options = frozenset(['num_events', 'timeframe', 'profile'])

    def __init__(self, *args):
        super(ProfiledFrequencyRule, self).__init__(*args)
        self.ts_field = self.rules.get('timestamp_field', '@timestamp')
        self.get_ts = new_get_event_ts(self.ts_field)
        self.attach_related = self.rules.get('attach_related', False)
        self._profile = {}
        self._profile_ts = 0.0
        self._update_ts = 0.0

    def timeframe(self, key):
        return self.profile.get(key, self.rules['timeframe'])

    @property
    def profile(self):
        profile_path = self.rules['profile']
        now = time.time()

        try:
            if now > self._update_ts + UPDATE_INTERVAL:
                ts = os.path.getmtime(profile_path)
            else:
                ts = self._profile_ts

            if ts > self._profile_ts:
                elastalert_logger.info('Reloading profile %s', profile_path)
                with open(profile_path, 'r') as profile_file:
                    profile = json.load(profile_file)
                    self._profile = {k: timedelta(seconds=profile[k])
                                     for k in profile}
                    self._profile_ts = ts
                    self._update_ts = now
        except (OSError, IOError, ValueError) as e:
            elastalert_logger.error('Cannot load profile %s: %s', profile_path, e)
        return self._profile

    def add_count_data(self, data):
        """ Add count data to the rule. Data should be of the form {ts: count}. """<commit_msg>Check if last updat timestamp is in the furure<commit_after># -*- coding: utf-8 -*-
import copy
from datetime import timedelta
import json
import os.path
import time

from elastalert.util import dt_to_ts
from elastalert.util import EAException
from elastalert.util import elastalert_logger
from elastalert.util import hashable
from elastalert.util import lookup_es_key
from elastalert.util import new_get_event_ts
from elastalert.util import pretty_ts
from elastalert.util import ts_to_dt
from elastalert.ruletypes import BlacklistRule, CompareRule
from elastalert.ruletypes import EventWindow
from elastalert.ruletypes import FrequencyRule


UPDATE_INTERVAL = 60.0


class BlacklistDurationRule(CompareRule):
    required_options = frozenset(['query_key', 'compound_compare_key',
                                  'ignore_null', 'blacklist', 'timeframe'])
    change_map = {}
    occurrence_time = {}

    def __init__(self, rules, args=None):
        super(BlacklistDurationRule, self).__init__(rules, args=None)
        self.expand_entries('blacklist')

    def compare(self, event):
        key = hashable(lookup_es_key(event, self.rules['query_key']))
        values = []
        elastalert_logger.debug(" Previous Values of compare keys  " + str(self.occurrences))
        for val in self.rules['compound_compare_key']:
            lookup_value = lookup_es_key(event, val)
            values.append(lookup_value)
        elastalert_logger.debug(" Current Values of compare keys   " + str(values))

        start = changed = False
        for val in values:
            if not isinstance(val, bool) and not val and self.rules['ignore_null']:
                return False
        # If we have seen this key before, compare it to the new value
        if key in self.occurrences:
            for idx, previous_values in enumerate(self.occurrences[key]):
                elastalert_logger.debug(" " + str(previous_values) + " " + str(values[idx]))
                start = (previous_values != values[idx] and
                         values[idx] in self.rules['blacklist'])
                changed = (previous_values != values[idx] and
                           previous_values in self.rules['blacklist'])
                if start or changed:
                    break
            if changed:
                entry = [self.occurrences[key], None, None]
                # If using timeframe, only return true if the time delta is < timeframe
                if key in self.occurrence_time:
                    changed = (event[self.rules['timestamp_field']] - self.occurrence_time[key] <=
                               self.rules['timeframe'])
                    old_time = self.occurrence_time[key]
                    new_time = event[self.rules['timestamp_field']]
                    duration = new_time - old_time
                    entry[1], entry[2] = [old_time, duration.total_seconds()]
                self.change_map[key] = tuple(entry)

        if key not in self.occurrences or start or changed:
            # Update the current value and time
            elastalert_logger.debug(" Setting current value of compare keys values " + str(values))
            self.occurrences[key] = values
            self.occurrence_time[key] = event[self.rules['timestamp_field']]
        elastalert_logger.debug("Final result of comparision between previous and current values " +
                                str(changed))
        return changed

    def add_match(self, match):
        # TODO this is not technically correct
        # if the term changes multiple times before an alert is sent
        # this data will be overwritten with the most recent change
        change = self.change_map.get(hashable(lookup_es_key(match, self.rules['query_key'])))
        extra = {}
        if change:
            extra = {'value': change[0],
                     'start_time': change[1],
                     'duration': change[2]}
            elastalert_logger.debug("Description of the changed records  " +
                                    str(dict(match.items() + extra.items())))
        super(BlacklistDurationRule, self).add_match(dict(match.items() + extra.items()))


class CompoundBlacklistRule(BlacklistRule):
    """ A CompareRule where the compare function checks a given key against a blacklist """
    def compare(self, event):
        terms = lookup_es_key(event, self.rules['compare_key'])
        if not isinstance(terms, list):
            terms = [terms]
        for term in terms:
            if term in self.rules['blacklist']:
                return True
        return False


class ProfiledFrequencyRule(FrequencyRule):
    """ A rule that matches if num_events number of events occur within a timeframe """
    required_options = frozenset(['num_events', 'timeframe', 'profile'])

    def __init__(self, *args):
        super(ProfiledFrequencyRule, self).__init__(*args)
        self.ts_field = self.rules.get('timestamp_field', '@timestamp')
        self.get_ts = new_get_event_ts(self.ts_field)
        self.attach_related = self.rules.get('attach_related', False)
        self._profile = {}
        self._profile_ts = 0.0
        self._update_ts = 0.0

    def timeframe(self, key):
        return self.profile.get(key, self.rules['timeframe'])

    @property
    def profile(self):
        profile_path = self.rules['profile']
        now = time.time()

        try:
            if not (self._update_ts <= now < self._update_ts + UPDATE_INTERVAL):
                ts = os.path.getmtime(profile_path)
            else:
                ts = self._profile_ts

            if ts > self._profile_ts:
                elastalert_logger.info('Reloading profile %s', profile_path)
                with open(profile_path, 'r') as profile_file:
                    profile = json.load(profile_file)
                    self._profile = {k: timedelta(seconds=profile[k])
                                     for k in profile}
                    self._profile_ts = ts
                    self._update_ts = now
        except (OSError, IOError, ValueError) as e:
            elastalert_logger.error('Cannot load profile %s: %s', profile_path, e)
        return self._profile

    def add_count_data(self, data):
        """ Add count data to the rule. Data should be of the form {ts: count}. """
{'fraction_non_alphanumeric': 0.09255132898296992, 'fraction_numerical': 0.0020690752825083558, 'mean_word_length': 2.253171110535853, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7130337', 'n_tokens_mistral': 3465, 'n_tokens_neox': 3268, 'n_words': 928}