<commit_before># -*- coding: utf-8 -*-
import datetime
import json
import logging
import subprocess
from email.mime.text import MIMEText
from smtplib import SMTP
from smtplib import SMTP_SSL
from smtplib import SMTPAuthenticationError
from smtplib import SMTPException
from socket import error

import boto.sns as sns
import requests
import simplejson
from jira.client import JIRA
from jira.exceptions import JIRAError
from requests.exceptions import RequestException
from staticconf.loader import yaml_loader
from util import EAException
from util import elastalert_logger
from util import lookup_es_key
from util import pretty_ts


class BasicMatchString(object):

    """ Creates a string containing fields in match for the given rule. """

    def __init__(self, rule, match):
        self.rule = rule
        self.match = match

    def _ensure_new_line(self):
        while self.text[-2:] != '\n\n':
            self.text += '\n'

    def _add_custom_alert_text(self):
        alert_text = self.rule.get('alert_text', '')
        if 'alert_text_args' in self.rule:
            alert_text_args = self.rule.get('alert_text_args')
            alert_text_values = [lookup_es_key(self.match, arg) for arg in alert_text_args]
            alert_text_values = ['<MISSING VALUE>' if val is None else val for val in alert_text_values]
            alert_text = alert_text.format(*alert_text_values)
        self.text += alert_text

    def _add_rule_text(self):
        self.text += self.rule['type'].get_match_str(self.match)

    def _add_top_counts(self):
        for key, counts in self.match.items():
            if key.startswith('top_events_'):
                self.text += '%s:\n' % (key[11:])
                top_events = counts.items()
                top_events.sort(key=lambda x: x[1], reverse=True)
                for term, count in top_events:
                    self.text += '%s: %s\n' % (term, count)
                self.text += '\n'

    def _add_match_items(self):
        match_items = self.match.items()
        match_items.sort(key=lambda x: x[0])
        for key, value in match_items:
            if key.startswith('top_events_'):
                continue
            value_str = str(value)
            if type(value) in [list, dict]:
                try:
                    value_str = self._pretty_print_as_json(value)
                except TypeError:
                    # Non serializable object, fallback to str
                    pass
            self.text += '%s: %s\n' % (key, value_str)

    def _pretty_print_as_json(self, blob):
        return simplejson.dumps(blob, sort_keys=True, indent=4)

    def __str__(self):
        self.text = self.rule['name'] + '\n\n'
        self._add_custom_alert_text()
        self._ensure_new_line()
        if self.rule.get('alert_text_type') != 'alert_text_only':
            self._add_rule_text()
            self._ensure_new_line()
            if self.rule.get('top_count_keys'):
                self._add_top_counts()
            if self.rule.get('alert_text_type') != 'exclude_fields':
                self._add_match_items()
        return self.text
<commit_msg>Added fallback for raw bytes in alert<commit_after># -*- coding: utf-8 -*-
import datetime
import json
import logging
import subprocess
from email.mime.text import MIMEText
from smtplib import SMTP
from smtplib import SMTP_SSL
from smtplib import SMTPAuthenticationError
from smtplib import SMTPException
from socket import error

import boto.sns as sns
import requests
import simplejson
from jira.client import JIRA
from jira.exceptions import JIRAError
from requests.exceptions import RequestException
from staticconf.loader import yaml_loader
from util import EAException
from util import elastalert_logger
from util import lookup_es_key
from util import pretty_ts


class BasicMatchString(object):

    """ Creates a string containing fields in match for the given rule. """

    def __init__(self, rule, match):
        self.rule = rule
        self.match = match

    def _ensure_new_line(self):
        while self.text[-2:] != '\n\n':
            self.text += '\n'

    def _add_custom_alert_text(self):
        alert_text = self.rule.get('alert_text', '')
        if 'alert_text_args' in self.rule:
            alert_text_args = self.rule.get('alert_text_args')
            alert_text_values = [lookup_es_key(self.match, arg) for arg in alert_text_args]
            alert_text_values = ['<MISSING VALUE>' if val is None else val for val in alert_text_values]
            alert_text = alert_text.format(*alert_text_values)
        self.text += alert_text

    def _add_rule_text(self):
        self.text += self.rule['type'].get_match_str(self.match)

    def _add_top_counts(self):
        for key, counts in self.match.items():
            if key.startswith('top_events_'):
                self.text += '%s:\n' % (key[11:])
                top_events = counts.items()
                top_events.sort(key=lambda x: x[1], reverse=True)
                for term, count in top_events:
                    self.text += '%s: %s\n' % (term, count)
                self.text += '\n'

    def _add_match_items(self):
        match_items = self.match.items()
        match_items.sort(key=lambda x: x[0])
        for key, value in match_items:
            if key.startswith('top_events_'):
                continue
            value_str = str(value)
            if type(value) in [list, dict]:
                try:
                    value_str = self._pretty_print_as_json(value)
                except TypeError:
                    # Non serializable object, fallback to str
                    pass
            self.text += '%s: %s\n' % (key, value_str)

    def _pretty_print_as_json(self, blob):
        try:
            return simplejson.dumps(blob, sort_keys=True, indent=4)
        except UnicodeDecodeError:
            # This blob contains non-unicode, so lets pretend it's Latin-1 to show something
            return simplejson.dumps(blob, sort_keys=True, indent=4, encoding='Latin-1')

    def __str__(self):
        self.text = self.rule['name'] + '\n\n'
        self._add_custom_alert_text()
        self._ensure_new_line()
        if self.rule.get('alert_text_type') != 'alert_text_only':
            self._add_rule_text()
            self._ensure_new_line()
            if self.rule.get('top_count_keys'):
                self._add_top_counts()
            if self.rule.get('alert_text_type') != 'exclude_fields':
                self._add_match_items()
        return self.text

{'fraction_non_alphanumeric': 0.09781111281187815, 'fraction_numerical': 0.0026021735802847085, 'mean_word_length': 2.371517027863777, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5407437', 'n_tokens_mistral': 1985, 'n_tokens_neox': 1926, 'n_words': 502}