<commit_before>#!/usr/bin/python

# -*- coding: utf-8 -*-

# Copyright (C) 2009-2013:
# <NAME>, <EMAIL>
# <NAME>, <EMAIL>
# <NAME>, <EMAIL>
# <NAME>, <EMAIL>
# <NAME>, <EMAIL>
# Savoir-Faire Linux inc.
#
# This file is part of Shinken.
#
# Shinken is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Shinken is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Shinken. If not, see <http://www.gnu.org/licenses/>.


# This Class is a plugin for the Shinken Broker. It is responsible
# for sending log broks to a raw socket destination.

import re
import socket
import time
import datetime

from shinken.basemodule import BaseModule
from shinken.log import logger

properties = {
    'daemons': ['broker'],
    'type': 'raw_socket',
    'external': False,
    'phases': ['running'],
}


# called by the plugin manager to get a broker
def get_instance(plugin):
    logger.info("Get a RawSocket broker for plugin %s" % plugin.get_name())

    #Catch errors
    #path = plugin.path
    instance = RawSocket_broker(plugin)
    return instance


# Get broks and send them to TCP Raw Socket listener
class RawSocket_broker(BaseModule):
    def __init__(self, modconf):
        BaseModule.__init__(self, modconf)
        self.host = getattr(modconf, 'host', 'localhost')
        self.port = int(getattr(modconf, 'port', '9514'))
        self.data = getattr(modconf, 'data', 'default')
        self.tick_limit = int(getattr(modconf, 'tick_limit', '3600'))
        # Buffer max size in bytes
        self.max_buffer_size = int(getattr(modconf, 'max_buffer_size', '60000'))
        self.buffer = []
        self.ticks = 0
        # Cache for business_impact
        self.dict_business_impact = {}

        # Number of lines to delete when the buffer is full
        self.lines_deleted = 30
        # For log brok only, to select only some event to send to socket.
        self.event_list = [("host_alert", "HARD"),
                           ("host_alert", "SOFT"),
                           ("service_alert", "HARD"),
                           ("service_alert", "SOFT"),
                           ("host_downtime_alert", "STARTED"),
                           ("host_downtime_alert", "STOPPED"),
                           ("service_downtime_alert", "STARTED"),
                           ("service_downtime_alert", "STOPPED"),
                           ("host_notification", "ACKNOWLEDGEMENT"),
                           ("service_notification", "ACKNOWLEDGEMENT"),
                           ("comment", None)]

        # properties for log event. Every log line has its own data. The list_param is listing them
        # the pattern is what the output looks like.
        self.parsing_properties = {
            'host_alert': {
                'pattern': 'event_type="%(event_type)s" hostname="%(hostname)s" state="%(state)s"'
                           ' business_impact="%(business_impact)d" output="%(output)s"',
                'list_params': ["hostname", "state", "state_type", "attempt", "output"]},
            'service_alert': {
                'pattern': 'event_type="%(event_type)s" hostname="%(hostname)s" '
                           'servicename="%(servicename)s" state="%(state)s"'
                           ' business_impact="%(business_impact)d" output="%(output)s"',
                'list_params': ["hostname", "servicename", "state", "state_type", "attempt", "output"]},
            'host_downtime_alert': {
                'pattern': 'event_type="%(event_type)s" hostname="%(hostname)s" state="%(state)s"'
                           ' business_impact="%(business_impact)d" output="%(output)s"',
                'list_params': ["hostname", "state", "output"]},
            'service_downtime_alert': {
                'pattern': 'event_type="%(event_type)s" hostname="%(hostname)s"'
                           ' servicename="%(servicename)s" state="%(state)s"'
                           ' business_impact="%(business_impact)d" output="%(output)s"',
                'list_params': ["hostname", "servicename", "state", "output"]},
            'host_flapping_alert': {
                'pattern': 'event_type="%(event_type)s" hostname="%(hostname)s" state="%(state)s"'
                           ' business_impact="%(business_impact)d" output="%(output)s"',
                'list_params': ["hostname", "state", "output"]},
            'service_flapping_alert': {
                'pattern': 'event_type="%(event_type)s" hostname="%(hostname)s"'
                           ' servicename="%(servicename)s" state="%(state)s"'
                           ' business_impact="%(business_impact)d" output="%(output)s"',
                'list_params': ["hostname", "servicename", "state", "output"]},
            'host_notification': {
                'pattern': 'event_type="%(event_type)s" contact="%(contact)s"'
                           ' hostname="%(hostname)s" ntype="%(ntype)s" command="%(command)s"'
                           ' business_impact="%(business_impact)d" output="%(output)s"',
                'list_params': ["contact", "hostname", "ntype", "command", "output"]},
            'service_notification': {
                'pattern': 'event_type="%(event_type)s" contact="%(contact)s"'
                           ' hostname="%(hostname)s" servicename="%(servicename)s"'
                           ' ntype="%(ntype)s" command="%(command)s"'
                           ' business_impact="%(business_impact)d" output="%(output)s"',
                'list_params': ["contact", "hostname", "servicename", "ntype", "command", "output"]},
            'add_svc_comment': {
                'pattern': 'event_type="%(event_type)s" message="Not Handled yet"',
                'list_params': []},
            'add_host_comment': {
                'pattern': 'event_type="%(event_type)s" message="Not Handled yet"',
                'list_params': []},
            'event_handler': {
                'pattern': 'event_type="%(event_type)s" message="Not Handled yet"',
                'list_params': []}
        }

    # Try to connect to host, port
    def init(self):
        try:
            self.con = socket.socket()
            self.con.connect((self.host, self.port))
        except Exception:
            logger.warning("[RawSocket broker] Failed to connect to host %s and port %d!"
                           % (self.host, self.port))

    # Parse the line and create and dict of key values.
    # Return the formatted line and timestamp
    def format_output(self, line, pattern, list_params):
        t, line = line.split('] ', 1)
        event_type, data = line.split(': ', 1)
        # Creates key-values dict
        l_params = dict(zip(list_params, data.split(';', max(len(list_params) - 1, 0))))
        # Add event type and business impact in the dict because all lines generated need it
        l_params["event_type"] = event_type
        key_search = l_params["hostname"]
        if "service_description" in l_params:
            key_search += "::" + l_params["service_description"]
        l_params["business_impact"] = self.dict_business_impact[key_search]
        return t, pattern % l_params

    # For log event only : return the event_name associated in the parsing properties
    def build_name(self, group):
        name = ""
        for elem in group:
            if elem:
                name += "_" + elem.lower()
        return name[1:]

    # For log event only : add to the buffer the converted log line into the new format
    def parse_event(self, name, etype, line):
        if self.data == 'all' or (name, etype) in self.event_list:
            if name in self.parsing_properties.keys():
                t, new_line = self.format_output(line, **self.parsing_properties[name])
                t = t[1:]
                formatted = datetime.datetime.fromtimestamp(int(t)).strftime('%Y-%m-%dT%H:%M:%S')
                tz = str.format('{0:05.2f}', float(time.timezone) / 3600).replace('.', ':')
                tz = '-' + tz if time.timezone > 0 else '+' + tz
                isodate = formatted + tz
                hostname = socket.gethostname()
                self.buffer.append("<0>" + isodate + " " + hostname + " " +
                                   socket.gethostbyname(hostname) + " " +
                                   self.name + "[0]: timestamp=" + str(t) + " " + new_line)

            else:
                logger.info("Can't parse event : %s. Skipping" % name)
        else:
            logger.info("Unhandled event : %s. I skipped the following line : %s"
                        % (name, etype))
<commit_msg>Add cancelled downtime to filter + fix log line<commit_after>#!/usr/bin/python

# -*- coding: utf-8 -*-

# Copyright (C) 2009-2013:
# <NAME>, <EMAIL>
# <NAME>, <EMAIL>
# <NAME>, <EMAIL>
# <NAME>, <EMAIL>
# <NAME>, <EMAIL>
# Savoir-Faire Linux inc.
#
# This file is part of Shinken.
#
# Shinken is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Shinken is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Shinken. If not, see <http://www.gnu.org/licenses/>.


# This Class is a plugin for the Shinken Broker. It is responsible
# for sending log broks to a raw socket destination.

import re
import socket
import time
import datetime

from shinken.basemodule import BaseModule
from shinken.log import logger

properties = {
    'daemons': ['broker'],
    'type': 'raw_socket',
    'external': False,
    'phases': ['running'],
}


# called by the plugin manager to get a broker
def get_instance(plugin):
    logger.info("Get a RawSocket broker for plugin %s" % plugin.get_name())

    #Catch errors
    #path = plugin.path
    instance = RawSocket_broker(plugin)
    return instance


# Get broks and send them to TCP Raw Socket listener
class RawSocket_broker(BaseModule):
    def __init__(self, modconf):
        BaseModule.__init__(self, modconf)
        self.host = getattr(modconf, 'host', 'localhost')
        self.port = int(getattr(modconf, 'port', '9514'))
        self.data = getattr(modconf, 'data', 'default')
        self.tick_limit = int(getattr(modconf, 'tick_limit', '3600'))
        # Buffer max size in bytes
        self.max_buffer_size = int(getattr(modconf, 'max_buffer_size', '60000'))
        self.buffer = []
        self.ticks = 0
        # Cache for business_impact
        self.dict_business_impact = {}

        # Number of lines to delete when the buffer is full
        self.lines_deleted = 30
        # For log brok only, to select only some event to send to socket.
        self.event_list = [("host_alert", "HARD"),
                           ("host_alert", "SOFT"),
                           ("service_alert", "HARD"),
                           ("service_alert", "SOFT"),
                           ("host_downtime_alert", "STARTED"),
                           ("host_downtime_alert", "STOPPED"),
                           ("host_downtime_alert", "CANCELLED"),
                           ("service_downtime_alert", "STARTED"),
                           ("service_downtime_alert", "STOPPED"),
                           ("service_downtime_alert", "CANCELLED"),
                           ("host_notification", "ACKNOWLEDGEMENT"),
                           ("service_notification", "ACKNOWLEDGEMENT"),
                           ("comment", None)]

        # properties for log event. Every log line has its own data. The list_param is listing them
        # the pattern is what the output looks like.
        self.parsing_properties = {
            'host_alert': {
                'pattern': 'event_type="%(event_type)s" hostname="%(hostname)s" state="%(state)s"'
                           ' business_impact="%(business_impact)d" output="%(output)s"',
                'list_params': ["hostname", "state", "state_type", "attempt", "output"]},
            'service_alert': {
                'pattern': 'event_type="%(event_type)s" hostname="%(hostname)s" '
                           'servicename="%(servicename)s" state="%(state)s"'
                           ' business_impact="%(business_impact)d" output="%(output)s"',
                'list_params': ["hostname", "servicename", "state", "state_type", "attempt", "output"]},
            'host_downtime_alert': {
                'pattern': 'event_type="%(event_type)s" hostname="%(hostname)s" state="%(state)s"'
                           ' business_impact="%(business_impact)d" output="%(output)s"',
                'list_params': ["hostname", "state", "output"]},
            'service_downtime_alert': {
                'pattern': 'event_type="%(event_type)s" hostname="%(hostname)s"'
                           ' servicename="%(servicename)s" state="%(state)s"'
                           ' business_impact="%(business_impact)d" output="%(output)s"',
                'list_params': ["hostname", "servicename", "state", "output"]},
            'host_flapping_alert': {
                'pattern': 'event_type="%(event_type)s" hostname="%(hostname)s" state="%(state)s"'
                           ' business_impact="%(business_impact)d" output="%(output)s"',
                'list_params': ["hostname", "state", "output"]},
            'service_flapping_alert': {
                'pattern': 'event_type="%(event_type)s" hostname="%(hostname)s"'
                           ' servicename="%(servicename)s" state="%(state)s"'
                           ' business_impact="%(business_impact)d" output="%(output)s"',
                'list_params': ["hostname", "servicename", "state", "output"]},
            'host_notification': {
                'pattern': 'event_type="%(event_type)s" contact="%(contact)s"'
                           ' hostname="%(hostname)s" ntype="%(ntype)s" command="%(command)s"'
                           ' business_impact="%(business_impact)d" output="%(output)s"',
                'list_params': ["contact", "hostname", "ntype", "command", "output"]},
            'service_notification': {
                'pattern': 'event_type="%(event_type)s" contact="%(contact)s"'
                           ' hostname="%(hostname)s" servicename="%(servicename)s"'
                           ' ntype="%(ntype)s" command="%(command)s"'
                           ' business_impact="%(business_impact)d" output="%(output)s"',
                'list_params': ["contact", "hostname", "servicename", "ntype", "command", "output"]},
            'add_svc_comment': {
                'pattern': 'event_type="%(event_type)s" message="Not Handled yet"',
                'list_params': []},
            'add_host_comment': {
                'pattern': 'event_type="%(event_type)s" message="Not Handled yet"',
                'list_params': []},
            'event_handler': {
                'pattern': 'event_type="%(event_type)s" message="Not Handled yet"',
                'list_params': []}
        }

    # Try to connect to host, port
    def init(self):
        try:
            self.con = socket.socket()
            self.con.connect((self.host, self.port))
        except Exception:
            logger.warning("[RawSocket broker] Failed to connect to host %s and port %d!"
                           % (self.host, self.port))

    # Parse the line and create and dict of key values.
    # Return the formatted line and timestamp
    def format_output(self, line, pattern, list_params):
        t, line = line.split('] ', 1)
        event_type, data = line.split(': ', 1)
        # Creates key-values dict
        l_params = dict(zip(list_params, data.split(';', max(len(list_params) - 1, 0))))
        # Add event type and business impact in the dict because all lines generated need it
        l_params["event_type"] = event_type
        key_search = l_params["hostname"]
        if "service_description" in l_params:
            key_search += "::" + l_params["service_description"]
        l_params["business_impact"] = self.dict_business_impact[key_search]
        return t, pattern % l_params

    # For log event only : return the event_name associated in the parsing properties
    def build_name(self, group):
        name = ""
        for elem in group:
            if elem:
                name += "_" + elem.lower()
        return name[1:]

    # For log event only : add to the buffer the converted log line into the new format
    def parse_event(self, name, etype, line):
        if self.data == 'all' or (name, etype) in self.event_list:
            if name in self.parsing_properties.keys():
                t, new_line = self.format_output(line, **self.parsing_properties[name])
                t = t[1:]
                formatted = datetime.datetime.fromtimestamp(int(t)).strftime('%Y-%m-%dT%H:%M:%S')
                tz = str.format('{0:05.2f}', float(time.timezone) / 3600).replace('.', ':')
                tz = '-' + tz if time.timezone > 0 else '+' + tz
                isodate = formatted + tz
                hostname = socket.gethostname()
                self.buffer.append("<0>" + isodate + " " + hostname + " " +
                                   socket.gethostbyname(hostname) + " " +
                                   self.name + "[0]: timestamp=" + str(t) + " " + new_line)

            else:
                logger.info("Can't parse event : %s. Skipping" % name)
        else:
            logger.info("Unhandled (event, type) : (%s, %s). I skipped the following line : %s"
                        % (name, etype, line))

{'fraction_non_alphanumeric': 0.1408057001918334, 'fraction_numerical': 0.004713620169909565, 'mean_word_length': 2.225954738330976, 'pattern_counts': {'":': 2, '<': 27, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1407275', 'n_tokens_mistral': 5505, 'n_tokens_neox': 5160, 'n_words': 1326}