<commit_before>#!/usr/bin/python

import os
import sys
import signal
import string
import re
import json
import time
import logging
from logging.handlers import SysLogHandler
from optparse import OptionParser, OptionGroup


# Add syslog levels to logging module.
logging.NOTICE = 25
logging.ALERT = 60
logging.EMERG = 70
logging.addLevelName(logging.NOTICE, 'NOTICE')
logging.addLevelName(logging.ALERT, 'ALERT')
logging.addLevelName(logging.EMERG, 'EMERG')
SysLogHandler.priority_map['NOTICE'] = 'notice'
SysLogHandler.priority_map['ALERT'] = 'alert'
SysLogHandler.priority_map['EMERG'] = 'emerg'
# Define data and message format according to RFC 5424.
rfc5424_format = '{version} {timestamp} {hostname} {appname} {procid}'\
                 ' {msgid} {structured_data} {msg}'
date_format = '%Y-%m-%dT%H:%M:%SZ'
# Define global semaphore.
sending_in_progress = 0
# Define file types.
msg_levels = {'ruby': {'regex': '(?P<level>[DIWEF]), \[[0-9-]{10}T',
                       'levels': {'D': logging.DEBUG,
                                  'I': logging.INFO,
                                  'W': logging.WARNING,
                                  'E': logging.ERROR,
                                  'F': logging.FATAL
                                  }
                       },
              'syslog': {'regex': ('[0-9-]{10}T[0-9:]{8}Z (?P<level>'
                                   'debug|info|notice|warning|err|crit|'
                                   'alert|emerg)'),
                         'levels': {'debug': logging.DEBUG,
                                    'info': logging.INFO,
                                    'notice': logging.NOTICE,
                                    'warning': logging.WARNING,
                                    'err': logging.ERROR,
                                    'crit': logging.CRITICAL,
                                    'alert': logging.ALERT,
                                    'emerg': logging.EMERG
                                    }
                         },
              'anaconda': {'regex': ('[0-9:]{8},[0-9]+ (?P<level>'
                                     'DEBUG|INFO|WARNING|ERROR|CRITICAL)'),
                           'levels': {'DEBUG': logging.DEBUG,
                                      'INFO': logging.INFO,
                                      'WARNING': logging.WARNING,
                                      'ERROR': logging.ERROR,
                                      'CRITICAL': logging.CRITICAL
                                      }
                           }
              }
# Create a main logger.
logging.basicConfig(format='%(levelname)s: %(message)s')
main_logger = logging.getLogger()
main_logger.setLevel(logging.NOTSET)


class WatchedFile:
    """ WatchedFile(filename) => Object that read lines from file if exist. """

    def __init__(self, name):
        self.name = name
        self.fo = None
        self.where = 0

    def reset(self):
        if self.fo:
            self.fo.close()
            self.fo = None
            self.where = 0

    def _checkRewrite(self):
        try:
            if os.stat(self.name)[6] < self.where:
                self.reset()
        except OSError:
            self.close()

    def readLines(self):
        """Return list of last append lines from file if exist. """

        self._checkRewrite()
        if not self.fo:
            try:
                self.fo = open(self.name, 'r')
            except IOError:
                return ()
        lines = self.fo.readlines()
        self.where = self.fo.tell()
        return lines

    def close(self):
        self.reset()


class WatchedGroup:
    """ Can send data from group of specified files to specified servers. """

    def __init__(self, servers, files, name):
        self.servers = servers
        self.files = files
        self.log_type = files.get('log_type', 'syslog')
        self.name = name
        self._createLogger()

    def _createLogger(self):
        self.watchedfiles = []
        logger = logging.getLogger(self.name)
        logger.setLevel(logging.NOTSET)
        logger.propagate = False
        # Create log formatter.
        format_dict = {'version': '1',
                       'timestamp': '%(asctime)s',
                       'hostname': config['hostname'],
                       'appname': self.files['tag'],
                       'procid': '-',
                       'msgid': '-',
                       'structured_data': '-',
                       'msg': '%(message)s'
                       }
        log_format = rfc5424_format.format(**format_dict)
        formatter = logging.Formatter(log_format, date_format)
        # Add log handler for each server.
        for server in self.servers:
            port = 'port' in server and server['port'] or 514
            syslog = SysLogHandler((server["host"], port))
            syslog.setFormatter(formatter)
            logger.addHandler(syslog)
        self.logger = logger
        # Create WatchedFile objects from list of files.
        for name in self.files['files']:
            self.watchedfiles.append(WatchedFile(name))

    def send(self):
        """ Send append data from files to servers. """

        for watchedfile in self.watchedfiles:
            for line in watchedfile.readLines():
                line = line.strip()
                level = self._get_msg_level(line, self.log_type)
                self.logger.log(level, line)
                main_logger and main_logger.log(
                    level,
                    'From file "%s" send: %s' % (watchedfile.name, line)
                )

    @staticmethod
    def _get_msg_level(line, log_type):
        if log_type in msg_levels:
            msg_type = msg_levels[log_type]
            regex = re.match(msg_type['regex'], line)
            if regex:
                return msg_type['levels'][regex.group('level')]
        return logging.INFO


def sig_handler(signum, frame):
    """ Send all new data when signal arrived. """

    if not sending_in_progress:
        send_all()
        exit(signum)
    else:<commit_msg>PRD-448 Fix anakonda log format<commit_after>#!/usr/bin/python

import os
import sys
import signal
import string
import re
import json
import time
import logging
from logging.handlers import SysLogHandler
from optparse import OptionParser, OptionGroup


# Add syslog levels to logging module.
logging.NOTICE = 25
logging.ALERT = 60
logging.EMERG = 70
logging.addLevelName(logging.NOTICE, 'NOTICE')
logging.addLevelName(logging.ALERT, 'ALERT')
logging.addLevelName(logging.EMERG, 'EMERG')
SysLogHandler.priority_map['NOTICE'] = 'notice'
SysLogHandler.priority_map['ALERT'] = 'alert'
SysLogHandler.priority_map['EMERG'] = 'emerg'
# Define data and message format according to RFC 5424.
rfc5424_format = '{version} {timestamp} {hostname} {appname} {procid}'\
                 ' {msgid} {structured_data} {msg}'
date_format = '%Y-%m-%dT%H:%M:%SZ'
# Define global semaphore.
sending_in_progress = 0
# Define file types.
msg_levels = {'ruby': {'regex': '(?P<level>[DIWEF]), \[[0-9-]{10}T',
                       'levels': {'D': logging.DEBUG,
                                  'I': logging.INFO,
                                  'W': logging.WARNING,
                                  'E': logging.ERROR,
                                  'F': logging.FATAL
                                  }
                       },
              'syslog': {'regex': ('[0-9-]{10}T[0-9:]{8}Z (?P<level>'
                                   'debug|info|notice|warning|err|crit|'
                                   'alert|emerg)'),
                         'levels': {'debug': logging.DEBUG,
                                    'info': logging.INFO,
                                    'notice': logging.NOTICE,
                                    'warning': logging.WARNING,
                                    'err': logging.ERROR,
                                    'crit': logging.CRITICAL,
                                    'alert': logging.ALERT,
                                    'emerg': logging.EMERG
                                    }
                         },
              'anaconda': {'regex': ('[0-9:]{8},[0-9]+ (?P<level>'
                                     'DEBUG|INFO|WARNING|ERROR|CRITICAL)'),
                           'levels': {'DEBUG': logging.DEBUG,
                                      'INFO': logging.INFO,
                                      'WARNING': logging.WARNING,
                                      'ERROR': logging.ERROR,
                                      'CRITICAL': logging.CRITICAL
                                      }
                           }
              }
# Create a main logger.
logging.basicConfig(format='%(levelname)s: %(message)s')
main_logger = logging.getLogger()
main_logger.setLevel(logging.NOTSET)


class WatchedFile:
    """ WatchedFile(filename) => Object that read lines from file if exist. """

    def __init__(self, name):
        self.name = name
        self.fo = None
        self.where = 0

    def reset(self):
        if self.fo:
            self.fo.close()
            self.fo = None
            self.where = 0

    def _checkRewrite(self):
        try:
            if os.stat(self.name)[6] < self.where:
                self.reset()
        except OSError:
            self.close()

    def readLines(self):
        """Return list of last append lines from file if exist. """

        self._checkRewrite()
        if not self.fo:
            try:
                self.fo = open(self.name, 'r')
            except IOError:
                return ()
        lines = self.fo.readlines()
        self.where = self.fo.tell()
        return lines

    def close(self):
        self.reset()


class WatchedGroup:
    """ Can send data from group of specified files to specified servers. """

    def __init__(self, servers, files, name):
        self.servers = servers
        self.files = files
        self.log_type = files.get('log_type', 'syslog')
        self.name = name
        self._createLogger()

    def _createLogger(self):
        self.watchedfiles = []
        logger = logging.getLogger(self.name)
        logger.setLevel(logging.NOTSET)
        logger.propagate = False
        # Create log formatter.
        format_dict = {'version': '1',
                       'timestamp': '%(asctime)s',
                       'hostname': config['hostname'],
                       'appname': self.files['tag'],
                       'procid': '-',
                       'msgid': '-',
                       'structured_data': '-',
                       'msg': '%(message)s'
                       }
        log_format = rfc5424_format.format(**format_dict)
        formatter = logging.Formatter(log_format, date_format)
        # Add log handler for each server.
        for server in self.servers:
            port = 'port' in server and server['port'] or 514
            syslog = SysLogHandler((server["host"], port))
            syslog.setFormatter(formatter)
            logger.addHandler(syslog)
        self.logger = logger
        # Create WatchedFile objects from list of files.
        for name in self.files['files']:
            self.watchedfiles.append(WatchedFile(name))

    def send(self):
        """ Send append data from files to servers. """

        for watchedfile in self.watchedfiles:
            for line in watchedfile.readLines():
                line = line.strip()
                level = self._get_msg_level(line, self.log_type)
                line = re.sub(
                    msg_levels[self.log_type]['regex'] + "\s*:?\s?",
                    "",
                    line
                )
                self.logger.log(level, line)
                main_logger and main_logger.log(
                    level,
                    'From file "%s" send: %s' % (watchedfile.name, line)
                )

    @staticmethod
    def _get_msg_level(line, log_type):
        if log_type in msg_levels:
            msg_type = msg_levels[log_type]
            regex = re.match(msg_type['regex'], line)
            if regex:
                return msg_type['levels'][regex.group('level')]
        return logging.INFO


def sig_handler(signum, frame):
    """ Send all new data when signal arrived. """

    if not sending_in_progress:
        send_all()
        exit(signum)
    else:
{'fraction_non_alphanumeric': 0.11554299555914413, 'fraction_numerical': 0.00702462656439241, 'mean_word_length': 1.4512170987532158, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 40, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2050846', 'n_tokens_mistral': 3472, 'n_tokens_neox': 3255, 'n_words': 764}