<commit_before>#!/usr/bin/env python
# coding: utf-8
"""
Postfix log parser.

This scripts parses a log file produced by postfix (or using the same
format). It looks for predefined events and build statistics about
their occurence rate.

At the, somes default graphics are generated using the grapher module.
(see grapher.py)

Predefined events are:
 * Per domain sent/received messages,
 * Per domain received bad messages (bounced, reject for now),
 * Per domain sent/received traffics size,
 * Global consolidation of all previous events.

"""
import time
import sys
import os
import re
import rrdtool
import string
from optparse import make_option
from django import db
from django.core.management.base import BaseCommand
from modoboa.lib import parameters
from modoboa.extensions.admin.models import Domain
from modoboa.extensions.stats import Stats
from modoboa.extensions.stats.grapher import str2Time, Grapher
from modoboa.extensions.stats.graph_templates import MailTraffic

rrdstep = 60
xpoints = 540
points_per_sample = 3
variables = ["sent", "recv", "bounced", "reject", "spam", "virus",
             "size_sent", "size_recv"]


class LogParser(object):

    def __init__(self, options, workdir, year=None):
        """Constructor
        """
        self.logfile = options["logfile"]
        self.debug = options["debug"]
        self.verbose = options["verbose"]
        try:
            self.f = open(self.logfile)
        except IOError as errno:
            self._dprint("%s" % errno)
            sys.exit(1)
        self.workdir = workdir
        self.__year = year
        self.cfs = ['AVERAGE', 'MAX']

        curtime = time.localtime()
        if not self.__year:
            self.__year = curtime.tm_year
        self.curmonth = curtime.tm_mon

        self.data = {}
        self.domains = []
        for dom in Domain.objects.all():
            self.domains += [str(dom.name)]
            self.data[str(dom.name)] = {}
        self.data["global"] = {}

        self.workdict = {}
        self.lupdates = {}
        self._s_date_expr = \
            re.compile(r"(?P<month>\w+)\s+(?P<day>\d+)\s+(?P<hour>\d+):(?P<min>\d+):(?P<sec>\d+)(?P<eol>.*)")
        self._hp_date_expr = \
            re.compile(r"(?P<year>\d+)-(?P<month>\d+)-(?P<day>\d+)T(?P<hour>\d+):(?P<min>\d+):(?P<sec>\d+)\.\d+\+\d+:\d+(?P<eol>.*)")
        self.date_expr = None
        self.line_expr = \
            re.compile(r"\s+([-\w]+)\s+(\w+)/?\w*[[](\d+)[]]:\s+(.*)")
        self._id_expr = re.compile(r"(\w+): (.*)")
        self._prev_se = -1
        self._prev_mi = -1
        self._prev_ho = -1
        self.cur_t = 0

    def _dprint(self, msg):
        """Print a debug message if required.

        :param str msg: debug message
        """
        if not self.debug:
            return
        print msg

    def _parse_date(self, line):
        """Try to match a date inside :kw:`line` and to convert it to<commit_msg>Load relay domain list if extension is activated.<commit_after>#!/usr/bin/env python
# coding: utf-8
"""
Postfix log parser.

This scripts parses a log file produced by postfix (or using the same
format). It looks for predefined events and build statistics about
their occurence rate.

At the, somes default graphics are generated using the grapher module.
(see grapher.py)

Predefined events are:
 * Per domain sent/received messages,
 * Per domain received bad messages (bounced, reject for now),
 * Per domain sent/received traffics size,
 * Global consolidation of all previous events.

"""
import time
import sys
import os
import re
import rrdtool
import string
from optparse import make_option

from django import db
from django.core.management.base import BaseCommand

from modoboa.core.extensions import exts_pool
from modoboa.extensions.admin.models import Domain
from modoboa.extensions.postfix_relay_domains.models import RelayDomain
from modoboa.extensions.stats import Stats
from modoboa.extensions.stats.grapher import str2Time, Grapher
from modoboa.extensions.stats.graph_templates import MailTraffic
from modoboa.lib import parameters

rrdstep = 60
xpoints = 540
points_per_sample = 3
variables = ["sent", "recv", "bounced", "reject", "spam", "virus",
             "size_sent", "size_recv"]


class LogParser(object):

    def __init__(self, options, workdir, year=None):
        """Constructor
        """
        self.logfile = options["logfile"]
        self.debug = options["debug"]
        self.verbose = options["verbose"]
        try:
            self.f = open(self.logfile)
        except IOError as errno:
            self._dprint("%s" % errno)
            sys.exit(1)
        self.workdir = workdir
        self.__year = year
        self.cfs = ['AVERAGE', 'MAX']

        curtime = time.localtime()
        if not self.__year:
            self.__year = curtime.tm_year
        self.curmonth = curtime.tm_mon

        self.data = {}
        self.domains = []
        self._load_domain_list()
        self.data["global"] = {}

        self.workdict = {}
        self.lupdates = {}
        self._s_date_expr = \
            re.compile(r"(?P<month>\w+)\s+(?P<day>\d+)\s+(?P<hour>\d+):(?P<min>\d+):(?P<sec>\d+)(?P<eol>.*)")
        self._hp_date_expr = \
            re.compile(r"(?P<year>\d+)-(?P<month>\d+)-(?P<day>\d+)T(?P<hour>\d+):(?P<min>\d+):(?P<sec>\d+)\.\d+\+\d+:\d+(?P<eol>.*)")
        self.date_expr = None
        self.line_expr = \
            re.compile(r"\s+([-\w]+)\s+(\w+)/?\w*[[](\d+)[]]:\s+(.*)")
        self._id_expr = re.compile(r"(\w+): (.*)")
        self._prev_se = -1
        self._prev_mi = -1
        self._prev_ho = -1
        self.cur_t = 0

    def _load_domain_list(self):
        """Load the list of allowed domains.

        Since the relay domains feature is an extension of the admin
        panel, we don't use an event to get the list of all supported
        domains...

        """
        for dom in Domain.objects.all():
            self.domains += [str(dom.name)]
            self.data[str(dom.name)] = {}
        if not exts_pool.is_extension_enabled("postfix_relay_domains"):
            return
        for rdom in RelayDomain.objects.all():
            self.domains += [str(rdom.name)]
            self.data[str(rdom.name)] = {}

    def _dprint(self, msg):
        """Print a debug message if required.

        :param str msg: debug message
        """
        if not self.debug:
            return
        print msg

    def _parse_date(self, line):
        """Try to match a date inside :kw:`line` and to convert it to
{'fraction_non_alphanumeric': 0.14149785144260282, 'fraction_numerical': 0.003990178023327195, 'mean_word_length': 2.5113146551724137, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6957824', 'n_tokens_mistral': 2132, 'n_tokens_neox': 2041, 'n_words': 535}