<commit_before>"""
Copyright 2016-2017 <NAME>

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import re
import datetime
from collections import namedtuple

ERR_FILE = 'file'
ERR_HEADER = 'header'
ERR_QSO = 'qso'

class Operator(object):
    """
    Keep operator callsign, info and logs path
    """
    callsign = None
    info = {}           # no idea what was this for :(
    logs = []           # list with Log() instances

    def __init__(self, callsign):
        self.callsign = callsign
        self.logs = []

    def add_log(self, path):
        self.logs.append(Log(path))


class Log(object):
    """
    Keep a single EDI log information:
    log path, log raw content, callsign, qth, band, section, qsos tuple (raw content)
    and a list with LogQso() instances

    errors format :
    {
        'file': [ (line or None, 'error: Cannot open file'), ... ],
        'header': [ (line or None, 'error: message'), ... ],
        'qso': [ (line, 'error: message'), ...],
    }
    """
    path = None
    rules = None
    use_as_checklog = False
    log_lines = None

    valid = None
    errors = {}

    callsign = None
    maidenhead_locator = None
    band = None
    section = None
    date = None

    qsos_tuple = namedtuple('qso_tuple', ['linenr', 'qso', 'valid', 'error'])
    qsos = []   # list with LogQso instances

    def __init__(self, path, rules=None, checklog=False):
        self.path = path
        self.rules = rules
        self.use_as_checklog = checklog

        self.validate()

        if not self.valid:
            return

        self.qsos = []
        self.get_qsos()

    def validate(self):
        """ Validate edi log.
        If errors are found they will be written in self.errors dictionary
        :return: True or False
        """

        self.errors[ERR_FILE] = []
        self.errors[ERR_HEADER] = []
        self.errors[ERR_QSO] = []
        self.valid = False

        try:
            self.log_lines = self.read_file_content(self.path)
        except Exception as e:
            self.errors[ERR_FILE].append((None, 'Cannot read edi log'))
            return

        if self.log_lines is None:
            self.errors[ERR_FILE].append((None, 'Log is empty'))
            return

        # get & validate callsign
        _callsign, line_nr = self.get_field('PCall')
        if not _callsign:
            self.errors[ERR_HEADER].append((line_nr, 'PCall field is not present'))
        elif len(_callsign) > 1:
            self.errors[ERR_HEADER].append((line_nr, 'PCall field is present multiple times'))
        else:
            self.callsign = _callsign[0]

        # get & validate maidenhead locator
        _qthlocator, line_nr = self.get_field('PWWLo')
        if not _qthlocator:
            self.errors[ERR_HEADER].append((line_nr, 'PWWLo field is not present'))
        elif len(_qthlocator) > 1:
            self.errors[ERR_HEADER].append((line_nr, 'PWWLo field is present multiple times'))
        elif not self.validate_qth_locator(_qthlocator[0]):
            self.errors[ERR_HEADER].append((line_nr, 'PWWLo field value is not valid'))
        else:
            self.maidenhead_locator = _qthlocator[0]

        # get & validate band based on generic rules and by custom rules if provided (rules.contest_band['regexp'])
        _band, line_nr = self.get_field('PBand')
        if not _band:
            self.errors[ERR_HEADER].append((line_nr, 'PBand field is not present'))
        elif len(_band) > 1:
            self.errors[ERR_HEADER].append((line_nr, 'PBand field is present multiple times'))
        elif not self.rules and not self.validate_band(_band[0]):
            self.errors[ERR_HEADER].append((line_nr, 'PBand field value is not valid'))
        elif self.rules and not self.rules_based_validate_band(_band[0], self.rules):
            self.errors[ERR_HEADER].append((line_nr, 'PBand field value has an invalid value ({}). '
                                                  'Not as defined in contest rules'.format(_band[0])))
        else:
            self.band = _band[0]

        # get & validate PSect based on generic rules and by custom rules if provided (rules.contest_category['regexp']
        _section, line_nr = self.get_field('PSect')
        if not _section:
            self.errors[ERR_HEADER].append((line_nr, 'PSect field is not present'))
        elif len(_section) > 1:
            self.errors[ERR_HEADER].append((line_nr, 'PSect field is present multiple times'))
        elif not self.rules and not self.validate_section(_section[0]):
            self.errors[ERR_HEADER].append((line_nr, 'PSect field value is not valid'))
        elif self.rules and not self.rules_based_validate_section(_section[0], self.rules):
            self.errors[ERR_HEADER].append((line_nr, 'PSect field value has an invalid value ({}). '
                                          'Not as defined in contest rules'.format(_section[0])))
        else:
            self.section = _section[0]

        # get & validate TDate based on generic rules format and by custom rules if provided
        # (rules.contest_begin_date & rules.contest_end_date)
        _date, line_nr = self.get_field('TDate')
        if not _date:
            self.errors[ERR_HEADER].append((line_nr, 'TDate field is not present'))
        elif len(_date) > 1:
            self.errors[ERR_HEADER].append((line_nr, 'TDate field is present multiple times'))
        elif not self.validate_date(_date[0]):
            self.errors[ERR_HEADER].append((line_nr, 'TDate field value is not valid ({})'.format(_date[0])))
        elif self.rules and not self.rules_based_validate_date(_date[0], self.rules):
            self.errors[ERR_HEADER].append((line_nr, 'TDate field value has an invalid value ({}). '
                                                  'Not as defined in contest rules'.format(_date[0])))
        else:
            self.date = _date[0]

        if all((self.callsign, self.maidenhead_locator, self.band, self.section, self.date)):
            self.valid = True

    @staticmethod
    def read_file_content(path):
        try:
            with open(path, 'r') as _file:
                content = _file.readlines()
        except IOError:
            raise
        except Exception:
            raise
        return content

    def get_field(self, field):
        """
        Search log content for a field
        :param field: field name (PCall, TDate, ...)
        :return: tuple(value, line_number or None)
        """
        value = []
        line_nr = None
        _field = str(field).upper() + '='
        for (nr, line_content) in enumerate(self.log_lines):
            if line_content.upper().startswith(_field):
                value.append(line_content.split('=', 1)[1].strip())
                line_nr = nr+1
        return value, line_nr

    def get_qsos(self):
        """
        Will read the self.log_content and will return a list of LogQso
        """
        qso_record_start = "[QSORECORDS"
        qso_record_end = "[END;"
        qso_lines = []
        do_read_qso = False

        # read qso lines
        for (index, line) in enumerate(self.log_lines):
            if line.upper().startswith(qso_record_start):
                do_read_qso = True
                continue
            if line.upper().startswith(qso_record_end):
                do_read_qso = False
                continue
            if do_read_qso:
                qso_lines.append((index, line.strip()))

        # validate qso lines
        for qso in qso_lines:
            message = LogQso.regexp_qso_validator(qso[1])
            self.qsos.append(
                # self.qsos_tuple(linenr=qso[0], qso=qso[1], valid=False if message else True, error=message)
                LogQso(qso[1], qso[0])  # LogQso(qso_line, qso_line_number_in_log)
            )

    @staticmethod
    def validate_qth_locator(qth):
        regex_maidenhead = r'^\s*([a-rA-R]{2}\d{2}[a-xA-X]{2})\s*$'
        res = re.match(regex_maidenhead, qth, re.IGNORECASE)
        return True if res else False

    @staticmethod
    def get_band(band):
        """
        This will parse the 'PBand=' field value
        and return the proper band
        :param band: the content of 'PBand=' field
        :return: The detected band (144,432,1296) or None
        """

        regexp_band = {144: ['144.*', '145.*'],
                       432: ['430.*', '432.*', '435.*'],
                       1296: ['1296.*', '1[.,][23].*']}
        for _band in regexp_band:
            for regexp in regexp_band[_band]:
                res = re.match(regexp, band)
                if res:
                    return _band
        return None

    @staticmethod
    def validate_band(band_value):
        """
        This will validate PBand based on generic rules
        """
        is_valid = False

        regexp_band_check = ['144.*', '145.*',
                             '430.*', '432.*', '435.*',
                             '1296.*', '1[.,][23].*']
        for _regex in regexp_band_check:
            res = re.match(_regex, band_value)
            if res:
                is_valid = True
        return is_valid

    @staticmethod
    def rules_based_validate_band(band_value, rules):
        """
        This will validate PBand based on Rules class instance
        """
        is_valid = False

        if rules is None:
            raise ValueError('No contest rules provided !')
        for _nr in range(1, rules.contest_bands_nr+1):
            _regex = r'\s*(' + rules.contest_band(_nr)['regexp'] + r')\s*'
            res = re.match(_regex, band_value)
            if res:
                is_valid = True
        return is_valid

    @staticmethod
    def validate_section(section_value):
        """
        This will validate PSect based on generic rules
        """
        is_valid = False

        regexpSectCheck = ['.*SOSB.*', '.*SOMB.*', '.*Single.*', '^SO$',
                           '.*MOSB.*', '.*MOMB.*', '.*Multi.*', '^MO$',
                           'check', 'checklog', 'check log']
        for _regex in regexpSectCheck:
            res = re.match(_regex, section_value, re.IGNORECASE)
            if res:
                is_valid = True
        return is_valid

    @staticmethod
    def rules_based_validate_section(section_value, rules):
        """
        This will validate PSect based on Rules class instance
        """
        is_valid = False

        if rules is None:
            raise ValueError('No contest rules provided !')
        for _nr in range(1, rules.contest_categories_nr+1):
            _regex = r'\s*(' + rules.contest_category(_nr)['regexp'] + r')\s*'
            res = re.match(_regex, section_value, re.IGNORECASE)
            if res:
                is_valid = True
        return is_valid

    @staticmethod
    def validate_date(date_value):
        """
        This will validate TDate based on generic rules
        """
        is_valid = False
        dates = date_value.split(';')
        if len(dates) != 2:
            return is_valid
        is_valid = True
        for _date in dates:
            try:
                datetime.datetime.strptime(_date, '%Y%m%d')
            except ValueError:
                is_valid = False
                break
        return is_valid

    def rules_based_validate_date(self, date_value, rules):
        """
        This will validate TDate based on Ruless class instance
        """
        is_valid = False

        if rules is None:
            raise ValueError('No contest rules provided !')
        _begin_date, _end_date = date_value.split(';')
        if _begin_date == rules.contest_begin_date and _end_date == rules.contest_end_date:
            is_valid = True
        return is_valid

    def dump_summary(self):
        """
        Based on the output format (text, html...) this will output a summary of the log
        """
        pass


class LogQso(object):
    """
    Keep a single QSO (in EDI format) and some info:
    qso line number, raw qso, is valid ? , error message if !valid, all qso fields
    """
    regexMinimalQsoCheck = '(?P<date>.*?);(?P<hour>.*?);(?P<call>.*?);(?P<mode>.*?);' \
                           '(?P<rst_sent>.*?);(?P<nr_sent>.*?);(?P<rst_recv>.*?);(?P<nr_recv>.*?);' \
                           '(?P<exchange_recv>.*?);(?P<wwl>.*?);(?P<points>.*?);' \
                           '(?P<new_exchange>.*?);(?P<new_wwl>.*?);(?P<new_dxcc>.*?);(?P<duplicate_qso>.*?)'
    regexMediumQsoCheck = r'^\d{6};\d{4};.*?;.?;\d{2,3}.?;\d{2,4};\d{2,3}.?;\d{2,4};.*?;' \
                          '[a-zA-Z]{2}\d{2}[a-zA-Z]{2};.*?;.*?;.*?;.*?;.*?'
    #                       date  time   id  m    rst       nr      rst       nr    .  qth  km  .   .   .   .

    qso_line_number = 0
    qso_line = None
    valid_qso = False
    error_message = None

    qsoFields = {'date': None,
                 'hour': None,
                 'call': None,
                 'mode': None,
                 'rst_sent': None,
                 'nr_sent': None,
                 'rst_recv': None,
                 'nr_recv': None,
                 'exchange_recv': None,
                 'wwl': None,
                 'points': None,
                 'new_exchange': None,
                 'new_wwl': None,
                 'new_dxcc': None,
                 'duplicate_qso': None,
                }

    def __init__(self, qso_line, qso_line_number, rules=None):
        self.qso_line = qso_line
        self.qso_line_number = qso_line_number
        self.error_message = self.regexp_qso_validator(qso_line) or None
        self.valid_qso = False if self.error_message else True

        if self.valid_qso:
            self.qso_parser()
            self.error_message = self.generic_qso_validator()
            if (self.error_message is None) and (rules is not None):
                self.error_message = self.rules_based_qso_validator(rules)
            self.valid_qso = False if self.error_message else True

    def qso_parser(self):
        """
        This should parse a qso based on log format
        """
        res = re.match(self.regexMinimalQsoCheck, self.qso_line)
        if res:
            for key in self.qsoFields:
                self.qsoFields[key] = res.group(key)

    @classmethod
    def regexp_qso_validator(cls, line):
        """
        This will validate the a line of qso from .edi log
        :param line:
        :return: None or error message
        """
        qso_min_line_length = 40

        if len(line) < qso_min_line_length:
            return 'QSO line is too short'
        res = re.match(cls.regexMinimalQsoCheck, line)<commit_msg>Added some TODO's for LogQso<commit_after>"""
Copyright 2016-2017 <NAME>

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import re
import datetime
from collections import namedtuple

ERR_FILE = 'file'
ERR_HEADER = 'header'
ERR_QSO = 'qso'

class Operator(object):
    """
    Keep operator callsign, info and logs path
    """
    callsign = None
    info = {}           # no idea what was this for :(
    logs = []           # list with Log() instances

    def __init__(self, callsign):
        self.callsign = callsign
        self.logs = []

    def add_log(self, path):
        self.logs.append(Log(path))


class Log(object):
    """
    Keep a single EDI log information:
    log path, log raw content, callsign, qth, band, section, qsos tuple (raw content)
    and a list with LogQso() instances

    errors format :
    {
        'file': [ (line or None, 'error: Cannot open file'), ... ],
        'header': [ (line or None, 'error: message'), ... ],
        'qso': [ (line, 'error: message'), ...],
    }
    """
    path = None
    rules = None
    use_as_checklog = False
    log_lines = None

    valid = None
    errors = {}

    callsign = None
    maidenhead_locator = None
    band = None
    section = None
    date = None

    qsos_tuple = namedtuple('qso_tuple', ['linenr', 'qso', 'valid', 'error'])
    qsos = []   # list with LogQso instances

    def __init__(self, path, rules=None, checklog=False):
        self.path = path
        self.rules = rules
        self.use_as_checklog = checklog

        self.validate_header()
        if not self.valid:
            return

        self.get_qsos()
        # TODO : validate individual QSO's
        # TODO ...

    def validate_header(self):
        """ Validate edi log header.
        If errors are found they will be written in self.errors dictionary
        :return: True or False
        """

        self.errors[ERR_FILE] = []
        self.errors[ERR_HEADER] = []
        self.errors[ERR_QSO] = []
        self.valid = False

        try:
            self.log_lines = self.read_file_content(self.path)
        except Exception as e:
            self.errors[ERR_FILE].append((None, 'Cannot read edi log'))
            return

        if self.log_lines is None:
            self.errors[ERR_FILE].append((None, 'Log is empty'))
            return

        # get & validate callsign
        _callsign, line_nr = self.get_field('PCall')
        if not _callsign:
            self.errors[ERR_HEADER].append((line_nr, 'PCall field is not present'))
        elif len(_callsign) > 1:
            self.errors[ERR_HEADER].append((line_nr, 'PCall field is present multiple times'))
        else:
            self.callsign = _callsign[0]

        # get & validate maidenhead locator
        _qthlocator, line_nr = self.get_field('PWWLo')
        if not _qthlocator:
            self.errors[ERR_HEADER].append((line_nr, 'PWWLo field is not present'))
        elif len(_qthlocator) > 1:
            self.errors[ERR_HEADER].append((line_nr, 'PWWLo field is present multiple times'))
        elif not self.validate_qth_locator(_qthlocator[0]):
            self.errors[ERR_HEADER].append((line_nr, 'PWWLo field value is not valid'))
        else:
            self.maidenhead_locator = _qthlocator[0]

        # get & validate band based on generic rules and by custom rules if provided (rules.contest_band['regexp'])
        _band, line_nr = self.get_field('PBand')
        if not _band:
            self.errors[ERR_HEADER].append((line_nr, 'PBand field is not present'))
        elif len(_band) > 1:
            self.errors[ERR_HEADER].append((line_nr, 'PBand field is present multiple times'))
        elif not self.rules and not self.validate_band(_band[0]):
            self.errors[ERR_HEADER].append((line_nr, 'PBand field value is not valid'))
        elif self.rules and not self.rules_based_validate_band(_band[0], self.rules):
            self.errors[ERR_HEADER].append((line_nr, 'PBand field value has an invalid value ({}). '
                                                  'Not as defined in contest rules'.format(_band[0])))
        else:
            self.band = _band[0]

        # get & validate PSect based on generic rules and by custom rules if provided (rules.contest_category['regexp']
        _section, line_nr = self.get_field('PSect')
        if not _section:
            self.errors[ERR_HEADER].append((line_nr, 'PSect field is not present'))
        elif len(_section) > 1:
            self.errors[ERR_HEADER].append((line_nr, 'PSect field is present multiple times'))
        elif not self.rules and not self.validate_section(_section[0]):
            self.errors[ERR_HEADER].append((line_nr, 'PSect field value is not valid'))
        elif self.rules and not self.rules_based_validate_section(_section[0], self.rules):
            self.errors[ERR_HEADER].append((line_nr, 'PSect field value has an invalid value ({}). '
                                          'Not as defined in contest rules'.format(_section[0])))
        else:
            self.section = _section[0]

        # get & validate TDate based on generic rules format and by custom rules if provided
        # (rules.contest_begin_date & rules.contest_end_date)
        _date, line_nr = self.get_field('TDate')
        if not _date:
            self.errors[ERR_HEADER].append((line_nr, 'TDate field is not present'))
        elif len(_date) > 1:
            self.errors[ERR_HEADER].append((line_nr, 'TDate field is present multiple times'))
        elif not self.validate_date(_date[0]):
            self.errors[ERR_HEADER].append((line_nr, 'TDate field value is not valid ({})'.format(_date[0])))
        elif self.rules and not self.rules_based_validate_date(_date[0], self.rules):
            self.errors[ERR_HEADER].append((line_nr, 'TDate field value has an invalid value ({}). '
                                                  'Not as defined in contest rules'.format(_date[0])))
        else:
            self.date = _date[0]

        if all((self.callsign, self.maidenhead_locator, self.band, self.section, self.date)):
            self.valid = True

    @staticmethod
    def read_file_content(path):
        try:
            with open(path, 'r') as _file:
                content = _file.readlines()
        except IOError:
            raise
        except Exception:
            raise
        return content

    def get_field(self, field):
        """
        Search log content for a field
        :param field: field name (PCall, TDate, ...)
        :return: tuple(value, line_number or None)
        """
        value = []
        line_nr = None
        _field = str(field).upper() + '='
        for (nr, line_content) in enumerate(self.log_lines):
            if line_content.upper().startswith(_field):
                value.append(line_content.split('=', 1)[1].strip())
                line_nr = nr+1
        return value, line_nr

    def get_qsos(self):
        """
        Will read the self.log_content and will return a list of LogQso
        """
        qso_record_start = "[QSORECORDS"
        qso_record_end = "[END;"
        qso_lines = []
        do_read_qso = False

        # read qso lines
        for (index, line) in enumerate(self.log_lines):
            if line.upper().startswith(qso_record_start):
                do_read_qso = True
                continue
            if line.upper().startswith(qso_record_end):
                do_read_qso = False
                continue
            if do_read_qso:
                qso_lines.append((index, line.strip()))

        # validate qso lines
        for qso in qso_lines:
            message = LogQso.regexp_qso_validator(qso[1])
            self.qsos.append(
                # self.qsos_tuple(linenr=qso[0], qso=qso[1], valid=False if message else True, error=message)
                LogQso(qso[1], qso[0])  # LogQso(qso_line, qso_line_number_in_log)
            )

    @staticmethod
    def validate_qth_locator(qth):
        regex_maidenhead = r'^\s*([a-rA-R]{2}\d{2}[a-xA-X]{2})\s*$'
        res = re.match(regex_maidenhead, qth, re.IGNORECASE)
        return True if res else False

    @staticmethod
    def get_band(band):
        """
        This will parse the 'PBand=' field value
        and return the proper band
        :param band: the content of 'PBand=' field
        :return: The detected band (144,432,1296) or None
        """

        regexp_band = {144: ['144.*', '145.*'],
                       432: ['430.*', '432.*', '435.*'],
                       1296: ['1296.*', '1[.,][23].*']}
        for _band in regexp_band:
            for regexp in regexp_band[_band]:
                res = re.match(regexp, band)
                if res:
                    return _band
        return None

    @staticmethod
    def validate_band(band_value):
        """
        This will validate PBand based on generic rules
        """
        is_valid = False

        regexp_band_check = ['144.*', '145.*',
                             '430.*', '432.*', '435.*',
                             '1296.*', '1[.,][23].*']
        for _regex in regexp_band_check:
            res = re.match(_regex, band_value)
            if res:
                is_valid = True
        return is_valid

    @staticmethod
    def rules_based_validate_band(band_value, rules):
        """
        This will validate PBand based on Rules class instance
        """
        is_valid = False

        if rules is None:
            raise ValueError('No contest rules provided !')
        for _nr in range(1, rules.contest_bands_nr+1):
            _regex = r'\s*(' + rules.contest_band(_nr)['regexp'] + r')\s*'
            res = re.match(_regex, band_value)
            if res:
                is_valid = True
        return is_valid

    @staticmethod
    def validate_section(section_value):
        """
        This will validate PSect based on generic rules
        """
        is_valid = False

        regexpSectCheck = ['.*SOSB.*', '.*SOMB.*', '.*Single.*', '^SO$',
                           '.*MOSB.*', '.*MOMB.*', '.*Multi.*', '^MO$',
                           'check', 'checklog', 'check log']
        for _regex in regexpSectCheck:
            res = re.match(_regex, section_value, re.IGNORECASE)
            if res:
                is_valid = True
        return is_valid

    @staticmethod
    def rules_based_validate_section(section_value, rules):
        """
        This will validate PSect based on Rules class instance
        """
        is_valid = False

        if rules is None:
            raise ValueError('No contest rules provided !')
        for _nr in range(1, rules.contest_categories_nr+1):
            _regex = r'\s*(' + rules.contest_category(_nr)['regexp'] + r')\s*'
            res = re.match(_regex, section_value, re.IGNORECASE)
            if res:
                is_valid = True
        return is_valid

    @staticmethod
    def validate_date(date_value):
        """
        This will validate TDate based on generic rules
        """
        is_valid = False
        dates = date_value.split(';')
        if len(dates) != 2:
            return is_valid
        is_valid = True
        for _date in dates:
            try:
                datetime.datetime.strptime(_date, '%Y%m%d')
            except ValueError:
                is_valid = False
                break
        return is_valid

    def rules_based_validate_date(self, date_value, rules):
        """
        This will validate TDate based on Ruless class instance
        """
        is_valid = False

        if rules is None:
            raise ValueError('No contest rules provided !')
        _begin_date, _end_date = date_value.split(';')
        if _begin_date == rules.contest_begin_date and _end_date == rules.contest_end_date:
            is_valid = True
        return is_valid

    def dump_summary(self):
        """
        Based on the output format (text, html...) this will output a summary of the log
        """
        pass


class LogQso(object):
    """
    Keep a single QSO (in EDI format) and some info:
    qso line number, raw qso, is valid ? , error message if !valid, all qso fields
    """
    regexMinimalQsoCheck = '(?P<date>.*?);(?P<hour>.*?);(?P<call>.*?);(?P<mode>.*?);' \
                           '(?P<rst_sent>.*?);(?P<nr_sent>.*?);(?P<rst_recv>.*?);(?P<nr_recv>.*?);' \
                           '(?P<exchange_recv>.*?);(?P<wwl>.*?);(?P<points>.*?);' \
                           '(?P<new_exchange>.*?);(?P<new_wwl>.*?);(?P<new_dxcc>.*?);(?P<duplicate_qso>.*?)'
    regexMediumQsoCheck = r'^\d{6};\d{4};.*?;.?;\d{2,3}.?;\d{2,4};\d{2,3}.?;\d{2,4};.*?;' \
                          '[a-zA-Z]{2}\d{2}[a-zA-Z]{2};.*?;.*?;.*?;.*?;.*?'
    #                       date  time   id  m    rst       nr      rst       nr    .  qth  km  .   .   .   .

    qso_line_number = 0
    qso_line = None
    valid_qso = False
    error_message = None

    qsoFields = {'date': None,
                 'hour': None,
                 'call': None,
                 'mode': None,
                 'rst_sent': None,
                 'nr_sent': None,
                 'rst_recv': None,
                 'nr_recv': None,
                 'exchange_recv': None,
                 'wwl': None,
                 'points': None,
                 'new_exchange': None,
                 'new_wwl': None,
                 'new_dxcc': None,
                 'duplicate_qso': None,
                }

    def __init__(self, qso_line, qso_line_number, rules=None):
        self.qso_line = qso_line
        self.qso_line_number = qso_line_number

        # TODO : call qso validator and add errors to self.error like it's in Log class

        self.error_message = self.regexp_qso_validator(qso_line) or None
        self.valid_qso = False if self.error_message else True

        if self.valid_qso:
            self.qso_parser()
            self.error_message = self.generic_qso_validator()
            if (self.error_message is None) and (rules is not None):
                self.error_message = self.rules_based_qso_validator(rules)
            self.valid_qso = False if self.error_message else True

    def qso_parser(self):
        """
        This should parse a qso based on log format
        """
        res = re.match(self.regexMinimalQsoCheck, self.qso_line)
        if res:
            for key in self.qsoFields:
                self.qsoFields[key] = res.group(key)

    @classmethod
    def regexp_qso_validator(cls, line):
        """
        This will validate the a line of qso from .edi log
        :param line:
        :return: None or error message
        """
        qso_min_line_length = 40

        if len(line) < qso_min_line_length:
            return 'QSO line is too short'
        res = re.match(cls.regexMinimalQsoCheck, line)
{'fraction_non_alphanumeric': 0.11633418921587992, 'fraction_numerical': 0.008492988346829943, 'mean_word_length': 1.9323359073359074, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5332201', 'n_tokens_mistral': 9321, 'n_tokens_neox': 8987, 'n_words': 2406}