<commit_before>#! /usr/bin/env python3
"""Transaction classifier Implementation."""

from naiveBayesClassifier import tokenizer
from naiveBayesClassifier.trainer import Trainer
from naiveBayesClassifier.classifier import Classifier as BayesClassifier
from yapsy.IPlugin import IPlugin
from itertools import groupby

from operator import itemgetter
import re

from pyledgertools.functions import amount_group, GCD

DOLLAR_REGEX = '([\$A-Z]+)?\s?([\-0-9]+.[0-9]{2,2})?'

# Allocation RegEx
ALLOC_REGEX = '\s+([A-Za-z0-9:_-]* ?[A-Za-z0-9:_-]*)\s*' + DOLLAR_REGEX
TRANS_REGEX = '^\d{4,4}[/-]{1,1}\d{2,2}[/-]{1,1}\d{2,2}\s+[!\*]?\s?(?P<payee>[&#\w\s]+)'


def train_journal(journal_string):
    """Generate training data from ledger entries."""
    blocks = [x.split('\n') for x in journal_string.decode('utf-8').split('\n\n')]
    training_data = []

    for tran in blocks:
        tran = [x.split('=')[0] for x in tran if x != '' and not re.match('\s*;',x)]
        if len(tran) > 0:
            result = re.search(TRANS_REGEX, tran[0])
            if result:
                payee = result.group('payee')

            result = re.findall(ALLOC_REGEX, ' '.join(tran[1:]))
            if len(result) > 1:
                alloc_list = list(result[1])
                alloc_list[2] = float(result[0][2])<commit_msg>Adjust naive bayes trainer regex.<commit_after>#! /usr/bin/env python3
"""Transaction classifier Implementation."""

from naiveBayesClassifier import tokenizer
from naiveBayesClassifier.trainer import Trainer
from naiveBayesClassifier.classifier import Classifier as BayesClassifier
from yapsy.IPlugin import IPlugin
from itertools import groupby

from operator import itemgetter
import re

from pyledgertools.functions import amount_group, GCD

DOLLAR_REGEX = '([\$A-Z]+)?\s?([\-0-9]+.[0-9]{2,2})?'

# Allocation RegEx
ALLOC_REGEX = '\s+([A-Za-z0-9:_\- ]*[A-Za-z0-9:_\-]+)\s*' + DOLLAR_REGEX
# ALLOC_REGEX = '\s+((?!.*  )[a-zA-Z0-9:_\- ]*)' + DOLLAR_REGEX
TRANS_REGEX = '^\d{4,4}[/-]{1,1}\d{2,2}[/-]{1,1}\d{2,2}\s+[!\*]?\s?(?P<payee>[&#\w\s]+)'


def train_journal(journal_string):
    """Generate training data from ledger entries."""
    blocks = [x.split('\n') for x in journal_string.decode('utf-8').split('\n\n')]
    training_data = []

    for tran in blocks:
        tran = [x.split('=')[0] for x in tran if x != '' and not re.match('\s*;',x)]
        if len(tran) > 0:
            result = re.search(TRANS_REGEX, tran[0])
            if result:
                payee = result.group('payee')

            result = re.findall(ALLOC_REGEX, ' '.join(tran[1:]))
            if len(result) > 1:
                alloc_list = list(result[1])
                alloc_list[2] = float(result[0][2])
{'fraction_non_alphanumeric': 0.18632162661737522, 'fraction_numerical': 0.02365988909426987, 'mean_word_length': 3.2413793103448274, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3824110', 'n_tokens_mistral': 1036, 'n_tokens_neox': 1004, 'n_words': 205}