<commit_before>#!/usr/bin/python

import os
import sys
from datetime import datetime
from datetime import date

class mt940e_account_info():
    def __init__(self, data):
        self.data = data
        self.ndata = len(data)
        self.offset = self.data.find('?')

    def __iter__(self):
        return self

    def next(self):
        if self.offset == -1:
            raise StopIteration

        next = self.data.find('?', self.offset+1)
        code = self.data[self.offset+1:self.offset+3]
        if next != -1:
            info = self.data[self.offset+3:next]
        else:
            info = self.data[self.offset+3:]
        self.offset = next
        return (code, info)

class mt940e_parser(object):
    _tagsep = ':'

    _code_map = {
        '20': 'type',
        '25': 'account_nb',
        '28': 'name',
    }

    def t_get_date(self, raw):
#        print(">>> parse date: _%s_" % (raw))
        return date(year=2000+int(raw[0:2]), month=int(raw[2:4]), day=int(raw[4:6]))

    def t_get_sign(self, raw):
#        print(">>> parse sign: _%s_" % (raw))
        return {
            'C': 1,
            'D': -1,
            #RC: ??
            #RD: ??
        }[raw.strip()]

    def t_get_amount(self, raw):
#        print(">>> parse amount: _%s_" % (raw))
        return float(raw.replace(',','.'))

    def t_get_swift_code(self, raw):
#        print(">>> parse swift code: _%s_" % (raw))
        if raw[0] != 'N':
            raise Exception("Invalid Swift Code, should start with N, %s" % (raw))
        return raw[1:4]

    def code_simple_store(self, code, data):
        self.data[self._code_map[code]] = data
        return True

    def p_opening_balance(self, code, data):
        sign = self.t_get_sign(data[0])
        d = self.t_get_date(data[1:7])
        currency = data[7:10]
        abs_amount = self.t_get_amount(data[10:])
        self.data.update({
            'date_start': d,
            'currency': currency,
            'balance_start': abs_amount * sign,
        })
#        print("Sign: %s" % (sign))
#        print("Data: %s" % (d))
#        print("Currency: %s" % (currency))
#        print("Abs Amount: %s" % (abs_amount))

    def p_ending_balance(self, code, data):
        sign = self.t_get_sign(data[0])
        d = self.t_get_date(data[1:7])
        currency = data[7:10]
        abs_amount = self.t_get_amount(data[10:])
#        print("Sign: %s" % (sign))
#        print("Data: %s" % (d))
#        print("Currency: %s" % (currency))
#        print("Abs Amount: %s" % (abs_amount))
        if self.data.get('currency') != currency:
            raise Exception("Invalid bank statement, opening and ending currency differs")
        self.data.update({
            'date_end': d,
            'balance_end': abs_amount * sign,
        })

    def p_transfert_statement(self, code, data):
        d = self.t_get_date(data[0:6]) # = 6
        entry_d = data[6:10] # = 4
        sign = self.t_get_sign(data[10:12]) # = 2
        amount = self.t_get_amount(data[12:27])
        swift_code = self.t_get_swift_code(data[27:31])
        owner_reference = data[31:47]
#        print("### SWIFT CODE: %s, OWNER REF: %s" % (swift_code, owner_reference))
        bank_reference = data[47:]
#        if not bank_reference.lstrip().startswith('//'):
#            raise Exception("Invalid bank reference _%s_" % (bank_reference))

        new_line = {
            'date': d,
            'entry_date': entry_d,
            'amount': sign * amount,
            'swift_code': swift_code,
            'owner_reference': owner_reference,
            'bank_reference': bank_reference,
        }
#        print("NEW LINE: %s" % (new_line))
        self.data['lines'].append(new_line)

    def p_transfert_accountinfo(self, code, data):
        gvc = int(data[0:2])
        _infocode = {
            '00': [27, 'reference', 'Trans. Descr. / Payment. Origin'],
            '20': [27, 'details', 'Debit/Credit AccountBankInfo with leading Zero'],
            '21': [10, 'details', ''],
            '22': [27, 'details', ''],
            #'23': [27, 'client_info', ''],
            #'24': [27, 'beneficiary', ''],
            #'25': [27, 'beneficiary', ''],
            '26': [27, 'original_amount', ''],
            '27': [27, 'charges', ''],
            '28': [27, 'exchange_rate', ''],
            #'29': [27, 'exchange_rate', ''],
            #'30': [8, 'bank_code', ''],
            '31': [24, 'communication', ''],
            '32': [27, 'beneficiary', ''],
            '33': [27, 'beneficiary', ''],
            '38': [31, 'iban', ''],
        }
        cline = data[2:]
        cdata = {
            'reference': '',
#            'trans_num': '',
            'details': '',
            'amount_currency': '',
            'communication': '',
            'charges': '',
            'exchange_rate': '',
#            'account_info': '',
#            'client_info': '',
            'beneficiary': '',
#            'bank_code': '',
            'original_amount': '',
#            'beneficiary_name': '',
            'iban': '',<commit_msg>[FIX] fix syntax for MT940E file format<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

import os
import sys
from datetime import datetime
from datetime import date

class mt940e_account_info():
    def __init__(self, data):
        self.data = data.replace('\n','').replace('\r','')
        self.ndata = len(data)
        self.offset = self.data.find('?')

    def __iter__(self):
        return self

    def next(self):
        if self.offset == -1:
            raise StopIteration

        next = self.data.find('?', self.offset+1)
        code = self.data[self.offset+1:self.offset+3]
        if next != -1:
            info = self.data[self.offset+3:next]
        else:
            info = self.data[self.offset+3:]
        self.offset = next
        return (code, info)

class mt940e_parser(object):
    _tagsep = ':'

    _code_map = {
        '20': 'type',
        '25': 'account_nb',
        '28': 'name',
    }

    def t_get_date(self, raw):
#        print(">>> parse date: _%s_" % (raw))
        return date(year=2000+int(raw[0:2]), month=int(raw[2:4]), day=int(raw[4:6]))

    def t_get_sign(self, raw):
#        print(">>> parse sign: _%s_" % (raw))
        return {
            'C': 1,
            'D': -1,
            #RC: ??
            #RD: ??
        }[raw.strip()]

    def t_get_amount(self, raw):
#        print(">>> parse amount: _%s_" % (raw))
        return float(raw.replace(',','.'))

    def t_get_swift_code(self, raw):
#        print(">>> parse swift code: _%s_" % (raw))
        if raw[0] != 'N':
            raise Exception("Invalid Swift Code, should start with N, %s" % (raw))
        return raw[1:4]

    def code_simple_store(self, code, data):
        self.data[self._code_map[code]] = data
        return True

    def p_opening_balance(self, code, data):
        sign = self.t_get_sign(data[0])
        d = self.t_get_date(data[1:7])
        currency = data[7:10]
        abs_amount = self.t_get_amount(data[10:])
        self.data.update({
            'date_start': d,
            'currency': currency,
            'balance_start': abs_amount * sign,
        })
#        print("Sign: %s" % (sign))
#        print("Data: %s" % (d))
#        print("Currency: %s" % (currency))
#        print("Abs Amount: %s" % (abs_amount))

    def p_ending_balance(self, code, data):
        sign = self.t_get_sign(data[0])
        d = self.t_get_date(data[1:7])
        currency = data[7:10]
        abs_amount = self.t_get_amount(data[10:])
#        print("Sign: %s" % (sign))
#        print("Data: %s" % (d))
#        print("Currency: %s" % (currency))
#        print("Abs Amount: %s" % (abs_amount))
        if self.data.get('currency') != currency:
            raise Exception("Invalid bank statement, opening and ending currency differs")
        self.data.update({
            'date_end': d,
            'balance_end': abs_amount * sign,
        })

    def p_transfert_statement(self, code, data):
        d = self.t_get_date(data[0:6]) # = 6
        entry_d = data[6:10] # = 4
        sign = self.t_get_sign(data[10:12]) # = 2
        amount = self.t_get_amount(data[12:27])
        swift_code = self.t_get_swift_code(data[27:31])
        owner_reference = data[31:47]
#        print("### SWIFT CODE: %s, OWNER REF: %s" % (swift_code, owner_reference))
        bank_reference = data[47:]
#        if not bank_reference.lstrip().startswith('//'):
#            raise Exception("Invalid bank reference _%s_" % (bank_reference))

        new_line = {
            'date': d,
            'entry_date': entry_d,
            'amount': sign * amount,
            'swift_code': swift_code,
            'owner_reference': owner_reference,
            'bank_reference': bank_reference,
        }
#        print("NEW LINE: %s" % (new_line))
        self.data['lines'].append(new_line)

    def p_transfert_accountinfo(self, code, data):
        gvc = int(data[0:2])
        _infocode = {
            '00': [27, 'reference', 'Trans. Descr. / Payment. Origin'],
            '20': [27, 'details', 'Debit/Credit AccountBankInfo with leading Zero'],
            '21': [10, 'details', ''],
            '22': [27, 'details', ''],
            '23': [27, 'details', ''],
            '24': [27, 'details', ''],
            '25': [27, 'details', ''],
            #'23': [27, 'client_info', ''],
            #'24': [27, 'beneficiary', ''],
            #'25': [27, 'beneficiary', ''],
            '26': [27, 'original_amount', ''],
            '27': [27, 'charges', ''],
            '28': [27, 'exchange_rate', ''],
            #'29': [27, 'exchange_rate', ''],
            #'30': [8, 'bank_code', ''],
            '31': [24, 'communication', ''], # numéro du compte bénénficiaire
            '32': [27, 'beneficiary', ''], # nom donneur d'ordre
            '33': [27, 'beneficiary', ''], # nom donneur d'ordre
            '38': [31, 'iban', ''],
        }
        cline = data[2:]
        cdata = {
            'reference': '',
#            'trans_num': '',
            'details': '',
            'amount_currency': '',
            'communication': '',
            'charges': '',
            'exchange_rate': '',
#            'account_info': '',
#            'client_info': '',
            'beneficiary': '',
#            'bank_code': '',
            'original_amount': '',
#            'beneficiary_name': '',
            'iban': '',
{'fraction_non_alphanumeric': 0.18019573978123202, 'fraction_numerical': 0.028785261945883708, 'mean_word_length': 1.9712086659064993, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2361122', 'n_tokens_mistral': 3635, 'n_tokens_neox': 3408, 'n_words': 723}