<commit_before># -*- encoding: utf-8 -*-
##############################################################################
#
#    Copyright (C) 2009 EduSense BV (<http://www.edusense.nl>)
#                  2011 Therp BV (<http://therp.nl>)
#    All Rights Reserved
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

'''
This parser follows the Dutch Banking Tools specifications which are
empirically recreated in this module.

Dutch Banking Tools uses the concept of 'Afschrift' or Bank Statement.
Every transaction is bound to a Bank Statement. As such, this module generates
Bank Statements along with Bank Transactions.
'''
from account_banking.parsers import models
from account_banking.parsers.convert import str2date
from account_banking.sepa import postalcode
from tools.translate import _
from osv import osv

import re
import csv

__all__ = ['parser']

bt = models.mem_bank_transaction

class transaction_message(object):
    '''
    A auxiliary class to validate and coerce read values
    '''
    attrnames = [
        'local_account', 'local_currency', 'date', 'u1', 'u2', 'date2',
        'transferred_amount', 'blob',
    ]

    def __init__(self, values, subno):
        '''
        Initialize own dict with attributes and coerce values to right type
        '''
        if len(self.attrnames) != len(values):
            raise ValueError, \
                    _('Invalid transaction line: expected %d columns, found '
                      '%d') % (len(self.attrnames), len(values))
        ''' Strip all values except the blob '''
        for (key, val) in zip(self.attrnames, values):
            self.__dict__[key] = key == 'blob' and val or val.strip()
        # for lack of a standardized locale function to parse amounts
        self.local_account = self.local_account.zfill(10)
        self.transferred_amount = float(
            self.transferred_amount.replace(',', '.'))
        self.execution_date = str2date(self.date, '%Y%m%d')
        self.effective_date = str2date(self.date, '%Y%m%d')
        # Set statement_id based on week number
        self.statement_id = self.effective_date.strftime('%Yw%W')
        self.id = str(subno).zfill(4)

class transaction(models.mem_bank_transaction):
    '''
    Implementation of transaction communication class for account_banking.
    '''
    attrnames = ['local_account', 'local_currency', 'transferred_amount',
                 'blob', 'execution_date', 'effective_date', 'id',
                ]

    type_map = {
        # retrieved from online help in the Triodos banking application
        'BEA': bt.PAYMENT_TERMINAL, # Pin
        'GEA': bt.BANK_TERMINAL, # ATM
        'COSTS': bt.BANK_COSTS,
        'BANK': bt.ORDER,
        'GIRO': bt.ORDER,
        'INTL': bt.ORDER, # international order
        'UNKN': bt.ORDER, # everything else
        'SEPA': bt.ORDER,
    }

    def __init__(self, line, *args, **kwargs):
        '''
        Initialize own dict with read values.
        '''
        super(transaction, self).__init__(*args, **kwargs)
        # Copy attributes from auxiliary class to self.
        for attr in self.attrnames:
            setattr(self, attr, getattr(line, attr))
        # Initialize other attributes
        self.transfer_type = 'UNKN'
        self.remote_account = ''
        self.remote_owner = ''
        self.reference = ''
        self.message = ''
        # Decompose structured messages
        self.parse_message()

    def is_valid(self):
        if not self.error_message:
            if not self.transferred_amount:
                self.error_message = "No transferred amount"
            elif not self.execution_date:
                self.error_message = "No execution date"
            elif not self.remote_account and self.transfer_type not in [
                'BEA', 'GEA', 'COSTS', 'UNKN',
                ]:
                self.error_message = _('No remote account for transaction type '
                                       '%s') % self.transfer_type
        if self.error_message:
            raise osv.except_osv(_('Error !'), _(self.error_message))
        return not self.error_message

    def parse_message(self):
        '''
        Parse structured message parts into appropriate attributes
        '''
        def split_blob(line):
            # here we split up the blob, which the last field in a tab
            # separated statement line the blob is a *space separated* fixed
            # field format with field length 32. Empty fields are ignored
            col = 0
            size = 33
            res = []
            while(len(line) > col * size):
                separation = (col + 1) * size - 1
                if line[col * size : separation].strip():
                    part = line[col * size : separation]
                    # If the separation character is not a space, add it anyway
                    # presumably for sepa feedback strings only
                    if (len(line) > separation
                        and line[separation] != ' '):
                        part += line[separation]
                    res.append(part)
                col += 1
            return res

        def get_sepa_dict(field):
            """
            Parses a subset of SEPA feedback strings as occur
            in this non-SEPA csv format.

            The string consists of slash separated KEY/VALUE pairs,
            but the slash is allowed to and known to occur in VALUE as well!
            """
            items = field[1:].split('/')  # skip leading slash
            assert len(items) > 1, (
                _('unable to parse SEPA string: %s - %s') %
                (field, _('too few items')))
            known_keys = [
                'TRTP', 'IBAN', 'BIC', 'NAME', 'RTRN', 'EREF', 'SWOC', 'REMI',
                'ADDR', 'BIC', 'CPRP', 'CREF', 'CSID', 'ISDT', 'MARF', 'NRTX',
                'NRTXR', 'PREF', 'PURP', 'REFOB', 'RREF', 'RTYP', 'SVCL',
                'SWOD'
            ]
            # Xtended keys have an extra part, separated from the first part
            # with a double slash. For example: 'ORIG//CSID'.
            # The double slash will make that the first part in items is
            # followed by an empty string item, and then the other part of the
            # key.
            extended_keys = [
                'BENM', 'ORDP', 'ORIG', 'ULTD', 'ULTB'
            ]
            known_keys.extend(extended_keys)
            assert items[0] in known_keys, (
                _('unable to parse SEPA string: %s - %s') %
                (field, _('First key unknown %s') % items[0]))
            sepa_dict = {}
            item_index = 0
            items_len = len(items)
            while item_index < items_len:
                sepa_key = items[item_index]
                sepa_values = []
                if  sepa_key in extended_keys:
                    item_index += 2
                    assert item_index < items_len, (
                        _('SEPA key %s missing extention') % sepa_key)
                    # For the moment no test on validity of extension
                    sepa_key = '//'.join([sepa_key, items[item_index]])
                item_index += 1
                while (item_index < items_len
                  and  items[item_index] not in known_keys):
                    sepa_values.append(items[item_index].strip())
                    item_index += 1
                sepa_dict[sepa_key] = '/'.join(sepa_values)
            return sepa_dict

        def parse_type(field):
            # here we process the first field, which identifies the statement type
            # and in case of certain types contains additional information
            transfer_type = 'UNKN'
            remote_account = False
            remote_owner = False
            if field.startswith('/TRTP/'):
                transfer_type = 'SEPA'
            elif field.startswith('GIRO '):
                transfer_type = 'GIRO'
                # field has markup 'GIRO ACCOUNT OWNER'<commit_msg>[FIX] Updated parsing of abnamro sepa string to handle even more unlikely cases.<commit_after># -*- encoding: utf-8 -*-
##############################################################################
#
#    Copyright (C) 2009 EduSense BV (<http://www.edusense.nl>)
#                  2011 Therp BV (<http://therp.nl>)
#    All Rights Reserved
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

'''
This parser follows the Dutch Banking Tools specifications which are
empirically recreated in this module.

Dutch Banking Tools uses the concept of 'Afschrift' or Bank Statement.
Every transaction is bound to a Bank Statement. As such, this module generates
Bank Statements along with Bank Transactions.
'''
from account_banking.parsers import models
from account_banking.parsers.convert import str2date
from account_banking.sepa import postalcode
from tools.translate import _
from osv import osv

import re
import csv

__all__ = ['parser']

bt = models.mem_bank_transaction

class transaction_message(object):
    '''
    A auxiliary class to validate and coerce read values
    '''
    attrnames = [
        'local_account', 'local_currency', 'date', 'u1', 'u2', 'date2',
        'transferred_amount', 'blob',
    ]

    def __init__(self, values, subno):
        '''
        Initialize own dict with attributes and coerce values to right type
        '''
        if len(self.attrnames) != len(values):
            raise ValueError, \
                    _('Invalid transaction line: expected %d columns, found '
                      '%d') % (len(self.attrnames), len(values))
        ''' Strip all values except the blob '''
        for (key, val) in zip(self.attrnames, values):
            self.__dict__[key] = key == 'blob' and val or val.strip()
        # for lack of a standardized locale function to parse amounts
        self.local_account = self.local_account.zfill(10)
        self.transferred_amount = float(
            self.transferred_amount.replace(',', '.'))
        self.execution_date = str2date(self.date, '%Y%m%d')
        self.effective_date = str2date(self.date, '%Y%m%d')
        # Set statement_id based on week number
        self.statement_id = self.effective_date.strftime('%Yw%W')
        self.id = str(subno).zfill(4)

class transaction(models.mem_bank_transaction):
    '''
    Implementation of transaction communication class for account_banking.
    '''
    attrnames = ['local_account', 'local_currency', 'transferred_amount',
                 'blob', 'execution_date', 'effective_date', 'id',
                ]

    type_map = {
        # retrieved from online help in the Triodos banking application
        'BEA': bt.PAYMENT_TERMINAL, # Pin
        'GEA': bt.BANK_TERMINAL, # ATM
        'COSTS': bt.BANK_COSTS,
        'BANK': bt.ORDER,
        'GIRO': bt.ORDER,
        'INTL': bt.ORDER, # international order
        'UNKN': bt.ORDER, # everything else
        'SEPA': bt.ORDER,
    }

    def __init__(self, line, *args, **kwargs):
        '''
        Initialize own dict with read values.
        '''
        super(transaction, self).__init__(*args, **kwargs)
        # Copy attributes from auxiliary class to self.
        for attr in self.attrnames:
            setattr(self, attr, getattr(line, attr))
        # Initialize other attributes
        self.transfer_type = 'UNKN'
        self.remote_account = ''
        self.remote_owner = ''
        self.reference = ''
        self.message = ''
        # Decompose structured messages
        self.parse_message()

    def is_valid(self):
        if not self.error_message:
            if not self.transferred_amount:
                self.error_message = "No transferred amount"
            elif not self.execution_date:
                self.error_message = "No execution date"
            elif not self.remote_account and self.transfer_type not in [
                'BEA', 'GEA', 'COSTS', 'UNKN',
                ]:
                self.error_message = _('No remote account for transaction type '
                                       '%s') % self.transfer_type
        if self.error_message:
            raise osv.except_osv(_('Error !'), _(self.error_message))
        return not self.error_message

    def parse_message(self):
        '''
        Parse structured message parts into appropriate attributes
        '''
        def split_blob(line):
            # here we split up the blob, which the last field in a tab
            # separated statement line the blob is a *space separated* fixed
            # field format with field length 32. Empty fields are ignored
            col = 0
            size = 33
            res = []
            while(len(line) > col * size):
                separation = (col + 1) * size - 1
                if line[col * size : separation].strip():
                    part = line[col * size : separation]
                    # If the separation character is not a space, add it anyway
                    # presumably for sepa feedback strings only
                    if (len(line) > separation
                        and line[separation] != ' '):
                        part += line[separation]
                    res.append(part)
                col += 1
            return res

        def get_sepa_dict(field):
            """
            Parses a subset of SEPA feedback strings as occur
            in this non-SEPA csv format.

            The string consists of slash separated KEY/VALUE pairs,
            but the slash is allowed to and known to occur in VALUE as well!
            """
            def _sepa_message(field, reason):
                return _(
                    'unable to parse SEPA string: %s - %s' % (field, reason))

            def _get_next_key(items, start):
                '''Find next key, starting from start, returns the key found,
                the start position in the array and the end position + 1'''
                known_keys = [
                    'TRTP', 'IBAN', 'BIC', 'NAME', 'RTRN', 'EREF', 'SWOC',
                    'REMI', 'ADDR', 'CPRP', 'CREF', 'CSID', 'ISDT', 'MARF',
                    'NRTX', 'NRTXR', 'PREF', 'PURP', 'REFOB', 'RREF', 'RTYP',
                    'SVCL', 'SWOD', 'BENM//ID', 'ORDP//ID', 'ORDP//RID',
                    'ORIG//CSID', 'ORIG//MARF', 'ULTD//NAME', 'ULTD//ID',
                    'ULTB//NAME', 'ULTB//ID'
                ]
                items_len = len(items)
                si = start
                # Search until start after end of items
                while si < items_len:
                    ei = si + 1
                    while ei < items_len:
                        key = '/'.join(items[si:ei])
                        if  key in known_keys:
                            return (key, si, ei)
                        ei += 1
                    si += 1
                return False

            items = field[1:].split('/')
            assert len(items) > 1, _sepa_message(field, _('too few items'))
            sepa_dict = {}
            item_index = 0
            items_len = len(items)
            key_info = _get_next_key(items, item_index)
            assert key_info, _sepa_message(
                field, _('no key found for start %d') % item_index)
            assert key_info[1] == 0, _sepa_message(
                field, _('invalid data found before key %s') % key_info[0])
            while key_info:
                sepa_key = key_info[0]
                item_index = key_info[2]
                # Find where next key - if any - starts
                key_info = _get_next_key(items, item_index)
                ve = (key_info and key_info[1]) or items_len
                sepa_value = (
                    (ve > item_index) and '/'.join(items[item_index:ve])) or ''
                sepa_dict[sepa_key] = sepa_value
            return sepa_dict

        def parse_type(field):
            # here we process the first field, which identifies the statement type
            # and in case of certain types contains additional information
            transfer_type = 'UNKN'
            remote_account = False
            remote_owner = False
            if field.startswith('/TRTP/'):
                transfer_type = 'SEPA'
            elif field.startswith('GIRO '):
                transfer_type = 'GIRO'
                # field has markup 'GIRO ACCOUNT OWNER'
{'fraction_non_alphanumeric': 0.11283673123623426, 'fraction_numerical': 0.004235613034393178, 'mean_word_length': 1.8948831126369134, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '393899', 'n_tokens_mistral': 4878, 'n_tokens_neox': 4650, 'n_words': 1554}