<commit_before># -*- coding: utf-8 -*-

# Copyright(C) 2010-2011 <NAME>
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.


from decimal import Decimal
import re
from datetime import date
from weboob.capabilities.bank import Account
from .base import CragrBasePage
from .tokenextractor import TokenExtractor
from weboob.capabilities.bank import Transaction


class AccountsList(CragrBasePage):
    """
        Unlike most pages used with the Browser class, this class represents
        several pages, notably accounts list, history and transfer. This is due
        to the Credit Agricole not having a clear pattern to identify a page
        based on its URL.
    """

    def is_accounts_list(self):
        """
            Returns True if the current page appears to be the page dedicated to
            list the accounts.
        """
        # we check for the presence of a "mes comptes titres" link_id
        link = self.document.xpath('/html/body//a[contains(text(), "comptes titres")]')
        return bool(link)

    def is_account_page(self):
        """
            Returns True if the current page appears to be a page dedicated to list
            the history of a specific account.
        """
        # tested on CA Lorraine, Paris, Toulouse
        title_spans = self.document.xpath('/html/body//div[@class="dv"]/span')
        for title_span in title_spans:
            title_text = title_span.text_content().strip().replace("\n", '')
            if (re.match('.*Compte.*n.*[0-9]+', title_text, flags=re.IGNORECASE)):
                return True
        return False

    def is_transfer_page(self):
        """
            Returns True if the current page appears to be the page dedicated to
            order transfers between accounts.
        """
        source_account_select_field = self.document.xpath('/html/body//form//select[@name="numCompteEmetteur"]')
        target_account_select_field  = self.document.xpath('/html/body//form//select[@name="numCompteBeneficiaire"]')
        return bool(source_account_select_field) and bool(target_account_select_field)

    def get_list(self):
        """
            Returns the list of available bank accounts
        """
        l = []
        for div in self.document.getiterator('div'):
            if div.attrib.get('class', '') in ('dv', 'headline') and div.getchildren()[0].tag in ('a', 'br'):
                self.logger.debug("Analyzing div %s" % div)
                # Step 1: extract text tokens
                tokens = []
                required_tokens = {}
                optional_tokens = {}
                token_extractor = TokenExtractor()
                for token in token_extractor.extract_tokens(div):
                    self.logger.debug('Extracted text token: "%s"' % token)
                    tokens.append(token)
                # Step 2: analyse tokens
                for token in tokens:
                    if self.look_like_account_number(token):
                        required_tokens['account_number'] = token
                    elif self.look_like_amount(token):
                        required_tokens['account_amount'] = token
                    elif self.look_like_account_name(token):
                        required_tokens['account_name'] = token
                    elif self.look_like_account_owner(token):
                        optional_tokens['account_owner'] = token
                # Step 3: create account objects
                if len(required_tokens) >= 3:
                    account = Account()
                    account.label = required_tokens['account_name']
                    account.id = required_tokens['account_number']
                    account.balance = self.clean_amount(required_tokens['account_amount'])
                    # we found almost all required information to create an account object
                    self.logger.debug('Found account %s with number %s and balance = %.2f' % (account.label, account.id, account.balance))
                    # we may have found the owner name too
                    if optional_tokens.get('account_owner') is not None:
                        # well, we could add it to the label, but is this really required?
                        self.logger.debug('  the owner appears to be %s' % optional_tokens['account_owner'])
                    # we simply lack the link to the account history... which remains optional
                    first_link = div.find('a')
                    if first_link is not None:
                        account._link_id = first_link.get('href')
                        self.logger.debug('  the history link appears to be %s' % account._link_id)
                    l.append(account)
        return l

    def get_history(self, start_index=0, start_offset=0):
        """
            Returns the history of a specific account. Note that this function
            expects the current page to be the one dedicated to this history.
            start_index is the id used for the first created operation.
            start_offset allows ignoring the `n' first Transactions on the page.
        """
        # tested on CA Lorraine, Paris, Toulouse
        # avoir parsing the page as an account-dedicated page if it is not the case
        if not self.is_account_page():
            return

        # Step 1: extract text tokens
        tokens = []
        token_extractor = TokenExtractor()
        for div in self.document.getiterator('div'):
            if div.attrib.get('class', '') in ('dv'):
                self.logger.debug("Analyzing div %s" % div)
                for token in token_extractor.extract_tokens(div):
                    self.logger.debug('Extracted text token: "%s"' % token)
                    tokens.append(token)

        # Step 2: convert tokens into operations
        # Notes:
        # * the code below expects pieces of information to be in the date-label-amount order;
        #   could we achieve a heuristic smart enough to guess this order?
        # * unlike the former code, we parse every operation
        operations = []
        current_operation = {}
        for token in tokens:
            self.logger.debug('Analyzing token: "%s"' % token)
            date_analysis = self.look_like_date_only(token)
            if date_analysis:
                current_operation = {}
                current_operation['date'] = date_analysis.groups()[0]
            else:
                date_desc_analysis = self.look_like_date_and_description(token)
                if date_desc_analysis:
                    current_operation = {}
                    current_operation['date'] = date_desc_analysis.groups()[0]
                    current_operation['label'] = date_desc_analysis.groups()[1]
                elif self.look_like_amount(token):
                    # we consider the amount is the last information we get for an operation
                    current_operation['amount'] = self.clean_amount(token)
                    if current_operation.get('label') is not None and current_operation.get('date') is not None:
                        self.logger.debug('Parsed operation: %s: %s: %s' % (current_operation['date'], current_operation['label'], current_operation['amount']))
                        operations.append(current_operation)
                        current_operation = {}
                else:
                    if current_operation.get('label') is not None:
                        current_operation['label'] = u'%s %s' % (current_operation['label'], token)
                    else:
                        current_operation['label'] = token

        # Step 3: yield adequate transactions
        index = start_index
        for op in operations[start_offset:]:
            self.logger.debug('will yield the following transaction with index %d: %s: %s: %s' % (index, op['date'], op['label'], op['amount']))
            transaction = Transaction(index)
            index += 1
            transaction.amount = op['amount']
            transaction.date = self.date_from_string(op['date'])
            transaction.raw = op['label']
            yield transaction

    def get_transfer_accounts(self, select_name):
        """
            Returns the accounts proposed for a transfer in a select field.
            This method assumes the current page is the one dedicated to transfers.
            select_name is the name of the select field to analyze
        """
        if not self.is_transfer_page():
            return False
        source_accounts = {}
        source_account_options = self.document.xpath('/html/body//form//select[@name="%s"]/option' % select_name)
        for option in source_account_options:
            source_account_value = option.get('value', -1)
            if (source_account_value != -1):
                matches = re.findall('^[A-Z0-9]+.*([0-9]{11}).*$', self.extract_text(option))
                if matches:
                    source_accounts[source_account_value] = matches[0]
        return source_accounts

    def get_transfer_source_accounts(self):
        return self.get_transfer_accounts('numCompteEmetteur')

    def get_transfer_target_accounts(self):
        return self.get_transfer_accounts('numCompteBeneficiaire')

    def expand_history_page_url(self):<commit_msg>CrAgr: transactions are now assigned types and categories through FrenchTransaction.<commit_after># -*- coding: utf-8 -*-

# Copyright(C) 2010-2011 <NAME>
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.


from decimal import Decimal
import re
from datetime import date
from weboob.capabilities.bank import Account
from .base import CragrBasePage
from .tokenextractor import TokenExtractor
from weboob.tools.capabilities.bank.transactions import FrenchTransaction

class Transaction(FrenchTransaction):
    PATTERNS = [
        (re.compile('^(Vp|Vt|Vrt|Virt|Vir(ement)?)\s*(?P<text>.*)'), FrenchTransaction.TYPE_TRANSFER),
        (re.compile('^(?P<text>(Tip|Plt|Prlv|Prelevement)\s*.*)'), FrenchTransaction.TYPE_ORDER),
        (re.compile('^Cheque\s*(?P<text>No.*)'), FrenchTransaction.TYPE_CHECK),
        (re.compile('^(?P<text>Rem\s*Chq\s*.*)'), FrenchTransaction.TYPE_DEPOSIT),
        (re.compile('^Ret(rait)?\s*Dab\s*(?P<text>.*)'), FrenchTransaction.TYPE_WITHDRAWAL),
        (re.compile('^Paiement\s*Carte\s*(?P<text>.*)'), FrenchTransaction.TYPE_CARD),
        (re.compile('^(?P<text>.*CAPITAL.*ECHEANCE.*)'), FrenchTransaction.TYPE_BANK),
        (re.compile('^\*\*(?P<text>(frais|cotis(ation)?)\s*.*)'), FrenchTransaction.TYPE_BANK),
        (re.compile('^(?P<text>Interets\s*.*)'), FrenchTransaction.TYPE_BANK),
        (re.compile('^(?P<text>Prelev\.\s*(C\.r\.d\.s\.|R\.s\.a\.|C\.a\.p\.s\.|C\.s\.g|P\.s\.))'), FrenchTransaction.TYPE_BANK),
    ]

class AccountsList(CragrBasePage):
    """
        Unlike most pages used with the Browser class, this class represents
        several pages, notably accounts list, history and transfer. This is due
        to the Credit Agricole not having a clear pattern to identify a page
        based on its URL.
    """

    def is_accounts_list(self):
        """
            Returns True if the current page appears to be the page dedicated to
            list the accounts.
        """
        # we check for the presence of a "mes comptes titres" link_id
        link = self.document.xpath('/html/body//a[contains(text(), "comptes titres")]')
        return bool(link)

    def is_account_page(self):
        """
            Returns True if the current page appears to be a page dedicated to list
            the history of a specific account.
        """
        # tested on CA Lorraine, Paris, Toulouse
        title_spans = self.document.xpath('/html/body//div[@class="dv"]/span')
        for title_span in title_spans:
            title_text = title_span.text_content().strip().replace("\n", '')
            if (re.match('.*Compte.*n.*[0-9]+', title_text, flags=re.IGNORECASE)):
                return True
        return False

    def is_transfer_page(self):
        """
            Returns True if the current page appears to be the page dedicated to
            order transfers between accounts.
        """
        source_account_select_field = self.document.xpath('/html/body//form//select[@name="numCompteEmetteur"]')
        target_account_select_field  = self.document.xpath('/html/body//form//select[@name="numCompteBeneficiaire"]')
        return bool(source_account_select_field) and bool(target_account_select_field)

    def get_list(self):
        """
            Returns the list of available bank accounts
        """
        l = []
        for div in self.document.getiterator('div'):
            if div.attrib.get('class', '') in ('dv', 'headline') and div.getchildren()[0].tag in ('a', 'br'):
                self.logger.debug("Analyzing div %s" % div)
                # Step 1: extract text tokens
                tokens = []
                required_tokens = {}
                optional_tokens = {}
                token_extractor = TokenExtractor()
                for token in token_extractor.extract_tokens(div):
                    self.logger.debug('Extracted text token: "%s"' % token)
                    tokens.append(token)
                # Step 2: analyse tokens
                for token in tokens:
                    if self.look_like_account_number(token):
                        required_tokens['account_number'] = token
                    elif self.look_like_amount(token):
                        required_tokens['account_amount'] = token
                    elif self.look_like_account_name(token):
                        required_tokens['account_name'] = token
                    elif self.look_like_account_owner(token):
                        optional_tokens['account_owner'] = token
                # Step 3: create account objects
                if len(required_tokens) >= 3:
                    account = Account()
                    account.label = required_tokens['account_name']
                    account.id = required_tokens['account_number']
                    account.balance = self.clean_amount(required_tokens['account_amount'])
                    # we found almost all required information to create an account object
                    self.logger.debug('Found account %s with number %s and balance = %.2f' % (account.label, account.id, account.balance))
                    # we may have found the owner name too
                    if optional_tokens.get('account_owner') is not None:
                        # well, we could add it to the label, but is this really required?
                        self.logger.debug('  the owner appears to be %s' % optional_tokens['account_owner'])
                    # we simply lack the link to the account history... which remains optional
                    first_link = div.find('a')
                    if first_link is not None:
                        account._link_id = first_link.get('href')
                        self.logger.debug('  the history link appears to be %s' % account._link_id)
                    l.append(account)
        return l

    def get_history(self, start_index=0, start_offset=0):
        """
            Returns the history of a specific account. Note that this function
            expects the current page to be the one dedicated to this history.
            start_index is the id used for the first created operation.
            start_offset allows ignoring the `n' first Transactions on the page.
        """
        # tested on CA Lorraine, Paris, Toulouse
        # avoir parsing the page as an account-dedicated page if it is not the case
        if not self.is_account_page():
            return

        # Step 1: extract text tokens
        tokens = []
        token_extractor = TokenExtractor()
        for div in self.document.getiterator('div'):
            if div.attrib.get('class', '') in ('dv'):
                self.logger.debug("Analyzing div %s" % div)
                for token in token_extractor.extract_tokens(div):
                    self.logger.debug('Extracted text token: "%s"' % token)
                    tokens.append(token)

        # Step 2: convert tokens into operations
        # Notes:
        # * the code below expects pieces of information to be in the date-label-amount order;
        #   could we achieve a heuristic smart enough to guess this order?
        # * unlike the former code, we parse every operation
        operations = []
        current_operation = {}
        for token in tokens:
            self.logger.debug('Analyzing token: "%s"' % token)
            date_analysis = self.look_like_date_only(token)
            if date_analysis:
                current_operation = {}
                current_operation['date'] = date_analysis.groups()[0]
            else:
                date_desc_analysis = self.look_like_date_and_description(token)
                if date_desc_analysis:
                    current_operation = {}
                    current_operation['date'] = date_desc_analysis.groups()[0]
                    current_operation['label'] = date_desc_analysis.groups()[1]
                elif self.look_like_amount(token):
                    # we consider the amount is the last information we get for an operation
                    current_operation['amount'] = self.clean_amount(token)
                    if current_operation.get('label') is not None and current_operation.get('date') is not None:
                        self.logger.debug('Parsed operation: %s: %s: %s' % (current_operation['date'], current_operation['label'], current_operation['amount']))
                        operations.append(current_operation)
                        current_operation = {}
                else:
                    if current_operation.get('label') is not None:
                        current_operation['label'] = u'%s %s' % (current_operation['label'], token)
                    else:
                        current_operation['label'] = token

        # Step 3: yield adequate transactions
        index = start_index
        for op in operations[start_offset:]:
            self.logger.debug('will yield the following transaction with index %d: %s: %s: %s' % (index, op['date'], op['label'], op['amount']))
            transaction = Transaction(index)
            index += 1
            transaction.amount = op['amount']
            transaction.parse(self.date_from_string(op['date']), re.sub('\s+', ' ', op['label']))
            yield transaction

    def get_transfer_accounts(self, select_name):
        """
            Returns the accounts proposed for a transfer in a select field.
            This method assumes the current page is the one dedicated to transfers.
            select_name is the name of the select field to analyze
        """
        if not self.is_transfer_page():
            return False
        source_accounts = {}
        source_account_options = self.document.xpath('/html/body//form//select[@name="%s"]/option' % select_name)
        for option in source_account_options:
            source_account_value = option.get('value', -1)
            if (source_account_value != -1):
                matches = re.findall('^[A-Z0-9]+.*([0-9]{11}).*$', self.extract_text(option))
                if matches:
                    source_accounts[source_account_value] = matches[0]
        return source_accounts

    def get_transfer_source_accounts(self):
        return self.get_transfer_accounts('numCompteEmetteur')

    def get_transfer_target_accounts(self):
        return self.get_transfer_accounts('numCompteBeneficiaire')

    def expand_history_page_url(self):
{'fraction_non_alphanumeric': 0.09913401272666379, 'fraction_numerical': 0.0034448112530500933, 'mean_word_length': 2.14410348977136, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6421725', 'n_tokens_mistral': 5502, 'n_tokens_neox': 5268, 'n_words': 1614}