<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

import re
import sys
import codecs
import tempfile
from subprocess import Popen, PIPE
import subprocess
import logging
import time
from datetime import datetime
from cStringIO import StringIO

def get_pdf_raw_data(filename):
    p = subprocess.Popen(['pdftotext', '-layout', filename, '-'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
    (out, err) = p.communicate()
    return out


period_re = ur'.*Période: ([0-9-]+) au ([0-9-]+).*Date: ([0-9-]+).*'
statement_number_re = ur'\s*Extrait n° (\S+)$'
amount_start_re = ur'.*Solde initial au ([0-9-]+)\s+([0-9,+-\.]+).*'
amount_end_re = ur'.*Solde final au ([0-9-]+)\s+([0-9,+-\.]+).*'
#line_re = ur'\s+([0-9-]+) (\S+?)\s{4,}(.+?)\s{4,}([0-9-]{10})\s+([0-9,\.+-]+)'
line_re = ur'\s*?([0-9-]+) (\S+?)\s{1,}(.+?)\s{4,}([0-9-]{10})\s+([0-9,\.+-]+)'
onlydesc_line_re = ur'^\s*(.+?)\s*$'
empty_line_re = ur'^\s*$'
account_iban_currency_re = ur'\s+Compte Courant : IBAN (.+)\s+?\((.+?)\)\s*'
report_re = ur'\s+Report\s+([0-9,\.+-]+)\s*'

ING_D_FORMAT = '%d-%m-%Y'

#XXX: make real logging
logging.basicConfig(level=logging.INFO)

class INGBankStatementParser(object):

    def __init__(self):
        self.iban = None
        self.currency = None
        self.mode = None
        self.period_start = None
        self.period_end = None
        self.date = None
        self.number = None
        self.amount_start = None
        self.amount_end = None
        self.lines = []

        self.set_mode('head')

    def set_mode(self, new_mode):
        assert new_mode in ('head', 'report', 'line_out', 'line_in', 'finished')
        self.mode = new_mode

    def to_monetary(self, txt):
        v = txt.strip().replace('.','').replace(',','.')
        return float(v)

    def is_valid(self):
        for k in ['period_start', 'period_end', 'amount_start', 'amount_end']:
            if getattr(self, k) is None:
                return False
        a = self.amount_start
        a += sum([ d['amount'] for d in self.lines ])
        if abs(a - self.amount_end) < 10 ** -4:
            return True
        return False

    def get_period_from_date(self, dt):
        #TODO: do a real search based on OERP periods
        return dt.strftime('%Y-%m')

    def is_multi_periods(self):
        periods = set([ self.get_period_from_date(d['maturity_date']) for d in self.lines ])
        return len(periods) > 1

    def parse(self, f):
        p = Popen(['pdftotext', '-layout', '-', '-'], stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True)
        (out, err) = p.communicate(input=f.read())
        self.parse_text(out)
        return self.get_statement_entries()

    def parse_text(self, text):
        c = StringIO(text)
        return self.parse_file(c)

    def parse_file(self, f):
        logger = logging.getLogger('parser')
        if isinstance(f, basestring):
            f = open(f, 'r')

        for i, line in enumerate(f.readlines()):
            line = unicode(line, 'utf-8')
            #print("[%s] line: %s" % (self.mode, line,))
            if self.mode == 'head':
                m = re.match(period_re, line)
                if m:
                    logger.debug("[match period] %s" % (m.groups(),))
                    self.period_start = datetime.strptime(m.groups()[0], ING_D_FORMAT)
                    self.period_end = datetime.strptime(m.groups()[1], ING_D_FORMAT)
                    self.date = datetime.strptime(m.groups()[1], ING_D_FORMAT)
                    continue
                m = re.match(statement_number_re, line)
                if m:
                    logger.debug("[match number] %s" % (m.groups(),))
                    self.number, = m.groups()
                    #self.set_mode('start')
                    continue<commit_msg>[FIX] account_payment_import_statement: use real file for 'pdftotext' call<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

from __future__ import with_statement
import re
import sys
import codecs
import tempfile
from subprocess import Popen, PIPE
import subprocess
import logging
import time
from datetime import datetime
from cStringIO import StringIO

def get_pdf_raw_data(filename):
    p = subprocess.Popen(['pdftotext', '-layout', filename, '-'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
    (out, err) = p.communicate()
    return out


period_re = ur'.*Période: ([0-9-]+) au ([0-9-]+).*Date: ([0-9-]+).*'
statement_number_re = ur'\s*Extrait n° (\S+)$'
amount_start_re = ur'.*Solde initial au ([0-9-]+)\s+([0-9,+-\.]+).*'
amount_end_re = ur'.*Solde final au ([0-9-]+)\s+([0-9,+-\.]+).*'
#line_re = ur'\s+([0-9-]+) (\S+?)\s{4,}(.+?)\s{4,}([0-9-]{10})\s+([0-9,\.+-]+)'
line_re = ur'\s*?([0-9-]+) (\S+?)\s{1,}(.+?)\s{4,}([0-9-]{10})\s+([0-9,\.+-]+)'
onlydesc_line_re = ur'^\s*(.+?)\s*$'
empty_line_re = ur'^\s*$'
account_iban_currency_re = ur'\s+Compte Courant : IBAN (.+)\s+?\((.+?)\)\s*'
report_re = ur'\s+Report\s+([0-9,\.+-]+)\s*'

ING_D_FORMAT = '%d-%m-%Y'

#XXX: make real logging
logging.basicConfig(level=logging.INFO)

class INGBankStatementParser(object):

    def __init__(self):
        self.iban = None
        self.currency = None
        self.mode = None
        self.period_start = None
        self.period_end = None
        self.date = None
        self.number = None
        self.amount_start = None
        self.amount_end = None
        self.lines = []

        self.set_mode('head')

    def set_mode(self, new_mode):
        assert new_mode in ('head', 'report', 'line_out', 'line_in', 'finished')
        self.mode = new_mode

    def to_monetary(self, txt):
        v = txt.strip().replace('.','').replace(',','.')
        return float(v)

    def is_valid(self):
        for k in ['period_start', 'period_end', 'amount_start', 'amount_end']:
            if getattr(self, k) is None:
                return False
        a = self.amount_start
        a += sum([ d['amount'] for d in self.lines ])
        if abs(a - self.amount_end) < 10 ** -4:
            return True
        return False

    def get_period_from_date(self, dt):
        #TODO: do a real search based on OERP periods
        return dt.strftime('%Y-%m')

    def is_multi_periods(self):
        periods = set([ self.get_period_from_date(d['maturity_date']) for d in self.lines ])
        return len(periods) > 1

    def parse(self, f):
        with tempfile.NamedTemporaryFile() as ftmp:
            ftmp.write(f.read())
            ftmp.flush()
            p = Popen(['pdftotext', '-layout', ftmp.name, '-'], stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True, bufsize=-1)
            (out, err) = p.communicate(input=f.read())
        self.parse_text(out)
        return self.get_statement_entries()

    def parse_text(self, text):
        c = StringIO(text)
        return self.parse_file(c)

    def parse_file(self, f):
        logger = logging.getLogger('parser')
        if isinstance(f, basestring):
            f = open(f, 'r')

        for i, line in enumerate(f.readlines()):
            line = unicode(line, 'utf-8')
            #print("[%s] line: %s" % (self.mode, line,))
            if self.mode == 'head':
                m = re.match(period_re, line)
                if m:
                    logger.debug("[match period] %s" % (m.groups(),))
                    self.period_start = datetime.strptime(m.groups()[0], ING_D_FORMAT)
                    self.period_end = datetime.strptime(m.groups()[1], ING_D_FORMAT)
                    self.date = datetime.strptime(m.groups()[1], ING_D_FORMAT)
                    continue
                m = re.match(statement_number_re, line)
                if m:
                    logger.debug("[match number] %s" % (m.groups(),))
                    self.number, = m.groups()
                    #self.set_mode('start')
                    continue
{'fraction_non_alphanumeric': 0.1797881842541789, 'fraction_numerical': 0.01161158606609672, 'mean_word_length': 2.33531914893617, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6032821', 'n_tokens_mistral': 2822, 'n_tokens_neox': 2728, 'n_words': 566}