<commit_before># -*- coding: utf-8 -*-

''' Read a CSV with revenues and insert them in the DB.

Usage:
    ./import_revenue [FILE] [LINES_PER_INSERT]
    ./import_revenue (-h | --help)

Options:
    -h --help   Show this message.
'''
import pandas as pd
from datetime import datetime, timedelta
import calendar
from sqlalchemy.sql.expression import insert
from docopt import docopt

from gastosabertos import create_app

from gastosabertos.contratos.models import Contrato 


def get_db():
    from gastosabertos.extensions import db
    app = create_app()
    db.app = app
    return db


def parse_money(money_string):
    return str(money_string).replace('.', '').replace(',', '.')
#        return -float(money_string[3:].replace('.', '').replace(',', '.'))
#    else:
#        return float(money_string[3:].replace('.', '').replace(',', '.'))


def parse_date(date_string):
    new_date = datetime.strptime(date_string, '%d/%m/%Y')
    return new_date


def parse_code(code_string):
    return [int(i) for i in code_string.split('.')]


def insert_rows(db, rows_data):
    ins = insert(Contrato.__table__, rows_data)
    db.session.execute(ins)
    db.session.commit()


def insert_all(db, csv_file='../data/contratos-2014.xls', lines_per_insert=100):
    data = pd.read_excel(csv_file)

    cache = {}
    to_insert = []
    total_lines = len(data)
    current_line = 0.0

    for row_i, row in data.iterrows():
        current_line += 1

        r = {}

        if len(to_insert) == lines_per_insert:
            insert_rows(db, to_insert)
            to_insert = []
            # Progress counter
            print(str(int(current_line/total_lines*100))+'%')

        r['numero'] = row_i + 1
        r['orgao'] = row['Orgao']
        r['data_assinatura'] = parse_date(row['Data da Assinatura'])
        r['vigencia'] = row['Vigencia']<commit_msg>Changing type to Python int in Contratos import script<commit_after># -*- coding: utf-8 -*-

''' Read a CSV with revenues and insert them in the DB.

Usage:
    ./import_revenue [FILE] [LINES_PER_INSERT]
    ./import_revenue (-h | --help)

Options:
    -h --help   Show this message.
'''
import pandas as pd
from datetime import datetime, timedelta
import calendar
from sqlalchemy.sql.expression import insert
from docopt import docopt

from gastosabertos import create_app

from gastosabertos.contratos.models import Contrato 


def get_db():
    from gastosabertos.extensions import db
    app = create_app()
    db.app = app
    return db


def parse_money(money_string):
    return str(money_string).replace('.', '').replace(',', '.')


def parse_date(date_string):
    new_date = datetime.strptime(date_string, '%d/%m/%Y')
    return new_date


def insert_rows(db, rows_data):
    ins = insert(Contrato.__table__, rows_data)
    db.session.execute(ins)
    db.session.commit()


def insert_all(db, csv_file='../data/contratos-2014.xls', lines_per_insert=100):
    data = pd.read_excel(csv_file)

    cache = {}
    to_insert = []
    total_lines = len(data)
    current_line = 0.0

    for row_i, row in data.iterrows():
        current_line += 1

        r = {}

        if len(to_insert) == lines_per_insert:
            insert_rows(db, to_insert)
            to_insert = []
            # Progress counter
            print(str(int(current_line/total_lines*100))+'%')

        r['numero'] = row_i + 1
        r['orgao'] = row['Orgao']
        r['data_assinatura'] = parse_date(row['Data da Assinatura'])
        r['vigencia'] = int(row['Vigencia'])
{'fraction_non_alphanumeric': 0.13793103448275862, 'fraction_numerical': 0.009119407238529496, 'mean_word_length': 3.0484429065743943, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '739639', 'n_tokens_mistral': 1233, 'n_tokens_neox': 1167, 'n_words': 274}