<commit_before># coding=utf-8
from basesource import *
from base.models import models
from base.models.models import Legislator, Supplier, Expense
from sqlalchemy import distinct

import re


Session = models.initialize(get_database_path('camara'))


class VerbaIndenizatoriaCamara(BaseCollector):

    legislators_uri = 'http://www2.camara.gov.br/transparencia/cota-para-exercicio-da-atividade-parlamentar/verba_indenizatoria_index'
    expenses_uri = 'http://www2.camara.gov.br/transparencia/cota-para-exercicio-da-atividade-parlamentar/verba_indenizatoria_detalheVerbaAnalitico?nuDeputadoId=%s&numMes=%s&numAno=%s'

    position = u'Deputado Federal'

    def update_legislators(self):
        if self.debug:
            print 'Retrieving legislators...'

        # Retrieving the select with legislators information
        select = self.get_element_from_uri (self.legislators_uri, 'select', {'id' : 'listaDep'})

        # We ignore the first one because it's a placeholder.
        options = select.findAll('option')[1:]

        # Turn the soup objects into a list of dictionaries
        legislators = []
        for item in options:
            item = item['value']

            # Parsing the value string
            #   <option value="[name]|[party]|[state]|[id]|[start_of_legislature]">[name with legislature information]</option>
            name, party, garbage, cod, garbage = item.split('|')

            legislators.append(dict(id=int(cod), name=name.strip(), party=party.strip()))

        # Obtain the existing ids
        session = Session()
        existing_ids = [item[0] for item in session.query(Legislator.id).filter(Legislator.position == self.position)]

        # Add legislators that do not exist yet
        for l in legislators:
            if l['id'] not in existing_ids:
               session.add(Legislator(l['id'], l['name'], l['party'], self.position))

        session.commit()

    def update_data(self, year = datetime.now().year):
        self.update_data_normal (year, 1712)
        #self.update_data_missing (year, 81)
        #self.update_data_manual (81, year)

    def update_data_missing(self, year = datetime.now().year, start = None):
        session = Session()

        existing_ids = session.query(distinct(Expense.legislator_id)).filter(and_(Expense.date >= '%s-01-01'%(year),
                                                                                  Expense.date <= '%s-12-31'%(year)));
        if start == None:
            legislators = session.query(Legislator.id).filter(Legislator.id.op('NOT IN')(existing_ids)).all()
        else:
            legislators = session.query(Legislator.id).filter(and_(Legislator.id.op('NOT IN')(existing_ids),
                                                                   Legislator.id >= start)).all()
        for l in legislators:
            for m in range(1, 13):
                self.update_data_for_id_period (l.id, year, m)

    def update_data_manual(self, id, year = datetime.now().year):
        session = Session()

        for month in range(1, 13):
            self.update_data_for_id_period(id, year, month)


    def update_data_normal(self, year = datetime.now().year, start = None):
        session = Session()
        if start == None:
            data = [item for item in session.query(Legislator.id, Legislator.name).filter(Legislator.position == self.position)]
        else:
            data = [item for item in session.query(Legislator.id, Legislator.name).filter(and_(Legislator.position == self.position,
                                                                                               Legislator.id >= start))]

        date = datetime.now()
        for legislator_id, name in data:
            try:
                if year == date.year:
                    for month in range(date.month-2, date.month+1):
                        self.update_data_for_id_period(legislator_id, year, month)
                else:
                    for month in range(1, 13):
                        self.update_data_for_id_period(legislator_id, year, month)
            except HTTPError as msg:
                print "Error retrieving expenses: %s\n" % (msg)
                continue

        return


    def update_data_for_id_period(self, id, year, month):
        session = Session()

        # Retrieving legislator
        legislator = session.query(Legislator).filter(Legislator.id == id).one()

        # Find the main content div. It looks like this:
        #
        # <div class="grid">
        #   <div class="grid-cell ...">
        #     <h4 class="header ...">[Category]</h4>
        #     <table class="tabela-1">[Table with expenses per supplier]</table>
        #   </div><commit_msg>Removing unnecessary functions from Camara<commit_after># coding=utf-8
from basesource import *
from base.models import models
from base.models.models import Legislator, Supplier, Expense
from sqlalchemy import distinct

import re


Session = models.initialize(get_database_path('camara'))


class VerbaIndenizatoriaCamara(BaseCollector):

    legislators_uri = 'http://www2.camara.gov.br/transparencia/cota-para-exercicio-da-atividade-parlamentar/verba_indenizatoria_index'
    expenses_uri = 'http://www2.camara.gov.br/transparencia/cota-para-exercicio-da-atividade-parlamentar/verba_indenizatoria_detalheVerbaAnalitico?nuDeputadoId=%s&numMes=%s&numAno=%s'

    position = u'Deputado Federal'

    def update_legislators(self):
        if self.debug:
            print 'Retrieving legislators...'

        # Retrieving the select with legislators information
        select = self.get_element_from_uri (self.legislators_uri, 'select', {'id' : 'listaDep'})

        # We ignore the first one because it's a placeholder.
        options = select.findAll('option')[1:]

        # Turn the soup objects into a list of dictionaries
        legislators = []
        for item in options:
            item = item['value']

            # Parsing the value string
            #   <option value="[name]|[party]|[state]|[id]|[start_of_legislature]">[name with legislature information]</option>
            name, party, garbage, cod, garbage = item.split('|')

            legislators.append(dict(id=int(cod), name=name.strip(), party=party.strip()))

        # Obtain the existing ids
        session = Session()
        existing_ids = [item[0] for item in session.query(Legislator.id).filter(Legislator.position == self.position)]

        # Add legislators that do not exist yet
        for l in legislators:
            if l['id'] not in existing_ids:
               session.add(Legislator(l['id'], l['name'], l['party'], self.position))

        session.commit()

    def update_data(self, year = datetime.now().year):
        session = Session()
        legislators = [item for item in session.query(Legislator.id).filter(Legislator.position == self.position)]

        date = datetime.now()
        for legislator in legislators:
            try:
                if year == date.year:
                    for month in range(date.month-2, date.month+1):
                        self.update_data_for_id_period(legislator.id, year, month)
                else:
                    for month in range(1, 13):
                        self.update_data_for_id_period(legislator.id, year, month)
            except HTTPError as msg:
                print "Error retrieving expenses: %s\n" % (msg)
                continue

        return

    def update_data_for_id_period(self, id, year, month):
        session = Session()

        # Retrieving legislator
        legislator = session.query(Legislator).filter(Legislator.id == id).one()

        # Find the main content div. It looks like this:
        #
        # <div class="grid">
        #   <div class="grid-cell ...">
        #     <h4 class="header ...">[Category]</h4>
        #     <table class="tabela-1">[Table with expenses per supplier]</table>
        #   </div>
{'fraction_non_alphanumeric': 0.11541850220264317, 'fraction_numerical': 0.006041535556954059, 'mean_word_length': 2.305324459234609, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2955440', 'n_tokens_mistral': 2367, 'n_tokens_neox': 2279, 'n_words': 521}