<commit_before># -*- coding: utf-8 -*-
#
# Copyright (c) 2015, <NAME> <<EMAIL>>
#
#  This program is free software: you can redistribute it and/or modify
#  it under the terms of the GNU Affero General Public License as
#  published by the Free Software Foundation, either version 3 of the
#  License, or (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU Affero General Public License for more details.
#
#  You should have received a copy of the GNU Affero General Public License
#  along with this program.  If not, see <http://www.gnu.org/licenses/>.

import csv
import logging
import multiprocessing
import os
import re
from datetime import datetime

from django.core.management.base import BaseCommand
from django.conf import settings
from django.utils.text import slugify

from politicians.management.commands._tse import TSE
from politicians.models import (
    PoliticalParty, Institution, PoliticalOffice, Politician,
    Mandate, Ethnicity, Education, PoliticianAlternativeName,
    PoliticianEventType, PoliticianEvent, PoliticianPoliticalParty,
    Candidacy, Election, CandidacyStatus, City, State, MaritalStatus,
    Nationality, Occupation
)


def call_it(instance, name, args=(), kwargs=None):
    if kwargs is None:
        kwargs = {}
    return getattr(instance, name)(*args, **kwargs)


class Base(object):

    @classmethod
    def set_options(cls, *args, **kwargs):
        if kwargs.get('log-level', '').lower() == 'debug':
            log_level = logging.DEBUG
        elif kwargs.get('log-level', '').lower() == 'info':
            log_level = logging.INFO
        else:
            log_level = logging.ERROR

        cls.logger = logging.getLogger('politicos_command')
        cls.logger.setLevel(log_level)

    # Inspired by http://miud.in/1H8y
    @classmethod
    def title_except(cls, s, exceptions=['do', 'de', 'da', 'dos', 'a', 'e']):
        word_list = re.split(' ', s.lower())  # re.split behaves as expected
        final = [word_list[0].title()]
        for word in word_list[1:]:
            final.append(word in exceptions and word or word.title())
        return ' '.join(final)

    @classmethod
    def create_siglum(cls, s):
        exceptions = ['do', 'dos', 'de', 'da', 'das', 'a', 'e']
        word_list = slugify(s).split('-')
        final = [word_list[0].title()[:1]]
        for w in word_list[1:]:
            final.append(w not in exceptions and w.title()[:1] or '')
        return ''.join(final)

    @classmethod
    def get_urls_in_text(cls, text):
        urls = re.finditer(
            r'(([a-z]{3,6}:\/\/)|(^|\s))([a-zA-Z0-9\-]+\.)+[a-z]{2,13}(?:\/[a-zA-Z0-9]{1,})*',
            text
        )
        return [x.group(0).strip() for x in urls]

    @classmethod
    def get_emails_in_text(cls, text):
        emails = re.findall(r'[\w\.-]+@[\w\.-]+', text)
        return emails if emails else None


class PoliticosCommand(BaseCommand, Base):

    def add_arguments(self, parser):
        parser.add_argument(
            '--log-level',
            dest='log-level',
            default='error',
            help='Debugger level'
        )

    def process_tse_data_by_year(self, politicos_class, year):
        tse = TSE(year)
        tse.download_and_extract()
        politicos_class.process_tse_files(tse, year)
        tse.remove_tmp_dir()

    @classmethod
    def raise_error(cls, text):
        cls.logger.error(text)
        raise


class Politicos(Base):

    @classmethod
    def raise_error(cls, text):
        cls.logger.error(text)
        raise

    @classmethod
    def get_picture(cls, *args, **kwargs):
        raise NotImplementedError('Subclasses should implement this!')

    @classmethod
    def convert_to_dict(cls, *args, **kwargs):
        raise NotImplementedError('Subclasses should implement this!')

    @classmethod
    def formatter(cls, text):
        try:
            return cls.title_except(text.decode('latin1')).strip()
        except Exception:
            return cls.title_except(text).strip()

    @classmethod
    def add_city(cls, item):
        if item.get('state_siglum') == 'BR':
            return None

        state = State.objects.cache().get(siglum=item.get('state_siglum'))

        city = None
        if item.get('city_name'):
            city, city_created = City.objects.cache().get_or_create(
                name=item.get('city_name'), state=state
            )
            if city_created:
                cls.logger.debug(
                    'Added City: %s (%s)', str(city), str(city.state)
                )

        return city

    @classmethod
    def add_institution_vereador(cls, item):
        city = cls.add_city(item)
        institution, created = Institution.objects.cache().get_or_create(
            name=u'Camara Municipal {0}'.format(city.name),
            siglum='CM{0}'.format(cls.create_siglum(city.name)),
            state=city.state,
        )
        if created:
            po = PoliticalOffice.get_by_name('Vereador')
            institution.political_offices.add(po)
            cls.logger.debug(
                'Addd Institution: "{0}"'.format(str(institution))
            )
        return institution

    @classmethod
    def add_institution_prefeito(cls,  item):
        city = cls.add_city(item)
        institution, created = Institution.objects.cache().get_or_create(
            name=u'Prefeitura Municipal {}'.format(city.name),
            siglum='PM{0}'.format(cls.create_siglum(city.name)),
            state=city.state,
        )
        if created:
            po1 = PoliticalOffice.get_by_name('Prefeito')
            po2 = PoliticalOffice.get_by_name('Vice-Prefeito')
            institution.political_offices.add(po1, po2)
            cls.logger.debug(
                'Added Institution: "{0}"'.format(str(institution))
            )
        return institution

    @classmethod
    def process_tse_files(cls, tse, year):
        try:
            election = Election.objects.cache().get(year=year)
        except Election.DoesNotExist:
            cls.raise_error('Election not found: {0}'.format(year))

        states = [(x.siglum, x.name) for x in State.objects.all()]
        states = ((u'BR', u'Brasil'),) + tuple(states)

        if settings.TSE_CONCURRENCY <= 1:
            for state in states:
                cls.process_tse_file(tse, year, state, election)
            return

        pool = multiprocessing.Pool(processes=settings.TSE_CONCURRENCY)
        async_results = [
            pool.apply_async(
                call_it,
                args=(cls, 'process_tse_file', (tse, year, state, election))
            ) for state in states
        ]
        pool.close()<commit_msg>process_tse_files: Fixed Election creation<commit_after># -*- coding: utf-8 -*-
#
# Copyright (c) 2015, <NAME> <<EMAIL>>
#
#  This program is free software: you can redistribute it and/or modify
#  it under the terms of the GNU Affero General Public License as
#  published by the Free Software Foundation, either version 3 of the
#  License, or (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU Affero General Public License for more details.
#
#  You should have received a copy of the GNU Affero General Public License
#  along with this program.  If not, see <http://www.gnu.org/licenses/>.

import csv
import logging
import multiprocessing
import os
import re
from datetime import datetime

from django.core.management.base import BaseCommand
from django.conf import settings
from django.utils.text import slugify

from politicians.management.commands._tse import TSE
from politicians.models import (
    PoliticalParty, Institution, PoliticalOffice, Politician,
    Mandate, Ethnicity, Education, PoliticianAlternativeName,
    PoliticianEventType, PoliticianEvent, PoliticianPoliticalParty,
    Candidacy, Election, CandidacyStatus, City, State, MaritalStatus,
    Nationality, Occupation
)


def call_it(instance, name, args=(), kwargs=None):
    if kwargs is None:
        kwargs = {}
    return getattr(instance, name)(*args, **kwargs)


class Base(object):

    @classmethod
    def set_options(cls, *args, **kwargs):
        if kwargs.get('log-level', '').lower() == 'debug':
            log_level = logging.DEBUG
        elif kwargs.get('log-level', '').lower() == 'info':
            log_level = logging.INFO
        else:
            log_level = logging.ERROR

        cls.logger = logging.getLogger('politicos_command')
        cls.logger.setLevel(log_level)

    # Inspired by http://miud.in/1H8y
    @classmethod
    def title_except(cls, s, exceptions=['do', 'de', 'da', 'dos', 'a', 'e']):
        word_list = re.split(' ', s.lower())  # re.split behaves as expected
        final = [word_list[0].title()]
        for word in word_list[1:]:
            final.append(word in exceptions and word or word.title())
        return ' '.join(final)

    @classmethod
    def create_siglum(cls, s):
        exceptions = ['do', 'dos', 'de', 'da', 'das', 'a', 'e']
        word_list = slugify(s).split('-')
        final = [word_list[0].title()[:1]]
        for w in word_list[1:]:
            final.append(w not in exceptions and w.title()[:1] or '')
        return ''.join(final)

    @classmethod
    def get_urls_in_text(cls, text):
        urls = re.finditer(
            r'(([a-z]{3,6}:\/\/)|(^|\s))([a-zA-Z0-9\-]+\.)+[a-z]{2,13}(?:\/[a-zA-Z0-9]{1,})*',
            text
        )
        return [x.group(0).strip() for x in urls]

    @classmethod
    def get_emails_in_text(cls, text):
        emails = re.findall(r'[\w\.-]+@[\w\.-]+', text)
        return emails if emails else None


class PoliticosCommand(BaseCommand, Base):

    def add_arguments(self, parser):
        parser.add_argument(
            '--log-level',
            dest='log-level',
            default='error',
            help='Debugger level'
        )

    def process_tse_data_by_year(self, politicos_class, year):
        tse = TSE(year)
        tse.download_and_extract()
        politicos_class.process_tse_files(tse, year)
        tse.remove_tmp_dir()

    @classmethod
    def raise_error(cls, text):
        cls.logger.error(text)
        raise


class Politicos(Base):

    @classmethod
    def raise_error(cls, text):
        cls.logger.error(text)
        raise

    @classmethod
    def get_picture(cls, *args, **kwargs):
        raise NotImplementedError('Subclasses should implement this!')

    @classmethod
    def convert_to_dict(cls, *args, **kwargs):
        raise NotImplementedError('Subclasses should implement this!')

    @classmethod
    def formatter(cls, text):
        try:
            return cls.title_except(text.decode('latin1')).strip()
        except Exception:
            return cls.title_except(text).strip()

    @classmethod
    def add_city(cls, item):
        if item.get('state_siglum') == 'BR':
            return None

        state = State.objects.cache().get(siglum=item.get('state_siglum'))

        city = None
        if item.get('city_name'):
            city, city_created = City.objects.cache().get_or_create(
                name=item.get('city_name'), state=state
            )
            if city_created:
                cls.logger.debug(
                    'Added City: %s (%s)', str(city), str(city.state)
                )

        return city

    @classmethod
    def add_institution_vereador(cls, item):
        city = cls.add_city(item)
        institution, created = Institution.objects.cache().get_or_create(
            name=u'Camara Municipal {0}'.format(city.name),
            siglum='CM{0}'.format(cls.create_siglum(city.name)),
            state=city.state,
        )
        if created:
            po = PoliticalOffice.get_by_name('Vereador')
            institution.political_offices.add(po)
            cls.logger.debug(
                'Addd Institution: "{0}"'.format(str(institution))
            )
        return institution

    @classmethod
    def add_institution_prefeito(cls,  item):
        city = cls.add_city(item)
        institution, created = Institution.objects.cache().get_or_create(
            name=u'Prefeitura Municipal {}'.format(city.name),
            siglum='PM{0}'.format(cls.create_siglum(city.name)),
            state=city.state,
        )
        if created:
            po1 = PoliticalOffice.get_by_name('Prefeito')
            po2 = PoliticalOffice.get_by_name('Vice-Prefeito')
            institution.political_offices.add(po1, po2)
            cls.logger.debug(
                'Added Institution: "{0}"'.format(str(institution))
            )
        return institution

    @classmethod
    def process_tse_files(cls, tse, year):
        try:
            election = Election.objects.cache().get_or_create(year=year)
        except Election.DoesNotExist:
            cls.raise_error('Election not found: {0}'.format(year))

        states = [(x.siglum, x.name) for x in State.objects.all()]
        states = ((u'BR', u'Brasil'),) + tuple(states)

        if settings.TSE_CONCURRENCY <= 1:
            for state in states:
                cls.process_tse_file(tse, year, state, election)
            return

        pool = multiprocessing.Pool(processes=settings.TSE_CONCURRENCY)
        async_results = [
            pool.apply_async(
                call_it,
                args=(cls, 'process_tse_file', (tse, year, state, election))
            ) for state in states
        ]
        pool.close()
{'fraction_non_alphanumeric': 0.12357262346352461, 'fraction_numerical': 0.005382209615244745, 'mean_word_length': 2.434065934065934, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4427713', 'n_tokens_mistral': 4262, 'n_tokens_neox': 4157, 'n_words': 994}