<commit_before># -*- coding: utf-8 -*-

import os
import csv
import math
import collections
from operator import itemgetter
from datetime import datetime
from itertools import groupby

import pipeline.db as db
import pipeline.models as models


class PartiesAndCoalitionsChanges(object):
    def run(self):
        path = 'parties_and_coalitions_changes.csv'
        result = self._legislators_groupped_by_party_and_coalition()

        with open(path, 'w', newline='') as csv_file:
            writer = csv.DictWriter(csv_file, fieldnames=result[0].keys())
            writer.writeheader()
            writer.writerows(result)

    def _legislators_groupped_by_party_and_coalition(self):
        """Returns a list of legislators with one entry per coalition change

        For example, if a legislators' party got in and out of the coalition,
        him will be returned twice (once when he was in the coalition, and once
        when he was out). If he switched parties but they both were in (or out)
        the coalition, he would be returned only once (the first one).
        """
        coalizoes = self._get_coalizoes()
        coalizoes_partidos = self._get_coalizoes_partidos()
        intermediary_results = []
        for coalizao in coalizoes:
            partidos_na_coalizao = coalizoes_partidos[coalizao["Id_Clz"]]

            start_date = coalizao["DataInicial"]
            end_date = coalizao["DataFinal"]
            votos_coalizao = self._get_parlamentares_between(start_date,
                                                             end_date)

            intermediary_results += self._add_extra_columns_and_convert_to_dict(votos_coalizao,
                                                                                partidos_na_coalizao)
        intermediary_results = sorted(intermediary_results,
                                      key=itemgetter("rollcall_date"))

        results = []
        for _, values in groupby(intermediary_results, itemgetter("legislature")):
            results += self._remove_uniques_and_convert_to_change_list(values)

        sort_keys = lambda v: (v["id"], v["rollcall_date"])
        return sorted(results, key=sort_keys)

    def _remove_uniques_and_convert_to_change_list(self, rows):
        result = []
        get_id = itemgetter("id")
        rows = sorted(rows, key=itemgetter("rollcall_date"))
        for _, elements in groupby(sorted(rows, key=get_id), get_id):
            elements = [r for r in elements]
            if len(elements) == 1:
                # Só estamos interessados em mudanças de coalizão, então
                # ignoramos quem nunca mudou
                continue
            for i in range(1, len(elements)):
                before = elements[i - 1]
                actual = elements[i]
                if before["coalizao"] == actual["coalizao"]:
                    continue
                result.append(collections.OrderedDict([
                    ("id", before["id"]),
                    ("name", before["name"]),
                    ("party_before", before["party"]),
                    ("party_after", actual["party"]),
                    ("rollcall_id", actual["rollcall_id"]),
                    ("rollcall_date", actual["rollcall_date"]),
                    ("coalition_before", before["coalizao"]),
                    ("coalition_after", actual["coalizao"]),
                    ("legislature", actual["legislature"]),
                    ("legislature_year", actual["legislature_year"]),
                ]))
        return result

    def _add_extra_columns_and_convert_to_dict(self, votos_coalizao, partidos_coalizao):
        result = [collections.OrderedDict(v) for v in votos_coalizao]
        for voto in result:
            voto["coalizao"] = voto["party"] in partidos_coalizao
            voto["legislature"] = self._get_legislature(voto["rollcall_date"])
            voto["legislature_year"] = self._get_legislature_year(voto["rollcall_date"])<commit_msg>Adiciona data de início da coalizão nos dados de mudança de coalizão<commit_after># -*- coding: utf-8 -*-

import os
import csv
import math
import collections
from operator import itemgetter
from datetime import datetime
from itertools import groupby

import pipeline.db as db
import pipeline.models as models


class PartiesAndCoalitionsChanges(object):
    def run(self):
        path = 'parties_and_coalitions_changes.csv'
        result = self._legislators_groupped_by_party_and_coalition()

        with open(path, 'w', newline='') as csv_file:
            writer = csv.DictWriter(csv_file, fieldnames=result[0].keys())
            writer.writeheader()
            writer.writerows(result)

    def _legislators_groupped_by_party_and_coalition(self):
        """Returns a list of legislators with one entry per coalition change

        For example, if a legislators' party got in and out of the coalition,
        him will be returned twice (once when he was in the coalition, and once
        when he was out). If he switched parties but they both were in (or out)
        the coalition, he would be returned only once (the first one).
        """
        coalizoes = self._get_coalizoes()
        coalizoes_partidos = self._get_coalizoes_partidos()
        intermediary_results = []
        for coalizao in coalizoes:
            partidos_na_coalizao = coalizoes_partidos[coalizao["Id_Clz"]]

            start_date = coalizao["DataInicial"]
            end_date = coalizao["DataFinal"]
            votos_coalizao = self._get_parlamentares_between(start_date,
                                                             end_date)

            data = self._add_extra_columns_and_convert_to_dict(votos_coalizao,
                                                               partidos_na_coalizao)
            for row in data:
                row["coalition_start_date"] = start_date

            intermediary_results += data

        intermediary_results = sorted(intermediary_results,
                                      key=itemgetter("coalition_start_date"))

        results = []
        for _, values in groupby(intermediary_results, itemgetter("legislature")):
            results += self._remove_uniques_and_convert_to_change_list(values)

        sort_keys = lambda v: (v["id"], v["coalition_start_date"], v["rollcall_date"])
        return sorted(results, key=sort_keys)

    def _remove_uniques_and_convert_to_change_list(self, rows):
        result = []
        get_id = itemgetter("id")
        sort_keys = lambda v: (v["id"], v["coalition_start_date"], v["rollcall_date"])
        rows = sorted(rows, key=sort_keys)
        for _, elements in groupby(rows, get_id):
            elements = [r for r in elements]
            if len(elements) == 1:
                # Só estamos interessados em mudanças de coalizão, então
                # ignoramos quem nunca mudou
                continue
            for i in range(1, len(elements)):
                before = elements[i - 1]
                actual = elements[i]
                if before["coalizao"] == actual["coalizao"]:
                    continue
                result.append(collections.OrderedDict([
                    ("id", before["id"]),
                    ("name", before["name"]),
                    ("party_before", before["party"]),
                    ("party_after", actual["party"]),
                    ("rollcall_id", actual["rollcall_id"]),
                    ("rollcall_date", actual["rollcall_date"]),
                    ("coalition_start_date", actual["coalition_start_date"]),
                    ("coalition_before", before["coalizao"]),
                    ("coalition_after", actual["coalizao"]),
                    ("legislature", actual["legislature"]),
                    ("legislature_year", actual["legislature_year"]),
                ]))
        return result

    def _add_extra_columns_and_convert_to_dict(self, votos_coalizao, partidos_coalizao):
        result = [collections.OrderedDict(v) for v in votos_coalizao]
        for voto in result:
            voto["coalizao"] = voto["party"] in partidos_coalizao
            voto["legislature"] = self._get_legislature(voto["rollcall_date"])
            voto["legislature_year"] = self._get_legislature_year(voto["rollcall_date"])
{'fraction_non_alphanumeric': 0.09515758966308417, 'fraction_numerical': 0.0012075836251660428, 'mean_word_length': 2.2001545595054095, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2223353', 'n_tokens_mistral': 2383, 'n_tokens_neox': 2256, 'n_words': 530}