<commit_before>import datetime
import logging

from django.db import models
from django.db import transaction
from django.utils import timezone

from person.models import Person

from document.models import Kamervraag
from document.models import Kamerantwoord
from document.models import Vote
from document.models import Voting
from document.models import Submitter

from government.models import Government
from government.models import Ministry
from parliament.models import PoliticalParty
from parliament.models import Parliament
from parliament.models import ParliamentMember

from stats import util

from stats.plots import PlotKamervraagVsTime
from stats.plots import PlotKamervraagVsTimePerParty
from stats.plots import PlotKamervraagReplyTimeContour
from stats.plots import PlotKamervraagReplyTimeHistogram
from stats.plots import PlotKamervraagVsTimePerPartySeat
from stats.plots import PlotPartySeatsVsTime
from stats.plots import kamervragen_reply_time_per_party
from stats.plots import kamervragen_reply_time_per_ministry
from stats.plots import kamervragen_reply_time_per_year


logger = logging.getLogger(__name__)


def update_all():
    logger.info('BEGIN')
    start_date = datetime.date(year=2010, month=1, day=1)
    SeatsPerParty.create_all(start_date)
    StatsVotingSubmitter.create()
    PartyVoteBehaviour.create_all()
    Plot.create()
    logger.info('END')


class PartyVoteBehaviour(models.Model):
    BILL = 'BILL'
    OTHER = 'OTHER'
    VOTING_TYPE_CHOICES = (
        (BILL, 'Wetsvoorstel'),
        (OTHER, 'Overig (Motie, Amendement)')
    )
    party = models.ForeignKey(PoliticalParty)
    submitter = models.ForeignKey(PoliticalParty, related_name='party_vote_behaviour_submitter', blank=True, null=True)
    government = models.ForeignKey(Government)
    voting_type = models.CharField(max_length=5, choices=VOTING_TYPE_CHOICES)
    votes_for = models.IntegerField()
    votes_against = models.IntegerField()
    votes_none = models.IntegerField()

    def total_votes(self):
        return self.votes_for + self.votes_against + self.votes_none

    @staticmethod
    def get_stats_party(party, government=None, voting_type=None):
        """
        Returns voting behaviour stats for a given party during a given government period.

        :param party: the party for which to get the stats
        :param government: (optional) the government period for which to get the stats
        :param voting_type: (optional) the type of voting
        :return: a dictionary with basic voting stats
        """
        vote_behaviours = PartyVoteBehaviour.objects.filter(submitter__isnull=True)
        vote_behaviours = PartyVoteBehaviour.filter_by_type_and_government(vote_behaviours, voting_type=voting_type, government=government)
        return PartyVoteBehaviour.get_stats_party_for_qs(party, vote_behaviours)

    @staticmethod
    def get_stats_party_for_submitter(party_voting, party_submitting, government=None, voting_type=None):
        """
        Returns voting behaviour stats for a given party, for a given submitting party, during a given government period.

        :param party_voting: the party for which to get the stats
        :param party_submitting: the party who submitted/initiated the voting (or related document or bill)
        :param government: the government period for which to get the stats
        :param voting_type: (optional) the type of voting
        :return: a dictionary with basic voting stats
        """
        vote_behaviours = PartyVoteBehaviour.objects.filter(submitter=party_submitting)
        vote_behaviours = PartyVoteBehaviour.filter_by_type_and_government(vote_behaviours, voting_type=voting_type, government=government)
        return PartyVoteBehaviour.get_stats_party_for_qs(party_voting, vote_behaviours)

    @staticmethod
    def filter_by_type_and_government(vote_behaviours, voting_type=None, government=None):
        if government:
            vote_behaviours = vote_behaviours.filter(government=government)
        if voting_type:
            vote_behaviours = vote_behaviours.filter(voting_type=voting_type)
        return vote_behaviours

    @staticmethod
    @transaction.atomic
    def create_all():
        logger.info('BEGIN')
        vote_submitters = StatsVotingSubmitter.objects.all()
        party_ids = list(vote_submitters.values_list('party__id', flat=True))
        PartyVoteBehaviour.objects.all().delete()
        parties = PoliticalParty.objects.filter(id__in=party_ids)
        for party in parties:
            PartyVoteBehaviour.create(party)
        logger.info('END, number of objects created: ' + str(PartyVoteBehaviour.objects.all().count()))

    @staticmethod
    @transaction.atomic
    def create(party):
        logger.info('BEGIN for party:', party)
        governments = Government.objects.all()
        party_votes_per_gov = []
        for gov in governments:
            party_votes_per_gov.append({'government': gov, 'party_votes': util.get_party_votes_for_government(gov)})
        stats = []
        parties = PoliticalParty.objects.all()
        for party_submitting in parties:
            PartyVoteBehaviour.create_for_submitting_party(party, party_submitting, party_votes_per_gov)
        PartyVoteBehaviour.create_for_submitting_party(party, None, party_votes_per_gov)
        logger.info('END for party:', party)
        return stats

    @staticmethod
    def create_for_submitting_party(party, party_submitting, party_votes_per_gov):
        for votes_for_gov in party_votes_per_gov:
            for voting_type in PartyVoteBehaviour.VOTING_TYPE_CHOICES:
                PartyVoteBehaviour.create_party_type_gov(
                    party=party,
                    party_votes=votes_for_gov['party_votes'],
                    party_submitting=party_submitting,
                    government=votes_for_gov['government'],
                    voting_type=voting_type[0]<commit_msg>Fix log statement<commit_after>import datetime
import logging

from django.db import models
from django.db import transaction
from django.utils import timezone

from person.models import Person

from document.models import Kamervraag
from document.models import Kamerantwoord
from document.models import Vote
from document.models import Voting
from document.models import Submitter

from government.models import Government
from government.models import Ministry
from parliament.models import PoliticalParty
from parliament.models import Parliament
from parliament.models import ParliamentMember

from stats import util

from stats.plots import PlotKamervraagVsTime
from stats.plots import PlotKamervraagVsTimePerParty
from stats.plots import PlotKamervraagReplyTimeContour
from stats.plots import PlotKamervraagReplyTimeHistogram
from stats.plots import PlotKamervraagVsTimePerPartySeat
from stats.plots import PlotPartySeatsVsTime
from stats.plots import kamervragen_reply_time_per_party
from stats.plots import kamervragen_reply_time_per_ministry
from stats.plots import kamervragen_reply_time_per_year


logger = logging.getLogger(__name__)


def update_all():
    logger.info('BEGIN')
    start_date = datetime.date(year=2010, month=1, day=1)
    SeatsPerParty.create_all(start_date)
    StatsVotingSubmitter.create()
    PartyVoteBehaviour.create_all()
    Plot.create()
    logger.info('END')


class PartyVoteBehaviour(models.Model):
    BILL = 'BILL'
    OTHER = 'OTHER'
    VOTING_TYPE_CHOICES = (
        (BILL, 'Wetsvoorstel'),
        (OTHER, 'Overig (Motie, Amendement)')
    )
    party = models.ForeignKey(PoliticalParty)
    submitter = models.ForeignKey(PoliticalParty, related_name='party_vote_behaviour_submitter', blank=True, null=True)
    government = models.ForeignKey(Government)
    voting_type = models.CharField(max_length=5, choices=VOTING_TYPE_CHOICES)
    votes_for = models.IntegerField()
    votes_against = models.IntegerField()
    votes_none = models.IntegerField()

    def total_votes(self):
        return self.votes_for + self.votes_against + self.votes_none

    @staticmethod
    def get_stats_party(party, government=None, voting_type=None):
        """
        Returns voting behaviour stats for a given party during a given government period.

        :param party: the party for which to get the stats
        :param government: (optional) the government period for which to get the stats
        :param voting_type: (optional) the type of voting
        :return: a dictionary with basic voting stats
        """
        vote_behaviours = PartyVoteBehaviour.objects.filter(submitter__isnull=True)
        vote_behaviours = PartyVoteBehaviour.filter_by_type_and_government(vote_behaviours, voting_type=voting_type, government=government)
        return PartyVoteBehaviour.get_stats_party_for_qs(party, vote_behaviours)

    @staticmethod
    def get_stats_party_for_submitter(party_voting, party_submitting, government=None, voting_type=None):
        """
        Returns voting behaviour stats for a given party, for a given submitting party, during a given government period.

        :param party_voting: the party for which to get the stats
        :param party_submitting: the party who submitted/initiated the voting (or related document or bill)
        :param government: the government period for which to get the stats
        :param voting_type: (optional) the type of voting
        :return: a dictionary with basic voting stats
        """
        vote_behaviours = PartyVoteBehaviour.objects.filter(submitter=party_submitting)
        vote_behaviours = PartyVoteBehaviour.filter_by_type_and_government(vote_behaviours, voting_type=voting_type, government=government)
        return PartyVoteBehaviour.get_stats_party_for_qs(party_voting, vote_behaviours)

    @staticmethod
    def filter_by_type_and_government(vote_behaviours, voting_type=None, government=None):
        if government:
            vote_behaviours = vote_behaviours.filter(government=government)
        if voting_type:
            vote_behaviours = vote_behaviours.filter(voting_type=voting_type)
        return vote_behaviours

    @staticmethod
    @transaction.atomic
    def create_all():
        logger.info('BEGIN')
        vote_submitters = StatsVotingSubmitter.objects.all()
        party_ids = list(vote_submitters.values_list('party__id', flat=True))
        PartyVoteBehaviour.objects.all().delete()
        parties = PoliticalParty.objects.filter(id__in=party_ids)
        for party in parties:
            PartyVoteBehaviour.create(party)
        logger.info('END, number of objects created: ' + str(PartyVoteBehaviour.objects.all().count()))

    @staticmethod
    @transaction.atomic
    def create(party):
        logger.info('BEGIN for party: {}'.format(party))
        governments = Government.objects.all()
        party_votes_per_gov = []
        for gov in governments:
            party_votes_per_gov.append({'government': gov, 'party_votes': util.get_party_votes_for_government(gov)})
        stats = []
        parties = PoliticalParty.objects.all()
        for party_submitting in parties:
            PartyVoteBehaviour.create_for_submitting_party(party, party_submitting, party_votes_per_gov)
        PartyVoteBehaviour.create_for_submitting_party(party, None, party_votes_per_gov)
        logger.info('END for party: {}'.format(party))
        return stats

    @staticmethod
    def create_for_submitting_party(party, party_submitting, party_votes_per_gov):
        for votes_for_gov in party_votes_per_gov:
            for voting_type in PartyVoteBehaviour.VOTING_TYPE_CHOICES:
                PartyVoteBehaviour.create_party_type_gov(
                    party=party,
                    party_votes=votes_for_gov['party_votes'],
                    party_submitting=party_submitting,
                    government=votes_for_gov['government'],
                    voting_type=voting_type[0]
{'fraction_non_alphanumeric': 0.06921522398113843, 'fraction_numerical': 0.0013472549680026945, 'mean_word_length': 3.8936959208899875, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '605004', 'n_tokens_mistral': 3578, 'n_tokens_neox': 3379, 'n_words': 783}