<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Load CandidateContest and related models with data scraped from the CAL-ACCESS website.
"""
from opencivicdata.core.models import Membership
from calaccess_processed.models import OCDRunoffProxy
from calaccess_processed.models import ScrapedCandidateProxy
from calaccess_processed.models import ScrapedCandidateElectionProxy
from calaccess_processed.management.commands import LoadOCDModelsCommand


class Command(LoadOCDModelsCommand):
    """
    Load CandidateContest and related models with data scraped from the CAL-ACCESS website.
    """
    help = 'Load CandidateContest and related models with data scraped from the CAL-ACCESS website.'

    def handle(self, *args, **options):
        """
        Make it happen.
        """
        super(Command, self).handle(*args, **options)
        self.header("Load Candidate Contests")

        # Load everything we can from the scrape
        self.load()

        # connect runoffs to their previously undecided contests
        if self.verbosity > 2:
            self.log(' Linking runoffs to previous contests')
        OCDRunoffProxy.objects.set_parents()

        self.success("Done!")

    def load(self):
        """
        Load OCD Election, CandidateContest and related models with data scraped from CAL-ACCESS website.
        """
        # See if we should bother checking incumbent status
        members_are_loaded = Membership.objects.exists()

        # Loop over scraped_elections
        for scraped_election in ScrapedCandidateElectionProxy.objects.all():

            # then over candidates in the scraped_election
            scraped_candidate_list = ScrapedCandidateProxy.objects.filter(election=scraped_election)
            for scraped_candidate in scraped_candidate_list:
                if self.verbosity > 2:
                    self.log(
                        ' Processing scraped Candidate.id {} ({})'.format(
                            scraped_candidate.id,
                            scraped_candidate
                        )
                    )

                #
                # Make a Candidacy
                #

                candidacy, candidacy_created = scraped_candidate.get_or_create_candidacy(
                    registration_status='qualified'
                )

                if candidacy_created and self.verbosity > 2:
                    template = ' Created new Candidacy: {0.candidate_name} in {0.post.label}'
                    self.log(template.format(candidacy))

                #
                # Dress it up with extra stuff
                #

                # add extra data from form501, if available
                form501 = scraped_candidate.get_form501_filing()

                if form501:
                    candidacy.link_form501(form501)
                    candidacy.update_from_form501(form501)

                    # if the scraped_candidate lacks a filer_id, add the
                    # Form501Filing.filer_id
                    if scraped_candidate.scraped_id == '':
                        candidacy.person.identifiers.get_or_create(
                            scheme='calaccess_filer_id',
                            identifier=form501.filer_id,
                        )

                # Fill the party if the candidacy doesn't have it
                # Get the candidate's party, looking in our correction file for any fixes
                if not candidacy.party:
                    party = scraped_candidate.get_party()
                    if party:
                        candidacy.party = party
                        candidacy.save()

                # always update the source for the candidacy
                candidacy.sources.update_or_create(
                    url=scraped_candidate.url,
                    note='Last scraped on {dt:%Y-%m-%d}'.format(
                        dt=scraped_candidate.last_modified,
                    )
                )

                # check incumbent status
                if members_are_loaded and candidacy.check_incumbency():
                    candidacy.is_incumbent = True
                    candidacy.save()
                    if self.verbosity > 2:
                        self.log(' Identified as incumbent.')
                    # set is_incumbent False for all other candidacies
                    candidacy.contest.candidacies.exclude(
                        is_incumbent=True
                    ).update(is_incumbent=False)
<commit_msg>try again<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Load CandidateContest and related models with data scraped from the CAL-ACCESS website.
"""
from opencivicdata.core.models import Membership
from calaccess_processed.models import OCDRunoffProxy
from calaccess_processed.models import ScrapedCandidateProxy
from calaccess_processed.models import ScrapedCandidateElectionProxy
from calaccess_processed.management.commands import LoadOCDModelsCommand


class Command(LoadOCDModelsCommand):
    """
    Load CandidateContest and related models with data scraped from the CAL-ACCESS website.
    """
    help = 'Load CandidateContest and related models with data scraped from the CAL-ACCESS website.'

    def handle(self, *args, **options):
        """
        Make it happen.
        """
        super(Command, self).handle(*args, **options)
        self.header("Load Candidate Contests")

        # Load everything we can from the scrape
        self.load()

        # connect runoffs to their previously undecided contests
        if self.verbosity > 2:
            self.log(' Linking runoffs to previous contests')
        OCDRunoffProxy.objects.set_parents()

        self.success("Done!")

    def load(self):
        """
        Load OCD Election, CandidateContest and related models with data scraped from CAL-ACCESS website.
        """
        # See if we should bother checking incumbent status
        members_are_loaded = Membership.objects.exists()

        # Loop over scraped_elections
        for scraped_election in ScrapedCandidateElectionProxy.objects.all():

            # then over candidates in the scraped_election
            scraped_candidate_list = ScrapedCandidateProxy.objects.filter(election=scraped_election)
            for scraped_candidate in scraped_candidate_list:
                if self.verbosity > 2:
                    self.log(
                        ' Processing scraped Candidate.id {} ({})'.format(
                            scraped_candidate.id,
                            scraped_candidate
                        )
                    )

                #
                # Make a Candidacy
                #

                candidacy, candidacy_created = scraped_candidate.get_or_create_candidacy(
                    registration_status='qualified'
                )

                if candidacy_created and self.verbosity > 2:
                    msg = ' Created new Candidacy: {0.candidate_name} in {0.post.label}'.format(candidacy)
                    self.log(msg)

                #
                # Dress it up with extra stuff
                #

                # add extra data from form501, if available
                form501 = scraped_candidate.get_form501_filing()

                if form501:
                    candidacy.link_form501(form501)
                    candidacy.update_from_form501(form501)

                    # if the scraped_candidate lacks a filer_id, add the
                    # Form501Filing.filer_id
                    if scraped_candidate.scraped_id == '':
                        candidacy.person.identifiers.get_or_create(
                            scheme='calaccess_filer_id',
                            identifier=form501.filer_id,
                        )

                # Fill the party if the candidacy doesn't have it
                # Get the candidate's party, looking in our correction file for any fixes
                if not candidacy.party:
                    party = scraped_candidate.get_party()
                    if party:
                        candidacy.party = party
                        candidacy.save()

                # always update the source for the candidacy
                candidacy.sources.update_or_create(
                    url=scraped_candidate.url,
                    note='Last scraped on {dt:%Y-%m-%d}'.format(
                        dt=scraped_candidate.last_modified,
                    )
                )

                # check incumbent status
                if members_are_loaded and candidacy.check_incumbency():
                    candidacy.is_incumbent = True
                    candidacy.save()
                    if self.verbosity > 2:
                        self.log(' Identified as incumbent.')
                    # set is_incumbent False for all other candidacies
                    candidacy.contest.candidacies.exclude(
                        is_incumbent=True
                    ).update(is_incumbent=False)

{'fraction_non_alphanumeric': 0.06513070447496677, 'fraction_numerical': 0.00819672131147541, 'mean_word_length': 1.6020172910662824, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '829301', 'n_tokens_mistral': 2373, 'n_tokens_neox': 2160, 'n_words': 563}