<commit_before>import csv
import os

from django.core.management.base import BaseCommand

import resultsbot
from elections.models import Election
from resultsbot.helpers import ResultsBot
from resultsbot.importers.modgov import ModGovImporter


class Command(BaseCommand):
    def handle(self, **options):
        id_to_url = {}

        path = os.path.join(
            os.path.dirname(resultsbot.__file__), "election_id_to_url.csv"
        )
        with open(path) as f:
            csv_file = csv.reader(f)
            for line in csv_file:
                if not line:
                    continue
                id_to_url[line[0]] = line[1]

        all_candidates = []

        for election_id, url in id_to_url.items():
            election = Election.objects.get(slug=election_id)
            if not election.current:
                continue
            print(election_id, url)
            importer = ModGovImporter(election_id, url=url)
            ballot_with_result = [
                hasattr(ballot, "resultset")
                for ballot in importer.election.ballot_set.all()
            ]
            if all(ballot_with_result):
                continue
            importer.get_data()
            for div in importer.divisions():
                if (
                    div.local_area.ballot_paper_id
                    == "local.tower-hamlets.whitechapel.2018-05-03"
                ):
                    continue
                if (
                    div.local_area.ballot_paper_id
                    == "local.eastleigh.eastleigh-north.2018-05-03"
                ):
                    continue

                if hasattr(div.local_area, "resultset"):
                    continue

                candidates = list(importer.candidates(div))
                candidates_len = len(candidates)
                if candidates_len != div.local_area.membership_set.count():
                    print(
                        f"Only found {candidates_len} candidates for {div.local_area.ballot_paper_id}, skipping"
                    )
                    continue

                # avoids storing a partial result
                has_all_votes = all([c.votes for c in candidates])
                if not has_all_votes:
                    print(
                        f"Couldn't find a vote count for every candidate, skipping {div.local_area.ballot_paper_id}"
                    )
                    continue

                all_candidates += candidates
                print("Adding results for {}".format(div.title))
                bot = ResultsBot()
                bot.add_results(
                    division=div,
                    candidate_list=candidates,<commit_msg>Skip berrylands ballot in resultsbot importer<commit_after>import csv
import os

from django.core.management.base import BaseCommand

import resultsbot
from elections.models import Election
from resultsbot.helpers import ResultsBot
from resultsbot.importers.modgov import ModGovImporter


class Command(BaseCommand):
    def handle(self, **options):
        id_to_url = {}

        path = os.path.join(
            os.path.dirname(resultsbot.__file__), "election_id_to_url.csv"
        )
        with open(path) as f:
            csv_file = csv.reader(f)
            for line in csv_file:
                if not line:
                    continue
                id_to_url[line[0]] = line[1]

        all_candidates = []

        for election_id, url in id_to_url.items():
            election = Election.objects.get(slug=election_id)
            if not election.current:
                continue
            print(election_id, url)
            importer = ModGovImporter(election_id, url=url)
            ballot_with_result = [
                hasattr(ballot, "resultset")
                for ballot in importer.election.ballot_set.all()
            ]
            if all(ballot_with_result):
                continue
            importer.get_data()
            for div in importer.divisions():
                if (
                    div.local_area.ballot_paper_id
                    == "local.tower-hamlets.whitechapel.2018-05-03"
                ):
                    continue
                if (
                    div.local_area.ballot_paper_id
                    == "local.eastleigh.eastleigh-north.2018-05-03"
                ):
                    continue

                if (
                    div.local_area.ballot_paper_id
                    == "local.kingston-upon-thames.berrylands.2022-05-05"
                ):
                    continue

                if hasattr(div.local_area, "resultset"):
                    continue

                candidates = list(importer.candidates(div))
                candidates_len = len(candidates)
                if candidates_len != div.local_area.membership_set.count():
                    print(
                        f"Only found {candidates_len} candidates for {div.local_area.ballot_paper_id}, skipping"
                    )
                    continue

                # avoids storing a partial result
                has_all_votes = all([c.votes for c in candidates])
                if not has_all_votes:
                    print(
                        f"Couldn't find a vote count for every candidate, skipping {div.local_area.ballot_paper_id}"
                    )
                    continue

                all_candidates += candidates
                print("Adding results for {}".format(div.title))
                bot = ResultsBot()
                bot.add_results(
                    division=div,
                    candidate_list=candidates,
{'fraction_non_alphanumeric': 0.07467704831003362, 'fraction_numerical': 0.007786232525216776, 'mean_word_length': 1.369811320754717, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1465454', 'n_tokens_mistral': 1570, 'n_tokens_neox': 1407, 'n_words': 298}