<commit_before>import json
import os

from collections import Counter
from django.core.management import call_command

from bulk_adding.models import RawPeople
from candidates.models import Ballot
from popolo.models import Membership
from sopn_parsing.helpers.command_helpers import BaseSOPNParsingCommand


class Command(BaseSOPNParsingCommand):

    CORRECT_EXACTLY = "correct_exactly"
    NUM_CORRECT_MISSING_PARTIES = "num_correct_some_parties_missing"
    NUM_INCORRECT = "num_incorrect"
    ZERO_CANDIDATES = "zero_candidates"

    def add_arguments(self, parser):
        super().add_arguments(parser)
        parser.add_argument("--loud", action="store_true", default=False)

    def handle(self, *args, **options):
        """
        - Check we have a baseline file to compare with
        - Prepare some OfficialDocuments
        - Re-parse the documents
        - Loop through the created RawPeople objects, comparing to our baseline
        to make sure that we are parsing at least as many people as before
        - If no asserts failed, use the data to write a new baseline file
        """

        self.loud = options.pop("loud")

        self.candidates_results = {
            "correct_exactly": [],
            "num_correct_some_parties_missing": [],
            "num_incorrect": [],
            "zero_candidates": [],
        }

        raw_people_file = "ynr/apps/sopn_parsing/tests/data/sopn_baseline.json"
        if not os.path.isfile(raw_people_file):
            call_command("sopn_tooling_write_baseline")
            self.stdout.write("Baseline file didn't exist so one was created")

        options.update({"testing": True})

        call_command("sopn_parsing_extract_page_numbers", *args, **options)
        call_command("sopn_parsing_extract_tables", *args, **options)
        call_command("sopn_parsing_parse_tables", *args, **options)

        with open(raw_people_file) as file:
            old_raw_people = json.loads(file.read())

        new_raw_people = {}
        for ballot in Ballot.objects.exclude(officialdocument__isnull=True):

            try:
                raw_people = ballot.rawpeople.data
            except RawPeople.DoesNotExist:
                raw_people = []

            old_raw_people_for_ballot = old_raw_people.get(
                ballot.ballot_paper_id, []
            )
            old_count = len(old_raw_people_for_ballot)
            new_count = len(raw_people)
            if new_count < old_count:
                self.stderr.write(
                    f"Uh oh, parsed people for {ballot.ballot_paper_id} decreased from {old_count} to {new_count}. Stopping."
                )<commit_msg>Clear existing objects before comparing SOPN parsing changes<commit_after>import json
import os

from collections import Counter
from django.core.management import call_command

from bulk_adding.models import RawPeople
from candidates.models import Ballot
from official_documents.models import OfficialDocument
from popolo.models import Membership
from sopn_parsing.helpers.command_helpers import BaseSOPNParsingCommand
from sopn_parsing.models import ParsedSOPN


class Command(BaseSOPNParsingCommand):

    CORRECT_EXACTLY = "correct_exactly"
    NUM_CORRECT_MISSING_PARTIES = "num_correct_some_parties_missing"
    NUM_INCORRECT = "num_incorrect"
    ZERO_CANDIDATES = "zero_candidates"

    def add_arguments(self, parser):
        super().add_arguments(parser)
        parser.add_argument("--loud", action="store_true", default=False)

    def handle(self, *args, **options):
        """
        - Check we have a baseline file to compare with
        - Prepare some OfficialDocuments
        - Re-parse the documents
        - Loop through the created RawPeople objects, comparing to our baseline
        to make sure that we are parsing at least as many people as before
        - If no asserts failed, use the data to write a new baseline file
        """

        self.loud = options.pop("loud")

        self.candidates_results = {
            "correct_exactly": [],
            "num_correct_some_parties_missing": [],
            "num_incorrect": [],
            "zero_candidates": [],
        }

        raw_people_file = "ynr/apps/sopn_parsing/tests/data/sopn_baseline.json"
        if not os.path.isfile(raw_people_file):
            call_command("sopn_tooling_write_baseline")
            self.stdout.write("Baseline file didn't exist so one was created")

        options.update({"testing": True})

        OfficialDocument.objects.update(relevant_pages="")
        call_command("sopn_parsing_extract_page_numbers", *args, **options)
        ParsedSOPN.objects.all().delete()
        call_command("sopn_parsing_extract_tables", *args, **options)
        RawPeople.objects.all().delete()
        call_command("sopn_parsing_parse_tables", *args, **options)

        with open(raw_people_file) as file:
            old_raw_people = json.loads(file.read())

        new_raw_people = {}
        for ballot in Ballot.objects.exclude(officialdocument__isnull=True):

            try:
                raw_people = ballot.rawpeople.data
            except RawPeople.DoesNotExist:
                raw_people = []

            old_raw_people_for_ballot = old_raw_people.get(
                ballot.ballot_paper_id, []
            )
            old_count = len(old_raw_people_for_ballot)
            new_count = len(raw_people)
            if new_count < old_count:
                self.stderr.write(
                    f"Uh oh, parsed people for {ballot.ballot_paper_id} decreased from {old_count} to {new_count}. Stopping."
                )
{'fraction_non_alphanumeric': 0.08397218755571403, 'fraction_numerical': 0.0, 'mean_word_length': 2.68836291913215, 'pattern_counts': {'":': 10, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '681916', 'n_tokens_mistral': 1683, 'n_tokens_neox': 1594, 'n_words': 364}