<commit_before>from official_documents.models import OfficialDocument
from sopn_parsing.helpers.pdf_helpers import SOPNDocument
from sopn_parsing.helpers.text_helpers import NoTextInDocumentError


def extract_pages_for_ballot(ballot, manual_upload=False):
    """
    Try to extract the page numbers for the latest SOPN document related to this
    ballot.

    Because documents can apply to more than one ballot, we also perform
    "drive by" parsing of other ballots contained in a given document.

    The manual_upload arg is used to force full page number extraction of from
    the document when it is uploaded manually, in case the document covers
    multiple ballots but this is the first ballot where it is being uploaded

    :type ballot: candidates.models.Ballot

    """
    # extract_pages_for_single_document(
    #     document=ballot.sopn, manual_upload=manual_upload
    # )

    # check if this is the only document with this source url and if so attempt
    # some optimisations before we try and parse the page numbers
    try:
        document = OfficialDocument.objects.get(
            source_url=ballot.sopn.source_url
        )
    except OfficialDocument.MultipleObjectsReturned:
        document = ballot.sopn
    else:
        # if this isn't a manual upload we assume all pages relate to the ballot
        if not manual_upload:
            document.relevant_pages = "all"
            return document.save()

    # otherwise parse as if we had multiple sources as the SOPN may cover
    # multiple ballots, but this is the first time parsing it
    try:
        sopn = SOPNDocument(
            file=document.uploaded_file, source_url=document.source_url<commit_msg>Remove assumption about by-elections<commit_after>from official_documents.models import OfficialDocument
from sopn_parsing.helpers.pdf_helpers import SOPNDocument
from sopn_parsing.helpers.text_helpers import NoTextInDocumentError


def extract_pages_for_ballot(ballot, manual_upload=False):
    """
    Try to extract the page numbers for the latest SOPN document related to this
    ballot.

    Because documents can apply to more than one ballot, we also perform
    "drive by" parsing of other ballots contained in a given document.

    The manual_upload arg is used to force full page number extraction of from
    the document when it is uploaded manually, in case the document covers
    multiple ballots but this is the first ballot where it is being uploaded

    :type ballot: candidates.models.Ballot

    """
    # check if this is the only document with this source url and if so attempt
    # some optimisations before we try and parse the page numbers
    try:
        document = OfficialDocument.objects.get(
            source_url=ballot.sopn.source_url
        )
        # if this isn't a manual upload we assume all pages relate to the ballot
        # and can return early
        if not manual_upload:
            document.relevant_pages = "all"
            return document.save()
    except OfficialDocument.MultipleObjectsReturned:
        document = ballot.sopn

    # otherwise parse as if we had multiple sources as the SOPN may cover
    # multiple ballots, but this is the first time parsing it
    try:
        sopn = SOPNDocument(
            file=document.uploaded_file, source_url=document.source_url
{'fraction_non_alphanumeric': 0.04259148170365927, 'fraction_numerical': 0.0, 'mean_word_length': 3.158354114713217, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3099054', 'n_tokens_mistral': 885, 'n_tokens_neox': 844, 'n_words': 362}