<commit_before>from __future__ import division
from django.conf import settings
from django.core.exceptions import ObjectDoesNotExist
from django.utils.timezone import localtime
from ballot_box.utils_files import Retriever
from ballot_box.utils_data import Framer
from ballot_box.utils_import import Saver
from ballot_box.models import ResultSource, Election
import logging
import time
import datetime
import os.path
import shutil
from bs4 import BeautifulSoup
from delorean import parse
from slugify import slugify

logger = logging.getLogger("kpcc_backroom_handshakes")


class BuildOcResults(object):
    """
    scaffolding to ingest secretary of state election results
    """

    retrieve = Retriever()

    data_directory = "%s/ballot_box/data_dump/" % (settings.BASE_DIR)

    sources = ResultSource.objects.filter(source_short="oc", source_active=True)

    race_ids = [
        "742",
        "743",
        "745",
        "746",
        "747",
        "748",
        "749",
        "750",
    ]

    def _init(self, *args, **kwargs):
        """
        """
        for src in self.sources:
            self.get_results_file(src, self.data_directory)
            self.parse_results_file(src, self.data_directory)
        self.retrieve._build_and_move_results()

    def get_results_file(self, src, data_directory):
        """
        """
        self.retrieve._request_results_and_save(src, data_directory)
        self.retrieve._create_directory_for_latest_file(src, data_directory)
        self.retrieve._copy_timestamped_file_to_latest(src, data_directory)
        self.retrieve._archive_downloaded_file(src, data_directory)
        self.retrieve._found_required_files(src, data_directory)
        self.retrieve._unzip_latest_file(src, data_directory)
        self.retrieve.log_message += "*** Ending Request ***\n"

    def parse_results_file(self, src, data_directory):
        """
        """
        saver = Saver()
        compiler = BuildResults()
        latest_directory = "%s%s_latest" % (data_directory, src.source_short)
        election = Election.objects.filter(
            test_results=True, electionid=src.election.electionid).first()
        for file in src.source_files.split(", "):
            latest_path = os.path.join(latest_directory, file)
            file_exists = os.path.isfile(latest_path)
            file_has_size = os.path.getsize(latest_path)
            if file_exists == True and file_has_size > 0:
                soup = BeautifulSoup(open(latest_path), "xml")
                file_timestring = unicode(soup.CumulativeRptHeader.ReportHeader.attrs["run_date"])
                file_timestamp = parse(file_timestring, dayfirst=False).datetime
                update_this = saver._eval_timestamps(file_timestamp, src.source_latest)
                update_this = election.test_results
                if update_this == False:
                    logger.debug(
                        "\n*****\nwe have newer data in the database so let's delete these files\n*****")
                    os.remove(latest_path)
                else:
                    logger.debug(
                        "\n*****\nwe have new data to save and we'll update timestamps in the database\n*****")
                    saver._update_result_timestamps(src, file_timestamp)
                    races = soup.find_all("ContestTotal")
                    race_log = "\n"
                    for race in races:
                        if race.attrs["race_id"] in self.race_ids:
                            if race.attrs["contest_type"] == "MS":
                                """
                                this is a proposition
                                """
                                result = compiler._compile_measure(race, election, src)
                                # race_log += saver.make_office(result.office)
                                # race_log += saver.make_contest(result.office, result.contest)
                                race_log += saver.make_measure(result.contest, result.measure)
                            else:
                                """
                                this is a non-judicial candidate
                                """
                                result = compiler._compile_candidate(race, election, src)
                                # race_log += saver.make_office(result.office)
                                # race_log += saver.make_contest(result.office, result.contest)
                                for candidate in result.candidates:
                                    race_log += saver.make_candidate(result.contest, candidate)
                    logger.debug(race_log)
                    os.remove(latest_path)
                    logger.debug("\n*****\nwe've finished processing sos results\n*****")
            else:
                logger.error("XML file to parse is not at expected location")


class BuildResults(object):
    """<commit_msg>restores contest save which stores the precinct reporting data for oc<commit_after>from __future__ import division
from django.conf import settings
from django.core.exceptions import ObjectDoesNotExist
from django.utils.timezone import localtime
from ballot_box.utils_files import Retriever
from ballot_box.utils_data import Framer
from ballot_box.utils_import import Saver
from ballot_box.models import ResultSource, Election
import logging
import time
import datetime
import os.path
import shutil
from bs4 import BeautifulSoup
from delorean import parse
from slugify import slugify

logger = logging.getLogger("kpcc_backroom_handshakes")


class BuildOcResults(object):
    """
    scaffolding to ingest secretary of state election results
    """

    retrieve = Retriever()

    data_directory = "%s/ballot_box/data_dump/" % (settings.BASE_DIR)

    sources = ResultSource.objects.filter(source_short="oc", source_active=True)

    race_ids = [
        "742",
        "743",
        "745",
        "746",
        "747",
        "748",
        "749",
        "750",
    ]

    def _init(self, *args, **kwargs):
        """
        """
        for src in self.sources:
            self.get_results_file(src, self.data_directory)
            self.parse_results_file(src, self.data_directory)
        self.retrieve._build_and_move_results()

    def get_results_file(self, src, data_directory):
        """
        """
        self.retrieve._request_results_and_save(src, data_directory)
        self.retrieve._create_directory_for_latest_file(src, data_directory)
        self.retrieve._copy_timestamped_file_to_latest(src, data_directory)
        self.retrieve._archive_downloaded_file(src, data_directory)
        self.retrieve._found_required_files(src, data_directory)
        self.retrieve._unzip_latest_file(src, data_directory)
        self.retrieve.log_message += "*** Ending Request ***\n"

    def parse_results_file(self, src, data_directory):
        """
        """
        saver = Saver()
        compiler = BuildResults()
        latest_directory = "%s%s_latest" % (data_directory, src.source_short)
        election = Election.objects.filter(
            test_results=True, electionid=src.election.electionid).first()
        for file in src.source_files.split(", "):
            latest_path = os.path.join(latest_directory, file)
            file_exists = os.path.isfile(latest_path)
            file_has_size = os.path.getsize(latest_path)
            if file_exists == True and file_has_size > 0:
                soup = BeautifulSoup(open(latest_path), "xml")
                file_timestring = unicode(soup.CumulativeRptHeader.ReportHeader.attrs["run_date"])
                file_timestamp = parse(file_timestring, dayfirst=False).datetime
                update_this = saver._eval_timestamps(file_timestamp, src.source_latest)
                update_this = election.test_results
                if update_this == False:
                    logger.debug(
                        "\n*****\nwe have newer data in the database so let's delete these files\n*****")
                    os.remove(latest_path)
                else:
                    logger.debug(
                        "\n*****\nwe have new data to save and we'll update timestamps in the database\n*****")
                    saver._update_result_timestamps(src, file_timestamp)
                    races = soup.find_all("ContestTotal")
                    race_log = "\n"
                    for race in races:
                        if race.attrs["race_id"] in self.race_ids:
                            if race.attrs["contest_type"] == "MS":
                                """
                                this is a proposition
                                """
                                result = compiler._compile_measure(race, election, src)
                                # race_log += saver.make_office(result.office)
                                race_log += saver.make_contest(result.office, result.contest)
                                race_log += saver.make_measure(result.contest, result.measure)
                            else:
                                """
                                this is a non-judicial candidate
                                """
                                result = compiler._compile_candidate(race, election, src)
                                # race_log += saver.make_office(result.office)
                                race_log += saver.make_contest(result.office, result.contest)
                                for candidate in result.candidates:
                                    race_log += saver.make_candidate(result.contest, candidate)
                    logger.debug(race_log)
                    os.remove(latest_path)
                    logger.debug("\n*****\nwe've finished processing sos results\n*****")
            else:
                logger.error("XML file to parse is not at expected location")


class BuildResults(object):
    """
{'fraction_non_alphanumeric': 0.09049049049049049, 'fraction_numerical': 0.005205205205205205, 'mean_word_length': 1.8570203031169574, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '949625', 'n_tokens_mistral': 2755, 'n_tokens_neox': 2588, 'n_words': 548}