<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Unittests for management commands.
"""
import os
import shutil
import requests_mock

# Time
import time
from email.utils import formatdate
from django.utils.timezone import now

# Django bits
from django.conf import settings
from django.core.management import call_command
from django.test import TestCase, override_settings, TransactionTestCase
from django.core.management.base import CommandError

# Models
from django.db import models
from calaccess_raw.models import RawDataVersion
from calaccess_processed.models import ProcessedDataVersion
from calaccess_scraped.models import Candidate as ScrapedCandidate
from calaccess_scraped.models import Proposition as ScrapedProposition
from opencivicdata.core.models import Person
from opencivicdata.elections.models import (
    BallotMeasureContest,
    Candidacy,
    CandidateContest
)


class NoProcessedDataTest(TestCase):
    """
    Tests to run with no data loaded.
    """
    def test_no_raw_data(self):
        """
        Confirm process command will not run without data.
        """
        with self.assertRaises(CommandError):
            call_command("loadcalaccessscrapeddata", verbosity=3)
            call_command("processcalaccessdata", verbosity=3)


@override_settings(CALACCESS_DATA_DIR=os.path.join(settings.BASE_DIR, 'test-data'))
@override_settings(MEDIA_ROOT=os.path.join(settings.BASE_DIR, 'test-data', ".media"))
@override_settings(CALACCESS_STORE_ARCHIVE=True)
class ProcessedDataTest(TransactionTestCase):
    """
    Run and test management commands.
    """
    fixtures = [
        'divisions.json',
        'candidate_election.json',
        'candidate.json',
        'incumbent_election.json',
        'incumbent.json',
        'proposition_election.json',
        'proposition.json',
    ]

    @classmethod
    @requests_mock.Mocker()
    def setUpClass(cls, m):
        """
        Load data for other tests.
        """
        super(ProcessedDataTest, cls).setUpClass()

        # fake a previous raw data download
        download_dir = os.path.join(settings.CALACCESS_DATA_DIR, 'download')
        os.path.exists(download_dir) or os.mkdir(download_dir)
        zip_path = os.path.join(
            settings.CALACCESS_DATA_DIR,
            'dbwebexport.zip',
        )
        shutil.copy(zip_path, download_dir)
        rdv = RawDataVersion.objects.create(
            release_datetime=now(),
            update_start_datetime=now(),
            download_start_datetime=now(),
            download_finish_datetime=now(),
            expected_size=os.stat(zip_path).st_size,
        )
        # mock an SoS HEAD response
        imf_datetime = formatdate(
            time.mktime(rdv.release_datetime.timetuple()),
            usegmt=True,
        )
        headers = {
            'Content-Length': str(rdv.expected_size),
            'Accept-Ranges': 'bytes',
            'Last-Modified': imf_datetime,
            'Connection': 'keep-alive',
            'Date': imf_datetime,
            'Content-Type': 'application/zip',
            'ETag': '2320c8-30619331-c54f7dc0',
            'Server': 'Apache/2.2.3 (Red Hat)',
        }
        m.register_uri(
            'HEAD',
            'https://campaignfinance.cdn.sos.ca.gov/dbwebexport.zip',
            headers=headers,
        )

        call_command("updatecalaccessrawdata", verbosity=3, noinput=True)
        call_command("processcalaccessdata", verbosity=3)

    def runTest(self):
        """
        Added to allow calling test_ methods on instance of this class in py2.
        """<commit_msg>Put it in the right place<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Unittests for management commands.
"""
import os
import shutil
import requests_mock

# Time
import time
from email.utils import formatdate
from django.utils.timezone import now

# Django bits
from django.conf import settings
from django.core.management import call_command
from django.test import TestCase, override_settings, TransactionTestCase
from django.core.management.base import CommandError

# Models
from django.db import models
from calaccess_raw.models import RawDataVersion
from calaccess_processed.models import ProcessedDataVersion
from calaccess_scraped.models import Candidate as ScrapedCandidate
from calaccess_scraped.models import Proposition as ScrapedProposition
from opencivicdata.core.models import Person
from opencivicdata.elections.models import (
    BallotMeasureContest,
    Candidacy,
    CandidateContest
)


class NoProcessedDataTest(TestCase):
    """
    Tests to run with no data loaded.
    """
    def test_no_raw_data(self):
        """
        Confirm process command will not run without data.
        """
        with self.assertRaises(CommandError):
            call_command("processcalaccessdata", verbosity=3)


@override_settings(CALACCESS_DATA_DIR=os.path.join(settings.BASE_DIR, 'test-data'))
@override_settings(MEDIA_ROOT=os.path.join(settings.BASE_DIR, 'test-data', ".media"))
@override_settings(CALACCESS_STORE_ARCHIVE=True)
class ProcessedDataTest(TransactionTestCase):
    """
    Run and test management commands.
    """
    fixtures = [
        'divisions.json',
        'candidate_election.json',
        'candidate.json',
        'incumbent_election.json',
        'incumbent.json',
        'proposition_election.json',
        'proposition.json',
    ]

    @classmethod
    @requests_mock.Mocker()
    def setUpClass(cls, m):
        """
        Load data for other tests.
        """
        super(ProcessedDataTest, cls).setUpClass()

        # fake a previous raw data download
        download_dir = os.path.join(settings.CALACCESS_DATA_DIR, 'download')
        os.path.exists(download_dir) or os.mkdir(download_dir)
        zip_path = os.path.join(
            settings.CALACCESS_DATA_DIR,
            'dbwebexport.zip',
        )
        shutil.copy(zip_path, download_dir)
        rdv = RawDataVersion.objects.create(
            release_datetime=now(),
            update_start_datetime=now(),
            download_start_datetime=now(),
            download_finish_datetime=now(),
            expected_size=os.stat(zip_path).st_size,
        )
        # mock an SoS HEAD response
        imf_datetime = formatdate(
            time.mktime(rdv.release_datetime.timetuple()),
            usegmt=True,
        )
        headers = {
            'Content-Length': str(rdv.expected_size),
            'Accept-Ranges': 'bytes',
            'Last-Modified': imf_datetime,
            'Connection': 'keep-alive',
            'Date': imf_datetime,
            'Content-Type': 'application/zip',
            'ETag': '2320c8-30619331-c54f7dc0',
            'Server': 'Apache/2.2.3 (Red Hat)',
        }
        m.register_uri(
            'HEAD',
            'https://campaignfinance.cdn.sos.ca.gov/dbwebexport.zip',
            headers=headers,
        )

        call_command("updatecalaccessrawdata", verbosity=3, noinput=True)
        call_command("loadcalaccessscrapeddata", verbosity=3)
        call_command("processcalaccessdata", verbosity=3)

    def runTest(self):
        """
        Added to allow calling test_ methods on instance of this class in py2.
        """
{'fraction_non_alphanumeric': 0.10035941387890517, 'fraction_numerical': 0.007188277578103401, 'mean_word_length': 2.7643080124869925, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1205214', 'n_tokens_mistral': 2130, 'n_tokens_neox': 2085, 'n_words': 435}