<commit_before># import_export_ballotpedia/views_admin.py
# Brought to you by We Vote. Be good.
# -*- coding: UTF-8 -*-

from .controllers import attach_ballotpedia_election_by_district_from_api, \
    retrieve_ballot_items_from_polling_location, \
    retrieve_ballotpedia_candidates_by_district_from_api, retrieve_ballotpedia_measures_by_district_from_api, \
    retrieve_ballotpedia_district_id_list_for_polling_location, retrieve_ballotpedia_offices_by_district_from_api
#    retrieve_ballotpedia_offices_by_election_from_api
from admin_tools.views import redirect_to_sign_in_page
from config.base import get_environment_variable
from datetime import date
from django.contrib import messages
from django.contrib.auth.decorators import login_required
from django.contrib.messages import get_messages
from django.core.urlresolvers import reverse
from django.db.models import Q
from django.http import HttpResponseRedirect
from django.shortcuts import redirect, render
from election.models import Election, ElectionManager
from import_export_batches.models import BatchSet, BATCH_SET_SOURCE_IMPORT_BALLOTPEDIA_BALLOT_ITEMS
from polling_location.models import PollingLocation
from voter.models import voter_has_authority
import wevote_functions.admin
from wevote_functions.functions import convert_to_int, is_valid_state_code, positive_value_exists

logger = wevote_functions.admin.get_logger(__name__)

BALLOTPEDIA_API_CONTAINS_URL = get_environment_variable("BALLOTPEDIA_API_CONTAINS_URL")

CANDIDATE = 'CANDIDATE'
CONTEST_OFFICE = 'CONTEST_OFFICE'
ELECTED_OFFICE = 'ELECTED_OFFICE'
IMPORT_BALLOT_ITEM = 'IMPORT_BALLOT_ITEM'
IMPORT_VOTER = 'IMPORT_VOTER'
MEASURE = 'MEASURE'
POLITICIAN = 'POLITICIAN'


@login_required
def import_ballot_items_for_location_view(request):
    """
    Reach out to Ballotpedia API to retrieve a short list of districts the voter can vote in.
    """
    # admin, partner_organization, political_data_manager, political_data_viewer, verified_volunteer
    authority_required = {'political_data_manager'}
    if not voter_has_authority(request, authority_required):
        return redirect_to_sign_in_page(request, authority_required)

    google_civic_election_id = convert_to_int(request.GET.get('google_civic_election_id', 0))
    polling_location_we_vote_id = request.GET.get('polling_location_we_vote_id', "")
    state_code = request.GET.get('state_code', "")

    results = retrieve_ballot_items_from_polling_location(
        google_civic_election_id, polling_location_we_vote_id, state_code=state_code)

    kind_of_batch = ""
    if 'kind_of_batch' in results:
        kind_of_batch = results['kind_of_batch']
    if not positive_value_exists(kind_of_batch):
        kind_of_batch = IMPORT_BALLOT_ITEM

    batch_header_id = 0
    if 'batch_saved' in results and results['batch_saved']:
        messages.add_message(request, messages.INFO, 'Ballot items import batch for {google_civic_election_id} '
                                                     'election saved.'
                                                     ''.format(google_civic_election_id=google_civic_election_id))
        batch_header_id = results['batch_header_id']
    elif 'batch_header_id' in results and results['batch_header_id']:
        messages.add_message(request, messages.INFO, 'Ballot items import batch for {google_civic_election_id} '
                                                     'election saved, batch_header_id.'
                                                     ''.format(google_civic_election_id=google_civic_election_id))
        batch_header_id = results['batch_header_id']
    else:
        messages.add_message(request, messages.ERROR, results['status'])

    if positive_value_exists(batch_header_id):
        # Go straight to the new batch
        return HttpResponseRedirect(reverse('import_export_batches:batch_action_list', args=()) +
                                    "?batch_header_id=" + str(batch_header_id) +
                                    "&kind_of_batch=" + str(kind_of_batch) +
                                    "&google_civic_election_id=" + str(google_civic_election_id))
    else:
        # Go to the ballot_item_list_edit page
        if positive_value_exists(polling_location_we_vote_id):
            return HttpResponseRedirect(reverse('ballot:ballot_item_list_by_polling_location_edit',
                                                args=(polling_location_we_vote_id,)) +
                                        "?google_civic_election_id=" + str(google_civic_election_id) +
                                        "&polling_location_we_vote_id=" + str(polling_location_we_vote_id) +
                                        "&state_code=" + str(state_code)
                                        )
        else:
            messages.add_message(request, messages.ERROR, "Missing polling_location_we_vote_id.")
            return HttpResponseRedirect(reverse('election:election_list', args=()) +
                                        "?google_civic_election_id=" + str(google_civic_election_id) +
                                        "&polling_location_we_vote_id=" + str(polling_location_we_vote_id) +
                                        "&state_code=" + str(state_code)
                                        )


@login_required
def import_export_ballotpedia_index_view(request):
    """
    Provide an index of import/export actions (for We Vote data maintenance)
    """
    messages_on_stage = get_messages(request)

    template_values = {
        'messages_on_stage':    messages_on_stage,
    }
    return render(request, 'import_export_ballotpedia/index.html', template_values)


@login_required
def attach_ballotpedia_election_view(request, election_local_id=0):
    """
    Reach out to Ballotpedia and retrieve the details about this election needed to make other API calls.
    :param request:
    :param election_local_id:
    :return:
    """
    # admin, partner_organization, political_data_manager, political_data_viewer, verified_volunteer
    authority_required = {'political_data_manager'}
    if not voter_has_authority(request, authority_required):
        return redirect_to_sign_in_page(request, authority_required)

    state_code = request.GET.get('state_code', '')
    force_district_retrieve_from_ballotpedia = request.GET.get('force_district_retrieve_from_ballotpedia', False)
    polling_location_list = []
    status = ""

    try:
        election_on_stage = Election.objects.get(id=election_local_id)
        google_civic_election_id = election_on_stage.google_civic_election_id
        election_state_code = election_on_stage.get_election_state()
        election_name = election_on_stage.election_name
        is_national_election = election_on_stage.is_national_election
    except Election.MultipleObjectsReturned as e:
        messages.add_message(request, messages.ERROR,
                             'Could not retrieve election data. More than one election found.')
        return HttpResponseRedirect(reverse('election:election_list', args=()))
    except Election.DoesNotExist:
        messages.add_message(request, messages.ERROR,
                             'Could not retrieve election data. Election could not be found.')
        return HttpResponseRedirect(reverse('election:election_list', args=()))

    # Check to see if we have polling location data related to the region(s) covered by this election
    # We request the ballot data for each polling location as a way to build up our local data
    if not positive_value_exists(state_code) and positive_value_exists(google_civic_election_id):
        state_code = election_state_code

    if positive_value_exists(is_national_election) and not positive_value_exists(state_code):
        messages.add_message(request, messages.ERROR,
                             'For National elections, a State Code is required in order to run any '
                             'Ballotpedia data preparation.')
        return HttpResponseRedirect(reverse('election:election_summary', args=(election_local_id,)))

    if not is_valid_state_code(state_code):
        messages.add_message(request, messages.ERROR,
                             '{state_code} is not a valid State Code'.format(state_code=state_code))
        return HttpResponseRedirect(reverse('election:election_summary', args=(election_local_id,)))

    try:
        polling_location_count_query = PollingLocation.objects.all()
        polling_location_count_query = polling_location_count_query.filter(state__iexact=state_code)
        polling_location_count_query = polling_location_count_query.filter(polling_location_deleted=False)
        polling_location_count_query = polling_location_count_query.exclude(
            Q(latitude__isnull=True) | Q(latitude__exact=0.0))
        polling_location_count_query = polling_location_count_query.exclude(
            Q(zip_long__isnull=True) | Q(zip_long__exact='0') | Q(zip_long__exact=''))
        polling_location_count = polling_location_count_query.count()

        if positive_value_exists(polling_location_count):
            polling_location_query = PollingLocation.objects.all()
            polling_location_query = polling_location_query.filter(state__iexact=state_code)
            polling_location_query = polling_location_query.filter(polling_location_deleted=False)
            polling_location_query = polling_location_query.exclude(
                Q(latitude__isnull=True) | Q(latitude__exact=0.0))
            polling_location_query = polling_location_query.exclude(
                Q(zip_long__isnull=True) | Q(zip_long__exact='0') | Q(zip_long__exact=''))
            # Ordering by "location_name" creates a bit of (locational) random order
            polling_location_list = polling_location_query.order_by('location_name')[:50]
    except PollingLocation.DoesNotExist:
        messages.add_message(request, messages.INFO,
                             'Could not retrieve polling location data for the {election_name}. '
                             'No polling locations exist for the state \'{state}\'. '
                             'Data needed from VIP.'.format(
                                 election_name=election_name,
                                 state=state_code))
        return HttpResponseRedirect(reverse('election:election_summary', args=(election_local_id,)) +
                                    "?state_code=" + str(state_code))
<commit_msg>Update views_admin.py<commit_after># import_export_ballotpedia/views_admin.py
# Brought to you by We Vote. Be good.
# -*- coding: UTF-8 -*-

from .controllers import attach_ballotpedia_election_by_district_from_api, \
    retrieve_ballot_items_from_polling_location, \
    retrieve_ballotpedia_candidates_by_district_from_api, retrieve_ballotpedia_measures_by_district_from_api, \
    retrieve_ballotpedia_district_id_list_for_polling_location, retrieve_ballotpedia_offices_by_district_from_api
#    retrieve_ballotpedia_offices_by_election_from_api
from admin_tools.views import redirect_to_sign_in_page
from config.base import get_environment_variable
from datetime import date
from django.contrib import messages
from django.contrib.auth.decorators import login_required
from django.contrib.messages import get_messages
from django.core.urlresolvers import reverse
from django.db.models import Q
from django.http import HttpResponseRedirect
from django.shortcuts import redirect, render
from election.models import Election, ElectionManager
from import_export_batches.models import BatchSet, BATCH_SET_SOURCE_IMPORT_BALLOTPEDIA_BALLOT_ITEMS
from polling_location.models import PollingLocation
from voter.models import voter_has_authority
import wevote_functions.admin
from wevote_functions.functions import convert_to_int, is_valid_state_code, positive_value_exists

logger = wevote_functions.admin.get_logger(__name__)

BALLOTPEDIA_API_CONTAINS_URL = get_environment_variable("BALLOTPEDIA_API_CONTAINS_URL")

CANDIDATE = 'CANDIDATE'
CONTEST_OFFICE = 'CONTEST_OFFICE'
ELECTED_OFFICE = 'ELECTED_OFFICE'
IMPORT_BALLOT_ITEM = 'IMPORT_BALLOT_ITEM'
IMPORT_VOTER = 'IMPORT_VOTER'
MEASURE = 'MEASURE'
POLITICIAN = 'POLITICIAN'


@login_required
def import_ballot_items_for_location_view(request):
    """
    Reach out to Ballotpedia API to retrieve a short list of districts the voter can vote in.
    """
    # admin, partner_organization, political_data_manager, political_data_viewer, verified_volunteer
    authority_required = {'political_data_manager'}
    if not voter_has_authority(request, authority_required):
        return redirect_to_sign_in_page(request, authority_required)

    google_civic_election_id = convert_to_int(request.GET.get('google_civic_election_id', 0))
    polling_location_we_vote_id = request.GET.get('polling_location_we_vote_id', "")
    state_code = request.GET.get('state_code', "")

    results = retrieve_ballot_items_from_polling_location(
        google_civic_election_id, polling_location_we_vote_id, state_code=state_code)

    kind_of_batch = ""
    if 'kind_of_batch' in results:
        kind_of_batch = results['kind_of_batch']
    if not positive_value_exists(kind_of_batch):
        kind_of_batch = IMPORT_BALLOT_ITEM

    batch_header_id = 0
    if 'batch_saved' in results and results['batch_saved']:
        messages.add_message(request, messages.INFO, 'Ballot items import batch for {google_civic_election_id} '
                                                     'election saved.'
                                                     ''.format(google_civic_election_id=google_civic_election_id))
        batch_header_id = results['batch_header_id']
    elif 'batch_header_id' in results and results['batch_header_id']:
        messages.add_message(request, messages.INFO, 'Ballot items import batch for {google_civic_election_id} '
                                                     'election saved, batch_header_id.'
                                                     ''.format(google_civic_election_id=google_civic_election_id))
        batch_header_id = results['batch_header_id']
    else:
        messages.add_message(request, messages.ERROR, results['status'])

    if positive_value_exists(batch_header_id):
        # Go straight to the new batch
        return HttpResponseRedirect(reverse('import_export_batches:batch_action_list', args=()) +
                                    "?batch_header_id=" + str(batch_header_id) +
                                    "&kind_of_batch=" + str(kind_of_batch) +
                                    "&google_civic_election_id=" + str(google_civic_election_id))
    else:
        # Go to the ballot_item_list_edit page
        if positive_value_exists(polling_location_we_vote_id):
            return HttpResponseRedirect(reverse('ballot:ballot_item_list_by_polling_location_edit',
                                                args=(polling_location_we_vote_id,)) +
                                        "?google_civic_election_id=" + str(google_civic_election_id) +
                                        "&polling_location_we_vote_id=" + str(polling_location_we_vote_id) +
                                        "&state_code=" + str(state_code)
                                        )
        else:
            messages.add_message(request, messages.ERROR, "Missing polling_location_we_vote_id.")
            return HttpResponseRedirect(reverse('election:election_list', args=()) +
                                        "?google_civic_election_id=" + str(google_civic_election_id) +
                                        "&polling_location_we_vote_id=" + str(polling_location_we_vote_id) +
                                        "&state_code=" + str(state_code)
                                        )


@login_required
def import_export_ballotpedia_index_view(request):
    """
    Provide an index of import/export actions (for We Vote data maintenance)
    """
    messages_on_stage = get_messages(request)

    template_values = {
        'messages_on_stage':    messages_on_stage,
    }
    return render(request, 'import_export_ballotpedia/index.html', template_values)


@login_required
def attach_ballotpedia_election_view(request, election_local_id=0):
    """
    Reach out to Ballotpedia and retrieve the details about this election needed to make other API calls.
    :param request:
    :param election_local_id:
    :return:
    """
    # admin, partner_organization, political_data_manager, political_data_viewer, verified_volunteer
    authority_required = {'political_data_manager'}
    if not voter_has_authority(request, authority_required):
        return redirect_to_sign_in_page(request, authority_required)

    state_code = request.GET.get('state_code', '')
    force_district_retrieve_from_ballotpedia = request.GET.get('force_district_retrieve_from_ballotpedia', False)
    polling_location_list = []
    status = ""

    try:
        election_on_stage = Election.objects.get(id=election_local_id)
        google_civic_election_id = election_on_stage.google_civic_election_id
        election_state_code = election_on_stage.get_election_state()
        election_name = election_on_stage.election_name
        is_national_election = election_on_stage.is_national_election
    except Election.MultipleObjectsReturned as e:
        messages.add_message(request, messages.ERROR,
                             'Could not retrieve election data. More than one election found.')
        return HttpResponseRedirect(reverse('election:election_list', args=()))
    except Election.DoesNotExist:
        messages.add_message(request, messages.ERROR,
                             'Could not retrieve election data. Election could not be found.')
        return HttpResponseRedirect(reverse('election:election_list', args=()))

    # Check to see if we have polling location data related to the region(s) covered by this election
    # We request the ballot data for each polling location as a way to build up our local data
    if not positive_value_exists(state_code) and positive_value_exists(google_civic_election_id):
        state_code = election_state_code

    if positive_value_exists(is_national_election) and not positive_value_exists(state_code):
        messages.add_message(request, messages.ERROR,
                             'For National elections, a State Code is required in order to run any '
                             'Ballotpedia data preparation.')
        return HttpResponseRedirect(reverse('election:election_summary', args=(election_local_id,)))

    if not is_valid_state_code(state_code):
        messages.add_message(request, messages.ERROR,
                             '{state_code} is not a valid State Code'.format(state_code=state_code))
        return HttpResponseRedirect(reverse('election:election_summary', args=(election_local_id,)))

    try:
        polling_location_count_query = PollingLocation.objects.all()
        polling_location_count_query = polling_location_count_query.filter(state__iexact=state_code)
        polling_location_count_query = polling_location_count_query.filter(polling_location_deleted=False)
        polling_location_count_query = polling_location_count_query.exclude(
            Q(latitude__isnull=True) | Q(latitude__exact=0.0))
        polling_location_count_query = polling_location_count_query.exclude(
            Q(zip_long__isnull=True) | Q(zip_long__exact='0') | Q(zip_long__exact=''))
        polling_location_count = polling_location_count_query.count()

        if positive_value_exists(polling_location_count):
            polling_location_query = PollingLocation.objects.all()
            polling_location_query = polling_location_query.filter(state__iexact=state_code)
            polling_location_query = polling_location_query.filter(polling_location_deleted=False)
            polling_location_query = polling_location_query.exclude(
                Q(latitude__isnull=True) | Q(latitude__exact=0.0))
            polling_location_query = polling_location_query.exclude(
                Q(zip_long__isnull=True) | Q(zip_long__exact='0') | Q(zip_long__exact=''))
            # Ordering by "location_name" creates a bit of (locational) random order
            polling_location_list = polling_location_query.order_by('location_name')[:1000]
    except PollingLocation.DoesNotExist:
        messages.add_message(request, messages.INFO,
                             'Could not retrieve polling location data for the {election_name}. '
                             'No polling locations exist for the state \'{state}\'. '
                             'Data needed from VIP.'.format(
                                 election_name=election_name,
                                 state=state_code))
        return HttpResponseRedirect(reverse('election:election_summary', args=(election_local_id,)) +
                                    "?state_code=" + str(state_code))

{'fraction_non_alphanumeric': 0.07159893150162183, 'fraction_numerical': 0.0012402213318069072, 'mean_word_length': 3.1888111888111887, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 50, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5437497', 'n_tokens_mistral': 5986, 'n_tokens_neox': 5817, 'n_words': 1075}