<commit_before>from __future__ import print_function, unicode_literals

import errno
import hashlib
import magic
import mimetypes
import os
from os.path import dirname, join, exists
import requests

from django.core.management.base import BaseCommand, CommandError
from django.core.files.storage import FileSystemStorage

from official_documents.models import OfficialDocument
from elections.models import Election
from popolo.models import Post, Area

from compat import StreamDictReader

allowed_mime_types = set([
    b'application/pdf',
    b'application/msword',
    b'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
])

PDF_COLUMN_HEADERS_TO_TRY = (
    'Statement of Persons Nominated (SOPN) URL',
    'Link to PDF',
)

POST_OR_AREA_COLUMN_HEADERS_TO_TRY = (
    'Region',
    'Constituency',
    'Ward',
)

def download_file_cached(url):
    url_hash = hashlib.md5(url).hexdigest()
    directory = join(dirname(__file__), '.noms-cache')
    try:
        os.mkdir(directory)
    except OSError as e:
        if e.errno != errno.EEXIST:
            raise
    filename = join(directory, url_hash)
    if exists(filename):
        return filename
    r = requests.get(url)
    with open(filename, 'w') as f:
        f.write(r.content)
    return filename


def get_column_header(possible_column_headers, row):
    return [
        ch for ch in possible_column_headers
        if ch in row
    ][0]


class Command(BaseCommand):
    help = "Import official documents for posts from a URL to a CSV file"

    args = "<CSV_URL>"

    def add_arguments(self, parser):
        parser.add_argument('--delete-existing', action='store_true')
        parser.add_argument('--election')

    def handle(self, *args, **options):

        csv_url, = args

        override_election = None
        override_election_slug = options['election']
        if override_election_slug:
            try:
                override_election = Election.objects.get(
                    slug=override_election_slug
                )
            except Election.DoesNotExist:
                msg = 'No election with slug {0} found'
                raise CommandError(msg.format(override_election_slug))

        election_name_to_election = {}

        mime_type_magic = magic.Magic(mime=True)
        storage = FileSystemStorage()

        r = requests.get(csv_url)
        r.encoding = 'utf-8'
        reader = StreamDictReader(r.text)
        for row in reader:
            post_or_area_header = get_column_header(
                POST_OR_AREA_COLUMN_HEADERS_TO_TRY, row
            )

            name = row[post_or_area_header]
            if not name:
                continue
            name = name.strip()

            # If there was no election specified, try to find it from
            # the 'Election' column (which has the election name):
            if override_election_slug:
                election = override_election
            else:
                election_name = row['Election']
                election = election_name_to_election.get(election_name)
                if election is None:
                    election = Election.objects.get(name=election_name)
                    election_name_to_election[election_name] = election

            try:
                post = Post.objects.get(
                    label=name,
                    extra__elections=election,
                )
            except Post.DoesNotExist:
                msg = "Failed to find the post {0}, guessing it might be the area name instead"
                print(msg.format(name))
                # If the post name isn't there, try getting it from
                # the area:
                try:<commit_msg>Give a more helpful error if it looks like --election is missing<commit_after>from __future__ import print_function, unicode_literals

import errno
import hashlib
import magic
import mimetypes
import os
from os.path import dirname, join, exists
import requests

from django.core.management.base import BaseCommand, CommandError
from django.core.files.storage import FileSystemStorage

from official_documents.models import OfficialDocument
from elections.models import Election
from popolo.models import Post, Area

from compat import StreamDictReader

allowed_mime_types = set([
    b'application/pdf',
    b'application/msword',
    b'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
])

PDF_COLUMN_HEADERS_TO_TRY = (
    'Statement of Persons Nominated (SOPN) URL',
    'Link to PDF',
)

POST_OR_AREA_COLUMN_HEADERS_TO_TRY = (
    'Region',
    'Constituency',
    'Ward',
)

def download_file_cached(url):
    url_hash = hashlib.md5(url).hexdigest()
    directory = join(dirname(__file__), '.noms-cache')
    try:
        os.mkdir(directory)
    except OSError as e:
        if e.errno != errno.EEXIST:
            raise
    filename = join(directory, url_hash)
    if exists(filename):
        return filename
    r = requests.get(url)
    with open(filename, 'w') as f:
        f.write(r.content)
    return filename


def get_column_header(possible_column_headers, row):
    return [
        ch for ch in possible_column_headers
        if ch in row
    ][0]


class Command(BaseCommand):
    help = "Import official documents for posts from a URL to a CSV file"

    args = "<CSV_URL>"

    def add_arguments(self, parser):
        parser.add_argument('--delete-existing', action='store_true')
        parser.add_argument('--election')

    def handle(self, *args, **options):

        csv_url, = args

        override_election = None
        override_election_slug = options['election']
        if override_election_slug:
            try:
                override_election = Election.objects.get(
                    slug=override_election_slug
                )
            except Election.DoesNotExist:
                msg = 'No election with slug {0} found'
                raise CommandError(msg.format(override_election_slug))

        election_name_to_election = {}

        mime_type_magic = magic.Magic(mime=True)
        storage = FileSystemStorage()

        r = requests.get(csv_url)
        r.encoding = 'utf-8'
        reader = StreamDictReader(r.text)
        for row in reader:
            post_or_area_header = get_column_header(
                POST_OR_AREA_COLUMN_HEADERS_TO_TRY, row
            )

            name = row[post_or_area_header]
            if not name:
                continue
            name = name.strip()

            # If there was no election specified, try to find it from
            # the 'Election' column (which has the election name):
            if override_election_slug:
                election = override_election
            else:
                if 'Election' not in row:
                    raise CommandError("There is no election name in the 'Election' column, so you must supply an election slug with --election")
                election_name = row['Election']
                election = election_name_to_election.get(election_name)
                if election is None:
                    election = Election.objects.get(name=election_name)
                    election_name_to_election[election_name] = election

            try:
                post = Post.objects.get(
                    label=name,
                    extra__elections=election,
                )
            except Post.DoesNotExist:
                msg = "Failed to find the post {0}, guessing it might be the area name instead"
                print(msg.format(name))
                # If the post name isn't there, try getting it from
                # the area:
                try:
{'fraction_non_alphanumeric': 0.07669231772531629, 'fraction_numerical': 0.001304291117777488, 'mean_word_length': 2.1817427385892114, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5835359', 'n_tokens_mistral': 2168, 'n_tokens_neox': 2065, 'n_words': 564}