<commit_before>from datetime import datetime
import os
import csv
import urllib2
import re
import json

from django.core.management.base import LabelCommand, CommandError
from django.conf import settings

from popit_api import PopIt
from slumber.exceptions import HttpServerError
import dateutil.parser

# For a version with only current parties, you can use this URL as an
# argument to the command:
#
#   https://raw.githubusercontent.com/DemocracyClub/UK-Political-Parties/gh-pages/data/parties.csv
#
# ... or for one which also includes de-registered parties, you can
# use this URL:
#
#   https://raw.githubusercontent.com/mhl/UK-Political-Parties/gh-pages/data/parties.csv

party_name_re = re.compile(
    r'^(.*?)(?:\s+\[De-registered\s+(\d{2}/\d{2}/\d{2})\])?\s*$'
)

class Command(LabelCommand):
    help = "Update parties from a CSV of party data"

    def handle_label(self, label, **options):
        self.api = PopIt(
            instance=settings.POPIT_INSTANCE,
            hostname=settings.POPIT_HOSTNAME,
            port=getattr(settings, 'POPIT_PORT', 80),
            api_version='v0.1',
            api_key=settings.POPIT_API_KEY
           )


        if os.path.exists(label):
            with open(label) as csv_content:
                self.parse_csv(csv_content)
        else:
            csv_content = urllib2.urlopen(label)
            self.parse_csv(csv_content)

    def parse_csv(self, csv_content):

        reader = csv.DictReader(csv_content)
        for line in reader:
            if line['Entity type'] != 'Political Party':
                continue
            party_id = self.clean_id(line['EC Reference Number'])
            register = line['Register']
            party_name, party_dissolved = self.clean_name(line['Entity name'], register)
            party_founded = self.clean_date(line['Date of registration / notification'])
            party_data = {
                'id': party_id,
                'name': party_name,<commit_msg>Also add a slug for each political party<commit_after>from datetime import datetime
import os
import csv
import urllib2
import re
import json
from slugify import slugify

from django.core.management.base import LabelCommand, CommandError
from django.conf import settings

from popit_api import PopIt
from slumber.exceptions import HttpServerError
import dateutil.parser

# For a version with only current parties, you can use this URL as an
# argument to the command:
#
#   https://raw.githubusercontent.com/DemocracyClub/UK-Political-Parties/gh-pages/data/parties.csv
#
# ... or for one which also includes de-registered parties, you can
# use this URL:
#
#   https://raw.githubusercontent.com/mhl/UK-Political-Parties/gh-pages/data/parties.csv

party_name_re = re.compile(
    r'^(.*?)(?:\s+\[De-registered\s+(\d{2}/\d{2}/\d{2})\])?\s*$'
)

class Command(LabelCommand):
    help = "Update parties from a CSV of party data"

    def handle_label(self, label, **options):
        self.api = PopIt(
            instance=settings.POPIT_INSTANCE,
            hostname=settings.POPIT_HOSTNAME,
            port=getattr(settings, 'POPIT_PORT', 80),
            api_version='v0.1',
            api_key=settings.POPIT_API_KEY
           )


        if os.path.exists(label):
            with open(label) as csv_content:
                self.parse_csv(csv_content)
        else:
            csv_content = urllib2.urlopen(label)
            self.parse_csv(csv_content)

    def parse_csv(self, csv_content):

        reader = csv.DictReader(csv_content)
        for line in reader:
            if line['Entity type'] != 'Political Party':
                continue
            party_id = self.clean_id(line['EC Reference Number'])
            register = line['Register']
            party_name, party_dissolved = self.clean_name(line['Entity name'], register)
            party_founded = self.clean_date(line['Date of registration / notification'])
            party_data = {
                'id': party_id,
                'name': party_name,
                'slug': slugify(party_name),
{'fraction_non_alphanumeric': 0.10700197238658778, 'fraction_numerical': 0.004437869822485207, 'mean_word_length': 2.648381294964029, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3630667', 'n_tokens_mistral': 1261, 'n_tokens_neox': 1193, 'n_words': 292}