<commit_before>import csv
import os
from codepoint.models import CodePoint

from django.core.management.base import BaseCommand


class Command(BaseCommand):
    help = "Import OS Code Point Open data"

    def add_arguments(self, parser):
        parser.add_argument('--datadir', nargs=1, required=False)

    def handle(self, *args, **options):

        if options['datadir']:
            datadir = options['datadir'][0]

        for root, dir, files in os.walk(datadir):
            for file in files:
                if file.endswith(".csv"):
                    with open(file, 'r') as csvfile:
                        cpreader = csv.reader(csvfile, delimiter=',', quotechar='"')
                        for cpline in cpreader:
                            if len(cpline) != 10:
                                print("Unknown line (skipped): %s" % cpline)
                                next

                            codepoint = CodePoint(postcode=cpline[0], pos_quality=cpline[1], eastings=cpline[2], northings=cpline[3], country_code=cpline[4], nhs_regional_ha_code=cpline[5], nhs_ha_code=cpline[6], admin_county_code=cpline[7], admin_district_code=cpline[8], admin_ward_code=cpline[9])
                            codepoint.save()<commit_msg>First working importer<commit_after>import csv
import os
from codepoint.models import CodePoint

from django.core.management.base import BaseCommand


class Command(BaseCommand):
    help = "Import OS Code Point Open data"

    def add_arguments(self, parser):
        parser.add_argument('--datadir', nargs=1, required=False)

    def handle(self, *args, **options):

        if options['datadir']:
            datadir = options['datadir'][0]

        for root, dir, files in os.walk(datadir):
            for file in files:
                if file.endswith(".csv"):
                    with open(os.path.join(root, file), 'r') as csvfile:
                        cpreader = csv.reader(csvfile, delimiter=',', quotechar='"')
                        for cpline in cpreader:
                            if len(cpline) != 10:
                                print("Unknown line (skipped): %s" % cpline)
                                next

                            codepoint = CodePoint(postcode=cpline[0], pos_quality=cpline[1], eastings=cpline[2], northings=cpline[3], country_code=cpline[4], nhs_regional_ha_code=cpline[5], nhs_ha_code=cpline[6], admin_county_code=cpline[7], admin_district_code=cpline[8], admin_ward_code=cpline[9])
                            codepoint.save()
{'fraction_non_alphanumeric': 0.11391407173827355, 'fraction_numerical': 0.01103665746945211, 'mean_word_length': 2.1645885286783044, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1454205', 'n_tokens_mistral': 792, 'n_tokens_neox': 765, 'n_words': 164}