<commit_before>"""
Usage:

    ./manage.py import_operators < NOC_db.csv
"""

from busstops.management.import_from_csv import ImportFromCSVCommand
from busstops.models import Operator


class Command(ImportFromCSVCommand):

    @staticmethod
    def get_region_id(region_id):
        if region_id in ('ADMIN', 'Admin', ''):
            return 'GB'
        elif region_id in ('SC', 'YO', 'WA', 'LO'):
            return region_id[0]

        return region_id

    @classmethod
    def handle_row(cls, row):
        "Given a CSV row (a list), returns an Operator object"
        if row['Duplicate'] != 'OK' and row['NOCCODE'] != 'FCYM': # First Cymru is marked as a duplicate even though it isn't
            return None

        operator_id = row['NOCCODE'].replace('=', '')

        if operator_id == 'TVSR':
            return None

        if row['OperatorPublicName'] in ('First', 'Arriva', 'Stagecoach') \
            or row['OperatorPublicName'].startswith('inc.') \
            or row['OperatorPublicName'].startswith('formerly'):
            name = row['RefNm']
        elif row['OperatorPublicName'] != '':
            name = row['OperatorPublicName']
        else:
            name = row['OpNm']

        name = name.replace('\'', u'\u2019') # Fancy apostrophe

        operator = Operator.objects.update_or_create(
            id=operator_id,
            defaults=dict(
                name=name.strip(),
                vehicle_mode=row['Mode'].lower().replace('ct operator', 'community transport').replace('drt', 'demand responsive transport'),
                region_id=cls.get_region_id(row['TLRegOwn']),
            )
        )
        return operator<commit_msg>Better avoid blank operator names, and ignore the unreliable Duplicate column more<commit_after>"""
Usage:

    ./manage.py import_operators < NOC_db.csv
"""

from busstops.management.import_from_csv import ImportFromCSVCommand
from busstops.models import Operator


class Command(ImportFromCSVCommand):

    @staticmethod
    def get_region_id(region_id):
        if region_id in ('ADMIN', 'Admin', ''):
            return 'GB'
        elif region_id in ('SC', 'YO', 'WA', 'LO'):
            return region_id[0]

        return region_id

    @staticmethod
    def get_name(row):
        if row['OperatorPublicName'] in ('First', 'Arriva', 'Stagecoach') \
            or row['OperatorPublicName'].startswith('inc.') \
            or row['OperatorPublicName'].startswith('formerly'):
            if row['RefNm'] != '':
                return row['RefNm']
            return row['OpNm']
        if row['OperatorPublicName'] != '':
            return row['OperatorPublicName']
        return row['OpNm']

    @classmethod
    def handle_row(cls, row):
        "Given a CSV row (a list), returns an Operator object"

        operator_id = row['NOCCODE'].replace('=', '')

        if operator_id == 'TVSR' or operator_id == 'FMAN' and row['Duplicate'] != 'OK':
            return None

        name = cls.get_name(row).replace('\'', u'\u2019') # Fancy apostrophe

        operator = Operator.objects.update_or_create(
            id=operator_id,
            defaults=dict(
                name=name.strip(),
                vehicle_mode=row['Mode'].lower().replace('ct operator', 'community transport').replace('drt', 'demand responsive transport'),
                region_id=cls.get_region_id(row['TLRegOwn']),
            )
        )
        return operator
{'fraction_non_alphanumeric': 0.13399359161083602, 'fraction_numerical': 0.0029129041654529565, 'mean_word_length': 2.4477911646586343, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3010803', 'n_tokens_mistral': 1030, 'n_tokens_neox': 973, 'n_words': 233}