<commit_before>"""
Usage:

    ./manage.py import_operator_contacts < nocrecords.xml
"""
import warnings
from io import open
from bs4 import BeautifulSoup
from django.core.exceptions import ValidationError
from django.core.management.base import BaseCommand
from ...models import Operator
from ...views import FIRST_OPERATORS


class Command(BaseCommand):
    input = 0

    @classmethod
    def format_address(cls, address):
        address_parts = address.split(', ')
        address_last_line_parts = address_parts[-1].split(' ')
        if len(address_last_line_parts) > 2:
            pre_postcode = ' '.join(address_last_line_parts[:-2])
            postcode = ' '.join(address_last_line_parts[-2:])
            address_parts[-1] = pre_postcode + '\n' + postcode
        return '\n'.join(address_parts)

    def handle(self, *args, **options):
        with open(self.input, encoding='cp1252') as input:
            soup = BeautifulSoup(input, 'html.parser')

        noc_codes = {
            record.pubnmid.string: record.noccode.string
            for record in soup.noctable.find_all('noctablerecord')
        }

        for public_name in soup.publicname.find_all('publicnamerecord'):
            noc_code = noc_codes.get(public_name.pubnmid.string)

            if not noc_code or len(noc_code) < 4:
                continue

            try:
                operator = Operator.objects.get(pk=noc_code.replace('=', ''))
            except Operator.DoesNotExist as e:
                warnings.warn('{} {}'.format(e, noc_code))
                continue

            if noc_code in FIRST_OPERATORS:
                operator.url = 'https://www.firstgroup.com/%s' % FIRST_OPERATORS[noc_code]
                operator.email = ''
                operator.phone = ''
                operator.save()
                continue

            website = public_name.website.string
            address = public_name.complenq.string
            email = public_name.ttrteenq.string
            phone = public_name.fareenq.string

            if website or address or email or phone:
                if website:
                    website = website.split('#')[-2]
                    if '.' in website and 'mailto:' not in website and ' ' not in website:<commit_msg>#51 update operator contacts import<commit_after>"""
Usage:

    ./manage.py import_operator_contacts < nocrecords.xml
"""
import warnings
from io import open
from bs4 import BeautifulSoup
from django.core.exceptions import ValidationError
from django.core.management.base import BaseCommand
from ...models import Operator
from ...views import FIRST_OPERATORS


class Command(BaseCommand):
    input = 0

    @classmethod
    def format_address(cls, address):
        address_parts = address.split(', ')
        address_last_line_parts = address_parts[-1].split(' ')
        if len(address_last_line_parts) > 2:
            pre_postcode = ' '.join(address_last_line_parts[:-2])
            postcode = ' '.join(address_last_line_parts[-2:])
            address_parts[-1] = pre_postcode + '\n' + postcode
        return '\n'.join(address_parts)

    def handle(self, *args, **options):
        with open(self.input, encoding='cp1252') as input:
            soup = BeautifulSoup(input, 'html.parser')

        noc_codes = {
            record.pubnmid.string: record.noccode.string
            for record in soup.noctable.find_all('noctablerecord')
        }

        for public_name in soup.publicname.find_all('publicnamerecord'):
            noc_code = noc_codes.get(public_name.pubnmid.string)

            if not noc_code or len(noc_code) != 4:
                continue

            try:
                operator = Operator.objects.get(operatorcode__code=noc_code.replace('=', ''),
                                                operatorcode__source__name='National Operator Codes')
            except Operator.DoesNotExist as e:
                warnings.warn('{} {}'.format(e, noc_code))
                continue

            if noc_code in FIRST_OPERATORS:
                operator.url = 'https://www.firstgroup.com/%s' % FIRST_OPERATORS[noc_code]
                operator.email = ''
                operator.phone = ''
                operator.save()
                continue

            website = public_name.website.string
            address = public_name.complenq.string
            email = public_name.ttrteenq.string
            phone = public_name.fareenq.string

            if website or address or email or phone:
                if website:
                    website = website.split('#')[-2]
                    if '.' in website and 'mailto:' not in website and ' ' not in website:
{'fraction_non_alphanumeric': 0.1016949152542373, 'fraction_numerical': 0.006007294571980262, 'mean_word_length': 2.0711462450592886, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1368253', 'n_tokens_mistral': 1354, 'n_tokens_neox': 1265, 'n_words': 290}