<commit_before># coding=utf8

import calendar
import datetime
import logging
import re

import bs4
import requests

from django.core.management import BaseCommand

from dmd.models import VMPP
from frontend.models import NCSOConcession
from gcutils.bigquery import Client
from openprescribing.slack import notify_slack

logger = logging.getLogger(__file__)


class Command(BaseCommand):
    def handle(self, *args, **kwargs):
        self.vmpps = VMPP.objects.values("id", "nm")
        num_before = NCSOConcession.objects.count()
        self.import_from_archive()
        self.import_from_current()
        self.reconcile()
        num_after = NCSOConcession.objects.count()
        num_unmatched = NCSOConcession.objects.filter(vmpp__isnull=True).count()

        Client("dmd").upload_model(NCSOConcession)

        if num_before == num_after:
            msg = "Found no new concessions to import"
        else:
            msg = "Imported %s new concessions" % (num_after - num_before)
        if num_unmatched:
            msg += "\nThere are %s unmatched concessions" % num_unmatched
        notify_slack(msg)

    def import_from_archive(self):
        logger.info("import_from_archive")

        doc = self.download_archive()
        for h2 in doc.find_all("h2", class_="trigger"):
            table = h2.find_next("table")
            date = self.date_from_heading(h2)
            drugs_and_pack_sizes = self.import_from_table(table, date)
            self.delete_missing(drugs_and_pack_sizes, date)

    def download_archive(self):
        url = "http://psnc.org.uk/dispensing-supply/supply-chain/generic-shortages/ncso-archive/"
        rsp = requests.get(url)
        return bs4.BeautifulSoup(rsp.content, "html.parser")

    def import_from_current(self):
        logger.info("import_from_current")

        doc = self.download_current()

        h1s = doc.find_all("h1", string=re.compile("\w+ \d{4}"))
        assert len(h1s) == 1

        date = self.date_from_heading(h1s[0])

        drugs_and_pack_sizes = set()

        for table in doc.find_all("table"):
            drugs_and_pack_sizes |= self.import_from_table(table, date)

        self.delete_missing(drugs_and_pack_sizes, date)

    def download_current(self):
        url = "http://psnc.org.uk/dispensing-supply/supply-chain/generic-shortages/"
        rsp = requests.get(url)
        return bs4.BeautifulSoup(rsp.content, "html.parser")

    def date_from_heading(self, heading):
        month_name, year = heading.text.strip().split()
        month_names = list(calendar.month_name)
        month = month_names.index(month_name)
        return datetime.date(int(year), month, 1)

    def import_from_table(self, table, date):
        """Creates NCSOConcession objects for any new records in the table.

        Returns set of strings containing drug name and pack size for all
        records in the table.
        """

        logger.info("import_from_table %s", date)

        if date < datetime.date(2014, 8, 1):
            # Data older than August 2018 is in a different format and we don't
            # need it at the moment.
            return set()

        trs = table.find_all("tr")
        records = [[td.text.strip() for td in tr.find_all("td")] for tr in trs]
<commit_msg>Fix NSCO importer<commit_after># coding=utf8

import calendar
import datetime
import logging
import re

import bs4
import requests

from django.core.management import BaseCommand

from dmd.models import VMPP
from frontend.models import NCSOConcession
from gcutils.bigquery import Client
from openprescribing.slack import notify_slack

logger = logging.getLogger(__file__)


class Command(BaseCommand):
    def handle(self, *args, **kwargs):
        self.vmpps = VMPP.objects.values("id", "nm")
        num_before = NCSOConcession.objects.count()
        self.import_from_archive()
        self.import_from_current()
        self.reconcile()
        num_after = NCSOConcession.objects.count()
        num_unmatched = NCSOConcession.objects.filter(vmpp__isnull=True).count()

        Client("dmd").upload_model(NCSOConcession)

        if num_before == num_after:
            msg = "Found no new concessions to import"
        else:
            msg = "Imported %s new concessions" % (num_after - num_before)
        if num_unmatched:
            msg += "\nThere are %s unmatched concessions" % num_unmatched
        notify_slack(msg)

    def import_from_archive(self):
        logger.info("import_from_archive")

        doc = self.download_archive()
        for h2 in doc.find_all("h2", class_="trigger"):
            table = h2.find_next("table")
            date = self.date_from_heading(h2)
            if date is None:
                continue
            drugs_and_pack_sizes = self.import_from_table(table, date)
            self.delete_missing(drugs_and_pack_sizes, date)

    def download_archive(self):
        url = "http://psnc.org.uk/dispensing-supply/supply-chain/generic-shortages/ncso-archive/"
        rsp = requests.get(url)
        return bs4.BeautifulSoup(rsp.content, "html.parser")

    def import_from_current(self):
        logger.info("import_from_current")

        doc = self.download_current()

        h1s = doc.find_all("h1", string=re.compile("\w+ \d{4}"))
        assert len(h1s) == 1

        date = self.date_from_heading(h1s[0])

        drugs_and_pack_sizes = set()

        for table in doc.find_all("table"):
            drugs_and_pack_sizes |= self.import_from_table(table, date)

        self.delete_missing(drugs_and_pack_sizes, date)

    def download_current(self):
        url = "http://psnc.org.uk/dispensing-supply/supply-chain/generic-shortages/"
        rsp = requests.get(url)
        return bs4.BeautifulSoup(rsp.content, "html.parser")

    def date_from_heading(self, heading):
        text = heading.text.strip()
        if re.match(r"\d{4}", text):
            return
        month_name, year = text.split()
        month_names = list(calendar.month_name)
        month = month_names.index(month_name)
        return datetime.date(int(year), month, 1)

    def import_from_table(self, table, date):
        """Creates NCSOConcession objects for any new records in the table.

        Returns set of strings containing drug name and pack size for all
        records in the table.
        """

        logger.info("import_from_table %s", date)

        if date < datetime.date(2014, 8, 1):
            # Data older than August 2018 is in a different format and we don't
            # need it at the moment.
            return set()

        trs = table.find_all("tr")
        records = [[td.text.strip() for td in tr.find_all("td")] for tr in trs]

{'fraction_non_alphanumeric': 0.1038103810381038, 'fraction_numerical': 0.00795079507950795, 'mean_word_length': 2.6977260122018856, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4157445', 'n_tokens_mistral': 2122, 'n_tokens_neox': 2043, 'n_words': 471}