<commit_before>import datetime
import logging

from django.conf import settings
from django.core.management.base import BaseCommand
from django.core.management.base import CommandError

from gcutils.bigquery import Client, TableExporter

logger = logging.getLogger(__name__)

TEMP_DATASET = 'tmp_eu'
TEMP_SOURCE_NAME = 'raw_nhs_digital_data'


class Command(BaseCommand):
    """There are two kinds of source we use to generate data.


    The legacy source (the code paths for which, once the new source
    has successfully been imported a few times, can be removed) is
    published erratically; the new "detailed" data source is published
    regularly, each month, so we now prefer that.

    The "detailed" source format has one iine for each presentation
    *and pack size*, so prescriptions of 28 paracetamol will be on a
    separate line from prescriptions of 100 paracetamol.

    The destination format has one line for paracetamol of any pack
    size.

    """
    args = ''
    help = 'Converts HSCIC data files into the format needed for our SQL COPY '
    help += 'statement. We use COPY because it is much faster than INSERT.'

    def add_arguments(self, parser):
        parser.add_argument('--filename')

    def handle(self, *args, **options):
        self.IS_VERBOSE = (options['verbosity'] > 1)

        if 'is_test' in options:
            self.IS_TEST = True
        else:
            self.IS_TEST = False

        filename = options['filename']

        if self.IS_VERBOSE:
            print "--------- Converting %s -----------" % filename

        filename_for_output = self.create_filename_for_output_file(filename)

        filename = filename.split('/prescribing/')[1]
        uri = 'gs://ebmdatalab/hscic/prescribing/' + filename
        # Grab date from file path
        try:
            date = datetime.datetime.strptime(
                uri.split("/")[-2] + "_01", "%Y_%m_%d"
            ).strftime('%Y_%m_%d')
        except ValueError as e:
            message = ('The file path must have a YYYY_MM '
                       'date component in the containing directory: ')
            message += e.message<commit_msg>Improve convert_hscic_prescribing docstring<commit_after>import datetime
import logging

from django.conf import settings
from django.core.management.base import BaseCommand
from django.core.management.base import CommandError

from gcutils.bigquery import Client, TableExporter

logger = logging.getLogger(__name__)

TEMP_DATASET = 'tmp_eu'
TEMP_SOURCE_NAME = 'raw_nhs_digital_data'


class Command(BaseCommand):
    """Processes manually-fetched detailed prescribing information CSV.

    * Appends prescribing information to the hscic.prescribing table in BQ
    * Downloads a converted CSV file for importing into relevant partition of
      frontend_partition table in Postgres by import_hscic_prescribing task

    The source CSV file contains "figures on the number of prescription items
    that are dispensed each month and information relating to costs for English
    practices".

    The source CSV file has one line for each presentation and pack size, (so
    prescriptions of 28 paracetamol will be on a separate line from
    prescriptions of 100 paracetamol).

    The converted CSV file has one line for paracetamol of any pack size.
    """

    def add_arguments(self, parser):
        parser.add_argument('--filename')

    def handle(self, *args, **options):
        self.IS_VERBOSE = (options['verbosity'] > 1)

        if 'is_test' in options:
            self.IS_TEST = True
        else:
            self.IS_TEST = False

        filename = options['filename']

        if self.IS_VERBOSE:
            print "--------- Converting %s -----------" % filename

        filename_for_output = self.create_filename_for_output_file(filename)

        filename = filename.split('/prescribing/')[1]
        uri = 'gs://ebmdatalab/hscic/prescribing/' + filename
        # Grab date from file path
        try:
            date = datetime.datetime.strptime(
                uri.split("/")[-2] + "_01", "%Y_%m_%d"
            ).strftime('%Y_%m_%d')
        except ValueError as e:
            message = ('The file path must have a YYYY_MM '
                       'date component in the containing directory: ')
            message += e.message
{'fraction_non_alphanumeric': 0.08738989337042188, 'fraction_numerical': 0.004636068613815485, 'mean_word_length': 2.6723404255319148, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '831439', 'n_tokens_mistral': 1236, 'n_tokens_neox': 1163, 'n_words': 411}