<commit_before>import json
import glob
import numpy as np
import os
import pandas as pd
import sys
import api.view_utils as utils
from datetime import datetime
from dateutil.parser import parse
from dateutil import relativedelta
from django.core.management.base import BaseCommand
from django.core.exceptions import ObjectDoesNotExist
from django.db import transaction
from django.db.models import Sum, Q
from frontend.models import Measure, MeasureGlobal
from frontend.models import MeasureValue, Practice, PCT
from scipy.stats import rankdata


class Command(BaseCommand):
    '''
    Supply either --end_date to load data for all months
    up to that date, or --month to load data for just one
    month. You can also supply --start_date.
    '''

    def add_arguments(self, parser):
        parser.add_argument('--month')
        parser.add_argument('--start_date')
        parser.add_argument('--end_date')
        parser.add_argument('--measure')
        parser.add_argument('--definitions_only', action='store_true')

    def handle(self, *args, **options):
        self.centiles = range(10, 100, 10)
        options = self.parse_options(options)
        for m in options['measure_ids']:
            measure_config = options['measures'][m]
            measure = self.create_or_update_measure(m, measure_config)
            if options['definitions_only']:
                continue
            for month in options['months']:
                MeasureValue.objects.filter(month=month)\
                            .filter(measure=measure).delete()
                MeasureGlobal.objects.filter(month=month)\
                    .filter(measure=measure).delete()

                # Create practice values and percentiles. Use these to
                # calculate global values and percentiles. Then
                # calculate cost savings for individual practices, and
                # globally.
                self.create_practice_measurevalues(
                    measure, month, measure_config)
                records = MeasureValue.objects.filter(
                    month=month, measure=measure).values()
                df = self.create_dataframe_with_ranks_and_percentiles(records)
                mg = self.create_or_update_measureglobal(
                    df, measure, month, 'practice')
                for i, row in df.iterrows():
                    self.set_percentile_and_savings(
                        row, measure, month, mg, 'practice')
                if measure.is_cost_based:
                    self.set_measureglobal_savings(mg, 'practice')

                # Now calculate CCG values, percentiles and cost savings.
                self.create_ccg_measurevalues(measure, month)
                ccg_records = MeasureValue.objects.filter(
                    month=month, measure=measure, practice=None).values()
                df = self.create_dataframe_with_ranks_and_percentiles(
                    ccg_records)
                mg = self.create_or_update_measureglobal(
                    df, measure, month, 'ccg')
                for i, row in df.iterrows():
                    self.set_percentile_and_savings(
                        row, measure, month, mg, 'ccg')
                if measure.is_cost_based:
                    self.set_measureglobal_savings(mg, 'ccg')

    def parse_options(self, options):
        self.IS_VERBOSE = False
        if options['verbosity'] > 1:
            self.IS_VERBOSE = True
        fpath = os.path.dirname(__file__)
        files = glob.glob(fpath + "/measure_definitions/*.json")
        options['measures'] = {}
        for fname in files:
            fname = os.path.join(fpath, fname)
            json_data = open(fname).read()
            d = json.loads(json_data)
            for k in d:
                if k in options['measures']:
                    sys.exit()
                    print "duplicate entry found!", k
                else:
                    options['measures'][k] = d[k]
        if 'measure' in options and options['measure']:
            options['measure_ids'] = [options['measure']]
        else:
            options['measure_ids'] = [k for k in options['measures']]

        # Get months to cover from options.
        if not options['month'] and not options['end_date']:
            err = 'You must supply either --month or --end_date '
            err += 'in the format YYYY-MM-DD. You can also '
            err += 'optionally supply a start date.'
            print err
            sys.exit()
        options['months'] = []
        if 'month' in options and options['month']:
            options['months'] = [options['month']]
        else:
            if 'start_date' in options and options['start_date']:<commit_msg>Add switch to compute measures based on file path<commit_after>import json
import glob
import numpy as np
import os
import pandas as pd
import sys
import re
import api.view_utils as utils
from datetime import datetime
from dateutil.parser import parse
from dateutil import relativedelta
from django.core.management.base import BaseCommand
from django.core.exceptions import ObjectDoesNotExist
from django.db import transaction
from django.db.models import Sum, Q
from frontend.models import Measure, MeasureGlobal
from frontend.models import MeasureValue, Practice, PCT
from scipy.stats import rankdata


class Command(BaseCommand):
    '''Supply either --end_date to load data for all months
    up to that date, or --month to load data for just one
    month.

    You can also supply --start_date, or supply a file path that
    includes a timestamp with --month_from_prescribing_filename

    '''

    def add_arguments(self, parser):
        parser.add_argument('--month')
        parser.add_argument('--month_from_prescribing_filename')
        parser.add_argument('--start_date')
        parser.add_argument('--end_date')
        parser.add_argument('--measure')
        parser.add_argument('--definitions_only', action='store_true')

    def handle(self, *args, **options):
        self.centiles = range(10, 100, 10)
        options = self.parse_options(options)
        for m in options['measure_ids']:
            measure_config = options['measures'][m]
            measure = self.create_or_update_measure(m, measure_config)
            if options['definitions_only']:
                continue
            for month in options['months']:
                MeasureValue.objects.filter(month=month)\
                            .filter(measure=measure).delete()
                MeasureGlobal.objects.filter(month=month)\
                    .filter(measure=measure).delete()

                # Create practice values and percentiles. Use these to
                # calculate global values and percentiles. Then
                # calculate cost savings for individual practices, and
                # globally.
                self.create_practice_measurevalues(
                    measure, month, measure_config)
                records = MeasureValue.objects.filter(
                    month=month, measure=measure).values()
                df = self.create_dataframe_with_ranks_and_percentiles(records)
                mg = self.create_or_update_measureglobal(
                    df, measure, month, 'practice')
                for i, row in df.iterrows():
                    self.set_percentile_and_savings(
                        row, measure, month, mg, 'practice')
                if measure.is_cost_based:
                    self.set_measureglobal_savings(mg, 'practice')

                # Now calculate CCG values, percentiles and cost savings.
                self.create_ccg_measurevalues(measure, month)
                ccg_records = MeasureValue.objects.filter(
                    month=month, measure=measure, practice=None).values()
                df = self.create_dataframe_with_ranks_and_percentiles(
                    ccg_records)
                mg = self.create_or_update_measureglobal(
                    df, measure, month, 'ccg')
                for i, row in df.iterrows():
                    self.set_percentile_and_savings(
                        row, measure, month, mg, 'ccg')
                if measure.is_cost_based:
                    self.set_measureglobal_savings(mg, 'ccg')

    def parse_options(self, options):
        self.IS_VERBOSE = False
        if options['verbosity'] > 1:
            self.IS_VERBOSE = True
        fpath = os.path.dirname(__file__)
        files = glob.glob(fpath + "/measure_definitions/*.json")
        options['measures'] = {}
        for fname in files:
            fname = os.path.join(fpath, fname)
            json_data = open(fname).read()
            d = json.loads(json_data)
            for k in d:
                if k in options['measures']:
                    sys.exit()
                    print "duplicate entry found!", k
                else:
                    options['measures'][k] = d[k]
        if 'measure' in options and options['measure']:
            options['measure_ids'] = [options['measure']]
        else:
            options['measure_ids'] = [k for k in options['measures']]

        # Get months to cover from options.
        if not options['month'] and not options['end_date'] \
           and not options['month_from_prescribing_filename']:
            err = 'You must supply either --month or --end_date '
            err += 'in the format YYYY-MM-DD, supply a path to a file which '
            err += 'includes the timetamp in the path. You can also '
            err += 'optionally supply a start date.'
            print err
            sys.exit()
        options['months'] = []
        if 'month' in options and options['month']:
            options['months'] = [options['month']]
        elif 'month_from_prescribing_filename' in options \
             and options['month_from_prescribing_filename']:
            filename = options['month_from_prescribing_filename']
            date_part = re.findall(r'/(\d{4}_\d{2})/', filename)[0]
            month = datetime.strptime(date_part + "_01", "%Y_%m_%d")
            options['month'] = month.strftime('%Y-%m-01')
        else:
            if 'start_date' in options and options['start_date']:
{'fraction_non_alphanumeric': 0.09372549019607843, 'fraction_numerical': 0.0022549019607843138, 'mean_word_length': 2.0073702830188678, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5043366', 'n_tokens_mistral': 2786, 'n_tokens_neox': 2611, 'n_words': 727}