<commit_before>import csv
import glob
from django.core.management.base import BaseCommand, CommandError


class Command(BaseCommand):
    args = ''
    help = 'Converts HSCIC data files into the format needed for our SQL COPY '
    help += 'statement. We use COPY because it is much faster than INSERT.'

    def add_arguments(self, parser):
        parser.add_argument('--filename')

    def handle(self, *args, **options):
        self.IS_VERBOSE = False
        if options['verbosity'] > 1:
            self.IS_VERBOSE = True

        if 'is_test' in options:
            self.IS_TEST = True
        else:
            self.IS_TEST = False

        if options['filename']:
            filenames = [options['filename']]
        else:
            filenames = glob.glob('./data/raw_data/T*PDPI+BNFT.CSV')

        for f in filenames:
            if self.IS_VERBOSE:
                print "--------- Converting %s -----------" % f
            reader = csv.reader(open(f, 'rU'))
            next(reader)
            filename_for_output = self.create_filename_for_output_file(f)
            writer = csv.writer(open(filename_for_output, 'wb'))
            for row in reader:
                if len(row) == 1:
                    continue
                data = self.format_row_for_sql_copy(row)
                writer.writerow(data)

    def create_filename_for_output_file(self, filename):
        if self.IS_TEST:
            return filename[:-4] + '_test.CSV'
        else:
            return filename[:-4] + '_formatted.CSV'

    def format_row_for_sql_copy(self, row):
        '''
        Transform the data into the format needed for COPY.
        '''
        row = [r.strip() for r in row]
        chemical_id = self.get_chemical_id(row[3])
        actual_cost = float(row[7])
        quantity = int(row[8])
        if quantity:
            price_per_unit = actual_cost / float(quantity)<commit_msg>Handle HSCIC's inconsistent file suffixes.<commit_after>import csv
import glob
from django.core.management.base import BaseCommand, CommandError


class Command(BaseCommand):
    args = ''
    help = 'Converts HSCIC data files into the format needed for our SQL COPY '
    help += 'statement. We use COPY because it is much faster than INSERT.'

    def add_arguments(self, parser):
        parser.add_argument('--filename')

    def handle(self, *args, **options):
        self.IS_VERBOSE = False
        if options['verbosity'] > 1:
            self.IS_VERBOSE = True

        if 'is_test' in options:
            self.IS_TEST = True
        else:
            self.IS_TEST = False

        if options['filename']:
            filenames = [options['filename']]
        else:
            filenames = glob.glob('./data/raw_data/T*PDPI+BNFT.*')

        for f in filenames:
            if self.IS_VERBOSE:
                print "--------- Converting %s -----------" % f
            reader = csv.reader(open(f, 'rU'))
            next(reader)
            filename_for_output = self.create_filename_for_output_file(f)
            writer = csv.writer(open(filename_for_output, 'wb'))
            for row in reader:
                if len(row) == 1:
                    continue
                data = self.format_row_for_sql_copy(row)
                writer.writerow(data)

    def create_filename_for_output_file(self, filename):
        if self.IS_TEST:
            return filename[:-4] + '_test.CSV'
        else:
            return filename[:-4] + '_formatted.CSV'

    def format_row_for_sql_copy(self, row):
        '''
        Transform the data into the format needed for COPY.
        '''
        row = [r.strip() for r in row]
        chemical_id = self.get_chemical_id(row[3])
        actual_cost = float(row[7])
        quantity = int(row[8])
        if quantity:
            price_per_unit = actual_cost / float(quantity)
{'fraction_non_alphanumeric': 0.11041503523884104, 'fraction_numerical': 0.0036543983294179067, 'mean_word_length': 1.9162861491628616, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3317979', 'n_tokens_mistral': 1149, 'n_tokens_neox': 1103, 'n_words': 281}