<commit_before>"""a work in progress"""
import datetime
from glob import glob
from json import dumps
from subprocess import call
import csv

from django.core.management.base import BaseCommand, CommandError
from disclosures.models import *
from disclosures.scripts import ipeds, prep_ipeds_csv
from django.conf import settings


DUMP = 'dumpdata --indent 4 disclosures > disclosures/fixtures/colleges.json'
DATA_ROOT = "%s/data/ipeds" % settings.PROJECT_ROOT


class Command(BaseCommand):
    args = '<path>'
    help = 'Load the specified CSV into the schools database'

    def handle(self, *args, **options):
        print('dumping backup fixture before starting ...')
        call('python manage.py %s' % DUMP, shell=True)
        print('dump finished; now downloading IPEDS data')
        ipeds.download_tuition(DATA_ROOT)
        print('download finished; now prepping csv')
        csvfile = prep_ipeds_csv()
        return "prepped %s" % csvfile
        starter = datetime.datetime.now()
        count = 0
        newcount = 0
        newaliascount = 0
        for path_spec in args:
            for path in glob(path_spec):
                print "loading %s" % path
                with open(path) as csv_file:
                    reader = csv.DictReader(csv_file)
                    for record in reader:<commit_msg>drop insecure 'shell=True' setting<commit_after>"""a work in progress"""
import datetime
from glob import glob
from json import dumps
from subprocess import call
import csv

from django.core.management.base import BaseCommand, CommandError
from paying_for_college.models import *
# from paying_for_college.disclosures.scripts import ipeds, prep_ipeds_csv
from django.conf import settings


DUMP = ['python',
        'manage.py',
        'dumpdata',
        '--indent',
        '4',
        'paying_for_college',
        '>',
        'paying_for_college/fixtures/colleges.json']
# DATA_ROOT = "%s/data/ipeds" % settings.PROJECT_ROOT


class Command(BaseCommand):
    args = '<path>'
    help = 'Load the specified CSV into the schools database'

    def handle(self, *args, **options):
        print('dumping backup fixture before starting ...')
        call(DUMP)
        # print('dump finished; now downloading IPEDS data')
        # ipeds.download_tuition(DATA_ROOT)
        # print('download finished; now prepping csv')
        # csvfile = prep_ipeds_csv()
        # print("prepped {0}".format(csvfile))
        starter = datetime.datetime.now()
        count = 0
        newcount = 0
        newaliascount = 0
        for path_spec in args:
            for path in glob(path_spec):
                print "loading %s" % path
                with open(path) as csv_file:
                    reader = csv.DictReader(csv_file)
                    for record in reader:
{'fraction_non_alphanumeric': 0.09334763948497854, 'fraction_numerical': 0.003218884120171674, 'mean_word_length': 2.5997425997425996, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3952385', 'n_tokens_mistral': 818, 'n_tokens_neox': 771, 'n_words': 240}