<commit_before>from django import db
from django.conf import settings
from django.core.management.base import NoArgsCommand
from data.models import AlternativeFuelVehicles
import csv

# National Priorities Project Data Repository
# import_alternative_fuel_vehicles.py
# Updated 5/17/2010, <NAME>, Sunlight Foundation

# Imports Department of Energy Alternative Fuel Vehicles Total
# source info: http://www.eia.doe.gov/cneaf/alternate/page/atftables/attf_v2.xls (accurate as of 5/17/2010)
# npp csv: http://assets.nationalpriorities.org/raw_data/doe.gov/attf_v2.csv (updated 5/17/2010)
# destination model:  AlternativeFuelVehicles

# HOWTO:
# 1) Download source files from url listed above
# 2) Convert source file to .csv with same formatting as npp csv
# 3) change SOURCE_FILE variable to the the path of the source file you just created
# 5) Run as Django management command from your project path "python manage.py import_alternative_fuel_vehicles

SOURCE_FILE = '%s/doe.gov/attf_v2.csv' % (settings.LOCAL_DATA_ROOT)

class Command(NoArgsCommand):
    
    def handle_noargs(self, **options):
        data_reader = csv.reader(open(SOURCE_FILE))
        first_year = 2003
        last_year = 2007
        total_years = last_year - first_year + 1

        #populate year array
        years = []
        for h in range(0, total_years):
            years.append(first_year + h)
        
        def clean_int(value):
            return int(value.replace(',', '').replace(' ', ''))
        
        for i, row in enumerate(data_reader):
            if i > 0:
                state = row[0]
                for j, col in enumerate(row):
                    if j > 0 and j < total_years + 1:
                        year = years[j-1]
                        value = clean_int(row[j])

                        record = AlternativeFuelVehicles(state=state, year=year, value=value)
                        record.save()<commit_msg>updates for alternative fuel vehicles<commit_after>from django import db
from django.conf import settings
from django.core.management.base import NoArgsCommand
from data.models import AlternativeFuelVehicles
import csv

# National Priorities Project Data Repository
# import_alternative_fuel_vehicles.py
# Updated 5/17/2010, <NAME>, Sunlight Foundation

# Imports Department of Energy Alternative Fuel Vehicles Total
# source info: http://www.eia.doe.gov/cneaf/alternate/page/atftables/attf_v2.xls (accurate as of 5/17/2010)
# npp csv: http://assets.nationalpriorities.org/raw_data/doe.gov/attf_v2.csv (updated 5/17/2010)
# destination model:  AlternativeFuelVehicles

# HOWTO:
# 1) Download source files from url listed above
# 2) Convert source file to .csv with same formatting as npp csv
# 3) change SOURCE_FILE variable to the the path of the source file you just created
# 5) Run as Django management command from your project path "python manage.py import_alternative_fuel_vehicles
#
# SAFE TO RE-RUN OR RELOAD A YEAR: NO

SOURCE_FILE = '%s/doe.gov/attf_v2.csv' % (settings.LOCAL_DATA_ROOT)

class Command(NoArgsCommand):
    
    def handle_noargs(self, **options):
        data_reader = csv.reader(open(SOURCE_FILE))
        first_year = 2004
        last_year = 2008
        total_years = last_year - first_year + 1

        #populate year array
        years = []
        for h in range(0, total_years):
            years.append(first_year + h)
        
        def clean_int(value):
            try:
                return int(value.replace(',', '').replace(' ', ''))
            except:
                print 'clean_int fail for value ' + str(value)
        
        for i, row in enumerate(data_reader):
            if i > 0:
                state = row[0]
                if len(state):
                    for j, col in enumerate(row):
                        if j > 0 and j < total_years + 1:
                            year = years[j-1]
                            value = clean_int(row[j])

                            record = AlternativeFuelVehicles(state=state, year=year, value=value)
                            record.save()
{'fraction_non_alphanumeric': 0.08731910718665685, 'fraction_numerical': 0.021093941623742948, 'mean_word_length': 2.4500846023688663, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4553455', 'n_tokens_mistral': 1292, 'n_tokens_neox': 1172, 'n_words': 353}