<commit_before>require 'set'
require 'csv'

DATE_RANGE = (2004..2010)
BATCH_SIZE = 1000

namespace :data do
  desc "Load crash data"
  task :create_indexes => :environment do
    Crash.ensure_index [[:accn, 1]], :unique => true
    Crash.ensure_index [[:city, 1], [:year, 1]]
    Crash.ensure_index [[:county, 1], [:year, 1]]
    Crash.ensure_index [[:day, 1]]
    Crash.ensure_index [[:month, 1]]
    Crash.ensure_index [[:year, 1]]
    Crash.ensure_index [[:weekday, 1]]
    Crash.ensure_index [["vehicles.vehcolor", 1]]
  end
  
  namespace :crashes do
    desc "Load crash data"
    task :load, [:year] => :environment do |t, args|
      year = args.year
      layout = process_layout Crash, "A7A9A9"
      puts "Loading crash data for #{year}"
      year_file = File.join Rails.root, "lib", "data", "crash", "mn-#{year}-acc.txt"
      crashes = []
      File.open(year_file).each_line do |line|
        crash = {}
        data = line.unpack layout[:pattern]
        puts "Loading #{year} crash for accn #{data[0]}"
        
        layout[:fields].each_with_index do |field, index|
          converter = case layout[:field_types][field.name]
            when "Integer" then "to_i"
            when "String" then "to_s"
          end
          crash[field.name.intern] = data[index].send(converter)
        end
        crash[:route_id] = crash[:rtsys][1..2] + pad_rtnumber(crash[:rtnumber])
        crash[:mile_point] = "#{crash[:truem1]}.#{crash[:truem3]}".to_f
        crash[:accdate] =~ /([\d]{2})\/([\d]{2})\/([\d]{4})/
        crash[:month] = $1
        crash[:day] = $2
        crash[:year] = $3
        crash[:weekday] = Date.strptime(crash[:accdate], "%m/%d/%Y").strftime("%A")
        crashes << crash
        if crashes.size % BATCH_SIZE == 0
          puts "Saving..."
          Crash.collection.insert crashes
          crashes = []
        end
      end
      puts "Saving..."
      Crash.collection.insert crashes
    end
    
    desc "Find, report, and remove duplicate crashes"
    task :remove_duplicates => :environment do
      duplicates_gone = false
      until duplicates_gone
        begin
          Crash.create_index([['accn', Mongo::ASCENDING]], :unique => true)
          duplicates_gone = true
        rescue Mongo::OperationFailure => e
          e.to_s.match /dup key: { : ([0-9]{8,10}) }/
          puts "Found duplicate accn: #{$1}"
          Crash.find_all_by_accn($1.to_i).each do |crash|
            puts "Deleting duplicate accn: #{$1}, accdate: #{crash.accdate}"
            crash.delete
          end
        end
      end
    end
    
    desc "Generate county files"
    task :generate_county => :environment do
      page_size = 1000
      last_page = Crash.count / page_size
      CSV.open("crash-location-data.csv", "wb") do |csv|
        current_page = 1
        until current_page > last_page
          crashes = Crash.paginate({:county => 62, :order => :accn, :per_page => page_size, :page => current_page})
          crashes.each do |crash|
            csv << [crash.accn, crash.route_id, crash.mile_point]
          end
          current_page += 1
        end
      end
    end
  end
  
  namespace :vehicles do
    desc "Load vehicle data"
    task :load, [:year] => :environment do |t, args|
      year = args.year
      layout = process_layout Vehicle, "A5A9A9"
      puts "Loading vehicle data for #{year}"
      year_file = File.join Rails.root, "lib", "data", "vehicle", "mn-#{year}-veh.txt"
      vehicles = []
      File.open(year_file).each_line do |line|<commit_msg>Finished crash location data file generation, now off to make a PostGIS app to do lat/lon conversion...<commit_after>require 'set'
require 'csv'

DATE_RANGE = (2004..2010)
BATCH_SIZE = 1000

namespace :data do
  desc "Create indexes"
  task :create_indexes => :environment do
    Crash.ensure_index [[:accn, 1]], :unique => true
    Crash.ensure_index [[:city, 1], [:year, 1]]
    Crash.ensure_index [[:county, 1], [:year, 1]]
    Crash.ensure_index [[:day, 1]]
    Crash.ensure_index [[:locrel, 1], [:year, 1]]
    Crash.ensure_index [[:month, 1]]
    Crash.ensure_index [[:weekday, 1]]
    Crash.ensure_index [[:year, 1]]
    Crash.ensure_index [["vehicles.vehcolor", 1]]
  end
  
  namespace :crashes do
    desc "Load crash data"
    task :load, [:year] => :environment do |t, args|
      year = args.year
      layout = process_layout Crash, "A7A9A9"
      puts "Loading crash data for #{year}"
      year_file = File.join Rails.root, "lib", "data", "crash", "mn-#{year}-acc.txt"
      crashes = []
      File.open(year_file).each_line do |line|
        crash = {}
        data = line.unpack layout[:pattern]
        puts "Loading #{year} crash for accn #{data[0]}"
        
        layout[:fields].each_with_index do |field, index|
          converter = case layout[:field_types][field.name]
            when "Integer" then "to_i"
            when "String" then "to_s"
          end
          crash[field.name.intern] = data[index].send(converter)
        end
        crash[:route_id] = crash[:rtsys][1..2] + pad_rtnumber(crash[:rtnumber])
        crash[:mile_point] = "#{crash[:truem1]}.#{crash[:truem3]}".to_f
        crash[:accdate] =~ /([\d]{2})\/([\d]{2})\/([\d]{4})/
        crash[:month] = $1
        crash[:day] = $2
        crash[:year] = $3
        crash[:weekday] = Date.strptime(crash[:accdate], "%m/%d/%Y").strftime("%A")
        crashes << crash
        if crashes.size % BATCH_SIZE == 0
          puts "Saving..."
          Crash.collection.insert crashes
          crashes = []
        end
      end
      puts "Saving..."
      Crash.collection.insert crashes
    end
    
    desc "Find, report, and remove duplicate crashes"
    task :remove_duplicates => :environment do
      duplicates_gone = false
      until duplicates_gone
        begin
          Crash.create_index([['accn', Mongo::ASCENDING]], :unique => true)
          duplicates_gone = true
        rescue Mongo::OperationFailure => e
          e.to_s.match /dup key: { : ([0-9]{8,10}) }/
          puts "Found duplicate accn: #{$1}"
          Crash.find_all_by_accn($1.to_i).each do |crash|
            puts "Deleting duplicate accn: #{$1}, accdate: #{crash.accdate}"
            crash.delete
          end
        end
      end
    end
    
    desc "Generate location file"
    task :generate_location_file, [:year] => :environment do |t, args|
      year = args.year
      page_size = 1000
      last_page = Crash.count(:year => year, :locrel => [1, 2, 3]) / page_size + 1
      puts "Generating crash location file for #{year}"
      CSV.open("mn-#{year}-loc.csv", "wb") do |csv|
        current_page = 1
        until current_page > last_page
          crashes = Crash.paginate({:year => year, :locrel => [1, 2, 3], :order => :accn, :per_page => page_size, :page => current_page})
          crashes.each do |crash|
            csv << [crash.accn, crash.route_id, crash.mile_point]
          end
          current_page += 1
        end
      end
    end
  end
  
  namespace :vehicles do
    desc "Load vehicle data"
    task :load, [:year] => :environment do |t, args|
      year = args.year
      layout = process_layout Vehicle, "A5A9A9"
      puts "Loading vehicle data for #{year}"
      year_file = File.join Rails.root, "lib", "data", "vehicle", "mn-#{year}-veh.txt"
      vehicles = []
      File.open(year_file).each_line do |line|
{'fraction_non_alphanumeric': 0.1486211112620568, 'fraction_numerical': 0.016166281755196306, 'mean_word_length': 2.3801652892561984, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '875585', 'n_tokens_mistral': 2537, 'n_tokens_neox': 2492, 'n_words': 605}