<commit_before>require File.dirname(__FILE__) +  '/data_loader'
namespace :naptan do
    
  namespace :load do
    
    include DataLoader
  
    desc "Loads stop data from a CSV file specified as FILE=filename"
    task :stops => :environment do 
      parse('stops', Parsers::NaptanParser)
    end 
    
    desc "Loads stop area data from a CSV file specified as FILE=filename"
    task :stop_areas => :environment do 
      parse('stop_areas', Parsers::NaptanParser)
    end
    
    desc "Loads stop area membership data from a CSV file specified as FILE=filename"
    task :stop_area_memberships => :environment do 
      parse('stop_area_memberships', Parsers::NaptanParser)
    end
    
    desc "Loads stop area hierarchy from a CSV file specified as FILE=filename"
    task :stop_area_hierarchy => :environment do 
      parse('stop_area_hierarchy', Parsers::NaptanParser)
    end
        
    desc "Loads stop type data from a CSV file specified as FILE=filename"
    task :stop_types => :environment do 
      parse('stop_types', Parsers::NaptanParser)
    end
    
    desc "Loads stop area type data from a CSV file specified as FILE=filename"
    task :stop_area_types => :environment do 
      parse('stop_area_types', Parsers::NaptanParser)
    end
  
    desc "Loads all data from CSV files in a directory specified as DIR=dirname"
    task :all => :environment do 
      unless ENV['DIR']
        usage_message "usage: rake naptan:load:all DIR=dirname"
      end
      puts "Loading data from #{ENV['DIR']}..."
      ENV['FILE'] = File.join(RAILS_ROOT, 'data', 'NaPTAN', 'StopTypes.csv')
      Rake::Task['naptan:load:stop_types'].execute
      ENV['FILE'] = File.join(RAILS_ROOT, 'data', 'NaPTAN', 'StopAreaTypes.csv')
      Rake::Task['naptan:load:stop_area_types'].execute
      ENV['FILE'] = File.join(ENV['DIR'], 'Stops.csv')
      Rake::Task['naptan:load:stops'].execute
      ENV['FILE'] = File.join(ENV['DIR'], 'StopAreas.csv')
      Rake::Task['naptan:load:stop_areas'].execute
      ENV['FILE'] = File.join(ENV['DIR'], 'StopsInArea.csv')
      Rake::Task['naptan:load:stop_area_memberships'].execute   
      ENV['FILE'] = File.join(ENV['DIR'], 'AreaHierarchy.csv')
      Rake::Task['naptan:load:stop_area_hierarchy'].execute   
    end
    
    desc 'Load new stop data from CSV file specified as FILE=filename'
    task :new_stops => :environment do 
      check_for_file
      puts "Loading new stops from #{ENV['FILE']}..."
      parser = Parsers::NaptanParser.new 
      stop_attributes = [:atco_code,
                         :naptan_code,
  		                   :plate_code,
                         :common_name,
                         :landmark,
                         :street,
                         :indicator,
                         :bearing,
                         :locality,
                         :town,
                         :suburb,
                         :locality_centre,<commit_msg>New naptan file names.<commit_after>require File.dirname(__FILE__) +  '/data_loader'
namespace :naptan do
    
  namespace :load do
    
    include DataLoader
  
    desc "Loads stop data from a CSV file specified as FILE=filename"
    task :stops => :environment do 
      parse('stops', Parsers::NaptanParser)
    end 
    
    desc "Loads stop area data from a CSV file specified as FILE=filename"
    task :stop_areas => :environment do 
      parse('stop_areas', Parsers::NaptanParser)
    end
    
    desc "Loads stop area membership data from a CSV file specified as FILE=filename"
    task :stop_area_memberships => :environment do 
      parse('stop_area_memberships', Parsers::NaptanParser)
    end
    
    desc "Loads stop area hierarchy from a CSV file specified as FILE=filename"
    task :stop_area_hierarchy => :environment do 
      parse('stop_area_hierarchy', Parsers::NaptanParser)
    end
        
    desc "Loads stop type data from a CSV file specified as FILE=filename"
    task :stop_types => :environment do 
      parse('stop_types', Parsers::NaptanParser)
    end
    
    desc "Loads stop area type data from a CSV file specified as FILE=filename"
    task :stop_area_types => :environment do 
      parse('stop_area_types', Parsers::NaptanParser)
    end
  
    desc "Loads all data from CSV files in a directory specified as DIR=dirname"
    task :all => :environment do 
      unless ENV['DIR']
        usage_message "usage: rake naptan:load:all DIR=dirname"
      end
      puts "Loading data from #{ENV['DIR']}..."
      ENV['FILE'] = File.join(RAILS_ROOT, 'data', 'NaPTAN', 'StopTypes.csv')
      Rake::Task['naptan:load:stop_types'].execute
      ENV['FILE'] = File.join(RAILS_ROOT, 'data', 'NaPTAN', 'StopAreaTypes.csv')
      Rake::Task['naptan:load:stop_area_types'].execute
      ENV['FILE'] = File.join(ENV['DIR'], 'Stops.csv')
      Rake::Task['naptan:load:stops'].execute
      ENV['FILE'] = File.join(ENV['DIR'], 'Groups.csv')
      Rake::Task['naptan:load:stop_areas'].execute
      ENV['FILE'] = File.join(ENV['DIR'], 'StopsInGroup.csv')
      Rake::Task['naptan:load:stop_area_memberships'].execute   
      ENV['FILE'] = File.join(ENV['DIR'], 'GroupsInGroup.csv')
      Rake::Task['naptan:load:stop_area_hierarchy'].execute   
    end
    
    desc 'Load new stop data from CSV file specified as FILE=filename'
    task :new_stops => :environment do 
      check_for_file
      puts "Loading new stops from #{ENV['FILE']}..."
      parser = Parsers::NaptanParser.new 
      stop_attributes = [:atco_code,
                         :naptan_code,
  		                   :plate_code,
                         :common_name,
                         :landmark,
                         :street,
                         :indicator,
                         :bearing,
                         :locality,
                         :town,
                         :suburb,
                         :locality_centre,
{'fraction_non_alphanumeric': 0.11129933931899035, 'fraction_numerical': 0.0, 'mean_word_length': 2.4186450492182976, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1318609', 'n_tokens_mistral': 1835, 'n_tokens_neox': 1754, 'n_words': 446}