<commit_before>namespace :db do 

  desc "Load all reference data into the database"
  task :populate => :environment do 
    
    # Load transport modes
    Rake::Task['db:seed'].execute
    
    # Load Localities, Regions, Districts, AdminAreas
    ENV['DIR'] = MySociety::Config.get('NPTG_DIR', '')
    Rake::Task['nptg:load:all'].execute
    Rake::Task['naptan:geo:convert_localities'].execute
    
    # Load Stops, StopAreas, give StopAreas lat/lon
    ENV['DIR'] = MySociety::Config.get('NAPTAN_DIR', '')
    Rake::Task['naptan:load:all'].execute
    Rake::Task['naptan:geo:convert_stop_areas'].execute

    # Load Operators
    ENV['FILE'] = File.join(MySociety::Config.get('NOC_DIR', ''), 'NOC_DB_31-03-2010.csv')
    Rake::Task['noc:load:operators'].execute
    <commit_msg>Stop areas have lat and lon fields already populated now.<commit_after>namespace :db do 

  desc "Load all reference data into the database"
  task :populate => :environment do 
    
    # Load transport modes
    Rake::Task['db:seed'].execute
    
    # Load Localities, Regions, Districts, AdminAreas
    ENV['DIR'] = MySociety::Config.get('NPTG_DIR', '')
    Rake::Task['nptg:load:all'].execute
    Rake::Task['naptan:geo:convert_localities'].execute
    
    # Load Stops, StopAreas, give StopAreas lat/lon
    ENV['DIR'] = MySociety::Config.get('NAPTAN_DIR', '')
    Rake::Task['naptan:load:all'].execute

    # Load Operators
    ENV['FILE'] = File.join(MySociety::Config.get('NOC_DIR', ''), 'NOC_DB_31-03-2010.csv')
    Rake::Task['noc:load:operators'].execute
    
{'fraction_non_alphanumeric': 0.15809768637532134, 'fraction_numerical': 0.010282776349614395, 'mean_word_length': 4.172757475083056, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1500343', 'n_tokens_mistral': 564, 'n_tokens_neox': 532, 'n_words': 96}