<commit_before>require 'progressbar'

namespace :scraper do
  desc 'Scrape clubs from the AGA web site'
  task :scrape_clubs => :environment do
    print 'Scraping clubs from usgo.org...'
    STDOUT.flush

    type_id = Type.find_by_name('Go Club').id
    count = 0

    ClubScraper.get_clubs_from_url('http://usgo.org/cgi-bin/chapters.cgi?state=ALL') do |club|
      location = Location.new

      location.type_id = type_id
      location.name = club[:name]
      location.street_address = club[:address]
      location.city = club[:city]
      #Set the country - default country is United States if none is set.
      location.country = club[:country] || 'US'
      #Set the state by two-letter code, or whatever the scraper gives us if we can't find it.
      location.state = STATE_TO_ABBREV[location.country][club[:state]] || club[:state]
      if club[:phone]
        location.phone_number = club[:phone][0][:number]
      end
      location.contacts = club[:contacts]
      location.url = club[:url]
      location.description = club[:info]
      location.hidden = false
      #location.is_aga = club[:is_aga?]

      location.save!

      count += 1
    end

    puts 'done!'
    puts "Got #{count} clubs from usgo.org"
  end

  desc 'Load clubs from MDB file'
  task :load_mdb => :environment do
    CsvLoader.load_mdb('db/chapclub.csv')
  end<commit_msg>Fixed issue 27 - club[:is_aga?] is now used properly, and the scraper creates affiliations.<commit_after>require 'progressbar'

namespace :scraper do
  desc 'Scrape clubs from the AGA web site'
  task :scrape_clubs => :environment do
    print 'Scraping clubs from usgo.org...'
    STDOUT.flush

    type_id = Type.find_by_name('Go Club').id
    count = 0

    ClubScraper.get_clubs_from_url('http://usgo.org/cgi-bin/chapters.cgi?state=ALL') do |club|
      location = Location.new

      location.type_id = type_id
      location.name = club[:name]
      location.street_address = club[:address]
      location.city = club[:city]
      #Set the country - default country is United States if none is set.
      location.country = club[:country] || 'US'
      #Set the state by two-letter code, or whatever the scraper gives us if we can't find it.
      location.state = STATE_TO_ABBREV[location.country][club[:state]] || club[:state]
      if club[:phone]
        location.phone_number = club[:phone][0][:number]
      end
      location.contacts = club[:contacts]
      location.url = club[:url]
      location.description = club[:info]
      location.hidden = false
      location.save!
      #If it is an AGA club, we create the affiliation, which requires some data massaging.
      if club[:is_aga?]
        aga = Affiliate.find_by_name('AGA')
        begin
          name = location.contacts[0][:name]
        rescue NoMethodError
          name = nil
        end
        
        begin
          email = location.contacts[0][:email]
        rescue NoMethodError
          email = nil
        end

        begin
          phone = location.contacts[0][:phone][0][:number]
        rescue NoMethodError
          phone = nil
        end
        affiliation = Affiliation.new(:location => location, :affiliate => aga, :expires => Time.now + 1.year,
                                 :contact_name => name, :contact_address => location.street_address,
                                 :contact_city => location.city, :contact_state => location.state,
                                 :contact_zip => location.zip_code, :contact_telephone => phone,
                                 :contact_email => email, :foreign_key => location.id)
        affiliation.save!
      end

      count += 1
    end

    puts 'done!'
    puts "Got #{count} clubs from usgo.org"
  end

  desc 'Load clubs from MDB file'
  task :load_mdb => :environment do
    CsvLoader.load_mdb('db/chapclub.csv')
  end
{'fraction_non_alphanumeric': 0.10808002078461938, 'fraction_numerical': 0.0033775006495193557, 'mean_word_length': 2.449820788530466, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3477311', 'n_tokens_mistral': 1187, 'n_tokens_neox': 1144, 'n_words': 310}