<commit_before>require 'mechanize'

namespace :craigslist_keywords do
  desc "Import Craigslist Keywords"
  task :import => :environment do
    cities = ["miami", "tampa", "dallas", "houston", "phoenix", "losangeles", "sfbay", "seattle", "minneapolis", "chicago", "philadelphia"]

    agent = Mechanize.new
    ad_urls = []

    for city in cities
      puts "Fetching #{city}"
      page = agent.get("http://#{city}.craigslist.org/apa/")

      20.times {|i|
        ad_links = page.links.reject{|l| !(l.href =~ /[0-9]{5,}[.]html$/) }
        ad_links.each{|l| ad_urls << l.href }
        page = page.link_with(:text => "next 100 postings").click
      }
    end

    ad_urls.uniq!

    words_hash = {}
    
    puts "Starting fetch of #{ad_urls.size} ads"
    i = 0
    print "0"
    ad_urls.each{|ad_url|
      begin
        ad = agent.get(ad_url)
        text = ad.at('div#userbody').text
        text.gsub!(/[^A-Za-z-]/, ' ')
        text.gsub!(/---*/, ' ')
        words = text.split(/\s\s*/)
        words.each{|word_in|
          word = word_in.downcase<commit_msg>Reduce number of page fetches<commit_after>require 'mechanize'

namespace :craigslist_keywords do
  desc "Import Craigslist Keywords"
  task :import => :environment do
    cities = ["miami", "tampa", "dallas", "houston", "phoenix", "losangeles", "sfbay", "seattle", "minneapolis", "chicago", "philadelphia"]

    agent = Mechanize.new
    ad_urls = []

    for city in cities
      puts "Fetching #{city}"
      page = agent.get("http://#{city}.craigslist.org/apa/")

      10.times {|i|
        ad_links = page.links.reject{|l| !(l.href =~ /[0-9]{5,}[.]html$/) }
        ad_links.each{|l| ad_urls << l.href }
        page = page.link_with(:text => "next 100 postings").click
      }
    end

    ad_urls.uniq!

    words_hash = {}
    
    puts "Starting fetch of #{ad_urls.size} ads"
    i = 0
    print "0"
    ad_urls.each{|ad_url|
      begin
        ad = agent.get(ad_url)
        text = ad.at('div#userbody').text
        text.gsub!(/[^A-Za-z-]/, ' ')
        text.gsub!(/---*/, ' ')
        words = text.split(/\s\s*/)
        words.each{|word_in|
          word = word_in.downcase
{'fraction_non_alphanumeric': 0.17946345975948197, 'fraction_numerical': 0.009250693802035153, 'mean_word_length': 2.384976525821596, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5812035', 'n_tokens_mistral': 819, 'n_tokens_neox': 802, 'n_words': 157}