<commit_before>namespace :geocode do
  desc "Geocode all objects without coordinates."
  task :all => :environment do
    class_name = ENV['CLASS'] || ENV['class']
    sleep_timer = ENV['SLEEP'] || ENV['sleep']
    batch = ENV['BATCH'] || ENV['batch']
    raise "Please specify a CLASS (model)" unless class_name
    klass = class_from_string(class_name)
    batch = batch.to_i unless batch.nil?

    klass.not_geocoded.find_each(batch_size: batch) do |obj|
      obj.geocode; obj.save
      sleep(sleep_timer.to_f) unless sleep_timer.nil?
    end
  end
end

##
# Get a class object from the string given in the shell environment.
# Similar to ActiveSupport's +constantize+ method.
#
def class_from_string(class_name)
  parts = class_name.split("::")
  constant = Object
  parts.each do |part|
    constant = constant.const_get(part)
  end
  constant
end
<commit_msg>#656 rake task for mongo<commit_after>namespace :geocode do
  desc "Geocode all objects without coordinates."
  task :all => :environment do
    class_name = ENV['CLASS'] || ENV['class']
    sleep_timer = ENV['SLEEP'] || ENV['sleep']
    batch = ENV['BATCH'] || ENV['batch']
    raise "Please specify a CLASS (model)" unless class_name
    klass = class_from_string(class_name)

    if batch.nil?
      klass.not_geocoded.each { |obj| geocode_obj(obj, sleep_timer) }
    else
      klass.not_geocoded.take(batch.to_i).each { |obj| geocode_obj(obj, sleep_timer) }
    end
    
  end
end

def geocode_obj(obj, sleep_timer)
  obj.geocode; obj.save
  sleep(sleep_timer.to_f) unless sleep_timer.nil?
end

##
# Get a class object from the string given in the shell environment.
# Similar to ActiveSupport's +constantize+ method.
#
def class_from_string(class_name)
  parts = class_name.split("::")
  constant = Object
  parts.each do |part|
    constant = constant.const_get(part)
  end
  constant
end

{'fraction_non_alphanumeric': 0.1143317230273752, 'fraction_numerical': 0.001610305958132045, 'mean_word_length': 3.931216931216931, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6779240', 'n_tokens_mistral': 639, 'n_tokens_neox': 635, 'n_words': 169}