<commit_before>module Couriers
  class Fusionable
    class InvalidFusionTableError < Exception; end 
  
    extend Resque::Plugins::HerokuAutoscaler
    
    @queue = 'couriers'
    
    class << self
      extend ActiveSupport::Memoizable

      attr_accessor :username, :password, :table_id
      
      def client
        GData::Client::FusionTables.new.tap do |c|
          c.clientlogin self.username, self.password
        end
      end
      memoize :client
      
      def execute id
        notify "Adding to Fusionable queue", Photo.find(id).key
        Resque.redis.rpush 'fusionable', id
      end
    
      def table
        puts "Selecting Fusion Table (#{self.table_id})"
        client.show_tables.select { |t| t.id == self.table_id }.first or raise InvalidFusionTableError
      end
      memoize :table
      
      def perform
        while id = Resque.redis.lpop('fusionable')
          begin
            photo = Photo.find id
          
            notify "Adding to Fusion Tables", photo.key
            table.select("ROWID", "WHERE name='#{photo.key}'").map(&:values).map(&:first).map { |id| table.delete id }
            table.insert [photo.to_fusion]
          rescue => e
            notify "Fusion Tables failed", id
          end
        end
      end
    end
  end
end
<commit_msg>Allow up to 5 queries per second<commit_after>module Couriers
  class Fusionable
    class InvalidFusionTableError < Exception; end 
  
    extend Resque::Plugins::HerokuAutoscaler
    
    @queue = 'couriers'
    
    class << self
      extend ActiveSupport::Memoizable

      attr_accessor :username, :password, :table_id
      
      def client
        GData::Client::FusionTables.new.tap do |c|
          c.clientlogin self.username, self.password
        end
      end
      memoize :client
      
      def execute id
        notify "Adding to Fusionable queue", Photo.find(id).key
        Resque.redis.rpush 'fusionable', id
      end
    
      def table
        puts "Selecting Fusion Table (#{self.table_id})"
        client.show_tables.select { |t| t.id == self.table_id }.first or raise InvalidFusionTableError
      end
      memoize :table
      
      def perform
        while id = Resque.redis.lpop('fusionable')
          begin
            photo = Photo.find id
          
            notify "Adding to Fusion Tables", photo.key
            table.select("ROWID", "WHERE name='#{photo.key}'").map(&:values).map(&:first).map { |id| table.delete id }
            table.insert [photo.to_fusion]
            sleep 0.6 # 5 queries per second
          rescue => e
            notify "Fusion Tables failed", id
          end
        end
      end
    end
  end
end

{'fraction_non_alphanumeric': 0.08996282527881042, 'fraction_numerical': 0.001486988847583643, 'mean_word_length': 1.9154929577464788, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1767377', 'n_tokens_mistral': 801, 'n_tokens_neox': 775, 'n_words': 210}