<commit_before>require 'restclient'
require 'json'
require 'base64'
require 'couch'
require 'actionpool'

class PopulateTask < Rake::TaskLib
  
  def initialize name = :populate
    @name = name
    define
  end
  
  def define
    desc "Populate a CLASSIFIER with existing data from COUCH."
    task @name do
      unless ENV['CLASSIFIER'] && ENV['COUCH']
        abort "You must set CLASSIFIER and COUCH environment variables!"
      end
      
      cla = ENV['CLASSIFIER'].sub(/\/?$/,'')
      couch = Couch.new(ENV['COUCH'].sub(/\/?$/,'/'))
      couch.create!
      
      count = couch.size.to_f
      progress = 0.0
      percent = 0.0

      start = Time.now.to_i
      pool = ActionPool::Pool.new :min_threads => 3, :max_threads => 10
      couch.each do |doc|
        data = JSON(doc['data'])
        id = doc['id']
        pool.process do
          RestClient.post "#{cla}/train/#{Base64.encode64(id)}", data
          puts "#{percent = (progress*100.0/count).floor}% geladen" if ((progress+=1)*100.0/count).floor > percent
        end
      end
      puts "done. #{(Time.now.to_i-start)} secs."
    end
  end

end<commit_msg>Revert "Thread pooling in populate task"<commit_after>require 'restclient'
require 'json'
require 'base64'
require 'couch'

class PopulateTask < Rake::TaskLib
  
  def initialize name = :populate
    @name = name
    define
  end
  
  def define
    desc "Populate a CLASSIFIER with existing data from COUCH."
    task @name do
      unless ENV['CLASSIFIER'] && ENV['COUCH']
        abort "You must set CLASSIFIER and COUCH environment variables!"
      end
      
      cla = ENV['CLASSIFIER'].sub(/\/?$/,'')
      couch = Couch.new(ENV['COUCH'].sub(/\/?$/,'/'))
      couch.create!
      
      count = couch.size.to_f
      progress = 0.0
      percent = 0.0

      start = Time.now.to_i
      couch.each do |doc|
        data = JSON(doc['data'])
        id = doc['id']
        RestClient.post "#{cla}/train/#{Base64.encode64(id)}", data
        progress += 1
        puts "#{percent = (progress*100.0/count).floor}% geladen" if (progress*100.0/count).floor > percent
      end
      puts "done. #{(Time.now.to_i-start)} secs."
    end
  end

end
{'fraction_non_alphanumeric': 0.1444901691815272, 'fraction_numerical': 0.018747142203932327, 'mean_word_length': 2.3609831029185866, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '300446', 'n_tokens_mistral': 773, 'n_tokens_neox': 740, 'n_words': 185}