<commit_before>require File.dirname(__FILE__) + '/redis_repeater/scheduler.rb'
require File.dirname(__FILE__) + '/redis_repeater/scheduler_job.rb'
require File.dirname(__FILE__) + '/redis_repeater/transfer_scheduler_job.rb'

require 'yaml'
require 'logger'
require 'fileutils'

require 'eventmachine'
require 'redis'

module RedisRepeater
 
  DefaultRedisHost = 'localhost'
  DefaultRedisPort = 6379
  LogDefaultFilename = File.dirname(__FILE__) + '/../log/redis_repeater.log'

  def self.start(config_dir)

    # Connect to redis
    config = YAML::load File.open("#{config_dir}/config.yml")
    redis_from = redis_configure(config, 'origin')
    redis_to = redis_configure(config, 'destination')

    # Load the queues from the config file
    queues = YAML::load File.open("#{config_dir}/queues.yml")

    # Logger
    if config.has_key?('log')
      log_filename = config['log']
      FileUtils.mkdir_p(File.dirname(log_filename))
      logger = Logger.new(log_filename)
    else
      logger = Logger.new(STDOUT)
    end

    # Load the queues into the scheduler
    scheduler = Scheduler.new(logger)
    EventMachine.run do
      queues.each do |name, timeout|
        scheduler << TransferSchedulerJob.new(name, timeout, logger, redis_from, redis_to)
      end
    end

    # Perform forever
    scheduler.perform # TODO make configurable

  end

  def self.redis_configure(config, name)
    host = DefaultRedisHost
    port = DefaultRedisPort
    if config.has_key?(name)
      host = config[name]['host'] if config[name].has_key?('host')
      port = config[name]['port'] if config[name].has_key?('port')
    end
    Redis.new(:host => host, :port => port)
  end

end<commit_msg> eventmachine typo.. (told you i was a noob) also puts'ng queue config on startup<commit_after>require File.dirname(__FILE__) + '/redis_repeater/scheduler.rb'
require File.dirname(__FILE__) + '/redis_repeater/scheduler_job.rb'
require File.dirname(__FILE__) + '/redis_repeater/transfer_scheduler_job.rb'

require 'yaml'
require 'logger'
require 'fileutils'

require 'eventmachine'
require 'redis'

module RedisRepeater
 
  DefaultRedisHost = 'localhost'
  DefaultRedisPort = 6379
  LogDefaultFilename = File.dirname(__FILE__) + '/../log/redis_repeater.log'

  def self.start(config_dir)

    # Connect to redis
    config = YAML::load File.open("#{config_dir}/config.yml")
    redis_from = redis_configure(config, 'origin')
    redis_to = redis_configure(config, 'destination')

    # Load the queues from the config file
    queues = YAML::load File.open("#{config_dir}/queues.yml")

    # Logger
    if config.has_key?('log')
      log_filename = config['log']
      FileUtils.mkdir_p(File.dirname(log_filename))
      logger = Logger.new(log_filename)
    else
      logger = Logger.new(STDOUT)
    end

    # Load the queues into the scheduler
    scheduler = Scheduler.new(logger)
    EventMachine::run do
      queues.each do |name, timeout|
        scheduler << TransferSchedulerJob.new(name, timeout, logger, redis_from, redis_to)
      end
    end

    # Perform forever
    scheduler.perform # TODO make configurable

  end

  def self.redis_configure(config, name)
    host = DefaultRedisHost
    port = DefaultRedisPort
    if config.has_key?(name)
      host = config[name]['host'] if config[name].has_key?('host')
      port = config[name]['port'] if config[name].has_key?('port')
    end
    Redis.new(:host => host, :port => port)
  end

end
{'fraction_non_alphanumeric': 0.11669085631349782, 'fraction_numerical': 0.0023222060957910013, 'mean_word_length': 3.772853185595568, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1164642', 'n_tokens_mistral': 1155, 'n_tokens_neox': 1117, 'n_words': 247}