<commit_before>require 'rubygems'
require 'erb'
require 'fog'
require 'eventmachine'
require 'right_popen'

class GrinderJob
  attr_accessor :status, :output, :logfile, :deployment, :rest_log, :trace_log, :no_resume, :verbose

  def link_to_rightscale
#    i = deployment.href.split(/\//).last
#    d = deployment.href.split(/\./).first.split(/\//).last
#    "https://#{d}.rightscale.com/deployments/#{i}#auditentries"
    deployment.href.gsub(/api\//,"") + "#auditentries"
  end

  # stdout hook for popen3
  def on_read_stdout(data)
    data_ary = data.split("\n")
    data_ary.each_index do |i|
      data_ary[i] = timestamp + data_ary[i]
      $stdout.syswrite("<#{deploy_id}>#{data_ary[i]}\n") if @verbose
    end
    File.open(@logfile, "a") { |f| f.write(data_ary.join("\n") + "\n") }
  end

  # stderr hook for popen3
  def on_read_stderr(data)
    data_ary = data.split("\n")
    data_ary.each_index do |i|
      data_ary[i] = timestamp + data_ary[i]
      $stdout.syswrite("<#{deploy_id}>#{data_ary[i]}\n") if @verbose
    end
    File.open(@logfile, "a") { |f| f.write(data_ary.join("\n") + "\n") }
  end

  def timestamp
    t = Time.now
    "#{t.strftime("[%m/%d/%Y %H:%M:%S.")}%-6d] " % t.usec
  end

  def deploy_id
    @id = deployment.nickname.match(/-[0-9]+-/)[0].match(/[0-9]+/)[0] unless @id
    @id
  end

  # Could be deprecated...
  def receive_data(data)
    data_ary = data.split("\n")
    data_ary.each_index do |i|
      data_ary[i] = timestamp + data_ary[i]
      $stdout.syswrite("<#{deploy_id}>#{data_ary[i]}\n") if @verbose
    end
    File.open(@logfile, "a") { |f| f.write(data_ary.join("\n") + "\n") }
  end

  # unbind hook for popen3
  def unbind
    @status = get_status.exitstatus
  end

  # on_exit hook for popen3
  def on_exit(status)
    @status = status.exitstatus
  end

  # Launch an asynchronous process
  def run(cmd)
    RightScale.popen3(:command        => cmd,
                      :target         => self,
                      :environment    => {"AWS_ACCESS_KEY_ID" => Fog.credentials[:aws_access_key_id],
                                          "AWS_SECRET_ACCESS_KEY" => Fog.credentials[:aws_secret_access_key],
                                          "REST_CONNECTION_LOG" => @rest_log},
                      :stdout_handler => :on_read_stdout,
                      :stderr_handler => :on_read_stderr,
                      :exit_handler   => :on_exit)
  end
end

class GrinderMonk
  attr_accessor :jobs
  attr_accessor :options
  # Runs a grinder test on a single Deployment
  # * deployment<~String> the nickname of the deployment
  # * feature<~String> the feature filename 
  def run_test(deployment, feature, test_ary)
    new_job = GrinderJob.new
    new_job.logfile = File.join(@log_dir, "#{deployment.nickname}.log")
    new_job.rest_log = File.join(@log_dir, "#{deployment.nickname}.rest_connection.log")
    new_job.deployment = deployment
#    new_job.trace_log = File.join(@log_dir, "#{File.basename(feature, ".rb")}.yaml")
    new_job.verbose = true if @options[:verbose]
#    break_point = @options[:breakpoint] if @options[:breakpoint] XXX DEPRECATED XXX
#    cmd = "bin/grinder #{feature} #{break_point}"
    cmd = "bin/grinder -f #{feature} -d \"#{deployment.nickname}\" -g -l #{new_job.logfile} -t "
    test_ary.each { |test| cmd += " \"#{test}\" " }
    cmd += " -n " if @options[:no_resume]
    @jobs << new_job
    puts "running #{cmd}"
    new_job.run(cmd)
  end

  def initialize()
    @started_at = Time.now
    @jobs = []
    @passed = []
    @failed = []
    @running = []
    dirname = Time.now.strftime(File.join("%Y", "%m", "%d", "%H-%M-%S"))
    @log_dir = File.join("log", dirname)
    @log_started = dirname
    FileUtils.mkdir_p(@log_dir)
    @feature_dir = File.join(File.dirname(__FILE__), '..', '..', 'features')
  end
 
  # runs a feature on an array of deployments
  # * deployments<~Array> array of strings containing the nicknames of the deployments
  # * feature_name<~String> the feature filename 
  def run_tests(deployments,cmd,set=[])
    test_case = VirtualMonkey::TestCase.new(@options[:feature], @options)
    total_keys = test_case.get_keys
    total_keys = total_keys - (total_keys - set) unless set.empty?
    keys_per_dep = (total_keys.length.to_f / deployments.length.to_f).ceil
   
    deployment_tests = []
    (keys_per_dep * deployments.length).times { |i|
      di = i % deployments.length
      deployment_tests[di] ||= []
      deployment_tests[di] << total_keys[i % total_keys.length]
    }

    deployments.each_with_index { |d,i| 
      run_test(d,cmd,deployment_tests[i]) <commit_msg>fix for tests were nil<commit_after>require 'rubygems'
require 'erb'
require 'fog'
require 'eventmachine'
require 'right_popen'

class GrinderJob
  attr_accessor :status, :output, :logfile, :deployment, :rest_log, :trace_log, :no_resume, :verbose

  def link_to_rightscale
#    i = deployment.href.split(/\//).last
#    d = deployment.href.split(/\./).first.split(/\//).last
#    "https://#{d}.rightscale.com/deployments/#{i}#auditentries"
    deployment.href.gsub(/api\//,"") + "#auditentries"
  end

  # stdout hook for popen3
  def on_read_stdout(data)
    data_ary = data.split("\n")
    data_ary.each_index do |i|
      data_ary[i] = timestamp + data_ary[i]
      $stdout.syswrite("<#{deploy_id}>#{data_ary[i]}\n") if @verbose
    end
    File.open(@logfile, "a") { |f| f.write(data_ary.join("\n") + "\n") }
  end

  # stderr hook for popen3
  def on_read_stderr(data)
    data_ary = data.split("\n")
    data_ary.each_index do |i|
      data_ary[i] = timestamp + data_ary[i]
      $stdout.syswrite("<#{deploy_id}>#{data_ary[i]}\n") if @verbose
    end
    File.open(@logfile, "a") { |f| f.write(data_ary.join("\n") + "\n") }
  end

  def timestamp
    t = Time.now
    "#{t.strftime("[%m/%d/%Y %H:%M:%S.")}%-6d] " % t.usec
  end

  def deploy_id
    @id = deployment.nickname.match(/-[0-9]+-/)[0].match(/[0-9]+/)[0] unless @id
    @id
  end

  # Could be deprecated...
  def receive_data(data)
    data_ary = data.split("\n")
    data_ary.each_index do |i|
      data_ary[i] = timestamp + data_ary[i]
      $stdout.syswrite("<#{deploy_id}>#{data_ary[i]}\n") if @verbose
    end
    File.open(@logfile, "a") { |f| f.write(data_ary.join("\n") + "\n") }
  end

  # unbind hook for popen3
  def unbind
    @status = get_status.exitstatus
  end

  # on_exit hook for popen3
  def on_exit(status)
    @status = status.exitstatus
  end

  # Launch an asynchronous process
  def run(cmd)
    RightScale.popen3(:command        => cmd,
                      :target         => self,
                      :environment    => {"AWS_ACCESS_KEY_ID" => Fog.credentials[:aws_access_key_id],
                                          "AWS_SECRET_ACCESS_KEY" => Fog.credentials[:aws_secret_access_key],
                                          "REST_CONNECTION_LOG" => @rest_log},
                      :stdout_handler => :on_read_stdout,
                      :stderr_handler => :on_read_stderr,
                      :exit_handler   => :on_exit)
  end
end

class GrinderMonk
  attr_accessor :jobs
  attr_accessor :options
  # Runs a grinder test on a single Deployment
  # * deployment<~String> the nickname of the deployment
  # * feature<~String> the feature filename 
  def run_test(deployment, feature, test_ary)
    new_job = GrinderJob.new
    new_job.logfile = File.join(@log_dir, "#{deployment.nickname}.log")
    new_job.rest_log = File.join(@log_dir, "#{deployment.nickname}.rest_connection.log")
    new_job.deployment = deployment
#    new_job.trace_log = File.join(@log_dir, "#{File.basename(feature, ".rb")}.yaml")
    new_job.verbose = true if @options[:verbose]
#    break_point = @options[:breakpoint] if @options[:breakpoint] XXX DEPRECATED XXX
#    cmd = "bin/grinder #{feature} #{break_point}"
    cmd = "bin/grinder -f #{feature} -d \"#{deployment.nickname}\" -g -l #{new_job.logfile} -t "
    test_ary.each { |test| cmd += " \"#{test}\" " }
    cmd += " -n " if @options[:no_resume]
    @jobs << new_job
    puts "running #{cmd}"
    new_job.run(cmd)
  end

  def initialize()
    @started_at = Time.now
    @jobs = []
    @passed = []
    @failed = []
    @running = []
    dirname = Time.now.strftime(File.join("%Y", "%m", "%d", "%H-%M-%S"))
    @log_dir = File.join("log", dirname)
    @log_started = dirname
    FileUtils.mkdir_p(@log_dir)
    @feature_dir = File.join(File.dirname(__FILE__), '..', '..', 'features')
  end
 
  # runs a feature on an array of deployments
  # * deployments<~Array> array of strings containing the nicknames of the deployments
  # * feature_name<~String> the feature filename 
  def run_tests(deployments,cmd,set=[])
    test_case = VirtualMonkey::TestCase.new(@options[:feature], @options)
    total_keys = test_case.get_keys
    total_keys = total_keys - (total_keys - set) unless set.nil? || set.empty?
    keys_per_dep = (total_keys.length.to_f / deployments.length.to_f).ceil
   
    deployment_tests = []
    (keys_per_dep * deployments.length).times { |i|
      di = i % deployments.length
      deployment_tests[di] ||= []
      deployment_tests[di] << total_keys[i % total_keys.length]
    }

    deployments.each_with_index { |d,i| 
      run_test(d,cmd,deployment_tests[i]) 
{'fraction_non_alphanumeric': 0.15857220118983234, 'fraction_numerical': 0.0025959978366684694, 'mean_word_length': 2.9512820512820515, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 35, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1665807', 'n_tokens_mistral': 3333, 'n_tokens_neox': 3276, 'n_words': 641}