<commit_before>require 'fileutils'
require 'tempfile'

require 'chef/shell_out'
require 'chef/mixin/shell_out'
include Chef::Mixin::ShellOut

PLATFORM_TEST_DIR = '/tmp/opscode-platform-test/'

def create_credentials_dir
  FileUtils.mkdir_p(PLATFORM_TEST_DIR) unless File.exist?(PLATFORM_TEST_DIR)
end

def create_local_test
  Chef::Log.info("Creating bootstrap user 'platform-superuser'")
  Chef::Log.debug "Tmpdir: #{PLATFORM_TEST_DIR}"
  path = File.join(OPSCODE_PROJECT_DIR, "opscode-account", OPSCODE_PROJECT_SUFFIX, "bin")
  Dir.chdir(path) do
    shell_out! "./account-whacker -c #{PLATFORM_TEST_DIR}/superuser.pem -d platform-superuser -e <EMAIL> -f PlatformSuperuser -l PlatformCukeSuperuser -m cuker -u platform-superuser -p <PASSWORD>"
    Chef::Log.info("Creating global containers")
    system("./global-containers platform-superuser")    
    output = create_public_user('local-test-user', 'Local', 'Test', 'User', 'Local Test User', '<EMAIL>')
    Chef::Log.debug(output)
    output = create_public_org("local-test-org", "Local Test Org", "platform-superuser", "#{PLATFORM_TEST_DIR}/superuser.pem", "local-test-user", "#{PLATFORM_TEST_DIR}/local-test-org-validator.pem")
    Chef::Log.debug(output)
  end
  FileUtils.copy("local-test-client.rb","/etc/chef/client.rb")
end

def create_public_org(org_name, org_fullname, opscode_username, opscode_pkey, customer_username, client_key_path)
  STDOUT.sync = true
  Chef::Log.info("Creating #{org_fullname} organization...")
  result = ""
  waiting = true
  while waiting
    result = `./createorgtool -a http://localhost:4042 -K "#{client_key_path}" -n "#{org_fullname}" -t Business -g "#{org_name}" -u "#{customer_username}" -p "#{opscode_pkey}" -o "#{opscode_username}"`
    if $?.exitstatus == 53
      Chef::Log.info("...")
      sleep 10
    elsif $?.exitstatus == 0
      Chef::Log.info("Created!")
      waiting = false
    else
      Chef::Log.debug("Error!")
      waiting = false
    end
  end
  result
end

def create_public_user(user_name, first_name, middle_name, last_name, display_name, email)
  Chef::Log.info("Creating user #{user_name}")
  o = shell_out!("./createobjecttool -a 'http://localhost:4042' -o 'platform-superuser' -p '#{PLATFORM_TEST_DIR}/superuser.pem' -w 'user' -n '#{user_name}' -f '#{first_name}' -m '#{middle_name}' -l '#{last_name}' -d '#{display_name}' -e '#{email}' -k '#{PLATFORM_TEST_DIR}/#{user_name}.pem' -s 'p@ssw0rd1'", :env=>{"DEBUG"=>"true"})
  puts o.format_for_exception if Chef::Log.debug?
end

def replace_platform_client
  STDERR.puts "Copying platform-client.rb to /etc/chef/client.rb"
  FileUtils.copy("platform-client.rb", "/etc/chef/client.rb")
end

def backup_platform_client
  if File.exists?("platform-client.rb")
    STDERR.puts "platform-client.rb already exists.  Doing nothing"
  else
    FileUtils.copy("/etc/chef/client.rb", "platform-client.rb")
  end
end

def cleanup_replicas
  c = Chef::REST.new(Chef::Config[:couchdb_url], nil, nil)
  c.get_rest('_all_dbs').each { |db| c.delete_rest("#{db}/") if db =~ /replica/ }
end

def cleanup_chefs
  c = Chef::REST.new(Chef::Config[:couchdb_url], nil, nil)
  begin
    c.get_rest('_all_dbs').each { |db| c.delete_rest("#{db}/") if db =~ /^chef_/ }
  rescue
    STDERR.puts "failed cleanup: #{db}, #{$!.message}"
  end
end

def cleanup_cookbooks
  c = Chef::REST.new("http://localhost/organizations/clownco", "clownco", "#{PLATFORM_TEST_DIR}/clownco.pem")
  cookbooks = c.get_rest("cookbooks").keys
  cookbooks.each do |cookbook|
    STDERR.puts c.delete_rest("cookbooks/#{cookbook}").inspect
  end
  cleanup_cookbook_tarballs
end

def setup_test_harness
  create_credentials_dir
  delete_databases
  cleanup_after_naughty_run
  create_account_databases
  create_organization
  org_db_names = create_chef_databases
  #prepare_feature_cookbooks
  create_test_harness_setup_database(org_db_names)
  replication_specs = (%w{authorization opscode_account opscode_account_internal} + org_db_names).map{|source_db| {:source_db => source_db,:target_db => "#{source_db}_integration"}}
  replicate_dbs(replication_specs)
end

def replicate_dbs(replication_specs)
  replication_specs = [replication_specs].flatten
  Chef::Log.debug "replication_specs = #{replication_specs.inspect}"
  c = Chef::REST.new(Chef::Config[:couchdb_url], nil, nil)
  replication_specs.each do |spec|
    source_db = spec[:source_db]
    target_db = spec[:target_db]

    Chef::Log.debug("Deleting #{target_db}, if exists")
    begin
      c.delete_rest("#{target_db}/")
    rescue Net::HTTPServerException => e
      raise unless e.message =~ /Not Found/
    end

    Chef::Log.debug("Creating #{target_db}")
    c.put_rest(target_db, nil)

    puts "REPLICATE: "
    puts ({ "source" => "#{Chef::Config[:couchdb_url]}/#{source_db}",
            "target" => "#{Chef::Config[:couchdb_url]}/#{target_db}" }.inspect)
    Chef::Log.debug("Replicating #{source_db} to #{target_db}")
    c.post_rest("_replicate", { "source" => "#{Chef::Config[:couchdb_url]}/#{source_db}", "target" => "#{Chef::Config[:couchdb_url]}/#{target_db}" })

  end
end

def cleanup_after_naughty_run
  %w{clownco-org-admin.pem clownco-org-validation.pem skynet-org-admin.pem skynet-org-validation.pem cooky.pem superuser.pem}.each do |pem_file|
    if File.exists?(File.join(PLATFORM_TEST_DIR, pem_file))
      File.unlink(File.join(PLATFORM_TEST_DIR,pem_file))
    end
  end
  cleanup_cookbook_tarballs
end

def cleanup_cookbook_tarballs
  fcpath = File.join(OPSCODE_PROJECT_DIR, "opscode-chef", OPSCODE_PROJECT_SUFFIX, "features", "data", "cookbooks")
  Dir.chdir(fcpath) do
    Dir[File.join(fcpath, '*.tar.gz')].each do |file|
      File.unlink(file)
    end
  end
end

def delete_databases
  c = Chef::REST.new(Chef::Config[:couchdb_url], nil, nil)
  %w{authorization authorization_integration opscode_account opscode_account_integration opscode_account_internal opscode_account_internal_integration test_harness_setup}.each do |db|
    begin
      c.delete_rest("#{db}/")
    rescue
    end
  end
  cleanup_replicas<commit_msg>now using bult-GET/POST replication for couch, instead of the builtin _replicate method, to avoid couch timeout barfing after _replicate<commit_after>require 'fileutils'
require 'tempfile'

require 'chef/shell_out'
require 'chef/mixin/shell_out'
include Chef::Mixin::ShellOut

PLATFORM_TEST_DIR = '/tmp/opscode-platform-test/'

def create_credentials_dir
  FileUtils.mkdir_p(PLATFORM_TEST_DIR) unless File.exist?(PLATFORM_TEST_DIR)
end

def create_local_test
  Chef::Log.info("Creating bootstrap user 'platform-superuser'")
  Chef::Log.debug "Tmpdir: #{PLATFORM_TEST_DIR}"
  path = File.join(OPSCODE_PROJECT_DIR, "opscode-account", OPSCODE_PROJECT_SUFFIX, "bin")
  Dir.chdir(path) do
    shell_out! "./account-whacker -c #{PLATFORM_TEST_DIR}/superuser.pem -d platform-superuser -e <EMAIL> -f PlatformSuperuser -l PlatformCukeSuperuser -m cuker -u platform-superuser -p p@ssw0rd1"
    Chef::Log.info("Creating global containers")
    system("./global-containers platform-superuser")    
    output = create_public_user('local-test-user', 'Local', 'Test', 'User', 'Local Test User', '<EMAIL>')
    Chef::Log.debug(output)
    output = create_public_org("local-test-org", "Local Test Org", "platform-superuser", "#{PLATFORM_TEST_DIR}/superuser.pem", "local-test-user", "#{PLATFORM_TEST_DIR}/local-test-org-validator.pem")
    Chef::Log.debug(output)
  end
  FileUtils.copy("local-test-client.rb","/etc/chef/client.rb")
end

def create_public_org(org_name, org_fullname, opscode_username, opscode_pkey, customer_username, client_key_path)
  STDOUT.sync = true
  Chef::Log.info("Creating #{org_fullname} organization...")
  result = ""
  waiting = true
  while waiting
    result = `./createorgtool -a http://localhost:4042 -K "#{client_key_path}" -n "#{org_fullname}" -t Business -g "#{org_name}" -u "#{customer_username}" -p "#{opscode_pkey}" -o "#{opscode_username}"`
    if $?.exitstatus == 53
      Chef::Log.info("...")
      sleep 10
    elsif $?.exitstatus == 0
      Chef::Log.info("Created!")
      waiting = false
    else
      Chef::Log.debug("Error!")
      waiting = false
    end
  end
  result
end

def create_public_user(user_name, first_name, middle_name, last_name, display_name, email)
  Chef::Log.info("Creating user #{user_name}")
  o = shell_out!("./createobjecttool -a 'http://localhost:4042' -o 'platform-superuser' -p '#{PLATFORM_TEST_DIR}/superuser.pem' -w 'user' -n '#{user_name}' -f '#{first_name}' -m '#{middle_name}' -l '#{last_name}' -d '#{display_name}' -e '#{email}' -k '#{PLATFORM_TEST_DIR}/#{user_name}.pem' -s 'p@ssw0rd1'", :env=>{"DEBUG"=>"true"})
  puts o.format_for_exception if Chef::Log.debug?
end

def replace_platform_client
  STDERR.puts "Copying platform-client.rb to /etc/chef/client.rb"
  FileUtils.copy("platform-client.rb", "/etc/chef/client.rb")
end

def backup_platform_client
  if File.exists?("platform-client.rb")
    STDERR.puts "platform-client.rb already exists.  Doing nothing"
  else
    FileUtils.copy("/etc/chef/client.rb", "platform-client.rb")
  end
end

def cleanup_replicas
  chef_rest.get_rest('_all_dbs').each { |db| c.delete_rest("#{db}/") if db =~ /replica/ }
end

def cleanup_chefs
  begin
    chef_rest.get_rest('_all_dbs').each { |db| c.delete_rest("#{db}/") if db =~ /^chef_/ }
  rescue
    STDERR.puts "failed cleanup: #{db}, #{$!.message}"
  end
end

def cleanup_cookbooks
  c = Chef::REST.new("http://localhost/organizations/clownco", "clownco", "#{PLATFORM_TEST_DIR}/clownco.pem")
  cookbooks = c.get_rest("cookbooks").keys
  cookbooks.each do |cookbook|
    STDERR.puts c.delete_rest("cookbooks/#{cookbook}").inspect
  end
  cleanup_cookbook_tarballs
end

def setup_test_harness
  create_credentials_dir
  delete_databases
  cleanup_after_naughty_run
  create_account_databases
  create_organization
  org_db_names = create_chef_databases
  #prepare_feature_cookbooks
  create_test_harness_setup_database(org_db_names)
  replication_specs = (%w{authorization opscode_account opscode_account_internal} + org_db_names).map{|source_db| {:source_db => source_db,:target_db => "#{source_db}_integration"}}
  replicate_dbs(replication_specs)
end

def chef_rest
  Chef::REST.new(Chef::Config[:couchdb_url], nil, nil)
end

def wipe_db(db)
  begin
    chef_rest.delete_rest("#{db}")
  rescue Net::HTTPServerException => e
    raise unless e.message =~ /Not Found/
  end
  
  chef_rest.put_rest(db, nil)
end

# Bulk GET all documents in the given db, using the given page size.
# Calls the required block for each page size, passing in an array of
# rows.
def bulk_get_paged(db, page_size)
  last_key = nil

  paged_rows = nil
  until (paged_rows && paged_rows.length == 0) do
    url = "#{db}/_all_docs?limit=100&include_docs=true"
    if last_key
      url += "&startkey=#{CGI.escape(last_key.to_json)}&skip=1"
    end
    #puts "replicate_manually: url = #{url}"
    
    paged_results = chef_rest.get_rest(url)
    paged_rows = paged_results['rows']

    if paged_rows.length > 0
      yield paged_rows
      last_key = paged_rows.last['key']
    end
  end
end

# Replicate a (set of) source databases to a (set of) target databases. Uses
# manual bulk GET/POST as Couch's internal _replicate endpoint crashes and
# starts to time out after some number of runs.
def replicate_dbs(replication_specs)
  replication_specs = [replication_specs].flatten
  
  replication_specs.each do |spec|
    source_db = spec[:source_db]
    target_db = spec[:target_db]

    wipe_db(target_db)
  
    bulk_get_paged(source_db, 100) do |paged_rows|
      #puts "incoming paged_rows is #{paged_rows.inspect}"
      paged_rows = paged_rows.map do |row|
        doc_in_row = row['doc']
        doc_in_row.delete '_rev'
        doc_in_row
      end

      #puts "now, paged_rows is #{paged_rows.inspect}"
      #pp({:_PAGED_ROWS => paged_rows})
    
      chef_rest.post_rest("#{target_db}/_bulk_docs", {"docs" => paged_rows})
    end
  end
end

def cleanup_after_naughty_run
  %w{clownco-org-admin.pem clownco-org-validation.pem skynet-org-admin.pem skynet-org-validation.pem cooky.pem superuser.pem}.each do |pem_file|
    if File.exists?(File.join(PLATFORM_TEST_DIR, pem_file))
      File.unlink(File.join(PLATFORM_TEST_DIR,pem_file))
    end
  end
  cleanup_cookbook_tarballs
end

def cleanup_cookbook_tarballs
  fcpath = File.join(OPSCODE_PROJECT_DIR, "opscode-chef", OPSCODE_PROJECT_SUFFIX, "features", "data", "cookbooks")
  Dir.chdir(fcpath) do
    Dir[File.join(fcpath, '*.tar.gz')].each do |file|
      File.unlink(file)
    end
  end
end

def delete_databases
  %w{authorization authorization_integration opscode_account opscode_account_integration opscode_account_internal opscode_account_internal_integration test_harness_setup}.each do |db|
    begin
      chef_rest.delete_rest("#{db}/")
    rescue
    end
  end
  cleanup_replicas
{'fraction_non_alphanumeric': 0.14130943102104443, 'fraction_numerical': 0.003195635229929852, 'mean_word_length': 4.7026666666666666, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '868552', 'n_tokens_mistral': 4807, 'n_tokens_neox': 4744, 'n_words': 885}