<commit_before># frozen_string_literal: true

require 'cluster_support'
require 'fileutils'
require 'jenkins'
require 'kubectl_helpers'
require 'name_generator'
require 'password_generator'
require 'securerandom'
require 'ssh'
require 'tfstate_file'
require 'tfvars_file'
require 'timeout'
require 'with_retries'
require 'open3'

# Cluster represents a k8s cluster
class Cluster
  attr_reader :tfvars_file, :kubeconfig, :manifest_path, :build_path,
              :tectonic_admin_email, :tectonic_admin_password, :tfstate_file

  def initialize(tfvars_file)
    @tfvars_file = tfvars_file

    # Enable local testers to specify a static cluster name
    # S3 buckets can only handle lower case names
    @name = ENV['CLUSTER'] || NameGenerator.generate(@tfvars_file.prefix)
    @tectonic_admin_email = ENV['TF_VAR_tectonic_admin_email'] || NameGenerator.generate_fake_email
    @tectonic_admin_password = ENV['TF_VAR_tectonic_admin_password'] || PasswordGenerator.generate_password

    @build_path = File.join(File.realpath('../../'), "build/#{@name}")
    @manifest_path = File.join(@build_path, 'generated')
    @kubeconfig = File.join(manifest_path, 'auth/kubeconfig')
    @tfstate_file = TFStateFile.new(@build_path)

    check_prerequisites
  end

  def plan(terraform_options = nil)
    env = env_variables
    env['TF_PLAN_OPTIONS'] = terraform_options unless terraform_options.nil?
    stdout, stderr, exit_status = Open3.capture3(env, 'make -C ../.. plan')
    [stdout, stderr, exit_status]
  end

  def start
    apply
    wait_til_ready
  end

  def update_cluster
    start
  end

  def stop
    if ENV.key?('TECTONIC_TESTS_DONT_CLEAN_UP')
      puts "*** Cleanup inhibiting flag set. Stopping here. ***\n"
      puts '*** Your email/password to use in the tectonic console is:'\
           "#{@tectonic_admin_email} / #{@tectonic_admin_password} ***\n"
      return
    end
    destroy
  end

  def check_prerequisites
    return if license_and_pull_secret_defined?
    raise 'Tectonic license and pull secret are not defined as environment'\
          'variables.'
  end

  def env_variables
    {
      'CLUSTER' => @name,
      'TF_VAR_tectonic_cluster_name' => @name,
      'TF_VAR_tectonic_admin_email' => @tectonic_admin_email,
      'TF_VAR_tectonic_admin_password' => <PASSWORD>
    }
  end

  def tf_var(v)
    tf_value "var.#{v}"
  end

  def tf_value(v)
    Dir.chdir(@build_path) do
      `echo '#{v}' | terraform console ../../platforms/#{env_variables['PLATFORM']}`.chomp
    end
  end

  def secret_files(namespace, secret)
    cmd = "get secret -n #{namespace} #{secret} -o go-template "\
          "\'--template={{range $key, $value := .data}}{{$key}}\n{{end}}\'"
    KubeCTL.run(@kubeconfig, cmd).split("\n")
  end

  def api_ip_addresses
    nodes = KubeCTL.run(
      @kubeconfig,
      'get node -l=node-role.kubernetes.io/master '\
      '-o jsonpath=\'{range .items[*]}'\
      '{@.metadata.name}{"\t"}{@.status.addresses[?(@.type=="ExternalIP")].address}'\
      '{"\n"}{end}\''
    )

    nodes = nodes.split("\n").map { |node| node.split("\t") }.to_h

    api_pods = KubeCTL.run(
      @kubeconfig,
      'get pod -n kube-system -l k8s-app=kube-apiserver '\
      '-o \'jsonpath={range .items[*]}'\
      '{@.metadata.name}{"\t"}{@.spec.nodeName}'\
      '{"\n"}{end}\''
    )

    api_pods
      .split("\n")
      .map { |pod| pod.split("\t") }
      .map { |pod| [pod[0], nodes[pod[1]]] }.to_h
  end

  def forensic
    save_kubernetes_events(@kubeconfig, @name)

    master_ip_addresses.each do |master_ip|
      save_docker_logs(master_ip, @name)

      ['bootkube', 'tectonic', 'kubelet', 'k8s-node-bootstrap'].each do |service|
        print_service_logs(master_ip, service, @name)
      end
    end

    worker_ip_addresses.each do |worker_ip|
      save_docker_logs(worker_ip, @name, master_ip_address)

      ['kubelet'].each do |service|
        print_service_logs(worker_ip, service, @name, master_ip_address)
      end
    end

    etcd_ip_addresses.each do |etcd_ip|
      ['etcd-member'].each do |service|
        print_service_logs(etcd_ip, service, @name, master_ip_address)
      end
    end
  end

  private

  def license_and_pull_secret_defined?
    license_path = 'TF_VAR_tectonic_license_path'
    pull_secret_path = 'TF_VAR_tectonic_pull_secret_path'

    EnvVar.set?([license_path, pull_secret_path])
  end

  def apply
    ::Timeout.timeout(30 * 60) do # 30 minutes
      3.times do |idx|
        env = env_variables
        env['TF_LOG'] = 'TRACE' if idx.positive?
        env['TF_APPLY_OPTIONS'] = '-no-color'
        env['TF_INIT_OPTIONS'] = '-no-color'
        return true if system(env, "make -C ../.. apply | tee ../../build/#{@name}/terraform-apply.log")
      end
    end<commit_msg>tests: call forensic to collect the logs when timeout happen during start (#2607)<commit_after># frozen_string_literal: true

require 'cluster_support'
require 'fileutils'
require 'jenkins'
require 'kubectl_helpers'
require 'name_generator'
require 'password_generator'
require 'securerandom'
require 'ssh'
require 'tfstate_file'
require 'tfvars_file'
require 'timeout'
require 'with_retries'
require 'open3'

# Cluster represents a k8s cluster
class Cluster
  attr_reader :tfvars_file, :kubeconfig, :manifest_path, :build_path,
              :tectonic_admin_email, :tectonic_admin_password, :tfstate_file

  def initialize(tfvars_file)
    @tfvars_file = tfvars_file

    # Enable local testers to specify a static cluster name
    # S3 buckets can only handle lower case names
    @name = ENV['CLUSTER'] || NameGenerator.generate(@tfvars_file.prefix)
    @tectonic_admin_email = ENV['TF_VAR_tectonic_admin_email'] || NameGenerator.generate_fake_email
    @tectonic_admin_password = ENV['TF_VAR_tectonic_admin_password'] || PasswordGenerator.generate_password

    @build_path = File.join(File.realpath('../../'), "build/#{@name}")
    @manifest_path = File.join(@build_path, 'generated')
    @kubeconfig = File.join(manifest_path, 'auth/kubeconfig')
    @tfstate_file = TFStateFile.new(@build_path)

    check_prerequisites
  end

  def plan(terraform_options = nil)
    env = env_variables
    env['TF_PLAN_OPTIONS'] = terraform_options unless terraform_options.nil?
    stdout, stderr, exit_status = Open3.capture3(env, 'make -C ../.. plan')
    [stdout, stderr, exit_status]
  end

  def start
    apply
    wait_til_ready
  end

  def update_cluster
    start
  end

  def stop
    if ENV.key?('TECTONIC_TESTS_DONT_CLEAN_UP')
      puts "*** Cleanup inhibiting flag set. Stopping here. ***\n"
      puts '*** Your email/password to use in the tectonic console is:'\
           "#{@tectonic_admin_email} / #{@tectonic_admin_password} ***\n"
      return
    end
    destroy
  end

  def check_prerequisites
    return if license_and_pull_secret_defined?
    raise 'Tectonic license and pull secret are not defined as environment'\
          'variables.'
  end

  def env_variables
    {
      'CLUSTER' => @name,
      'TF_VAR_tectonic_cluster_name' => @name,
      'TF_VAR_tectonic_admin_email' => @tectonic_admin_email,
      'TF_VAR_tectonic_admin_password' => <PASSWORD>
    }
  end

  def tf_var(v)
    tf_value "var.#{v}"
  end

  def tf_value(v)
    Dir.chdir(@build_path) do
      `echo '#{v}' | terraform console ../../platforms/#{env_variables['PLATFORM']}`.chomp
    end
  end

  def secret_files(namespace, secret)
    cmd = "get secret -n #{namespace} #{secret} -o go-template "\
          "\'--template={{range $key, $value := .data}}{{$key}}\n{{end}}\'"
    KubeCTL.run(@kubeconfig, cmd).split("\n")
  end

  def api_ip_addresses
    nodes = KubeCTL.run(
      @kubeconfig,
      'get node -l=node-role.kubernetes.io/master '\
      '-o jsonpath=\'{range .items[*]}'\
      '{@.metadata.name}{"\t"}{@.status.addresses[?(@.type=="ExternalIP")].address}'\
      '{"\n"}{end}\''
    )

    nodes = nodes.split("\n").map { |node| node.split("\t") }.to_h

    api_pods = KubeCTL.run(
      @kubeconfig,
      'get pod -n kube-system -l k8s-app=kube-apiserver '\
      '-o \'jsonpath={range .items[*]}'\
      '{@.metadata.name}{"\t"}{@.spec.nodeName}'\
      '{"\n"}{end}\''
    )

    api_pods
      .split("\n")
      .map { |pod| pod.split("\t") }
      .map { |pod| [pod[0], nodes[pod[1]]] }.to_h
  end

  def forensic
    save_kubernetes_events(@kubeconfig, @name)

    master_ip_addresses.each do |master_ip|
      save_docker_logs(master_ip, @name)

      ['bootkube', 'tectonic', 'kubelet', 'k8s-node-bootstrap'].each do |service|
        print_service_logs(master_ip, service, @name)
      end
    end

    worker_ip_addresses.each do |worker_ip|
      save_docker_logs(worker_ip, @name, master_ip_address)

      ['kubelet'].each do |service|
        print_service_logs(worker_ip, service, @name, master_ip_address)
      end
    end

    etcd_ip_addresses.each do |etcd_ip|
      ['etcd-member'].each do |service|
        print_service_logs(etcd_ip, service, @name, master_ip_address)
      end
    end
  end

  private

  def license_and_pull_secret_defined?
    license_path = 'TF_VAR_tectonic_license_path'
    pull_secret_path = 'TF_VAR_tectonic_pull_secret_path'

    EnvVar.set?([license_path, pull_secret_path])
  end

  def apply
    ::Timeout.timeout(30 * 60) do # 30 minutes
      3.times do |idx|
        env = env_variables
        env['TF_LOG'] = 'TRACE' if idx.positive?
        env['TF_APPLY_OPTIONS'] = '-no-color'
        env['TF_INIT_OPTIONS'] = '-no-color'
        return true if system(env, "make -C ../.. apply | tee ../../build/#{@name}/terraform-apply.log")
      end
    end
  rescue Timeout::Error
    forensic
{'fraction_non_alphanumeric': 0.1366330456483311, 'fraction_numerical': 0.003743371113652906, 'mean_word_length': 3.2899197145405887, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7558933', 'n_tokens_mistral': 3466, 'n_tokens_neox': 3365, 'n_words': 694}