<commit_before>#    Copyright 2013 Mirantis, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

module Astute
  class Orchestrator
    def initialize(deploy_engine=nil, log_parsing=false)
      @deploy_engine = deploy_engine || Astute::DeploymentEngine::NailyFact
      @log_parsing = log_parsing
    end

    def node_type(reporter, task_id, nodes, timeout=nil)
      context = Context.new(task_id, reporter)
      uids = nodes.map {|n| n['uid']}
      systemtype = MClient.new(context, "systemtype", uids, check_result=false, timeout)
      systems = systemtype.get_type
      systems.map do |n|
        {
          'uid'       => n.results[:sender],
          'node_type' => n.results[:data][:node_type].chomp
        }
      end
    end

    def deploy(up_reporter, task_id, deployment_info)
      proxy_reporter = ProxyReporter::DeploymentProxyReporter.new(up_reporter)
      log_parser = @log_parsing ? LogParser::ParseDeployLogs.new : LogParser::NoParsing.new
      context = Context.new(task_id, proxy_reporter, log_parser)
      deploy_engine_instance = @deploy_engine.new(context)
      Astute.logger.info "Using #{deploy_engine_instance.class} for deployment."
      begin
        log_parser.prepare(deployment_info)
      rescue Exception => e
        Astute.logger.warn "Some error occurred when prepare LogParser: #{e.message}, trace: #{e.format_backtrace}"
      end
      deploy_engine_instance.deploy(deployment_info)
      return SUCCESS
    end

    def fast_provision(reporter, engine_attrs, nodes)
      raise "Nodes to provision are not provided!" if nodes.empty?

      engine = create_engine(engine_attrs, reporter)

      begin
        reboot_events = reboot_nodes(engine, nodes)
        failed_nodes  = check_reboot_nodes(engine, reboot_events)

      rescue RuntimeError => e
        Astute.logger.error("Error occured while provisioning: #{e.inspect}")
        reporter.report({
                          'status' => 'error',
                          'error' => 'Cobbler error',
                          'progress' => 100
                        })
        raise StopIteration
      ensure
        engine.sync
      end

      if failed_nodes.empty?
        report_result({}, reporter)
        return SUCCESS
      else
        Astute.logger.error("Nodes failed to reboot: #{failed_nodes.inspect}")
        reporter.report({
                          'status' => 'error',
                          'error' => "Nodes failed to reboot: #{failed_nodes.inspect}",
                          'progress' => 100
                        })
        raise StopIteration
      end
    end

    def provision(reporter, task_id, nodes_up)
      raise "Nodes to provision are not provided!" if nodes_up.empty?

      # We need only those which are not ready/provisioned yet
      nodes = []
      nodes_up.each do |n|
        nodes << n unless ['provisioned', 'ready'].include?(n['status'])
      end

      nodes_uids = nodes.map { |n| n['uid'] }

      provisionLogParser = @log_parsing ? LogParser::ParseProvisionLogs.new : LogParser::NoParsing.new
      proxy_reporter = ProxyReporter::DeploymentProxyReporter.new(reporter)
      sleep_not_greater_than(10) do # Wait while nodes going to reboot
        Astute.logger.info "Starting OS provisioning for nodes: #{nodes_uids.join(',')}"
        begin
          provisionLogParser.prepare(nodes)
        rescue => e
          Astute.logger.warn "Some error occurred when prepare LogParser: #{e.message}, trace: #{e.format_backtrace}"
        end
      end
      nodes_not_booted = nodes_uids.clone
      begin
        Timeout.timeout(Astute.config.PROVISIONING_TIMEOUT) do  # Timeout for booting target OS
          catch :done do
            while true
              sleep_not_greater_than(5) do
                types = node_type(proxy_reporter, task_id, nodes, 2)
                types.each { |t| Astute.logger.debug("Got node types: uid=#{t['uid']} type=#{t['node_type']}") }

                Astute.logger.debug("Not target nodes will be rejected")
                target_uids = types.reject{|n| n['node_type'] != 'target'}.map{|n| n['uid']}
                nodes_not_booted -= types.map { |n| n['uid'] }
                Astute.logger.debug "Not provisioned: #{nodes_not_booted.join(',')}, got target OSes: #{target_uids.join(',')}"

                if nodes.length == target_uids.length
                  Astute.logger.info "All nodes #{target_uids.join(',')} are provisioned."
                  throw :done
                else
                  Astute.logger.debug("Nodes list length is not equal to target nodes list length: #{nodes.length} != #{target_uids.length}")
                end

                report_about_progress(proxy_reporter, provisionLogParser, nodes_uids, target_uids, nodes)
              end
            end
          end
          # We are here if jumped by throw from while cycle
        end
      rescue Timeout::Error
        msg = "Timeout of provisioning is exceeded."
        Astute.logger.error msg
        error_nodes = nodes_not_booted.map { |n| {'uid' => n,
                                                  'status' => 'error',
                                                  'error_msg' => msg,
                                                  'progress' => 100,
                                                  'error_type' => 'provision'} }
        proxy_reporter.report({'status' => 'error', 'error' => msg, 'nodes' => error_nodes})
        return FAIL
      end

      nodes_progress = nodes.map do |n|
        {'uid' => n['uid'], 'progress' => 100, 'status' => 'provisioned'}
      end
      proxy_reporter.report({'nodes' => nodes_progress})
      return SUCCESS
    end

    def remove_nodes(reporter, task_id, nodes)
      NodesRemover.new(Context.new(task_id, reporter), nodes).remove
    end

    def verify_networks(reporter, task_id, nodes)
      Network.check_network(Context.new(task_id, reporter), nodes)
    end

    def download_release(up_reporter, task_id, release_info)
      raise "Release information not provided!" if release_info.empty?
      attrs = {'deployment_mode' => 'rpmcache',
               'deployment_id' => 'rpmcache'}
      facts = {'rh_username' => release_info['username'],
               'rh_password' => release_info['password']}
      facts.merge!(attrs)
      if release_info['license_type'] == 'rhn'
        facts.merge!(
          {'use_satellite' => 'true',
           'sat_hostname' => release_info['satellite'],
           'activation_key' => release_info['activation_key']})
      end
      nodes = [{'uid' => 'master', 'facts' => facts}]
      proxy_reporter = ProxyReporter::DLReleaseProxyReporter.new(up_reporter, nodes.size)
      #FIXME: These parameters should be propagated from Nailgun. Maybe they should be saved
      #       in Release.json.
      nodes_to_parser = [
        {:uid => 'master',
         :path_items => [
            {:max_size => 1111280705, :path => '/var/www/nailgun/rhel', :weight => 3},
            {:max_size => 195900000, :path => '/var/cache/yum/x86_64/6Server', :weight => 1},
         ]}
      ]
      log_parser = @log_parsing ? LogParser::DirSizeCalculation.new(nodes_to_parser) : LogParser::NoParsing.new
      context = Context.new(task_id, proxy_reporter, log_parser)
      deploy_engine_instance = @deploy_engine.new(context)
      Astute.logger.info "Using #{deploy_engine_instance.class} for release download."
      deploy_engine_instance.deploy(nodes, attrs)
      proxy_reporter.report({'status' => 'ready', 'progress' => 100})
    end

    def check_redhat_credentials(reporter, task_id, credentials)
      ctx = Context.new(task_id, reporter)<commit_msg>fix downloading task<commit_after>#    Copyright 2013 Mirantis, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

module Astute
  class Orchestrator
    def initialize(deploy_engine=nil, log_parsing=false)
      @deploy_engine = deploy_engine || Astute::DeploymentEngine::NailyFact
      @log_parsing = log_parsing
    end

    def node_type(reporter, task_id, nodes, timeout=nil)
      context = Context.new(task_id, reporter)
      uids = nodes.map {|n| n['uid']}
      systemtype = MClient.new(context, "systemtype", uids, check_result=false, timeout)
      systems = systemtype.get_type
      systems.map do |n|
        {
          'uid'       => n.results[:sender],
          'node_type' => n.results[:data][:node_type].chomp
        }
      end
    end

    def deploy(up_reporter, task_id, deployment_info)
      proxy_reporter = ProxyReporter::DeploymentProxyReporter.new(up_reporter)
      log_parser = @log_parsing ? LogParser::ParseDeployLogs.new : LogParser::NoParsing.new
      context = Context.new(task_id, proxy_reporter, log_parser)
      deploy_engine_instance = @deploy_engine.new(context)
      Astute.logger.info "Using #{deploy_engine_instance.class} for deployment."
      begin
        log_parser.prepare(deployment_info)
      rescue Exception => e
        Astute.logger.warn "Some error occurred when prepare LogParser: #{e.message}, trace: #{e.format_backtrace}"
      end
      deploy_engine_instance.deploy(deployment_info)
      return SUCCESS
    end

    def fast_provision(reporter, engine_attrs, nodes)
      raise "Nodes to provision are not provided!" if nodes.empty?

      engine = create_engine(engine_attrs, reporter)

      begin
        reboot_events = reboot_nodes(engine, nodes)
        failed_nodes  = check_reboot_nodes(engine, reboot_events)

      rescue RuntimeError => e
        Astute.logger.error("Error occured while provisioning: #{e.inspect}")
        reporter.report({
                          'status' => 'error',
                          'error' => 'Cobbler error',
                          'progress' => 100
                        })
        raise StopIteration
      ensure
        engine.sync
      end

      if failed_nodes.empty?
        report_result({}, reporter)
        return SUCCESS
      else
        Astute.logger.error("Nodes failed to reboot: #{failed_nodes.inspect}")
        reporter.report({
                          'status' => 'error',
                          'error' => "Nodes failed to reboot: #{failed_nodes.inspect}",
                          'progress' => 100
                        })
        raise StopIteration
      end
    end

    def provision(reporter, task_id, nodes_up)
      raise "Nodes to provision are not provided!" if nodes_up.empty?

      # We need only those which are not ready/provisioned yet
      nodes = []
      nodes_up.each do |n|
        nodes << n unless ['provisioned', 'ready'].include?(n['status'])
      end

      nodes_uids = nodes.map { |n| n['uid'] }

      provisionLogParser = @log_parsing ? LogParser::ParseProvisionLogs.new : LogParser::NoParsing.new
      proxy_reporter = ProxyReporter::DeploymentProxyReporter.new(reporter)
      sleep_not_greater_than(10) do # Wait while nodes going to reboot
        Astute.logger.info "Starting OS provisioning for nodes: #{nodes_uids.join(',')}"
        begin
          provisionLogParser.prepare(nodes)
        rescue => e
          Astute.logger.warn "Some error occurred when prepare LogParser: #{e.message}, trace: #{e.format_backtrace}"
        end
      end
      nodes_not_booted = nodes_uids.clone
      begin
        Timeout.timeout(Astute.config.PROVISIONING_TIMEOUT) do  # Timeout for booting target OS
          catch :done do
            while true
              sleep_not_greater_than(5) do
                types = node_type(proxy_reporter, task_id, nodes, 2)
                types.each { |t| Astute.logger.debug("Got node types: uid=#{t['uid']} type=#{t['node_type']}") }

                Astute.logger.debug("Not target nodes will be rejected")
                target_uids = types.reject{|n| n['node_type'] != 'target'}.map{|n| n['uid']}
                nodes_not_booted -= types.map { |n| n['uid'] }
                Astute.logger.debug "Not provisioned: #{nodes_not_booted.join(',')}, got target OSes: #{target_uids.join(',')}"

                if nodes.length == target_uids.length
                  Astute.logger.info "All nodes #{target_uids.join(',')} are provisioned."
                  throw :done
                else
                  Astute.logger.debug("Nodes list length is not equal to target nodes list length: #{nodes.length} != #{target_uids.length}")
                end

                report_about_progress(proxy_reporter, provisionLogParser, nodes_uids, target_uids, nodes)
              end
            end
          end
          # We are here if jumped by throw from while cycle
        end
      rescue Timeout::Error
        msg = "Timeout of provisioning is exceeded."
        Astute.logger.error msg
        error_nodes = nodes_not_booted.map { |n| {'uid' => n,
                                                  'status' => 'error',
                                                  'error_msg' => msg,
                                                  'progress' => 100,
                                                  'error_type' => 'provision'} }
        proxy_reporter.report({'status' => 'error', 'error' => msg, 'nodes' => error_nodes})
        return FAIL
      end

      nodes_progress = nodes.map do |n|
        {'uid' => n['uid'], 'progress' => 100, 'status' => 'provisioned'}
      end
      proxy_reporter.report({'nodes' => nodes_progress})
      return SUCCESS
    end

    def remove_nodes(reporter, task_id, nodes)
      NodesRemover.new(Context.new(task_id, reporter), nodes).remove
    end

    def verify_networks(reporter, task_id, nodes)
      Network.check_network(Context.new(task_id, reporter), nodes)
    end

    def download_release(up_reporter, task_id, release_info)
      raise "Release information not provided!" if release_info.empty?

      attrs = {'deployment_mode' => 'rpmcache',
               'deployment_id' => 'rpmcache'}
      facts = {'rh_username' => release_info['username'],
               'rh_password' => release_info['password']}
      facts.merge!(attrs)

      if release_info['license_type'] == 'rhn'
        facts.merge!(
          {'use_satellite' => 'true',
           'sat_hostname' => release_info['satellite'],
           'activation_key' => release_info['activation_key']})
      end
      facts['uid'] = 'master'
      facts = [facts]
      proxy_reporter = ProxyReporter::DLReleaseProxyReporter.new(up_reporter, facts.size)
      #FIXME: These parameters should be propagated from Nailgun. Maybe they should be saved
      #       in Release.json.
      nodes_to_parser = [
        {:uid => 'master',
         :path_items => [
            {:max_size => 1111280705, :path => '/var/www/nailgun/rhel', :weight => 3},
            {:max_size => 195900000, :path => '/var/cache/yum/x86_64/6Server', :weight => 1},
         ]}
      ]
      log_parser = @log_parsing ? LogParser::DirSizeCalculation.new(nodes_to_parser) : LogParser::NoParsing.new
      context = Context.new(task_id, proxy_reporter, log_parser)
      deploy_engine_instance = @deploy_engine.new(context)
      Astute.logger.info "Using #{deploy_engine_instance.class} for release download."
      deploy_engine_instance.deploy_piece(facts, 0)
      proxy_reporter.report({'status' => 'ready', 'progress' => 100})
    end

    def check_redhat_credentials(reporter, task_id, credentials)
      ctx = Context.new(task_id, reporter)
{'fraction_non_alphanumeric': 0.10869167429094236, 'fraction_numerical': 0.00652637999390058, 'mean_word_length': 2.349540347293156, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 85, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 21, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5869837', 'n_tokens_mistral': 4813, 'n_tokens_neox': 4621, 'n_words': 1146}