<commit_before>###############################################################################
# Copyright (c) 2018 Cisco and/or its affiliates.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
###############################################################################
# This presuite takes several optional parameters through environment variables
# to support different test scenarios for master/agent interactions. For agentless
# testing, the usual .fixtures.yml and Gemfile control the versions used.
#
#   * NETDEV_STDLIB_URL, NETDEV_STDLIB_REF: the source git URL and git ref (branch)
#     to load the netdev_stdlib module; if no NETDEV_STDLIB_URL is specified, the
#     last released version from the module forge is used
#   * MODULE_URL, MODULE_REF: the source git URL and git ref (branch)
#     to load the cisco-network-puppet-module module; if no MODULE_URL is specified, the
#     local checkout is used
#   * RSAPI_URL, RSAPI_REF: the source git URL and git ref (branch)
#     to load the puppetlabs-resource_api module; if no RSAPI_URL is specified, the
#     last released version from the module forge is used
#   * NODE_GEM_URL, NODE_GEM_REF: the source git URL and git ref (branch)
#     to load the cisco_node_utils gem; if no NODE_GEM_URL is specified, the
#     last released gem version
#   * RSAPI_GEM_URL, RSAPI_GEM_REF: the source git URL and git ref (branch)
#     to load the puppet-resource_api gem; if no RSAPI_GEM_URL is specified, the
#     last released gem version.
#
# If a URL, but no REF is specified, the `master` branch will be used.
#

require 'beaker-pe'
require 'shellwords'

test_name 'Prep Masters & Install Puppet' do
  # collect host objects from the nodeset. code further down needs to be prepared that one of those lists is empty
  # in regular testing at puppet there is either a master and an agent for agentful testing, or a default node for agentless testing
  # Get all hosts with role master or compile_master
  masters = select_hosts(roles: ['master', 'compile_master'])
  # Get all hosts with role agent
  agents = select_hosts(roles: ['agent'])

  unless masters.empty?
    step 'install PE on masters' do
      masters.each do |_node|
        install_pe_on(masters, {})
      end
    end
  end

  def mod_line_from_env(module_name, env_prefix=nil)
    if env_prefix && ENV["#{env_prefix}_URL"]
      "mod '#{module_name}', git: '#{ENV["#{env_prefix}_URL"]}', ref: '#{ENV["#{env_prefix}_REF"] || 'master'}'"
    else
      "mod '#{module_name}'"
    end
  end

  unless masters.empty?
    step 'install Cisco Module on master' do
      masters.each do |node|
        on(node, "echo #{Shellwords.escape(mod_line_from_env('puppetlabs-resource_api', 'RSAPI'))} >> /root/Puppetfile")
        on(node, "echo #{Shellwords.escape(mod_line_from_env('puppetlabs-puppetserver_gem', 'RSAPI'))} >> /root/Puppetfile")
        on(node, "echo #{Shellwords.escape(mod_line_from_env('puppetlabs-netdev_stdlib', 'NETDEV_STDLIB'))} >> /root/Puppetfile")
        on(node, "echo #{Shellwords.escape(mod_line_from_env('puppetlabs-ciscopuppet', 'MODULE'))} >> /root/Puppetfile")
        on(node, '/opt/puppetlabs/puppet/bin/r10k puppetfile install /root/Puppetfile -v --moduledir /etc/puppetlabs/code/environments/production/modules', acceptable_exit_codes: [0])
        on(node, puppet('plugin', 'download'), acceptable_exit_codes: [0, 1])
      end
    end
  end

  def clone_and_build_gem(gem_name, env_prefix=nil)
    # nothing to do when no URL set
    return unless env_prefix && ENV["#{env_prefix}_URL"]

    `git clone #{ENV["#{env_prefix}_URL"]} #{gem_name}`
    Dir.chdir(gem_name) do
      `git checkout #{ENV["#{env_prefix}_REF"] || 'master'}`
      `gem build #{gem_name}.gemspec`
    end
  end

  if ENV['RSAPI_GEM_URL']
    step 'Build Resource API gem' do
      clone_and_build_gem('puppet-resource_api', 'RSAPI_GEM')
    end
  end

  if ENV['NODE_GEM_URL']
    step 'Build Cisco Node Utils gem' do
      clone_and_build_gem('cisco_node_utils', 'NODE_GEM')
    end
  end

  def upload_gem_to(node, target_path, gem_name, env_prefix=nil)
    # nothing to upload if gem should be installed from rubygems
    return unless env_prefix && ENV["#{env_prefix}_URL"]

    gem_path = Dir.glob("#{gem_name}/*.gem")[0]
    gem_file = "#{gem_name}.gem"
    scp_to node, gem_path, "#{target_path}/#{gem_file}"
  end

  def install_server_gem(node, gem_name, env_prefix=nil)
    upload_gem_to(node, '/tmp', gem_name, env_prefix)
    gem_location = if env_prefix && ENV["#{env_prefix}_URL"]
                     "/tmp/#{gem_name}.gem"
                   else
                     gem_name
                   end
    on(node, "/opt/puppetlabs/bin/puppetserver gem install #{gem_location}")
  end

  unless masters.empty?
    step 'install Resource API on masters' do
      masters.each do |node|
        install_server_gem(node, 'puppet-resource_api', 'RSAPI_GEM')
        on(node, '/usr/bin/systemctl restart pe-puppetserver.service', acceptable_exit_codes: [0])
      end
    end
  end

  def install_agent_gem(node, gem_name, env_prefix=nil)
    upload_gem_to(node, '/var/volatile/tmp', gem_name, env_prefix)
    gem_location = if env_prefix && ENV["#{env_prefix}_URL"]
                     "/var/volatile/tmp/#{gem_name}.gem"
                   else
                     gem_name
                   end
    on(node, "/opt/puppetlabs/puppet/bin/gem install #{gem_location}")
  end

  unless agents.empty?
    step 'Install agent on switches, sign certificates' do
      opts = {
        puppet_collection: 'PC1',
        puppet_agent_sha: ENV['SHA'],
        puppet_agent_version: ENV['SUITE_VERSION'] || ENV['SHA']
      }
      agents.each do |switch|
        next unless switch['platform'] =~ /cisco_/
        on(switch, 'yum -y erase puppet-agent', acceptable_exit_codes: [0, 1])
        on(switch, 'rpm -qa | grep -i puppetlabs | xargs rpm -e', acceptable_exit_codes: [0, 123])
        on(switch, 'rm -rf /var/cache/yum/puppetlabs-pc1/packages/puppet*', acceptable_exit_codes: [0, 1])
        on(switch, 'rm -rf /var/cache/yum/pl-puppet-agent*', acceptable_exit_codes: [0, 1])
        on(switch, 'rm -rf /etc/yum/repos.d/pl-puppet-agent*', acceptable_exit_codes: [0, 1])
        on(switch, 'rm -rf /var/volatile/log/puppet*', acceptable_exit_codes: [0, 1])
        on(switch, 'find /var/volatile/tmp -type f -delete')
        install_puppet_agent_dev_repo_on(switch, opts)
        install_agent_gem(switch, 'puppet-resource_api', 'RSAPI_GEM')
        install_agent_gem(switch, 'cisco_node_utils', 'NODE_GEM')
        on(switch, 'find /etc/puppetlabs/puppet/ssl/ -type f -print0 |xargs -0r sudo rm')
        on(switch, 'rm /etc/puppetlabs/puppet/puppet.conf')
        on(switch, 'touch /etc/puppetlabs/puppet/puppet.conf')
        on(switch, 'chmod a+w /etc/puppetlabs/puppet/puppet.conf')
        on(switch, 'echo -e "[main]" >> /etc/puppetlabs/puppet/puppet.conf')
        on(switch, "echo -e \"  server = #{master.hostname}\n\" >> /etc/puppetlabs/puppet/puppet.conf")
        on(switch, 'echo -e "[agent]" >> /etc/puppetlabs/puppet/puppet.conf')
        on(switch, 'echo -e "  pluginsync  = true" >> /etc/puppetlabs/puppet/puppet.conf')
        on(switch, "/opt/puppetlabs/bin/puppet agent -t --server #{master.hostname}", acceptable_exit_codes: [1])
        unless masters.empty?
          on(master, puppet('cert', 'sign', switch.to_s), acceptable_exit_codes: [0, 1])
        end
        on(switch, '/opt/puppetlabs/bin/puppet agent -t --waitforcert 60', acceptable_exit_codes: [0, 2])
      end
    end
  end
end
<commit_msg>(maint) fix for the failures on the presuite deploy<commit_after>###############################################################################
# Copyright (c) 2018 Cisco and/or its affiliates.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
###############################################################################
# This presuite takes several optional parameters through environment variables
# to support different test scenarios for master/agent interactions. For agentless
# testing, the usual .fixtures.yml and Gemfile control the versions used.
#
#   * NETDEV_STDLIB_URL, NETDEV_STDLIB_REF: the source git URL and git ref (branch)
#     to load the netdev_stdlib module; if no NETDEV_STDLIB_URL is specified, the
#     last released version from the module forge is used
#   * MODULE_URL, MODULE_REF: the source git URL and git ref (branch)
#     to load the cisco-network-puppet-module module; if no MODULE_URL is specified, the
#     local checkout is used
#   * RSAPI_URL, RSAPI_REF: the source git URL and git ref (branch)
#     to load the puppetlabs-resource_api module; if no RSAPI_URL is specified, the
#     last released version from the module forge is used
#   * NODE_GEM_URL, NODE_GEM_REF: the source git URL and git ref (branch)
#     to load the cisco_node_utils gem; if no NODE_GEM_URL is specified, the
#     last released gem version
#   * RSAPI_GEM_URL, RSAPI_GEM_REF: the source git URL and git ref (branch)
#     to load the puppet-resource_api gem; if no RSAPI_GEM_URL is specified, the
#     last released gem version.
#
# If a URL, but no REF is specified, the `master` branch will be used.
#

require 'beaker-pe'
require 'shellwords'

test_name 'Prep Masters & Install Puppet' do
  # collect host objects from the nodeset. code further down needs to be prepared that one of those lists is empty
  # in regular testing at puppet there is either a master and an agent for agentful testing, or a default node for agentless testing
  # Get all hosts with role master or compile_master
  masters = select_hosts(roles: ['master', 'compile_master'])
  # Get all hosts with role agent
  agents = select_hosts(roles: ['agent'])

  unless masters.empty?
    step 'install PE on masters' do
      masters.each do |_node|
        install_pe_on(masters, {})
      end
    end
  end

  def mod_line_from_env(module_name, env_prefix=nil)
    if env_prefix && ENV["#{env_prefix}_URL"]
      "mod '#{module_name}', git: '#{ENV["#{env_prefix}_URL"]}', ref: '#{ENV["#{env_prefix}_REF"] || 'master'}'"
    else
      "mod '#{module_name}'"
    end
  end

  unless masters.empty?
    step 'install Cisco Module on master' do
      masters.each do |node|
        on(node, "echo #{Shellwords.escape(mod_line_from_env('puppetlabs-resource_api', 'RSAPI'))} >> /root/Puppetfile")
        on(node, "echo #{Shellwords.escape(mod_line_from_env('puppetlabs-puppetserver_gem', 'RSAPI'))} >> /root/Puppetfile")
        on(node, "echo #{Shellwords.escape(mod_line_from_env('puppetlabs-netdev_stdlib', 'NETDEV_STDLIB'))} >> /root/Puppetfile")
        on(node, "echo #{Shellwords.escape(mod_line_from_env('puppetlabs-ciscopuppet', 'MODULE'))} >> /root/Puppetfile")
        on(node, '/opt/puppetlabs/puppet/bin/r10k puppetfile install /root/Puppetfile -v --moduledir /etc/puppetlabs/code/environments/production/modules', acceptable_exit_codes: [0])
        on(node, puppet('plugin', 'download'), acceptable_exit_codes: [0, 1])
      end
    end
  end

  def clone_and_build_gem(gem_name, env_prefix=nil)
    # nothing to do when no URL set
    return unless env_prefix && ENV["#{env_prefix}_URL"]

    `git clone #{ENV["#{env_prefix}_URL"]} #{gem_name}`
    Dir.chdir(gem_name) do
      `git checkout #{ENV["#{env_prefix}_REF"] || 'master'}`
      `gem build #{gem_name}.gemspec`
    end
  end

  if ENV['RSAPI_GEM_URL']
    step 'Build Resource API gem' do
      clone_and_build_gem('puppet-resource_api', 'RSAPI_GEM')
    end
  end

  if ENV['NODE_GEM_URL']
    step 'Build Cisco Node Utils gem' do
      clone_and_build_gem('cisco_node_utils', 'NODE_GEM')
    end
  end

  def upload_gem_to(node, target_path, gem_name, env_prefix=nil)
    # nothing to upload if gem should be installed from rubygems
    return unless env_prefix && ENV["#{env_prefix}_URL"]

    gem_path = Dir.glob("#{gem_name}/*.gem")[0]
    gem_file = "#{gem_name}.gem"
    scp_to node, gem_path, "#{target_path}/#{gem_file}"
  end

  def install_server_gem(node, gem_name, env_prefix=nil)
    upload_gem_to(node, '/tmp', gem_name, env_prefix)
    gem_location = if env_prefix && ENV["#{env_prefix}_URL"]
                     "/tmp/#{gem_name}.gem"
                   else
                     gem_name
                   end
    on(node, "/opt/puppetlabs/bin/puppetserver gem install #{gem_location}")
  end

  unless masters.empty?
    step 'install Resource API on masters' do
      masters.each do |node|
        install_server_gem(node, 'puppet-resource_api', 'RSAPI_GEM')
        on(node, '/usr/bin/systemctl restart pe-puppetserver.service', acceptable_exit_codes: [0])
      end
    end
  end

  def install_agent_gem(node, gem_name, env_prefix=nil)
    upload_gem_to(node, '/var/volatile/tmp', gem_name, env_prefix)
    gem_location = if env_prefix && ENV["#{env_prefix}_URL"]
                     "/var/volatile/tmp/#{gem_name}.gem"
                   else
                     gem_name
                   end
    on(node, "/opt/puppetlabs/puppet/bin/gem install #{gem_location}")
  end

  unless agents.empty?
    step 'Install agent on switches, sign certificates' do
      opts = {
        puppet_collection: 'PC1',
        puppet_agent_sha: ENV['SHA'],
        puppet_agent_version: ENV['SUITE_VERSION'] || ENV['SHA']
      }
      agents.each do |switch|
        next unless switch['platform'] =~ /cisco_/
        on(switch, 'yum -y erase puppet-agent', acceptable_exit_codes: [0, 1])
        on(switch, 'rpm -qa | grep -i puppetlabs | xargs rpm -e', acceptable_exit_codes: [0, 123])
        on(switch, 'rm -rf /var/cache/yum/puppetlabs-pc1/packages/puppet*', acceptable_exit_codes: [0, 1])
        on(switch, 'rm -rf /var/cache/yum/pl-puppet-agent*', acceptable_exit_codes: [0, 1])
        on(switch, 'rm -rf /etc/yum/repos.d/pl-puppet-agent*', acceptable_exit_codes: [0, 1])
        on(switch, 'rm -rf /var/volatile/log/puppet*', acceptable_exit_codes: [0, 1])
        on(switch, 'find /var/volatile/tmp -type f -delete')
        install_puppet_agent_dev_repo_on(switch, opts)
        install_agent_gem(switch, 'puppet-resource_api', 'RSAPI_GEM')
        install_agent_gem(switch, 'cisco_node_utils', 'NODE_GEM')
        on(switch, 'find /etc/puppetlabs/puppet/ssl/ -type f -print0 |xargs -0r sudo rm')
        on(switch, 'rm /etc/puppetlabs/puppet/puppet.conf')
        on(switch, 'touch /etc/puppetlabs/puppet/puppet.conf')
        on(switch, 'chmod a+w /etc/puppetlabs/puppet/puppet.conf')
        on(switch, "/opt/puppetlabs/bin/puppet config set server #{master.hostname}")
        on(switch, '/opt/puppetlabs/bin/puppet agent -t', acceptable_exit_codes: [1])
        unless masters.empty?
          on(master, puppet('cert', 'sign', switch.to_s), acceptable_exit_codes: [0, 1])
        end
        on(switch, '/opt/puppetlabs/bin/puppet agent -t --waitforcert 60', acceptable_exit_codes: [0, 2])
      end
    end
  end
end

{'fraction_non_alphanumeric': 0.13584905660377358, 'fraction_numerical': 0.005031446540880503, 'mean_word_length': 3.449076664801343, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5442943', 'n_tokens_mistral': 5370, 'n_tokens_neox': 5258, 'n_words': 1452}