<commit_before>#
# Author:: <NAME> (<<EMAIL>>)
#
# Copyright:: 2011-2013, Opscode, Inc <<EMAIL>>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

module Spiceweasel
  class Nodes

    include CommandHelper

    PROVIDERS = %w{bluebox clodo cs digital_ocean ec2 gandi hp joyent kvm linode lxc openstack rackspace slicehost terremark vagrant voxel vsphere}

    attr_reader :create, :delete

    def initialize(nodes, cookbooks, environments, roles, knifecommands)
      @create = Array.new
      @delete = Array.new
      chefclient = Array.new
      create_command_options = {}
      if nodes
        Spiceweasel::Log.debug("nodes: #{nodes}")
        nodes.each do |node|
          name = node.keys.first
          names = name.split
          Spiceweasel::Log.debug("node: '#{name}' '#{node[name]}'")
          # get the node's run_list and options
          if node[name]
            run_list = process_run_list(node[name]['run_list'])
            Spiceweasel::Log.debug("node: '#{name}' run_list: '#{run_list}'")
            validate_run_list(name, run_list, cookbooks, roles) unless Spiceweasel::Config[:novalidation]
            options = node[name]['options'] || ''
            Spiceweasel::Log.debug("node: '#{name}' options: '#{options}'")
            validate_options(name, options, environments) unless Spiceweasel::Config[:novalidation]
            %w(allow_create_failure timeout).each do |key|
              if(node[name].has_key?(key))
                create_command_options[key] = node[name][key]
              end
            end
            additional_commands = node[name]['additional_commands'] || []
          end
          if Spiceweasel::Config[:chefclient]
            chefclient.push(process_chef_client(names, options, run_list))
          elsif Spiceweasel::Config[:node_only]
            nodenames = []
            if PROVIDERS.member?(names[0])
              count = names.length == 2 ? names[1] : 1
              options.split().each do |opt|
                if opt =~ /^-N|^--node-name/
                  optname = opt.sub(/-N|--node-name/,'').lstrip
                  optname = options.split[options.split.find_index(opt)+1] if optname.empty?
                  count.to_i.times do |i|
                    nodenames.push(optname.gsub(/\{\{n\}\}/, (i + 1).to_s))
                  end
                end
              end
            elsif names[0].start_with?("windows_")
              nodenames.push(names[1..-1])
            else #standard nodes
              nodenames.push(names)
            end
            nodenames.flatten.each do |nodeonly|
              if File.directory?("nodes/")
                if File.exists?("nodes/#{nodeonly}.json")
                  # validate individual node files
                  servercommand = "knife node from file #{nodeonly}.json #{Spiceweasel::Config[:knife_options]}".rstrip
                else
                  STDERR.puts "'nodes/#{nodeonly}.json' not found, unable to validate or load node. Using 'knife node create' instead."
                  servercommand = "knife node create #{nodeonly} #{Spiceweasel::Config[:knife_options]}".rstrip
                end
              else
                STDERR.puts "'nodes' directory not found, unable to validate or load nodes. Using 'knife node create' instead."
                servercommand = "knife node create #{nodeonly} #{Spiceweasel::Config[:knife_options]}".rstrip
              end
              servercommand += " -r '#{run_list}'" unless run_list.empty?
              create_command(servercommand, create_command_options)
              delete_command("knife node#{Spiceweasel::Config[:knife_options]} delete #{nodeonly} -y")
              delete_command("knife client#{Spiceweasel::Config[:knife_options]} delete #{nodeonly} -y")
              # knife node run list add NAME
            end
          else #create/delete
            #provider support
            if PROVIDERS.member?(names[0])
              count = names.length == 2 ? names[1] : 1
              process_providers(names[0], count, node[name]['name'], options, run_list, create_command_options, knifecommands)
            elsif names[0].start_with?("windows_")
              #windows node bootstrap support
              protocol = names.shift.split('_') #split on 'windows_ssh' etc
              names.each do |server|
                servercommand = "knife bootstrap #{protocol[0]} #{protocol[1]}#{Spiceweasel::Config[:knife_options]} #{server} #{options}"
                servercommand += " -r '#{run_list}'" unless run_list.empty?
                create_command(servercommand, create_command_options)
                delete_command("knife node#{Spiceweasel::Config[:knife_options]} delete #{server} -y")
                delete_command("knife client#{Spiceweasel::Config[:knife_options]} delete #{server} -y")
              end
            else
              #node bootstrap support
              name.split.each_with_index do |server, i|
                servercommand = "knife bootstrap#{Spiceweasel::Config[:knife_options]} #{server} #{options}".gsub(/\{\{n\}\}/, (i + 1).to_s)
                servercommand += " -r '#{run_list}'" unless run_list.empty?
                create_command(servercommand, create_command_options)
                delete_command("knife node#{Spiceweasel::Config[:knife_options]} delete #{server} -y")
                delete_command("knife client#{Spiceweasel::Config[:knife_options]} delete #{server} -y")
              end
            end
            unless additional_commands.empty?
              additional_commands.each do |cmd|
                create_command(cmd, create_command_options)
              end
            end
          end
        end
        if Spiceweasel::Config[:bulkdelete]
          delete_command("knife node#{Spiceweasel::Config[:knife_options]} bulk delete .* -y")
        end
        #remove repeats in chefclient and push into create_command
        chefclient.flatten.each_with_index {|x,i| create_command(x, create_command_options) unless x.eql?(chefclient[i-1])} if Spiceweasel::Config[:chefclient]
        #nodeonly
      end
    end

    #ensure run_list contents are listed previously.
    def validate_run_list(node, run_list, cookbooks, roles)
      run_list.split(',').each do |item|
        if item.start_with?("recipe[")
          #recipe[foo] or recipe[foo::bar]
          cb = item.split(/\[|\]/)[1].split(':')[0]
          unless cookbooks.member?(cb)
            STDERR.puts "ERROR: '#{node}' run list cookbook '#{cb}' is missing from the list of cookbooks in the manifest."
            exit(-1)
          end
        elsif item.start_with?("role[")
          #role[blah]
          role = item.split(/\[|\]/)[1]
          unless roles.member?(role)
            STDERR.puts "ERROR: '#{node}' run list role '#{role}' is missing from the list of roles in the manifest."
            exit(-1)
          end
        else
          STDERR.puts "ERROR: '#{node}' run list '#{item}' is an invalid run list entry in the manifest."
          exit(-1)
        end
      end
    end

    #for now, just check that -E is legit
    def validate_options(node, options, environments)
      if options =~ /-E/ #check for environments
        env = options.split('-E')[1].split()[0]
        unless environments.member?(env)
          STDERR.puts "ERROR: '#{node}' environment '#{env}' is missing from the list of environments in the manifest."
          exit(-1)
        end
      end
    end

    #manage all the provider logic
    def process_providers(provider, count, name, options, run_list, create_command_options, knifecommands)
      validate_provider(provider, knifecommands) unless Spiceweasel::Config[:novalidation]<commit_msg>extract process_nodes_only and vlidate_node_file into separate methods<commit_after>#
# Author:: <NAME> (<<EMAIL>>)
#
# Copyright:: 2011-2013, Opscode, Inc <<EMAIL>>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

module Spiceweasel
  class Nodes

    include CommandHelper

    PROVIDERS = %w{bluebox clodo cs digital_ocean ec2 gandi hp joyent kvm linode lxc openstack rackspace slicehost terremark vagrant voxel vsphere}

    attr_reader :create, :delete

    def initialize(nodes, cookbooks, environments, roles, knifecommands)
      @create = Array.new
      @delete = Array.new
      chefclient = Array.new
      create_command_options = {}
      if nodes
        Spiceweasel::Log.debug("nodes: #{nodes}")
        nodes.each do |node|
          name = node.keys.first
          names = name.split
          Spiceweasel::Log.debug("node: '#{name}' '#{node[name]}'")
          # get the node's run_list and options
          if node[name]
            run_list = process_run_list(node[name]['run_list'])
            Spiceweasel::Log.debug("node: '#{name}' run_list: '#{run_list}'")
            validate_run_list(name, run_list, cookbooks, roles) unless Spiceweasel::Config[:novalidation]
            options = node[name]['options'] || ''
            Spiceweasel::Log.debug("node: '#{name}' options: '#{options}'")
            validate_options(name, options, environments) unless Spiceweasel::Config[:novalidation]
            %w(allow_create_failure timeout).each do |key|
              if(node[name].has_key?(key))
                create_command_options[key] = node[name][key]
              end
            end
            additional_commands = node[name]['additional_commands'] || []
          end
          if Spiceweasel::Config[:chefclient]
            chefclient.push(process_chef_client(names, options, run_list))
          elsif Spiceweasel::Config[:node_only]
            process_nodes_only(names, options, run_list, create_command_options)
          else #create/delete
            #provider support
            if PROVIDERS.member?(names[0])
              count = names.length == 2 ? names[1] : 1
              process_providers(names[0], count, node[name]['name'], options, run_list, create_command_options, knifecommands)
            elsif names[0].start_with?("windows_")
              #windows node bootstrap support
              protocol = names.shift.split('_') #split on 'windows_ssh' etc
              names.each do |server|
                servercommand = "knife bootstrap #{protocol[0]} #{protocol[1]}#{Spiceweasel::Config[:knife_options]} #{server} #{options}"
                servercommand += " -r '#{run_list}'" unless run_list.empty?
                create_command(servercommand, create_command_options)
                delete_command("knife node#{Spiceweasel::Config[:knife_options]} delete #{server} -y")
                delete_command("knife client#{Spiceweasel::Config[:knife_options]} delete #{server} -y")
              end
            else
              #node bootstrap support
              name.split.each_with_index do |server, i|
                servercommand = "knife bootstrap#{Spiceweasel::Config[:knife_options]} #{server} #{options}".gsub(/\{\{n\}\}/, (i + 1).to_s)
                servercommand += " -r '#{run_list}'" unless run_list.empty?
                create_command(servercommand, create_command_options)
                delete_command("knife node#{Spiceweasel::Config[:knife_options]} delete #{server} -y")
                delete_command("knife client#{Spiceweasel::Config[:knife_options]} delete #{server} -y")
              end
            end
            unless additional_commands.empty?
              additional_commands.each do |cmd|
                create_command(cmd, create_command_options)
              end
            end
          end
        end
        if Spiceweasel::Config[:bulkdelete]
          delete_command("knife node#{Spiceweasel::Config[:knife_options]} bulk delete .* -y")
        end
        #remove repeats in chefclient and push into create_command
        chefclient.flatten.each_with_index {|x,i| create_command(x, create_command_options) unless x.eql?(chefclient[i-1])} if Spiceweasel::Config[:chefclient]
        #nodeonly
      end
    end

    #ensure run_list contents are listed previously.
    def validate_run_list(node, run_list, cookbooks, roles)
      run_list.split(',').each do |item|
        if item.start_with?("recipe[")
          #recipe[foo] or recipe[foo::bar]
          cb = item.split(/\[|\]/)[1].split(':')[0]
          unless cookbooks.member?(cb)
            STDERR.puts "ERROR: '#{node}' run list cookbook '#{cb}' is missing from the list of cookbooks in the manifest."
            exit(-1)
          end
        elsif item.start_with?("role[")
          #role[blah]
          role = item.split(/\[|\]/)[1]
          unless roles.member?(role)
            STDERR.puts "ERROR: '#{node}' run list role '#{role}' is missing from the list of roles in the manifest."
            exit(-1)
          end
        else
          STDERR.puts "ERROR: '#{node}' run list '#{item}' is an invalid run list entry in the manifest."
          exit(-1)
        end
      end
    end

    #for now, just check that -E is legit
    def validate_options(node, options, environments)
      if options =~ /-E/ #check for environments
        env = options.split('-E')[1].split()[0]
        unless environments.member?(env)
          STDERR.puts "ERROR: '#{node}' environment '#{env}' is missing from the list of environments in the manifest."
          exit(-1)
        end
      end
    end

    # handle --nodes-only
    def process_nodes_only(names, options, run_list, create_command_options)
      nodenames = []
      if PROVIDERS.member?(names[0])
        count = names.length == 2 ? names[1] : 1
        options.split().each do |opt|
          if opt =~ /^-N|^--node-name/
            optname = opt.sub(/-N|--node-name/,'').lstrip
            optname = options.split[options.split.find_index(opt)+1] if optname.empty?
            count.to_i.times do |i|
              nodenames.push(optname.gsub(/\{\{n\}\}/, (i + 1).to_s))
            end
          end
        end
      elsif names[0].start_with?("windows_")
        nodenames.push(names[1..-1])
      else #standard nodes
        nodenames.push(names)
      end
      nodenames.flatten.each do |node|
        if File.directory?("nodes/")
          if File.exists?("nodes/#{node}.json")
            validate_node_file(node) unless Spiceweasel::Config[:novalidation]
            servercommand = "knife node from file #{node}.json #{Spiceweasel::Config[:knife_options]}".rstrip
          else
            STDERR.puts "'nodes/#{node}.json' not found, unable to validate or load node. Using 'knife node create' instead."
            servercommand = "knife node create -d #{node} #{Spiceweasel::Config[:knife_options]}".rstrip
          end
        else
          STDERR.puts "'nodes' directory not found, unable to validate or load nodes. Using 'knife node create' instead."
          servercommand = "knife node create -d #{node} #{Spiceweasel::Config[:knife_options]}".rstrip
        end
        create_command(servercommand, create_command_options)
        create_command("knife node run_list set #{node} '#{run_list}'", create_command_options) unless run_list.empty?
        delete_command("knife node#{Spiceweasel::Config[:knife_options]} delete #{node} -y")
        delete_command("knife client#{Spiceweasel::Config[:knife_options]} delete #{node} -y")
      end
    end

    # validate individual node files
    def validate_node_file(name)
      # read in the file
      node = Chef::JSONCompat.from_json(IO.read("nodes/#{name}.json"))
      # check the node name vs. contents of the file
      if(node['name'] != name)
        STDERR.puts "ERROR: Node '#{name}' listed in the manifest does not match the name '#{node['name']}' within the nodes/#{name}.json file."
        exit(-1)
      end
    end

    #manage all the provider logic
    def process_providers(provider, count, name, options, run_list, create_command_options, knifecommands)
      validate_provider(provider, knifecommands) unless Spiceweasel::Config[:novalidation]
{'fraction_non_alphanumeric': 0.12020854316013982, 'fraction_numerical': 0.004917352923751407, 'mean_word_length': 2.4604346043460437, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3915905', 'n_tokens_mistral': 4911, 'n_tokens_neox': 4819, 'n_words': 1248}