<commit_before>require 'elasticsearch/persistence/model'
require 'hashie'
require 'trollop'

class Noodle::Node
    include Elasticsearch::Persistence::Model

    attribute :name,   String
    attribute :fqdn,   String, default: :name
    attribute :facts,  Hashie::Mash, mapping: { type: 'object' }, default: {}
    attribute :params, Hashie::Mash, mapping: { type: 'object' }, default: {}

    validates_each :params do |record, attr, value|
        # TODO: Don't get options every single time
        # Get default options
        options = Noodle::Option.get

        # Check for required params
        options.required_params.each do |param|
            record.errors.add attr, "#{param} must be provided but is not." if value[param].nil?
        end

        # Check per-param liits
        options.limits.each do |param,limit|
            case limit.class.to_s
            when 'Array'
                record.errors.add attr, "#{param} is not one of these: #{limit.join(',')}.  It is #{value[param]}." unless limit.include?(value[param])
            # cf TODO in option.rb
            when 'String'
                record.errors.add attr, "#{param} is not a(n) #{limit}" unless value[param].nil? or value[param].class.to_s == limit
            end
        end
    end

    def to_puppet
        r = {}
        # TODO: Get class list from node/options
        r['classes']    = ['baseclass']
        r['parameters'] = @params
        r.to_yaml.strip
    end

    def full
        r = []
        r << "Name:   " + @name
        r << "Params: " ; r << @params.map {|term,value| "  #{term}=#{value}"}
        r << "Facts:  " ; r << @facts.map  {|term,value| "  #{term}=#{value}"}
        r.join("\n")
    end

    # Update a node based on options.
    # TODO: Catch errors
    # TODO: Referring to myself must be wrong?
    def update(options)
        options.each_pair do |key,value|
            # TODO: Yuck?
            # TODO: Elsewhere isn't status a param?
            if [:status, :ilk].include?(key)
                # ilk and status are just strings,
                self.send("#{key}=", value)
            else
                # facts and params are Hashie::Mash
                self.send("#{key}=", self.send(key).deep_merge(value))
            end 
            self.save
        end
        self
    end

    def self.all_names
        body = self.all.results.collect{|hit| hit.name}.sort.join("\n")
        [body, 200]
    end

    # Magic:
    #
    # Make everything return either JSON or pretty text, defaul based on
    # either user-agent or accepts or both or more :)
    #
    # Duplicate existing:
    #
    # All parts of the query are ANDed by default
    #
    # Implemented:
    # x=y
    # x=~y
    # @x=y AKA -x=y
    # x?
    # x?=
    # x=
    # full
    # json # Implies full
    # hostname (partial or full)
    # hostnames (partial or full)
    #
    # Still TODO:
    # jmm :)  Maybe by some extensible plugin thing?
    #
    # New ideas:
    # - A way to say OR or switch the whole query to be ORed instead of ANDed
    # - Make precedence explicit:
    #   - if both fact and param, param wins.
    #   - And bare words are highest (or tunable?)
    # - Make order explicit?  Needed?
    # - ~x=y  That is, regexp on the fact/param name
    # - barewords=paramname1[,paramname2,factname1,...] (needs better name)
    #   Allow a list of fact/param names the values of which can be used 
    #   as bare words in queries.
    #
    #   For example, if 'prodlevel' were in the list then 'prod'
    #   could be used in a search to mean prodlevel=prod
    def self.magic(query)
        search = Noodle::Search.new(Noodle::Node)
        show   = []
        format = :default

        # NOTE: Order below should be preserved in case statement
        term_present                = Regexp.new '\?$'
        term_present_and_show_value = Regexp.new '\?=$'
        term_does_not_equal         = Regexp.new '^[-@][^=]+=.+'
        term_show_value             = Regexp.new '=$'
        term_matches_regexp         = Regexp.new '=~'
        term_equals                 = Regexp.new '='
        query.split(/\s+/).each do |part|
            case part
            when term_present
                 term = part.sub(/\?$/,'')
                 search.exists(term)

            when term_present_and_show_value
                 term = part.sub(/\?=$/,'')
                 search.exists(term)
                 show << term

            when term_does_not_equal
                 term,value = part.sub(/^[-@]/,'').split(/=/,2)
                 search.not_equal(term,value)

            when term_show_value
                 show << part.chop

            when term_matches_regexp
                 term,value = part.split(/=~/,2)
                 search.match(term,value)

            when term_equals
                 term,value = part.split(/=/,2)
                 search.equals(term,value)

            when 'full'
                 format = :full

            when 'json'
                 format = :json

            else
                 # Assume everything else is a hostname (or partial hostname)
                 # TODO: Maybe this is a bit awkward when bare words are used with
                 # other magic operators?
                 search.match_names(part)
                 format = :yaml
            end
        end

        # TODO: Don't get options every single time
        # Get default options
        options = Noodle::Option.get
        search.equals('ilk',   options.default_ilk)    unless search.search_terms.include?('ilk')
        search.equals('status',options.default_status) unless search.search_terms.include?('status')

        status = 200
        found = search.go
        case format
        when :json
            body = found.results.to_json + "\n"
        when :yaml
            body = found.results.map{|one| one.to_puppet}.join("\n") + "\n"
        when :full
            body = found.results.map{|one| one.full}.join("\n") + "\n"
        else
            ['',200] if found.response.hits.empty?
            # Always show name. Show term=value pairs for anything in 'show'
            body = []
            found.results.each do |hit|
                add = hit.name
                show.each do |term|
                    if !hit.params.nil?   and hit.params[term]
                        add << " #{term}=#{hit.params[term]}"
                    elsif !hit.facts.nil? and hit.facts[term]
                        add << " #{term}=#{hit.facts[term]}"
                    end
                end
                body << add + "\n"
            end
            body = body.sort.join
        end
        [body,status]
    end

    ## noodlin
    #
    # Below HOST can be short name (as long as it's unique) or FQDN
    # noodlin param|fact -r NAME HOST1 [HOST2 ...]     # Remove param or fact named NAME from listed HOSTs
    # noodlin param|fact NAME=VALUE HOST1 [HOST2 ...]  # Set param or fact named NAME to VALUE for listed HOSTs
    # noodlin param|fact NAME+=VALUE HOST1 [HOST2 ...] # Add    VALUE to   NAME (which must be an array) for listed HOSTs
    # noodlin param|fact NAME-=VALUE HOST1 [HOST2 ...] # Remove VALUE from NAME (which must be an array) for listed HOSTs
    #
    # TODO: Maybe extend this to cover every possible status.
    # noodlin enable  HOST1 [HOST2 ...] # shorthand for noodlin param status=enabled HOST1 [HOST2 ...]
    # noodlin surplus HOST1 [HOST2 ...] # shorthand for noodlin param status=surplus HOST1 [HOST2 ...]
    # noodlin future  HOST1 [HOST2 ...] # shorthand for noodlin param status=future HOST1 [HOST2 ...]
    #
    # Remove requires FQDNs for safety:
    # noodlin remove FQDN1 [FQDN2]      # Remove node(s)
    #
    # Don't forget to specify the required params:
    # noodlin create [-a PARAM=VALUE ...] [-f FACT=VALUE ...] FQDN
    #
    # For historical convenience?
    # noodlin create -i ILK -s STATUS -p PROJECT -P PRODLEVEL -s SITE [-a PARAM=VALUE ...] [-f FACT=VALUE ...] FQDN
    #
    # What else?
    def self.noodlin(changes)
        # Default to success
        status = 200
        body = 'ok'

        # TODO prettier?
        command,rest = changes.split(/\s+/,2)
        rest = rest.split(/\s+/)

        p = Trollop::Parser.new do
            opt :remove,   "thing to remove (used with fact, param)", :type => :string
            opt :param,    "Add param paramname=value",               :type => :string, :multi => true, :short => 'a'
            opt :fact,     "Add fact  factname=value",                :type => :string, :multi => true
            opt :ilk,      "Set ilk at create",                       :type => :string
            opt :site,     "Set site at create",                      :type => :string
            opt :status,   "Set status at create",                    :type => :string
            opt :project,  "Set site at create",                      :type => :string
            opt :prodlevel,"Set prodlevel at create",                 :type => :string, :short => 'P'
        end
        opts = p.parse(rest)
        nodes = rest

        # Unless creating, must be able to find all nodes
        return false unless command == 'create' or found =
            Noodle::Search.new(Noodle::Node).match_names(nodes).go({:minimum => nodes.size})

        # TODO: This should be the list of available statuses
        allowed_statuses = %w{enable future surplus}
        # TODO:
        default_ilk = 'host'
        default_status = 'enabled'

        # TODO: Error when "at create" argument given but not
        # creating.  Maybe easiest if switch to gli :)
        case command
        when 'create'
            # TODO: Create more than one at a time?
            nodes.each do |name|
                args = {
                    name:    name,
                    id:      name,
                }
                facts  = Hash.new
                params = Hash.new

                # Convert special opts into params:
                params[:ilk]       = opts[:ilk]    #|| default_ilk,    # TODO
                params[:project]   = opts[:project]
                params[:prodlevel] = opts[:prodlevel]
                params[:site]      = opts[:site]
                params[:status]    = opts[:status] || default_status  # TODO

                # Merge in the rest
                opts[:fact].map{|pair| name,value = pair.split(/=/); facts[name] = value}
                opts[:param].map{|pair| name,value = pair.split(/=/); facts[name] = value}
                args[:facts]  = facts
                args[:params] = params
                node = Noodle::Node.create_one(args)

                if defined?(node.keys) and node.keys.member?(:errors)
                    body = node[:errors]
                    status = 444
                end
            end
        when 'fact','param'
            sym = "#{command}s".to_sym
            opts[sym].each do |change|
                name,op,value = change.match(/^([^-+=]+)([-+]*=)(.*)$/)[1..3]
                # TODO: Error check fact names and values
                # TODO: Do something with the error strings below :)
                case op
                when '='
                    found.each do |node|
                        node.send(name)[fact] = value
                        node.save
                    end
                when '+=','-='
                    method = op == '+=' ? :push : :delete
                    found.each do |node|
                        if node.send(name)[fact].kind_of?(Array)
                            node.send(name)[fact].send(method,value)
                            node.save
                        else
                            "#{fact} is not an array for #{node.name}"
                        end
                    end
                else
                    "unknown op: #{op}"
                end
            end
        when *allowed_statuses
            found.each do |node|
                node.params['status'] = command
                node.save
                # TODO: Error check
            end
        when 'remove'
            found.map{|node| node.destroy}<commit_msg>More of noodlin works.<commit_after>require 'elasticsearch/persistence/model'
require 'hashie'
require 'trollop'

class Noodle::Node
    include Elasticsearch::Persistence::Model

    attribute :name,   String
    attribute :fqdn,   String, default: :name
    attribute :facts,  Hashie::Mash, mapping: { type: 'object' }, default: {}
    attribute :params, Hashie::Mash, mapping: { type: 'object' }, default: {}

    validates_each :params do |record, attr, value|
        # TODO: Don't get options every single time
        # Get default options
        options = Noodle::Option.get

        # Check for required params
        options.required_params.each do |param|
            record.errors.add attr, "#{param} must be provided but is not." if value[param].nil?
        end

        # Check per-param liits
        options.limits.each do |param,limit|
            case limit.class.to_s
            when 'Array'
                record.errors.add attr, "#{param} is not one of these: #{limit.join(',')}.  It is #{value[param]}." unless limit.include?(value[param])
            # cf TODO in option.rb
            when 'String'
                record.errors.add attr, "#{param} is not a(n) #{limit}" unless value[param].nil? or value[param].class.to_s == limit
            end
        end
    end

    def to_puppet
        r = {}
        # TODO: Get class list from node/options
        r['classes']    = ['baseclass']
        r['parameters'] = @params
        r.to_yaml.strip
    end

    def full
        r = []
        r << "Name:   " + @name
        r << "Params: " ; r << @params.map {|term,value| "  #{term}=#{value}"}
        r << "Facts:  " ; r << @facts.map  {|term,value| "  #{term}=#{value}"}
        r.join("\n")
    end

    # Update a node based on options.
    # TODO: Catch errors
    # TODO: Referring to myself must be wrong?
    def update(options)
        options.each_pair do |key,value|
            # TODO: Yuck?
            # TODO: Elsewhere isn't status a param?
            if [:status, :ilk].include?(key)
                # ilk and status are just strings,
                self.send("#{key}=", value)
            else
                # facts and params are Hashie::Mash
                self.send("#{key}=", self.send(key).deep_merge(value))
            end 
            self.save
        end
        self
    end

    def self.all_names
        body = self.all.results.collect{|hit| hit.name}.sort.join("\n")
        [body, 200]
    end

    # Magic:
    #
    # Make everything return either JSON or pretty text, defaul based on
    # either user-agent or accepts or both or more :)
    #
    # Duplicate existing:
    #
    # All parts of the query are ANDed by default
    #
    # Implemented:
    # x=y
    # x=~y
    # @x=y AKA -x=y
    # x?
    # x?=
    # x=
    # full
    # json # Implies full
    # hostname (partial or full)
    # hostnames (partial or full)
    #
    # Still TODO:
    # jmm :)  Maybe by some extensible plugin thing?
    #
    # New ideas:
    # - A way to say OR or switch the whole query to be ORed instead of ANDed
    # - Make precedence explicit:
    #   - if both fact and param, param wins.
    #   - And bare words are highest (or tunable?)
    # - Make order explicit?  Needed?
    # - ~x=y  That is, regexp on the fact/param name
    # - barewords=paramname1[,paramname2,factname1,...] (needs better name)
    #   Allow a list of fact/param names the values of which can be used 
    #   as bare words in queries.
    #
    #   For example, if 'prodlevel' were in the list then 'prod'
    #   could be used in a search to mean prodlevel=prod
    def self.magic(query)
        search = Noodle::Search.new(Noodle::Node)
        show   = []
        format = :default

        # NOTE: Order below should be preserved in case statement
        term_present                = Regexp.new '\?$'
        term_present_and_show_value = Regexp.new '\?=$'
        term_does_not_equal         = Regexp.new '^[-@][^=]+=.+'
        term_show_value             = Regexp.new '=$'
        term_matches_regexp         = Regexp.new '=~'
        term_equals                 = Regexp.new '='
        query.split(/\s+/).each do |part|
            case part
            when term_present
                 term = part.sub(/\?$/,'')
                 search.exists(term)

            when term_present_and_show_value
                 term = part.sub(/\?=$/,'')
                 search.exists(term)
                 show << term

            when term_does_not_equal
                 term,value = part.sub(/^[-@]/,'').split(/=/,2)
                 search.not_equal(term,value)

            when term_show_value
                 show << part.chop

            when term_matches_regexp
                 term,value = part.split(/=~/,2)
                 search.match(term,value)

            when term_equals
                 term,value = part.split(/=/,2)
                 search.equals(term,value)

            when 'full'
                 format = :full

            when 'json'
                 format = :json

            else
                 # Assume everything else is a hostname (or partial hostname)
                 # TODO: Maybe this is a bit awkward when bare words are used with
                 # other magic operators?
                 search.match_names(part)
                 format = :yaml
            end
        end

        # TODO: Don't get options every single time
        # Get default options
        options = Noodle::Option.get
        search.equals('ilk',   options.default_ilk)    unless search.search_terms.include?('ilk')
        search.equals('status',options.default_status) unless search.search_terms.include?('status')

        status = 200
        found = search.go
        case format
        when :json
            body = found.results.to_json + "\n"
        when :yaml
            body = found.results.map{|one| one.to_puppet}.join("\n") + "\n"
        when :full
            body = found.results.map{|one| one.full}.join("\n") + "\n"
        else
            ['',200] if found.response.hits.empty?
            # Always show name. Show term=value pairs for anything in 'show'
            body = []
            found.results.each do |hit|
                add = hit.name
                show.each do |term|
                    if !hit.params.nil?   and hit.params[term]
                        add << " #{term}=#{hit.params[term]}"
                    elsif !hit.facts.nil? and hit.facts[term]
                        add << " #{term}=#{hit.facts[term]}"
                    end
                end
                body << add + "\n"
            end
            body = body.sort.join
        end
        [body,status]
    end

    ## noodlin
    #
    # Below HOST can be short name (as long as it's unique) or FQDN
    # noodlin param|fact -r NAME HOST1 [HOST2 ...]     # Remove param or fact named NAME from listed HOSTs
    # noodlin param|fact NAME=VALUE HOST1 [HOST2 ...]  # Set param or fact named NAME to VALUE for listed HOSTs
    # noodlin param|fact NAME+=VALUE HOST1 [HOST2 ...] # Add    VALUE to   NAME (which must be an array) for listed HOSTs
    # noodlin param|fact NAME-=VALUE HOST1 [HOST2 ...] # Remove VALUE from NAME (which must be an array) for listed HOSTs
    #
    # TODO: Maybe extend this to cover every possible status.
    # noodlin enable  HOST1 [HOST2 ...] # shorthand for noodlin param status=enabled HOST1 [HOST2 ...]
    # noodlin surplus HOST1 [HOST2 ...] # shorthand for noodlin param status=surplus HOST1 [HOST2 ...]
    # noodlin future  HOST1 [HOST2 ...] # shorthand for noodlin param status=future HOST1 [HOST2 ...]
    #
    # Remove requires FQDNs for safety:
    # noodlin remove FQDN1 [FQDN2]      # Remove node(s)
    #
    # Don't forget to specify the required params:
    # noodlin create [-a PARAM=VALUE ...] [-f FACT=VALUE ...] FQDN
    #
    # For historical convenience?
    # noodlin create -i ILK -s STATUS -p PROJECT -P PRODLEVEL -s SITE [-a PARAM=VALUE ...] [-f FACT=VALUE ...] FQDN
    #
    # What else?
    def self.noodlin(changes)
        # Default to success
        status = 200
        body = ''

        # TODO prettier?
        command,rest = changes.split(/\s+/,2)
        rest = rest.split(/\s+/)

        p = Trollop::Parser.new do
            opt :remove,   "thing to remove (used with fact, param)", :type => :string
            opt :param,    "Add param paramname=value",               :type => :string, :multi => true, :short => 'a'
            opt :fact,     "Add fact  factname=value",                :type => :string, :multi => true
            opt :ilk,      "Set ilk at create",                       :type => :string
            opt :site,     "Set site at create",                      :type => :string
            opt :status,   "Set status at create",                    :type => :string
            opt :project,  "Set site at create",                      :type => :string
            opt :prodlevel,"Set prodlevel at create",                 :type => :string, :short => 'P'
        end
        opts = p.parse(rest)
        # At this point rest contains node(s) and possibly key=value
        # pairs for the fact or param sub-commands

        # Now split into nodes and pairs
        # TODO: Prettier
        pairs = []
        rest.each do |elem|
            pairs << rest.delete(elem) if elem.match('=')
        end
        nodes = rest

        # Unless creating, must be able to find all nodes
        return false unless command == 'create' or found =
            Noodle::Search.new(Noodle::Node).match_names(nodes).go({:minimum => nodes.size})

        # TODO: This should be the list of available statuses
        allowed_statuses = %w{enable future surplus}
        # TODO:
        default_ilk = 'host'
        default_status = 'enabled'

        # TODO: Error when "at create" argument given but not
        # creating.  Maybe easiest if switch to gli :)
        case command
        when 'create'
            # TODO: Create more than one at a time?
            nodes.each do |name|
                args = {
                    name:    name,
                    id:      name,
                }
                facts  = Hash.new
                params = Hash.new

                # Convert special opts into params:
                params[:ilk]       = opts[:ilk]    #|| default_ilk,    # TODO
                params[:project]   = opts[:project]
                params[:prodlevel] = opts[:prodlevel]
                params[:site]      = opts[:site]
                params[:status]    = opts[:status] || default_status  # TODO

                # Merge in the rest
                opts[:fact].map{|pair| name,value = pair.split(/=/); facts[name] = value}
                opts[:param].map{|pair| name,value = pair.split(/=/); facts[name] = value}
                args[:facts]  = facts
                args[:params] = params
                node = Noodle::Node.create_one(args)

                if defined?(node.keys) and node.keys.member?(:errors)
                    body = node[:errors]
                    status = 444
                end
            end
        when 'fact','param'
            which = "#{command}s"
            [opts[command.to_sym] + pairs].flatten.each do |change|
                name,op,value = change.match(/^([^-+=]+)([-+]*=)(.*)$/)[1..3]

                # TODO: Error check fact names and values
                # TODO: Do something with the error strings below :)
                case op
                when '='
                    found.each do |node|
                        node.send(which)[name] = value
                        node.save
                    end
                when '+=','-='
                    method = op == '+=' ? :push : :delete
                    found.each do |node|
                        if node.send(which)[name].kind_of?(Array)
                            node.send(which)[name].send(method,value)
                            node.save
                        else
                            "#{fact} is not an array for #{node.name}"
                        end
                    end
                else
                    "unknown op: #{op}"
                end
            end
        when *allowed_statuses
            found.each do |node|
                node.params['status'] = command
                node.save
                # TODO: Error check
            end
        when 'remove'
            found.map{|node| node.destroy}
{'fraction_non_alphanumeric': 0.11403829735333577, 'fraction_numerical': 0.003740293531731512, 'mean_word_length': 1.5069302894415002, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1955588', 'n_tokens_mistral': 7140, 'n_tokens_neox': 6823, 'n_words': 2067}