<commit_before>(ns com.puppetlabs.puppetdb.test.http.facts
  (:require [com.puppetlabs.puppetdb.scf.storage :as scf-store]
            [com.puppetlabs.http :as pl-http]
            [cheshire.core :as json]
            [clojure.java.jdbc :as sql]
            [com.puppetlabs.puppetdb.http.server :as server]
            [clojure.java.io :as io]
            [flatland.ordered.map :as omap]
            [puppetlabs.kitchensink.core :as ks])
  (:use clojure.test
        ring.mock.request
        [com.puppetlabs.puppetdb.fixtures]
        [com.puppetlabs.puppetdb.examples]
        [clj-time.core :only [now]]
        [com.puppetlabs.puppetdb.testutils :only [get-request assert-success! paged-results paged-results*]]
        [com.puppetlabs.jdbc :only [with-transacted-connection]]))

(def v2-endpoint "/v2/facts")
(def v3-endpoint "/v3/facts")
(def endpoints [v2-endpoint v3-endpoint])

(defixture super-fixture :each with-test-db with-http-app)

(def c-t pl-http/json-response-content-type)

(defn get-response
  ([endpoint]      (get-response endpoint nil))
  ([endpoint query] (*app* (get-request endpoint query)))
  ([endpoint query params] (*app* (get-request endpoint query params))))

(defn is-query-result
  [endpoint query results]
  (let [request (get-request endpoint (json/generate-string query))
        {:keys [status body]} (*app* request)]
    (is (= (try
             (if (string? body)
               (json/parse-string body true)
               (json/parse-string (slurp body) true))
             (catch Throwable e
               body)) results))
    (is (= status pl-http/status-ok))))

(def common-subquery-tests
  (omap/ordered-map
   ["and"
    ["=" "name" "ipaddress"]
    ["in" "certname" ["extract" "certname" ["select-resources"
                                            ["and"
                                             ["=" "type" "Class"]
                                             ["=" "title" "Apache"]]]]]]

   [{:certname "foo" :name "ipaddress" :value "192.168.1.100"}
    {:certname "bar" :name "ipaddress" :value "192.168.1.101"}]

   ;; "not" matching resources
   ["and"
    ["=" "name" "ipaddress"]
    ["not"
     ["in" "certname" ["extract" "certname" ["select-resources"
                                             ["and"
                                              ["=" "type" "Class"]
                                              ["=" "title" "Apache"]]]]]]]

   [{:certname "baz" :name "ipaddress" :value "192.168.1.102"}]

   ;; Multiple matching resources
   ["and"
    ["=" "name" "ipaddress"]
    ["in" "certname" ["extract" "certname" ["select-resources"
                                            ["=" "type" "Class"]]]]]

   [{:certname "foo" :name "ipaddress" :value "192.168.1.100"}
    {:certname "bar" :name "ipaddress" :value "192.168.1.101"}
    {:certname "baz" :name "ipaddress" :value "192.168.1.102"}]

   ;; Multiple facts
   ["and"
    ["or"
     ["=" "name" "ipaddress"]
     ["=" "name" "operatingsystem"]]
    ["in" "certname" ["extract" "certname" ["select-resources"
                                            ["and"
                                             ["=" "type" "Class"]
                                             ["=" "title" "Apache"]]]]]]

   [{:certname "bar" :name "ipaddress" :value "192.168.1.101"}
    {:certname "bar" :name "operatingsystem" :value "Ubuntu"}
    {:certname "foo" :name "ipaddress" :value "192.168.1.100"}
    {:certname "foo" :name "operatingsystem" :value "Debian"}]

   ;; Multiple subqueries
   ["and"
    ["=" "name" "ipaddress"]
    ["or"
     ["in" "certname" ["extract" "certname" ["select-resources"
                                             ["and"
                                              ["=" "type" "Class"]
                                              ["=" "title" "Apache"]]]]]
     ["in" "certname" ["extract" "certname" ["select-resources"
                                             ["and"
                                              ["=" "type" "Class"]
                                              ["=" "title" "Main"]]]]]]]

   [{:certname "foo" :name "ipaddress" :value "192.168.1.100"}
    {:certname "bar" :name "ipaddress" :value "192.168.1.101"}
    {:certname "baz" :name "ipaddress" :value "192.168.1.102"}]

   ;; No matching resources
   ["and"
    ["=" "name" "ipaddress"]
    ["in" "certname" ["extract" "certname" ["select-resources"
                                            ["=" "type" "NotRealAtAll"]]]]]
   []

   ;; No matching facts
   ["and"
    ["=" "name" "nosuchfact"]
    ["in" "certname" ["extract" "certname" ["select-resources"
                                            ["=" "type" "Class"]]]]]
   []

   ;; Fact subquery
   ["and"
    ["=" "name" "ipaddress"]
    ["in" "certname" ["extract" "certname" ["select-facts"
                                            ["and"
                                             ["=" "name" "osfamily"]
                                             ["=" "value" "Debian"]]]]]]

   [{:certname "foo" :name "ipaddress" :value "192.168.1.100"}
    {:certname "bar" :name "ipaddress" :value "192.168.1.101"}]

   ;; Using a different column
   ["in" "name" ["extract" "name" ["select-facts"
                                   ["=" "name" "osfamily"]]]]

   [{:certname "bar" :name "osfamily" :value "Debian"}
    {:certname "baz" :name "osfamily" :value "RedHat"}
    {:certname "foo" :name "osfamily" :value "Debian"}]

   ;; Nested fact subqueries
   ["and"
    ["=" "name" "ipaddress"]
    ["in" "certname" ["extract" "certname" ["select-facts"
                                            ["and"
                                             ["=" "name" "osfamily"]
                                             ["=" "value" "Debian"]
                                             ["in" "certname" ["extract" "certname" ["select-facts"
                                                                                     ["and"
                                                                                      ["=" "name" "uptime_seconds"]
                                                                                      [">" "value" 10000]]]]]]]]]]

   [{:certname "foo" :name "ipaddress" :value "192.168.1.100"}]

   ;; Multiple fact subqueries
   ["and"
    ["=" "name" "ipaddress"]
    ["in" "certname" ["extract" "certname" ["select-facts"
                                            ["and"
                                             ["=" "name" "osfamily"]
                                             ["=" "value" "Debian"]]]]]
    ["in" "certname" ["extract" "certname" ["select-facts"
                                            ["and"
                                             ["=" "name" "uptime_seconds"]
                                             [">" "value" 10000]]]]]]

   [{:certname "foo" :name "ipaddress" :value "192.168.1.100"}]))

(def versioned-subqueries
  (omap/ordered-map
   "/v2/facts"
   (merge common-subquery-tests
          (omap/ordered-map
           ;; Subqueries using file/line
           ["and"
            ["=" "name" "ipaddress"]
            ["in" "certname" ["extract" "certname" ["select-resources"
                                                    ["and"
                                                     ["=" "sourcefile" "/etc/puppet/modules/settings/manifests/init.pp"]
                                                     ["=" "sourceline" 1]]]]]]

           [{:certname "foo" :name "ipaddress" :value "192.168.1.100"}
            {:certname "bar" :name "ipaddress" :value "192.168.1.101"}
            {:certname "baz" :name "ipaddress" :value "192.168.1.102"}]))

   "/v3/facts"
   (merge common-subquery-tests
          (omap/ordered-map
           ;; Subqueries using file/line
           ["and"
            ["=" "name" "ipaddress"]
            ["in" "certname" ["extract" "certname" ["select-resources"
                                                    ["and"
                                                     ["=" "file" "/etc/puppet/modules/settings/manifests/init.pp"]
                                                     ["=" "line" 1]]]]]]

           [{:certname "foo" :name "ipaddress" :value "192.168.1.100"}
            {:certname "bar" :name "ipaddress" :value "192.168.1.101"}
            {:certname "baz" :name "ipaddress" :value "192.168.1.102"}]))))

(def versioned-invalid-subqueries
  (omap/ordered-map
   "/v2/facts" (omap/ordered-map
                ;; Extract using an invalid field should throw an error
                ["in" "certname" ["extract" "nothing" ["select-resources"
                                                       ["=" "type" "Class"]]]]
                "Can't extract unknown resource field 'nothing'. Acceptable fields are: catalog, certname, exported, resource, sourcefile, sourceline, tags, title, type"

                ;; In-query for invalid fields should throw an error
                ["in" "nothing" ["extract" "certname" ["select-resources"
                                                       ["=" "type" "Class"]]]]<commit_msg>Switched fact subquery tests from vector results to sets.<commit_after>(ns com.puppetlabs.puppetdb.test.http.facts
  (:require [com.puppetlabs.puppetdb.scf.storage :as scf-store]
            [com.puppetlabs.http :as pl-http]
            [cheshire.core :as json]
            [clojure.java.jdbc :as sql]
            [com.puppetlabs.puppetdb.http.server :as server]
            [clojure.java.io :as io]
            [flatland.ordered.map :as omap]
            [puppetlabs.kitchensink.core :as ks])
  (:use clojure.test
        ring.mock.request
        [com.puppetlabs.puppetdb.fixtures]
        [com.puppetlabs.puppetdb.examples]
        [clj-time.core :only [now]]
        [com.puppetlabs.puppetdb.testutils :only [get-request assert-success! paged-results paged-results*]]
        [com.puppetlabs.jdbc :only [with-transacted-connection]]))

(def v2-endpoint "/v2/facts")
(def v3-endpoint "/v3/facts")
(def endpoints [v2-endpoint v3-endpoint])

(defixture super-fixture :each with-test-db with-http-app)

(def c-t pl-http/json-response-content-type)

(defn get-response
  ([endpoint]      (get-response endpoint nil))
  ([endpoint query] (*app* (get-request endpoint query)))
  ([endpoint query params] (*app* (get-request endpoint query params))))

(defn parse-result
  "Stringify (if needed) then parse the response"
  [body]
  (try
    (if (string? body)
      (json/parse-string body true)
      (json/parse-string (slurp body) true))
    (catch Throwable e
      body)))

(defn is-query-result
  [endpoint query expected-results]
  (let [request (get-request endpoint (json/generate-string query))
        {:keys [status body]} (*app* request)
        actual-result (parse-result body)]
    (is (= (count actual-result) (count expected-results)))
    (is (= (set actual-result) expected-results))
    (is (= status pl-http/status-ok))))

(def common-subquery-tests
  (omap/ordered-map
   ["and"
    ["=" "name" "ipaddress"]
    ["in" "certname" ["extract" "certname" ["select-resources"
                                            ["and"
                                             ["=" "type" "Class"]
                                             ["=" "title" "Apache"]]]]]]

   #{{:certname "foo" :name "ipaddress" :value "192.168.1.100"}
     {:certname "bar" :name "ipaddress" :value "192.168.1.101"}}

   ;; "not" matching resources
   ["and"
    ["=" "name" "ipaddress"]
    ["not"
     ["in" "certname" ["extract" "certname" ["select-resources"
                                             ["and"
                                              ["=" "type" "Class"]
                                              ["=" "title" "Apache"]]]]]]]

   #{{:certname "baz" :name "ipaddress" :value "192.168.1.102"}}

   ;; Multiple matching resources
   ["and"
    ["=" "name" "ipaddress"]
    ["in" "certname" ["extract" "certname" ["select-resources"
                                            ["=" "type" "Class"]]]]]

   #{{:certname "foo" :name "ipaddress" :value "192.168.1.100"}
     {:certname "bar" :name "ipaddress" :value "192.168.1.101"}
     {:certname "baz" :name "ipaddress" :value "192.168.1.102"}}

   ;; Multiple facts
   ["and"
    ["or"
     ["=" "name" "ipaddress"]
     ["=" "name" "operatingsystem"]]
    ["in" "certname" ["extract" "certname" ["select-resources"
                                            ["and"
                                             ["=" "type" "Class"]
                                             ["=" "title" "Apache"]]]]]]

   #{{:certname "bar" :name "ipaddress" :value "192.168.1.101"}
     {:certname "bar" :name "operatingsystem" :value "Ubuntu"}
     {:certname "foo" :name "ipaddress" :value "192.168.1.100"}
     {:certname "foo" :name "operatingsystem" :value "Debian"}}

   ;; Multiple subqueries
   ["and"
    ["=" "name" "ipaddress"]
    ["or"
     ["in" "certname" ["extract" "certname" ["select-resources"
                                             ["and"
                                              ["=" "type" "Class"]
                                              ["=" "title" "Apache"]]]]]
     ["in" "certname" ["extract" "certname" ["select-resources"
                                             ["and"
                                              ["=" "type" "Class"]
                                              ["=" "title" "Main"]]]]]]]

   #{{:certname "foo" :name "ipaddress" :value "192.168.1.100"}
     {:certname "bar" :name "ipaddress" :value "192.168.1.101"}
     {:certname "baz" :name "ipaddress" :value "192.168.1.102"}}

   ;; No matching resources
   ["and"
    ["=" "name" "ipaddress"]
    ["in" "certname" ["extract" "certname" ["select-resources"
                                            ["=" "type" "NotRealAtAll"]]]]]
   #{}

   ;; No matching facts
   ["and"
    ["=" "name" "nosuchfact"]
    ["in" "certname" ["extract" "certname" ["select-resources"
                                            ["=" "type" "Class"]]]]]
   #{}

   ;; Fact subquery
   ["and"
    ["=" "name" "ipaddress"]
    ["in" "certname" ["extract" "certname" ["select-facts"
                                            ["and"
                                             ["=" "name" "osfamily"]
                                             ["=" "value" "Debian"]]]]]]

   #{{:certname "foo" :name "ipaddress" :value "192.168.1.100"}
     {:certname "bar" :name "ipaddress" :value "192.168.1.101"}}

   ;; Using a different column
   ["in" "name" ["extract" "name" ["select-facts"
                                   ["=" "name" "osfamily"]]]]

   #{{:certname "bar" :name "osfamily" :value "Debian"}
     {:certname "baz" :name "osfamily" :value "RedHat"}
     {:certname "foo" :name "osfamily" :value "Debian"}}

   ;; Nested fact subqueries
   ["and"
    ["=" "name" "ipaddress"]
    ["in" "certname" ["extract" "certname" ["select-facts"
                                            ["and"
                                             ["=" "name" "osfamily"]
                                             ["=" "value" "Debian"]
                                             ["in" "certname" ["extract" "certname" ["select-facts"
                                                                                     ["and"
                                                                                      ["=" "name" "uptime_seconds"]
                                                                                      [">" "value" 10000]]]]]]]]]]

   #{{:certname "foo" :name "ipaddress" :value "192.168.1.100"}}

   ;; Multiple fact subqueries
   ["and"
    ["=" "name" "ipaddress"]
    ["in" "certname" ["extract" "certname" ["select-facts"
                                            ["and"
                                             ["=" "name" "osfamily"]
                                             ["=" "value" "Debian"]]]]]
    ["in" "certname" ["extract" "certname" ["select-facts"
                                            ["and"
                                             ["=" "name" "uptime_seconds"]
                                             [">" "value" 10000]]]]]]

   #{{:certname "foo" :name "ipaddress" :value "192.168.1.100"}}))

(def versioned-subqueries
  (omap/ordered-map
   "/v2/facts"
   (merge common-subquery-tests
          (omap/ordered-map
           ;; Subqueries using file/line
           ["and"
            ["=" "name" "ipaddress"]
            ["in" "certname" ["extract" "certname" ["select-resources"
                                                    ["and"
                                                     ["=" "sourcefile" "/etc/puppet/modules/settings/manifests/init.pp"]
                                                     ["=" "sourceline" 1]]]]]]

           #{{:certname "foo" :name "ipaddress" :value "192.168.1.100"}
             {:certname "bar" :name "ipaddress" :value "192.168.1.101"}
             {:certname "baz" :name "ipaddress" :value "192.168.1.102"}}))

   "/v3/facts"
   (merge common-subquery-tests
          (omap/ordered-map
           ;; Subqueries using file/line
           ["and"
            ["=" "name" "ipaddress"]
            ["in" "certname" ["extract" "certname" ["select-resources"
                                                    ["and"
                                                     ["=" "file" "/etc/puppet/modules/settings/manifests/init.pp"]
                                                     ["=" "line" 1]]]]]]

           #{{:certname "foo" :name "ipaddress" :value "192.168.1.100"}
             {:certname "bar" :name "ipaddress" :value "192.168.1.101"}
             {:certname "baz" :name "ipaddress" :value "192.168.1.102"}}))))

(def versioned-invalid-subqueries
  (omap/ordered-map
   "/v2/facts" (omap/ordered-map
                ;; Extract using an invalid field should throw an error
                ["in" "certname" ["extract" "nothing" ["select-resources"
                                                       ["=" "type" "Class"]]]]
                "Can't extract unknown resource field 'nothing'. Acceptable fields are: catalog, certname, exported, resource, sourcefile, sourceline, tags, title, type"

                ;; In-query for invalid fields should throw an error
                ["in" "nothing" ["extract" "certname" ["select-resources"
                                                       ["=" "type" "Class"]]]]
{'fraction_non_alphanumeric': 0.1666757241454269, 'fraction_numerical': 0.025107331123308516, 'mean_word_length': 1.4477254588986432, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 42, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 180, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4325776', 'n_tokens_mistral': 5595, 'n_tokens_neox': 5136, 'n_words': 1157}