<commit_before>(ns bzzz.core-test
  (:import (java.io File))
  (:use clojure.test
        bzzz.core
        bzzz.index))

(def test-index-name "lein-test-testing-index")

(defn should-work []
  (let [ret (search :index test-index-name
                    :query {:query-parser {:query "<NAME>"
                                           :default-operator :and
                                           :default-field "name"}})]
    (println ret)
    (is (= 1 (:total ret)))
    (is (= "<NAME>" (:name (first (:hits ret)))))))

(deftest test-app
  (testing "cleanup"
    (delete-all test-index-name)
    (refresh-search-managers))

  (testing "store"
    (let [ret-0 (store test-index-name [{:name "<NAME>"}
                                        {:name "<NAME>"}
                                        {:id "baz bar"
                                         :name "duplicate",
                                         :name_no_norms "bar baz"
                                         :name_no_store "with space"}]
                       {:name_no_norms {:type "keyword" }})
          ret-1 (store test-index-name [{:id "WS baz bar"
                                         :name "duplicate"
                                         :name_no_norms "bar baz"
                                         :name_ngram_no_norms "andurilXX"
                                         :name_edge_ngram_no_norms "andurilXX"
                                         :name_keyword_no_norms "hello worldXX"
                                         :name_no_html_no_norms "bzbzXX<br><html>"
                                         :name_no_store "with space"}]
                       {:name_no_norms {:type "whitespace" }
                        :name_keyword_no_norms {:type "custom"
                                                            :tokenizer "keyword"
                                                            :char-filter [{:type "pattern-replace"
                                                                           :pattern "X+",
                                                                           :replacement "ZZ"}]}
                        :name_no_html_no_norms {:type "custom"
                                                            :tokenizer "whitespace"
                                                            :char-filter [{:type "pattern-replace"
                                                                           :pattern "X+",
                                                                           :replacement "ZZ"}
                                                                          {:type "html-strip",:escaped-tags ["br"]}]}
                        :name_edge_ngram_no_norms {:type "custom"
                                                               :tokenizer "edge-ngram"
                                                               :char-filter [{:type "pattern-replace"
                                                                              :pattern "X+",
                                                                              :replacement "ZZ"}]
                                                               :min_gram 1
                                                               :max_gram 8}
                        :name_ngram_no_norms {:type "custom"
                                                          :tokenizer "ngram"
                                                          :filter [{:type "lowercase"}]
                                                          :char-filter [{:type "pattern-replace"
                                                                        :pattern "X+",
                                                                        :replacement "ZZ"}]
                                                          :min_gram 2
                                                          :max_gram 4}})]
      (refresh-search-managers)
      (is (= true (ret-0 test-index-name)))
      (is (= true (ret-1 test-index-name)))))

  (testing "search-bool"
    (let [ret (search :index test-index-name
                      :query {:bool {:must [{:term {:field "id", :value "baz bar"}}]}})]
      (is (= 1 (:total ret)))
      (is (nil? (:name_no_store (first (:hits ret)))))
      (is (= "baz bar" (:id (first (:hits ret)))))))

  (testing "search-filter-query"
    (let [ret (search :index test-index-name
                      :query {:filtered {:filter {:bool {:must [{:term {:field "id", :value "baz bar"}}]}}
                                         :query {:bool {:must [{:term {:field "name", :value "duplicate"}}]}}}})
          ret-2 (search :index test-index-name
                      :query {:bool {:must [{:term {:field "name", :value "duplicate"}}]}})]
      (is (= 1 (:total ret)))
      (is (= 2 (:total ret-2)))
      (is (= (:_score (first (:hits ret))) (:_score (first (:hits ret-2)))))
      (is (= "baz bar" (:id (first (:hits ret)))))))

  (testing "search-constant-score-query"
    (let [ret (search :index test-index-name
                      :query {:constant-score {:boost 10
                                               :query {:bool {:must [{:term {:field "name", :value "duplicate"}}]}}}})]
      (is (= 2 (:total ret)))
      (is (= (= (:_score (first (:hits ret))) (:_score (last (:hits ret))))) 10)))


  (testing "search-no-html"<commit_msg>test the match-all query<commit_after>(ns bzzz.core-test
  (:import (java.io File))
  (:use clojure.test
        bzzz.core
        bzzz.index))

(def test-index-name "lein-test-testing-index")

(defn should-work []
  (let [ret (search :index test-index-name
                    :query {:query-parser {:query "<NAME>"
                                           :default-operator :and
                                           :default-field "name"}})]
    (println ret)
    (is (= 1 (:total ret)))
    (is (= "<NAME>" (:name (first (:hits ret)))))))

(deftest test-app
  (testing "cleanup"
    (delete-all test-index-name)
    (refresh-search-managers))

  (testing "store"
    (let [ret-0 (store test-index-name [{:name "<NAME>"}
                                        {:name "<NAME>"}
                                        {:id "baz bar"
                                         :name "duplicate",
                                         :name_no_norms "bar baz"
                                         :name_no_store "with space"}]
                       {:name_no_norms {:type "keyword" }})
          ret-1 (store test-index-name [{:id "WS baz bar"
                                         :name "duplicate"
                                         :name_no_norms "bar baz"
                                         :name_ngram_no_norms "andurilXX"
                                         :name_edge_ngram_no_norms "andurilXX"
                                         :name_keyword_no_norms "hello worldXX"
                                         :name_no_html_no_norms "bzbzXX<br><html>"
                                         :name_no_store "with space"}]
                       {:name_no_norms {:type "whitespace" }
                        :name_keyword_no_norms {:type "custom"
                                                            :tokenizer "keyword"
                                                            :char-filter [{:type "pattern-replace"
                                                                           :pattern "X+",
                                                                           :replacement "ZZ"}]}
                        :name_no_html_no_norms {:type "custom"
                                                            :tokenizer "whitespace"
                                                            :char-filter [{:type "pattern-replace"
                                                                           :pattern "X+",
                                                                           :replacement "ZZ"}
                                                                          {:type "html-strip",:escaped-tags ["br"]}]}
                        :name_edge_ngram_no_norms {:type "custom"
                                                               :tokenizer "edge-ngram"
                                                               :char-filter [{:type "pattern-replace"
                                                                              :pattern "X+",
                                                                              :replacement "ZZ"}]
                                                               :min_gram 1
                                                               :max_gram 8}
                        :name_ngram_no_norms {:type "custom"
                                                          :tokenizer "ngram"
                                                          :filter [{:type "lowercase"}]
                                                          :char-filter [{:type "pattern-replace"
                                                                        :pattern "X+",
                                                                        :replacement "ZZ"}]
                                                          :min_gram 2
                                                          :max_gram 4}})]
      (refresh-search-managers)
      (is (= true (ret-0 test-index-name)))
      (is (= true (ret-1 test-index-name)))))

  (testing "search-bool"
    (let [ret (search :index test-index-name
                      :query {:bool {:must [{:term {:field "id", :value "baz bar"}}]}})]
      (is (= 1 (:total ret)))
      (is (nil? (:name_no_store (first (:hits ret)))))
      (is (= "baz bar" (:id (first (:hits ret)))))))

  (testing "search-filter-query"
    (let [ret (search :index test-index-name
                      :query {:filtered {:filter {:bool {:must [{:term {:field "id", :value "baz bar"}}]}}
                                         :query {:bool {:must [{:term {:field "name", :value "duplicate"}}]}}}})
          ret-2 (search :index test-index-name
                      :query {:bool {:must [{:term {:field "name", :value "duplicate"}}]}})]
      (is (= 1 (:total ret)))
      (is (= 2 (:total ret-2)))
      (is (= (:_score (first (:hits ret))) (:_score (first (:hits ret-2)))))
      (is (= "baz bar" (:id (first (:hits ret)))))))

  (testing "search-constant-score-query"
    (let [ret (search :index test-index-name
                      :query {:constant-score {:boost 10
                                               :query {:bool {:must [{:term {:field "name", :value "duplicate"}}]}}}})]
      (is (= 2 (:total ret)))
      (is (= (= (:_score (first (:hits ret))) (:_score (last (:hits ret))))) 10)))

  (testing "search-match-all"
    (let [ret (search :index test-index-name
                      :query {:match-all {}})
          num-docs (:docs (get (index-stat) test-index-name))]
      (is (= (:total ret) num-docs))
      (is (> num-docs 0))))

  (testing "search-no-html"
{'fraction_non_alphanumeric': 0.12294192147638863, 'fraction_numerical': 0.003709064592002895, 'mean_word_length': 0.8629929221435794, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 168, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2937957', 'n_tokens_mistral': 2589, 'n_tokens_neox': 2379, 'n_words': 619}