<commit_before>(ns braid.search.lucene
  (:require
   [braid.core.server.conf :refer [config]]
   [clucie.analysis :as analysis]
   [clucie.core :as clucie]
   [clucie.store :as store]
   [clucie.query :as query]
   [clojure.string :as string]
   [datomic.api :as d]
   [braid.core.server.db :as db]
   [braid.core.server.db.common :as common])
  (:import
   (org.apache.lucene.index IndexNotFoundException)
   (org.apache.lucene.analysis.core LowerCaseFilter)
   (org.apache.lucene.analysis.icu ICUNormalizer2CharFilterFactory)
   (org.apache.lucene.analysis.standard StandardTokenizer)))

(extend-protocol query/FormParsable
  java.util.UUID
  (parse-formt [u opts]
    (query/parse-formt (str u) opts)))

(def analyzer
  (analysis/build-analyzer
    (StandardTokenizer.)
    :char-filter-factories [(ICUNormalizer2CharFilterFactory.
                              (java.util.HashMap. {"name" "nfkc"
                                                   "mode" "compose"}))]
    :token-filters [(LowerCaseFilter.)]))

(defonce -store (atom nil))
(defn store
  []
  (if-let [s @-store]
    s
    (let [s (if-let [store-path (config :lucene-store-location)]
              (store/disk-store store-path)
              (store/memory-store))]<commit_msg>Fix how lucene interprets UUIDs to mean "match exactly"<commit_after>(ns braid.search.lucene
  (:require
   [braid.core.server.conf :refer [config]]
   [clucie.analysis :as analysis]
   [clucie.core :as clucie]
   [clucie.store :as store]
   [clucie.query :as query]
   [clucie.utils]
   [clojure.string :as string]
   [datomic.api :as d]
   [braid.core.server.db :as db]
   [braid.core.server.db.common :as common])
  (:import
   (org.apache.lucene.analysis.core LowerCaseFilter)
   (org.apache.lucene.analysis.icu ICUNormalizer2CharFilterFactory)
   (org.apache.lucene.analysis.standard StandardTokenizer)
   (org.apache.lucene.index IndexNotFoundException)
   (org.apache.lucene.search BooleanClause$Occur)
   (org.apache.lucene.util QueryBuilder)))

(extend-protocol query/FormParsable
  java.util.UUID
  (parse-formt [u {:keys [^QueryBuilder builder key]}]
    ;; UUIDs should always match exactly
    (.createBooleanQuery builder (clucie.utils/stringify-value key) (str u)
                         BooleanClause$Occur/MUST)))

(def analyzer
  (analysis/build-analyzer
    (StandardTokenizer.)
    :char-filter-factories [(ICUNormalizer2CharFilterFactory.
                              (java.util.HashMap. {"name" "nfkc"
                                                   "mode" "compose"}))]
    :token-filters [(LowerCaseFilter.)]))

(defonce -store (atom nil))
(defn store
  []
  (if-let [s @-store]
    s
    (let [s (if-let [store-path (config :lucene-store-location)]
              (store/disk-store store-path)
              (store/memory-store))]
{'fraction_non_alphanumeric': 0.13475935828877006, 'fraction_numerical': 0.00142602495543672, 'mean_word_length': 2.791891891891892, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6858723', 'n_tokens_mistral': 958, 'n_tokens_neox': 937, 'n_words': 185}