<commit_before>(ns pathfinder.data.elasticsearch
  (:require [clojurewerkz.elastisch.rest :as esr]
            [clojurewerkz.elastisch.rest.document :as esd]
            [clojurewerkz.elastisch.rest.index :as esi]
            [pathfinder.data.data :refer [Data]]))

(defn- search-query [params]
  {:bool
   {:must [{:term {:project (:project params)}}
           {:prefix {:path (:path params)}}]}})

(defn- doc-id [data]
  (format "%s/%s"
          (get-in data [:meta :project] "UNKNOWN")
          (get-in data [:meta :path] "UNKNOWN")))

(defn- extract-results [results]
  {:found (get-in results [:hits :total])
   :results (->> results :hits :hits (map :_source))})

(deftype ElasticSearch [conn]
  Data
  (stash! [this data]
    (esd/create conn "docs" "doc" data :id (doc-id data))) ;; TODO: set a ttl?
  (search [this params]
    (extract-results (esd/search conn "docs" "doc" :query (search-query params)))))

(defn- setup-index! [conn]
  (let [mappings {:doc {:properties
                        {:meta {:properties
                                {:path {:type "string" :index "not_analyzed"}}}}}}]
    (when-not (esi/exists? conn "docs")
      (esi/create conn "docs" :mappings mappings))))

(defn ->ElasticSearch [es-config]
  (let [es (esr/connect (:endpoint es-config))]
    (setup-index! es)
    (ElasticSearch. es)))<commit_msg>Project should not be analyzed<commit_after>(ns pathfinder.data.elasticsearch
  (:require [clojurewerkz.elastisch.rest :as esr]
            [clojurewerkz.elastisch.rest.document :as esd]
            [clojurewerkz.elastisch.rest.index :as esi]
            [pathfinder.data.data :refer [Data]]))

(defn- search-query [params]
  {:bool
   {:must [{:term {:project (:project params)}}
           {:prefix {:path (:path params)}}]}})

(defn- doc-id [data]
  (format "%s/%s"
          (get-in data [:meta :project] "UNKNOWN")
          (get-in data [:meta :path] "UNKNOWN")))

(defn- extract-results [results]
  {:found (get-in results [:hits :total])
   :results (->> results :hits :hits (map :_source))})

(deftype ElasticSearch [conn]
  Data
  (stash! [this data]
    (esd/create conn "docs" "doc" data :id (doc-id data))) ;; TODO: set a ttl?
  (search [this params]
    (extract-results (esd/search conn "docs" "doc" :query (search-query params)))))

(defn- setup-index! [conn]
  (let [mappings {:doc {:properties
                        {:meta {:properties
                                {:path {:type "string" :index "not_analyzed"}
                                 :project {:type "string" :index "not_analyzed"}}}}}}]
    (when-not (esi/exists? conn "docs")
      (esi/create conn "docs" :mappings mappings))))

(defn ->ElasticSearch [es-config]
  (let [es (esr/connect (:endpoint es-config))]
    (setup-index! es)
    (ElasticSearch. es)))
{'fraction_non_alphanumeric': 0.17526881720430107, 'fraction_numerical': 0.0, 'mean_word_length': 2.8496551724137933, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4995996', 'n_tokens_mistral': 984, 'n_tokens_neox': 964, 'n_words': 252}