<commit_before>(ns protoml.elastic-search
  (:use [clojure.tools.logging :only (info)])
  (:require [protoml.config :as config]
            [clojurewerkz.elastisch.rest          :as esr]
            [clojurewerkz.elastisch.rest.document :as esd]
            [clojurewerkz.elastisch.query         :as q]
            [clojurewerkz.elastisch.rest.response :as esrsp]
            [clojure.pprint :as pp]))

; http://clojureelasticsearch.info/articles/getting_started.html

(defn create-new-transform [document]
  "create a document in elastic search"
  (esr/connect! config/elastic-search-url)
  (info "Input protoml.elastic-search/create-new-transform: " document)
  (let [result (esd/create "protoml" "new-transform" document)]
    (info "Output protoml.elastic-search/create-new-transform: " result)
    result))<commit_msg>elastic search client code<commit_after>(ns protoml.elastic-search
  (:use [clojure.tools.logging :only (info)])
  (:require [protoml.config :as config]
            [clojurewerkz.elastisch.rest          :as esr]
            [clojurewerkz.elastisch.rest.document :as esd]
            [clojurewerkz.elastisch.query         :as q]
            [clojurewerkz.elastisch.rest.response :as esrsp]
            [clojure.pprint :as pp]))

; http://clojureelasticsearch.info/articles/getting_started.html

(defn create-new-transform [document]
  "create a document in elastic search"
  (esr/connect! config/elastic-search-url)
  (info "Input protoml.elastic-search/create-new-transform: " document)
  (let [result (esd/create "protoml" "new-transform" document)]
    (info "Output protoml.elastic-search/create-new-transform: " result)
    result))

(defn extract-source [results]
  "extracts a list of source documents from search results"
  (if (results :timed_out) [nil "Unable to extract documents, searching timed out"]
    [(->> results
         :hits
         :hits
         (map #(% :_source))) nil]))

(defn search-new-transform [key value]
  "searches the new-transform output for a value in a key"
  (let [results (esd/search "protoml" "new-transform" :query (q/term key value))]
    (extract-source results)))
{'fraction_non_alphanumeric': 0.12711069418386492, 'fraction_numerical': 0.0, 'mean_word_length': 3.266, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4015874', 'n_tokens_mistral': 723, 'n_tokens_neox': 711, 'n_words': 160}