<commit_before>(ns searchbot.server
  (:require [clojure.java.io :as io]
            [clojure.edn]
            [searchbot.dev :refer [is-dev? inject-devmode-html browser-repl start-figwheel]]
            [compojure.core :refer [GET POST defroutes]]
            [compojure.route :refer [resources not-found]]
            [net.cgrand.enlive-html :refer [deftemplate]]
            [net.cgrand.reload :refer [auto-reload]]
            [ring.middleware.reload :as reload]
            [ring.middleware.defaults :refer [wrap-defaults api-defaults site-defaults]]
            [ring.middleware.json :as mid-json]
            [ring.util.response :refer [resource-response response redirect]]
            [ring.util.json-response :as jresp]
            [environ.core :refer [env]]
            [org.httpkit.server :refer [run-server]]
            [searchbot.es :as es])
  (:gen-class))

(defn- has-file?
  [filename]
  (.exists (io/file filename)))

(defn- read-edn
  [ednfile & {:keys [default]}]
  (if (has-file? ednfile)
    (slurp ednfile)
    default))

(def ^:private es-host
  (atom {:host "127.0.0.1" :port "9200"}))

(defn get-es-host
  []
  (str "http://"
       (or (get-in (clojure.edn/read-string (read-edn "server.edn")) [:es :host])
           (:host @es-host))
       ":"
       (or (get-in (clojure.edn/read-string (read-edn "server.edn")) [:es :port])
           (:port @es-host))))

(deftemplate init-page (io/resource "index.html") []
  [:body] (if is-dev? inject-devmode-html identity))

(def default-app-state
  {:header-text "searchbot"
   :menu {:top-open? false :sub-open? false :selected {} :cm nil}
   :nav [{:view "app-state" :label "Dashboard" :active true}]
   :es-count 0
   :agg {:div {:width "90%" :height 300}}
   :aggregators []
   :widgets []})

(defroutes routes
  (resources "/react" {:root "react"})
  (GET "/" req (apply str (init-page)))
  (GET "/_init" req (response (clojure.edn/read-string
                               (read-edn "app-state.edn" :default (pr-str default-app-state)))))
  (GET "/_init/:view" [view] (response (clojure.edn/read-string
                                        (read-edn (str view ".edn") :default (pr-str default-app-state)))))
  (GET "/es/:idx/_count" [idx]
       (response (es/es-count (get-es-host) idx)))
  (POST "/es/:idx/_search" [idx :as req]
        (response (es/es-search (get-es-host) idx (:body req))))
  (GET "/es/:idx/:idxType/_count" [idx idxType]
       (response (es/es-count (get-es-host) idx idxType)))
  (POST "/es/:idx/:idxType/_search" [idx idxType :as req]
        (response (es/es-search (get-es-host) idx idxType (:body req))))
  (resources "/")
  (not-found "Not Found"))
<commit_msg>extract common fn<commit_after>(ns searchbot.server
  (:require [clojure.java.io :as io]
            [clojure.edn]
            [searchbot.dev :refer [is-dev? inject-devmode-html browser-repl start-figwheel]]
            [compojure.core :refer [GET POST defroutes]]
            [compojure.route :refer [resources not-found]]
            [net.cgrand.enlive-html :refer [deftemplate]]
            [net.cgrand.reload :refer [auto-reload]]
            [ring.middleware.reload :as reload]
            [ring.middleware.defaults :refer [wrap-defaults api-defaults site-defaults]]
            [ring.middleware.json :as mid-json]
            [ring.util.response :refer [resource-response response redirect]]
            [ring.util.json-response :as jresp]
            [environ.core :refer [env]]
            [org.httpkit.server :refer [run-server]]
            [searchbot.es :as es])
  (:gen-class))

(defn- has-file?
  [filename]
  (.exists (io/file filename)))

(defn- read-edn
  [ednfile & {:keys [default]}]
  (if (has-file? ednfile)
    (slurp ednfile)
    default))

(def ^:private es-host
  (atom {:host "127.0.0.1" :port "9200"}))

(defn get-es-host
  []
  (str "http://"
       (or (get-in (clojure.edn/read-string (read-edn "server.edn")) [:es :host])
           (:host @es-host))
       ":"
       (or (get-in (clojure.edn/read-string (read-edn "server.edn")) [:es :port])
           (:port @es-host))))

(deftemplate init-page (io/resource "index.html") []
  [:body] (if is-dev? inject-devmode-html identity))

(def default-app-state
  {:header-text "searchbot"
   :menu {:top-open? false :sub-open? false :selected {} :cm nil}
   :nav [{:view "app-state" :label "Dashboard" :active true}]
   :es-count 0
   :agg {:div {:width "90%" :height 300}}
   :aggregators []
   :widgets []})

(defn load-view-edn [view]
  (clojure.edn/read-string
   (read-edn (str view ".edn") :default (pr-str default-app-state))))

(defroutes routes
  (resources "/react" {:root "react"})
  (GET "/" req (apply str (init-page)))
  (GET "/_init" req (response (load-view-edn "app-state")))
  (GET "/_init/:view" [view] (response (load-view-edn view)))
  (GET "/es/:idx/_count" [idx]
       (response (es/es-count (get-es-host) idx)))
  (POST "/es/:idx/_search" [idx :as req]
        (response (es/es-search (get-es-host) idx (:body req))))
  (GET "/es/:idx/:idxType/_count" [idx idxType]
       (response (es/es-count (get-es-host) idx idxType)))
  (POST "/es/:idx/:idxType/_search" [idx idxType :as req]
        (response (es/es-search (get-es-host) idx idxType (:body req))))
  (resources "/")
  (not-found "Not Found"))

{'fraction_non_alphanumeric': 0.18407582938388625, 'fraction_numerical': 0.006066350710900474, 'mean_word_length': 2.6562716562716564, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5368486', 'n_tokens_mistral': 1929, 'n_tokens_neox': 1908, 'n_words': 441}