<commit_before>(ns doctopus.db
  (:require [camel-snake-kebab.core :refer [->snake_case_keyword ->kebab-case-keyword]]
            [camel-snake-kebab.extras :refer [transform-keys]]
            [clj-time.coerce :refer [to-sql-time]]
            [clj-time.core :as clj-time]
            [clojure.string :as str :refer [split-lines]]
            [doctopus.configuration :refer [server-config]]
            [korma.core :refer :all]
            [korma.db :refer [defdb postgres default-connection]]
            [taoensso.timbre :as log]))

(defn- now
  []
  "returns a string for 'now' in SQL Timestamp format"
  (to-sql-time (clj-time/now)))

(defn- add-updated
  "adds updated field to a map with given time"
  ([fields] (add-updated fields (now)))
  ([fields now-string] (assoc fields :updated now-string)))

(defn- ->kebab-keys
  "converts all keys in a map to kebab-case keywords"
  [fields]
  (transform-keys ->kebab-case-keyword fields))

(defn- ->snake-keys
  "converts all keys in a map to snake-case keywords"
  [fields]
  (transform-keys ->snake_case_keyword fields))

(defn- ->shallow-snake-keys
  "Converts only the outermost keys of a map to snake-case keywords"
  [m]
  (doall (into {} (map (fn [[k v]][(->snake_case_keyword k) v]) m))))

(defdb main-db (postgres
             (select-keys (get-in (server-config) [:database :main])
                          [:db :user :password :host :port])))
(defdb test-db (postgres
                (select-keys (get-in (server-config) [:database :test])
                             [:db :user :password :host :port])))

(default-connection (if (= "dev" (:nomad/environment (server-config)))
                      test-db
                      main-db))

(declare head-tentacle-mappings documents)
(defentity tentacles
  (pk :name)
  (has-many head-tentacle-mappings {:fk :tentacle_name})
  (has-one documents {:fk :tentacle_name})
  (prepare add-updated)
  (prepare (fn [{html-commands :html-commands :as tentacle}]
             (if html-commands
               (assoc tentacle :html-commands (str/join "\n" html-commands))
               tentacle)))
  (prepare ->snake-keys)
  (transform ->kebab-keys)
  (transform (fn [{html-commands :html-commands :as tentacle}]
               (if html-commands
                 (assoc tentacle :html-commands (split-lines html-commands))
                 tentacle))))


(defentity heads
  (pk :name)
  (prepare add-updated)
  (prepare ->snake-keys)
  (transform ->kebab-keys)
  (has-many head-tentacle-mappings {:fk :head_name}))

;; ### Head <-> Tentacle Join Table
;; Heads and Tentacles can share the notorious SQL many-to-many
;; relationship. Further, having either a head or a tentacle record _on the
;; database_ storing information about the other gets conceptually weird. Korma
;; will automatically handle this by using two queries, but I'm unconvinced by
;; that as a data modeling approach. Enter: the join table.
;;
;; Join tables capture many-to-many relationships by inserting themselves in the
;; middle, effectively creating a one-to-many relationship between itself and
;; both of the other tables you're attempting to look up. Getting from Heads to
;; Tentacles (or vice-versa) is now a set of left-joins. Easy peasy!
(defentity head-tentacle-mappings
  (table :head_tentacle_mappings)
  (prepare ->snake-keys)
  (transform ->kebab-keys)
  (belongs-to heads)
  (belongs-to tentacles))

(defentity documents
  (table :documents)
  (prepare ->snake-keys)
  (prepare add-updated)
  (transform ->kebab-keys)
  (belongs-to tentacles)
  (entity-fields :name :uri :tentacle_name :body))<commit_msg>Rejigger the prepare statement for use with document index updatinc<commit_after>(ns doctopus.db
  (:require [camel-snake-kebab.core :refer [->snake_case_keyword ->kebab-case-keyword]]
            [camel-snake-kebab.extras :refer [transform-keys]]
            [clj-time.coerce :refer [to-sql-time]]
            [clj-time.core :as clj-time]
            [clojure.string :as str :refer [split-lines]]
            [doctopus.configuration :refer [server-config]]
            [korma.core :refer :all]
            [korma.db :refer [defdb postgres default-connection]]
            [taoensso.timbre :as log]))

(defn- now
  []
  "returns a string for 'now' in SQL Timestamp format"
  (to-sql-time (clj-time/now)))

(defn- add-updated
  "adds updated field to a map with given time"
  ([fields] (add-updated fields (now)))
  ([fields now-string] (assoc fields :updated now-string)))

(defn- ->kebab-keys
  "converts all keys in a map to kebab-case keywords"
  [fields]
  (transform-keys ->kebab-case-keyword fields))

(defn- ->snake-keys
  "converts all keys in a map to snake-case keywords"
  [fields]
  (transform-keys ->snake_case_keyword fields))

(defn- ->shallow-snake-keys
  "Converts only the outermost keys of a map to snake-case keywords"
  [m]
  (doall (into {} (map (fn [[k v]][(->snake_case_keyword k) v]) m))))

(defdb main-db (postgres
             (select-keys (get-in (server-config) [:database :main])
                          [:db :user :password :host :port])))
(defdb test-db (postgres
                (select-keys (get-in (server-config) [:database :test])
                             [:db :user :password :host :port])))

(default-connection (if (= "dev" (:nomad/environment (server-config)))
                      test-db
                      main-db))

(declare head-tentacle-mappings documents)
(defentity tentacles
  (pk :name)
  (has-many head-tentacle-mappings {:fk :tentacle_name})
  (has-one documents {:fk :tentacle_name})
  (prepare add-updated)
  (prepare (fn [{html-commands :html-commands :as tentacle}]
             (if html-commands
               (assoc tentacle :html-commands (str/join "\n" html-commands))
               tentacle)))
  (prepare ->snake-keys)
  (transform ->kebab-keys)
  (transform (fn [{html-commands :html-commands :as tentacle}]
               (if html-commands
                 (assoc tentacle :html-commands (split-lines html-commands))
                 tentacle))))


(defentity heads
  (pk :name)
  (prepare add-updated)
  (prepare ->snake-keys)
  (transform ->kebab-keys)
  (has-many head-tentacle-mappings {:fk :head_name}))

;; ### Head <-> Tentacle Join Table
;; Heads and Tentacles can share the notorious SQL many-to-many
;; relationship. Further, having either a head or a tentacle record _on the
;; database_ storing information about the other gets conceptually weird. Korma
;; will automatically handle this by using two queries, but I'm unconvinced by
;; that as a data modeling approach. Enter: the join table.
;;
;; Join tables capture many-to-many relationships by inserting themselves in the
;; middle, effectively creating a one-to-many relationship between itself and
;; both of the other tables you're attempting to look up. Getting from Heads to
;; Tentacles (or vice-versa) is now a set of left-joins. Easy peasy!
(defentity head-tentacle-mappings
  (table :head_tentacle_mappings)
  (prepare ->snake-keys)
  (transform ->kebab-keys)
  (belongs-to heads)
  (belongs-to tentacles))

(defentity documents
  (table :documents)
  ;; Because of the `raw' call used in update-document-index', this table
  ;; specifically needs to *not* have any but its top level keys munged by
  ;; ->snake-keys
  (prepare ->shallow-snake-keys)
  (transform ->kebab-keys)
  (belongs-to tentacles)
  (entity-fields :name :uri :tentacle_name :body))
{'fraction_non_alphanumeric': 0.12900175604484668, 'fraction_numerical': 0.0, 'mean_word_length': 3.0, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 38, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1314206', 'n_tokens_mistral': 2414, 'n_tokens_neox': 2358, 'n_words': 728}