<commit_before>(ns metabase.driver.generic-sql.util
  "Shared functions for our generic-sql query processor."
  (:require [clojure.core.memoize :as memo]
            [clojure.java.jdbc :as jdbc]
            [clojure.tools.logging :as log]
            [colorize.core :as color]
            [korma.core :as korma]
            [korma.db :as kdb]
            [metabase.driver :as driver]
            [metabase.driver.query-processor :as qp]))

;; Cache the Korma DB connections for a given Database for 60 seconds instead of creating new ones every single time
(defn- db->connection-spec [database]
  (let [driver                              (driver/engine->driver (:engine database))
        database->connection-details        (:database->connection-details driver)
        connection-details->connection-spec (:connection-details->connection-spec driver)]
    (-> database database->connection-details connection-details->connection-spec)))

(def ^{:arglists '([database])} db->korma-db
  "Return a Korma database definition for DATABASE.
   This does a little bit of smart caching (for 60 seconds) to avoid creating new connections when unneeded."
  (let [-db->korma-db (memo/ttl (fn [database]
                                  (log/debug (color/red "Creating a new DB connection..."))
                                  (assoc (kdb/create-db (db->connection-spec database))
                                         :make-pool? true))
                                :ttl/threshold (* 60 1000))]
    ;; only :engine and :details are needed for driver/connection so just pass those so memoization works as expected
    (fn [database]
      (-db->korma-db (select-keys database [:engine :details])))))

(def ^:dynamic ^java.sql.DatabaseMetaData *jdbc-metadata*
  "JDBC metadata object for a database. This is set by `with-jdbc-metadata`."
  nil)

(defn -with-jdbc-metadata
  "Internal implementation. Don't use this directly; use `with-jdbc-metadata`."
  [database f]
  (if *jdbc-metadata* (f *jdbc-metadata*)
                      (jdbc/with-db-metadata [md (db->connection-spec database)]
                        (binding [*jdbc-metadata* md]
                          (f *jdbc-metadata*)))))

(defmacro with-jdbc-metadata
  "Execute BODY with the jdbc metadata for DATABASE bound to BINDING.
   This will reuse `*jdbc-metadata*` if it's already set (to avoid opening extra connections).
   Otherwise it will open a new metadata connection and bind `*jdbc-metadata*` so it can be reused by subsequent calls to `with-jdbc-metadata` within BODY.

    (with-jdbc-metadata [^java.sql.DatabaseMetaData md (sel :one Database :id 1)] ; (1)
      (-> (.getPrimaryKeys md nil nil nil)
          jdbc/result-set-seq                                                     ; (2)
          doall))                                                                 ; (3)

   NOTES

   1.  You should tag BINDING to avoid reflection.
   2.  Use `jdbc/result-set-seq` to convert JDBC `ResultSet` into something we can use in Clojure
   3.  Make sure to realize the lazy sequence within the BODY before connection is closed."
  [[binding database] & body]
  {:pre [(symbol? binding)]}
  `(-with-jdbc-metadata ~database
     (fn [~binding]
       ~@body)))

(defn korma-entity
  "Return a Korma entity for [DB and] TABLE .

    (-> (sel :one Table :id 100)
        korma-entity
        (select (aggregate (count :*) :count)))"
  ([{db-delay :db, :as table}]
   {:pre [(delay? db-delay)]}
   (korma-entity @db-delay table))
  ([db {table-name :name}]
   {:pre [(map? db)]}
   {:table table-name
    :pk    :id
    :db    (db->korma-db db)}))
<commit_msg>Need the keep a connection pool open in order to unCLOB-er CLOBs<commit_after>(ns metabase.driver.generic-sql.util
  "Shared functions for our generic-sql query processor."
  (:require [clojure.core.memoize :as memo]
            [clojure.java.jdbc :as jdbc]
            [clojure.tools.logging :as log]
            [colorize.core :as color]
            [korma.core :as korma]
            [korma.db :as kdb]
            [metabase.driver :as driver]
            [metabase.driver.query-processor :as qp]))

;; Cache the Korma DB connections for a given Database for 60 seconds instead of creating new ones every single time
(defn- db->connection-spec [database]
  (let [driver                              (driver/engine->driver (:engine database))
        database->connection-details        (:database->connection-details driver)
        connection-details->connection-spec (:connection-details->connection-spec driver)]
    (-> database
        database->connection-details
        connection-details->connection-spec
        (assoc :make-pool? true))))     ; need to make a pool or the connection will be closed before we get a change to unCLOB-er the results during JSON serialization

(def ^{:arglists '([database])}
  db->korma-db
  "Return a Korma database definition for DATABASE.
   This does a little bit of smart caching (for 60 seconds) to avoid creating new connections when unneeded."
  (let [-db->korma-db (memo/ttl (fn [database]
                                  (log/debug (color/red "Creating a new DB connection..."))
                                  (kdb/create-db (db->connection-spec database)))
                                :ttl/threshold (* 60 1000))]
    ;; only :engine and :details are needed for driver/connection so just pass those so memoization works as expected
    (fn [database]
      (let [r (-db->korma-db (select-keys database [:engine :details]))]
        (println "R:\n" (metabase.util/pprint-to-str 'green r))
        r))))

(def ^:dynamic ^java.sql.DatabaseMetaData *jdbc-metadata*
  "JDBC metadata object for a database. This is set by `with-jdbc-metadata`."
  nil)

(defn -with-jdbc-metadata
  "Internal implementation. Don't use this directly; use `with-jdbc-metadata`."
  [database f]
  (if *jdbc-metadata* (f *jdbc-metadata*)
                      (jdbc/with-db-metadata [md (db->connection-spec database)]
                        (binding [*jdbc-metadata* md]
                          (f *jdbc-metadata*)))))

(defmacro with-jdbc-metadata
  "Execute BODY with the jdbc metadata for DATABASE bound to BINDING.
   This will reuse `*jdbc-metadata*` if it's already set (to avoid opening extra connections).
   Otherwise it will open a new metadata connection and bind `*jdbc-metadata*` so it can be reused by subsequent calls to `with-jdbc-metadata` within BODY.

    (with-jdbc-metadata [^java.sql.DatabaseMetaData md (sel :one Database :id 1)] ; (1)
      (-> (.getPrimaryKeys md nil nil nil)
          jdbc/result-set-seq                                                     ; (2)
          doall))                                                                 ; (3)

   NOTES

   1.  You should tag BINDING to avoid reflection.
   2.  Use `jdbc/result-set-seq` to convert JDBC `ResultSet` into something we can use in Clojure
   3.  Make sure to realize the lazy sequence within the BODY before connection is closed."
  [[binding database] & body]
  {:pre [(symbol? binding)]}
  `(-with-jdbc-metadata ~database
     (fn [~binding]
       ~@body)))

(defn korma-entity
  "Return a Korma entity for [DB and] TABLE .

    (-> (sel :one Table :id 100)
        korma-entity
        (select (aggregate (count :*) :count)))"
  {:arglists '([table] [db table])}
  ([{db-delay :db, :as table}]
   {:pre [(delay? db-delay)]}
   (korma-entity @db-delay table))
  ([db {table-name :name}]
   {:pre [(map? db)]}
   {:table table-name
    :pk    :id
    :db    (db->korma-db db)}))

{'fraction_non_alphanumeric': 0.11971924248443915, 'fraction_numerical': 0.005297311614355715, 'mean_word_length': 2.2692640692640693, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '238618', 'n_tokens_mistral': 2293, 'n_tokens_neox': 2154, 'n_words': 736}