<commit_before>(ns pdok.featured.extracts
   (:require [pdok.featured.mustache  :as m]
             [pdok.postgres :as pg]
             [pdok.featured.tiles :as tiles]
             [pdok.featured.core :as core]
             [pdok.featured.json-reader :as json-reader]
             [clojure.edn :as edn]
             [clojure.java.jdbc :as j]))

(defn features-for-extract [dataset feature-type features template-dir]
  "Returns the rendered representation of the collection of features for a the given feature-type inclusive tiles-set"
  (let [template-dir (if (empty? template-dir) "" (str template-dir "/"))
        template (str template-dir dataset "-" feature-type ".template")
        partials (edn/read-string (slurp (str template-dir dataset ".partials")))]
    (map #(vector (tiles/nl (:geometry %)) (m/render-resource template partials %)) features)))

(defn create-extract-collection [db dataset feature-type]
  (let [table feature-type]
    (do (when (not (pg/schema-exists? db dataset))
          (pg/create-schema db dataset))
        (when (not (pg/table-exists? db dataset table))
          (pg/create-table db dataset feature-type
                     [:id "bigserial" :primary :key]
                     [:feature_type "text"]
                     [:valid_from "timestamp without time zone"]
                     [:valid_to "timestamp without time zone"]
                     [:tiles "text"]
                     [:xml "text"]
                     [:created_on "timestamp without time zone"]
                     )
          ))))


(defn- jdbc-insert
  ([db feature-type valid-from valid-to tiles xml created_on]
   (jdbc-insert db feature-type (list [feature-type valid-from valid-to tiles xml created_on])))
  ([db feature-type entries]
   (try (j/with-db-connection [c db]
          (apply
           (partial j/insert! c (str "bgtextract." feature-type) :transaction? false
                    [:feature_type :valid_from :valid_to :tiles :xml :created_on])
           entries)
            )
        (catch java.sql.SQLException e (j/print-sql-exception-chain e)))))


(defn add-extract-records [dataset feature-type rendered-features]
  "Inserts the xml-features and tile-set in an extract schema based on dataset and feature-type,
   if schema or table doesn't exists it will be created."
  (do
   (create-extract-collection core/data-db (str dataset "extract") feature-type)
   (doseq [[tiles xml-feature] rendered-features]
     (jdbc-insert core/data-db feature-type nil nil (vec tiles) xml-feature nil))))


(defn file-to-features [path dataset]
  "Helper function to read features from a file.<commit_msg>bugfix: config moved to config, also for extracts<commit_after>(ns pdok.featured.extracts
   (:require [pdok.featured.mustache  :as m]
             [pdok.postgres :as pg]
             [pdok.featured.tiles :as tiles]
             [pdok.featured.config :as config]
             [pdok.featured.json-reader :as json-reader]
             [clojure.edn :as edn]
             [clojure.java.jdbc :as j]))

(defn features-for-extract [dataset feature-type features template-dir]
  "Returns the rendered representation of the collection of features for a the given feature-type inclusive tiles-set"
  (let [template-dir (if (empty? template-dir) "" (str template-dir "/"))
        template (str template-dir dataset "-" feature-type ".template")
        partials (edn/read-string (slurp (str template-dir dataset ".partials")))]
    (map #(vector (tiles/nl (:geometry %)) (m/render-resource template partials %)) features)))

(defn create-extract-collection [db dataset feature-type]
  (let [table feature-type]
    (do (when (not (pg/schema-exists? db dataset))
          (pg/create-schema db dataset))
        (when (not (pg/table-exists? db dataset table))
          (pg/create-table db dataset feature-type
                     [:id "bigserial" :primary :key]
                     [:feature_type "text"]
                     [:valid_from "timestamp without time zone"]
                     [:valid_to "timestamp without time zone"]
                     [:tiles "text"]
                     [:xml "text"]
                     [:created_on "timestamp without time zone"]
                     )
          ))))


(defn- jdbc-insert
  ([db feature-type valid-from valid-to tiles xml created_on]
   (jdbc-insert db feature-type (list [feature-type valid-from valid-to tiles xml created_on])))
  ([db feature-type entries]
   (try (j/with-db-connection [c db]
          (apply
           (partial j/insert! c (str "bgtextract." feature-type) :transaction? false
                    [:feature_type :valid_from :valid_to :tiles :xml :created_on])
           entries)
            )
        (catch java.sql.SQLException e (j/print-sql-exception-chain e)))))


(defn add-extract-records [dataset feature-type rendered-features]
  "Inserts the xml-features and tile-set in an extract schema based on dataset and feature-type,
   if schema or table doesn't exists it will be created."
  (do
   (create-extract-collection config/data-db (str dataset "extract") feature-type)
   (doseq [[tiles xml-feature] rendered-features]
     (jdbc-insert config/data-db feature-type nil nil (vec tiles) xml-feature nil))))


(defn file-to-features [path dataset]
  "Helper function to read features from a file.
{'fraction_non_alphanumeric': 0.11864406779661017, 'fraction_numerical': 0.0, 'mean_word_length': 2.453185955786736, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 14}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2879519', 'n_tokens_mistral': 1561, 'n_tokens_neox': 1501, 'n_words': 474}