<commit_before>(ns kixi.datastore.structural-validation
  (:require [byte-streams :as bs]
            [clojure-csv.core :as csv :refer [parse-csv]]
            [clojure.spec :as s]
            [com.stuartsierra.component :as component]
            [kixi.comms :as c
             :refer [attach-event-handler!]]
            [kixi.datastore.filestore
             :refer [retrieve]]
            [kixi.datastore.file
             :refer [temp-file]]
            [kixi.datastore.schemastore
             :as ss]
            [kixi.datastore.schemastore.validator
             :as sv]
            [kixi.datastore.metadatastore :as ms]
            [taoensso.timbre :as timbre :refer [error info infof]]
            [clojure.java.io :as io])
  (:import [java.io File]))

(defn metadata->file
  [filestore metadata]
  (let [id (::ms/id metadata)
        f (temp-file id)]
    (bs/transfer
     (retrieve filestore id)
     f)
    f))

(defn csv-schema-test
  [schemastore schema-id file]
  (let [line-count (atom 0)]
    (try
      (with-open [contents (io/reader file)]
        (let [parser (parse-csv contents :strict true)
              explains (keep (fn [line]
                               (swap! line-count inc)
                               (sv/explain-data schemastore schema-id line))
                             (rest parser))]
          (if (seq explains)
            {::ms/valid false
             ::ms/explain (doall (take 100 explains))} ;This should be some sort of %age of file size
            {::ms/valid true})))
      (catch Exception e
        {::ms/valid false
         :line (inc @line-count)
         :e e}))))

(defn structural-validator
  [filestore schemastore]
  (fn [metadata]
    (let [^File file (metadata->file filestore metadata)]
      (try
        {:kixi.comms.event/key :kixi.datastore/file-metadata-updated
         :kixi.comms.event/version "1.0.0"
         :kixi.comms.event/payload {::ms/file-metadata-update-type ::ms/file-metadata-structural-validation-checked
                                    ::ms/id (::ms/id metadata)
                                    ::ms/structural-validation
                                    (case (::ms/type metadata)
                                      "csv" (csv-schema-test schemastore (::ss/id metadata) file))}}
        (finally
          (.delete file))))))

(defprotocol IStructuralValidator)

(defrecord StructuralValidator
    [communications filestore schemastore structural-validator-fn]
    IStructuralValidator
    component/Lifecycle
    (start [component]
      (if-not structural-validator-fn<commit_msg>Add better logging for when a file is unavailable<commit_after>(ns kixi.datastore.structural-validation
  (:require [byte-streams :as bs]
            [clojure-csv.core :as csv :refer [parse-csv]]
            [clojure.spec :as s]
            [com.stuartsierra.component :as component]
            [kixi.comms :as c
             :refer [attach-event-handler!]]
            [kixi.datastore.filestore
             :refer [retrieve]]
            [kixi.datastore.file
             :refer [temp-file]]
            [kixi.datastore.schemastore
             :as ss]
            [kixi.datastore.schemastore.validator
             :as sv]
            [kixi.datastore.metadatastore :as ms]
            [taoensso.timbre :as timbre :refer [error info infof]]
            [clojure.java.io :as io])
  (:import [java.io File]))

(defn metadata->file
  [filestore metadata]
  (let [id (::ms/id metadata)
        ^File f (temp-file id)]
    (if (.exists f)
      (do (bs/transfer
           (retrieve filestore id)
           f)
          f)
      (error "File does exist: " id))))

(defn csv-schema-test
  [schemastore schema-id file]
  (let [line-count (atom 0)]
    (try
      (with-open [contents (io/reader file)]
        (let [parser (parse-csv contents :strict true)
              explains (keep (fn [line]
                               (swap! line-count inc)
                               (sv/explain-data schemastore schema-id line))
                             (rest parser))]
          (if (seq explains)
            {::ms/valid false
             ::ms/explain (doall (take 100 explains))} ;This should be some sort of %age of file size
            {::ms/valid true})))
      (catch Exception e
        {::ms/valid false
         :line (inc @line-count)
         :e e}))))

(defn structural-validator
  [filestore schemastore]
  (fn [metadata]
    (when-let [^File file (metadata->file filestore metadata)]
      (try
        {:kixi.comms.event/key :kixi.datastore/file-metadata-updated
         :kixi.comms.event/version "1.0.0"
         :kixi.comms.event/payload {::ms/file-metadata-update-type ::ms/file-metadata-structural-validation-checked
                                    ::ms/id (::ms/id metadata)
                                    ::ms/structural-validation
                                    (case (::ms/type metadata)
                                      "csv" (csv-schema-test schemastore (::ss/id metadata) file))}}
        (finally
          (.delete file))))))

(defprotocol IStructuralValidator)

(defrecord StructuralValidator
    [communications filestore schemastore structural-validator-fn]
    IStructuralValidator
    component/Lifecycle
    (start [component]
      (if-not structural-validator-fn
{'fraction_non_alphanumeric': 0.11859939759036145, 'fraction_numerical': 0.0026355421686746986, 'mean_word_length': 1.7528497409326425, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4323039', 'n_tokens_mistral': 1655, 'n_tokens_neox': 1614, 'n_words': 381}