<commit_before>(ns kixi.datastore.transport-specs
  (:require [clojure.spec :as s]
            [kixi.datastore.schemastore :as ss]
            [kixi.datastore.metadatastore :as ms]
            [kixi.datastore.schemastore.conformers :refer [uuid]]))

(s/def ::schema-id uuid)

(s/def ::filemetadata-transport
  (s/keys :req-un [::schema-id
                   ::ms/name]
          :opt-un [::ms/type
                   ::ms/header]))

(s/def ::file-details
  (s/keys :req [::ms/id ::ms/size-bytes ::ms/provenance]))

(def file-type->default-metadata
  {"csv" {::ms/headers true}})

(def default-primary-metadata
  {::ms/type "csv"})

(s/fdef filemetadata-transport->internal
        :args (s/cat :meta ::filemetadata-transport
                     :details ::file-details)
        :ret ::ms/file-metadata)

(def key-mapping
  {:name ::ms/name
   :header ::ms/header
   :type ::ms/type
   :schema-id ::ss/id})

(defn filemetadata-transport->internal
  [transport file-details]
  (let [mapped (zipmap (map key-mapping (keys transport))
                       (vals transport))
        with-primaries (merge default-primary-metadata
                              mapped)
        with-file-type (merge (get file-type->default-metadata
                                   (::ms/type with-primaries))
                              with-primaries)]
    (merge with-file-type<commit_msg>Added fn properties to fdef<commit_after>(ns kixi.datastore.transport-specs
  (:require [clojure.spec :as s]
            [kixi.datastore.schemastore :as ss]
            [kixi.datastore.metadatastore :as ms]
            [kixi.datastore.schemastore.conformers :refer [uuid]]))

(s/def ::schema-id uuid)

(s/def ::filemetadata-transport
  (s/keys :req-un [::schema-id
                   ::ms/name]
          :opt-un [::ms/type
                   ::ms/header]))

(s/def ::file-details
  (s/keys :req [::ms/id ::ms/size-bytes ::ms/provenance]))

(def file-type->default-metadata
  {"csv" {::ms/header true}})

(def default-primary-metadata
  {::ms/type "csv"})

(s/fdef filemetadata-transport->internal
        :args (s/cat :meta ::filemetadata-transport
                     :details ::file-details)
        :fn #(let [{:keys [meta details]} (get % :args)
                   file-metadata (get % :ret)]
               (and (= (::ms/id details)
                       (::ms/id file-metadata))
                    (= (::ms/size-bytes details)
                       (::ms/size-bytes file-metadata))
                    (= (::ms/provenance details)
                       (::ms/provenance file-metadata))
                    (= (:schema-id meta)
                       (::ss/id file-metadata))
                    (= (:name meta)
                       (::ms/name file-metadata))
                    (case (or (:type meta) 
                              (::ms/type file-metadata)) 
                      "csv" (if-not (nil? (:header meta))
                              (= (:header meta) 
                                 (::ms/header file-metadata))
                              (true? (::ms/header file-metadata)))
                      true)))
        :ret ::ms/file-metadata)

(def key-mapping
  {:name ::ms/name
   :header ::ms/header
   :type ::ms/type
   :schema-id ::ss/id})

(defn filemetadata-transport->internal
  [transport file-details]
  (let [mapped (zipmap (map key-mapping (keys transport))
                       (vals transport))
        with-primaries (merge default-primary-metadata
                              mapped)
        with-file-type (merge (get file-type->default-metadata
                                   (::ms/type with-primaries))
                              with-primaries)]
    (merge with-file-type
{'fraction_non_alphanumeric': 0.1471698113207547, 'fraction_numerical': 0.0, 'mean_word_length': 1.6356534090909092, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1772706', 'n_tokens_mistral': 1148, 'n_tokens_neox': 1113, 'n_words': 231}