<commit_before>(ns grafter.tabular.common
  {:no-doc true}
  (:require [clj-excel.core :as xls]
            [grafter.sequences :as seqs]
            [clojure.java.io :as io]
            [incanter.core :as inc]
            [me.raynes.fs :as fs])
  (:import [org.apache.poi.ss.usermodel Sheet]
           [incanter.core Dataset]
           [java.io File InputStream]
           [java.net URI URL]))


(defn mapply
  "Like apply, but f takes keyword arguments and the last argument is
  not a seq but a map with the arguments for f"
  [f & args]
  {:pre [(let [kwargs (last args)] (or (map? kwargs) (nil? kwargs)))]}
  (apply f (apply concat
                  (butlast args) (last args))))

(defn move-first-row-to-header
  "For use with make-dataset.  Moves the first row of data into the
  header, removing it from the source data."
  [[first-row & other-rows]]

  [first-row other-rows])

(defn- fill-gaps-with-nil [rows headers]
  (let [blank-row (zipmap headers (repeat (count headers) nil))
        pad-with-nils (fn [row] (let [row-map (if (map? row) row (zipmap headers row))]
                                  (merge blank-row row-map)))]
    (map pad-with-nils rows)))

(defn make-dataset
  "Like incanter's dataset function except it can take a lazy-sequence
  of column names which will get mapped to the source data.

  Works by inspecting the amount of columns in the first row, and
  taking that many column names from the sequence.

  Inspects the first row of data to determine the number of columns,
  and creates an incanter dataset with columns named alphabetically as
  by grafter.sequences/column-names-seq."

  ([]
   (inc/dataset []))

  ([data]
   (let [columns (take (-> data first count) (seqs/alphabetical-column-names))]
     (make-dataset data columns)))

  ([data columns-or-f]
   (let [data-seq (if (inc/dataset? data) (inc/to-list data) data)
         [column-headers rows] (if (fn? columns-or-f)
                                 (columns-or-f data-seq)
                                 [columns-or-f data-seq])
         full-data (fill-gaps-with-nil rows column-headers)]
     (-> (inc/dataset column-headers full-data)
         (with-meta (meta full-data))))))

(defn dataset?
  "Predicate function to test whether the supplied argument is a
  dataset or not."
  [ds]
  (or (instance? incanter.core.Dataset ds)
      (and (map? ds)
           (:rows ds)
           (:column-names ds))))

(def column-names
  "If given a dataset, it returns its column names. If given a dataset and a sequence
  of column names, it returns a dataset with the given column names."
  inc/col-names)

(defn pass-rows
  "Passes the function f the collection of raw rows from the dataset
   and returns a new dataset containing (f rows) as its data.

  f should expect a collection of row maps and return a collection of
  rows.

  This function is intended to be used by Grafter itself and Grafter
  library authors.  It's not recommended to by users of the DSL
  because users of this function need to be aware of Dataset
  implementation details."
  [dataset f]
  (-> (make-dataset (->> dataset :rows f)
                    (column-names dataset))
      (with-meta (meta dataset))))

(defn ^:no-doc extension
  "Gets the extension for the given file name as a keyword, or nil if the file has no extension"
  [f]
  (when-let [^String ext (-> f fs/extension)]
    (-> ext
        (.substring 1)
        keyword)))

(defn format-or-type [ds {:keys [format]}]
  (if (#{File String} (class ds))
    (or format (extension ds))
    (class ds)))

(defn- get-format [source {:keys [format] :as opts}]
  format)

(defn assoc-data-source-meta [output-ds data-source]
  "Adds metadata about where the dataset was loaded from to the object."
  (cond
    (#{String File URI URL} (class data-source)) (with-meta output-ds {:grafter.tabular/data-source data-source})
    :else (with-meta output-ds {:grafter.tabular/data-source :datasource-unknown})))

(defmulti ^:no-doc read-dataset*
  "Multimethod for adapter implementers to hook custom dataset readers
  into grafter.

  API users should use the front end function read-dataset instead of
  calling this."

  get-format)

(defprotocol DatasetFormat
  "Represents a type from which it may be possible to infer the format
  of the contained data."
  (infer-format [source]
    "Attempt to infer the data format of the given source. Should
    return a keyword if the format was inferred, or nil if the
    inference failed."))

(extend-protocol DatasetFormat
  String
  (infer-format [s] (extension s))

  File
  (infer-format [f] (extension (.getName f)))

  java.net.URL
  (infer-format [url] (extension (.getPath url)))

  java.net.URI
  (infer-format [uri] (extension (.getPath uri)))

  nil
  (infer-format [_] nil))

(defn- ^:no-doc infer-format-of
  "Attempt to infer the format of the given data source. Returns nil
  if the format could not be inferred."
  [source]
  (if (satisfies? DatasetFormat source)
    (infer-format source)))

(defmulti read-dataset-source
  "Opens a dataset from a datasetable thing e.g. a filename or an existing Dataset.
  The multi-method dispatches based upon the type of the source.

  Supplied options are passed to the individual handler methods and they may
  have their own requirements on the options provided."
  ;; NOTE: This is not a protocol, because protocols don't give you a :default
  ;; option for dispatch.
  (fn [src opts]
    (class src)))

(defn- ^:no-doc dispatch-with-format-option
  "Takes a function to call, a data source and an options hash containing an
  optional :format key.

  If :format is not provided then an attempt will be made to infer it from the
  data source via the DatasetFormat protocol.

  Once the format is resolved it then the target function is called with the
  data source and options map."
  [f source {:keys [format] :as opts}]
  (if-let [format (or format (infer-format-of source))]
    (f source (assoc opts :format format))
    (throw (IllegalArgumentException. (str "Please specify a format, it could not be from the source: " source)))))

(defmethod read-dataset-source Dataset [ds opts] ds)

(defmethod read-dataset-source :default [source opts]
  (dispatch-with-format-option read-dataset* source opts))

(defn read-dataset [source & {:as opts}]
  (-> (read-dataset-source source opts)
      (assoc-data-source-meta source)))
<commit_msg>Preserve metadata on datasets when given a dataset in make-dataset<commit_after>(ns grafter.tabular.common
  {:no-doc true}
  (:require [clj-excel.core :as xls]
            [grafter.sequences :as seqs]
            [clojure.java.io :as io]
            [incanter.core :as inc]
            [me.raynes.fs :as fs])
  (:import [org.apache.poi.ss.usermodel Sheet]
           [incanter.core Dataset]
           [java.io File InputStream]
           [java.net URI URL]))


(defn mapply
  "Like apply, but f takes keyword arguments and the last argument is
  not a seq but a map with the arguments for f"
  [f & args]
  {:pre [(let [kwargs (last args)] (or (map? kwargs) (nil? kwargs)))]}
  (apply f (apply concat
                  (butlast args) (last args))))

(defn move-first-row-to-header
  "For use with make-dataset.  Moves the first row of data into the
  header, removing it from the source data."
  [[first-row & other-rows]]

  [first-row other-rows])

(defn- fill-gaps-with-nil [rows headers]
  (let [blank-row (zipmap headers (repeat (count headers) nil))
        pad-with-nils (fn [row] (let [row-map (if (map? row) row (zipmap headers row))]
                                  (merge blank-row row-map)))]
    (map pad-with-nils rows)))

(defn make-dataset
  "Like incanter's dataset function except it can take a lazy-sequence
  of column names which will get mapped to the source data.

  Works by inspecting the amount of columns in the first row, and
  taking that many column names from the sequence.

  Inspects the first row of data to determine the number of columns,
  and creates an incanter dataset with columns named alphabetically as
  by grafter.sequences/column-names-seq."

  ([]
   (inc/dataset []))

  ([data]
   (let [columns (take (-> data first count) (seqs/alphabetical-column-names))]
     (make-dataset data columns)))

  ([data columns-or-f]
   (let [data-seq (if (inc/dataset? data) (inc/to-list data) data)
         [column-headers rows] (if (fn? columns-or-f)
                                 (columns-or-f data-seq)
                                 [columns-or-f data-seq])
         full-data (fill-gaps-with-nil rows column-headers)]
     (-> (inc/dataset column-headers full-data)
         (with-meta (meta data))))))

(defn dataset?
  "Predicate function to test whether the supplied argument is a
  dataset or not."
  [ds]
  (or (instance? incanter.core.Dataset ds)
      (and (map? ds)
           (:rows ds)
           (:column-names ds))))

(def column-names
  "If given a dataset, it returns its column names. If given a dataset and a sequence
  of column names, it returns a dataset with the given column names."
  inc/col-names)

(defn pass-rows
  "Passes the function f the collection of raw rows from the dataset
   and returns a new dataset containing (f rows) as its data.

  f should expect a collection of row maps and return a collection of
  rows.

  This function is intended to be used by Grafter itself and Grafter
  library authors.  It's not recommended to by users of the DSL
  because users of this function need to be aware of Dataset
  implementation details."
  [dataset f]
  (-> (make-dataset (->> dataset :rows f)
                    (column-names dataset))
      (with-meta (meta dataset))))

(defn ^:no-doc extension
  "Gets the extension for the given file name as a keyword, or nil if the file has no extension"
  [f]
  (when-let [^String ext (-> f fs/extension)]
    (-> ext
        (.substring 1)
        keyword)))

(defn format-or-type [ds {:keys [format]}]
  (if (#{File String} (class ds))
    (or format (extension ds))
    (class ds)))

(defn- get-format [source {:keys [format] :as opts}]
  format)

(defn assoc-data-source-meta [output-ds data-source]
  "Adds metadata about where the dataset was loaded from to the object."
  (cond
    (#{String File URI URL} (class data-source)) (with-meta output-ds {:grafter.tabular/data-source data-source})
    (instance? incanter.core.Dataset data-source) (with-meta output-ds (meta data-source))
    :else (with-meta output-ds {:grafter.tabular/data-source :datasource-unknown})))

(defmulti ^:no-doc read-dataset*
  "Multimethod for adapter implementers to hook custom dataset readers
  into grafter.

  API users should use the front end function read-dataset instead of
  calling this."

  get-format)

(defprotocol DatasetFormat
  "Represents a type from which it may be possible to infer the format
  of the contained data."
  (infer-format [source]
    "Attempt to infer the data format of the given source. Should
    return a keyword if the format was inferred, or nil if the
    inference failed."))

(extend-protocol DatasetFormat
  String
  (infer-format [s] (extension s))

  File
  (infer-format [f] (extension (.getName f)))

  java.net.URL
  (infer-format [url] (extension (.getPath url)))

  java.net.URI
  (infer-format [uri] (extension (.getPath uri)))

  nil
  (infer-format [_] nil))

(defn- ^:no-doc infer-format-of
  "Attempt to infer the format of the given data source. Returns nil
  if the format could not be inferred."
  [source]
  (if (satisfies? DatasetFormat source)
    (infer-format source)))

(defmulti read-dataset-source
  "Opens a dataset from a datasetable thing e.g. a filename or an existing Dataset.
  The multi-method dispatches based upon the type of the source.

  Supplied options are passed to the individual handler methods and they may
  have their own requirements on the options provided."
  ;; NOTE: This is not a protocol, because protocols don't give you a :default
  ;; option for dispatch.
  (fn [src opts]
    (class src)))

(defn- ^:no-doc dispatch-with-format-option
  "Takes a function to call, a data source and an options hash containing an
  optional :format key.

  If :format is not provided then an attempt will be made to infer it from the
  data source via the DatasetFormat protocol.

  Once the format is resolved it then the target function is called with the
  data source and options map."
  [f source {:keys [format] :as opts}]
  (if-let [format (or format (infer-format-of source))]
    (f source (assoc opts :format format))
    (throw (IllegalArgumentException. (str "Please specify a format, it could not be inferred from the source: " source)))))

(defmethod read-dataset-source Dataset [ds opts] ds)

(defmethod read-dataset-source :default [source opts]
  (dispatch-with-format-option read-dataset* source opts))

(defn read-dataset [source & {:as opts}]
  (-> (read-dataset-source source opts)
      (assoc-data-source-meta source)))

{'fraction_non_alphanumeric': 0.105218333849489, 'fraction_numerical': 0.0001548467017652524, 'mean_word_length': 3.0164800995024876, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5093318', 'n_tokens_mistral': 3973, 'n_tokens_neox': 3861, 'n_words': 1593}