<commit_before>(ns ^{:doc
      "A *client* entry point library to help with evaluating machine learning
models.  This library not only wraps the Weka library but also provides
additional functionality like a two pass cross
validation (see [[*two-pass-config*]])."
      :author "<NAME>"}
    zensols.model.eval-classifier
  (:use [clojure.java.io :as io :only [input-stream output-stream file]])
  (:use [clojure.pprint :only [pprint]])
  (:require [clojure.tools.logging :as log]
            [clojure.string :as str]
            [clojure.stacktrace :refer (print-stack-trace)]
            [clojure.data.csv :as csv])
  (:require [zensols.tabres.display-results :as dr])
  (:require [zensols.model.execute-classifier :refer (model-config) :as exc]
            [zensols.model.weka :as weka]
            [zensols.model.classifier :as cl]))

(def ^:dynamic *cross-fold-count*
  "The default number of folds to use during cross fold
  validation (see [[cmpile-results]])."
  10)

(def ^:dynamic *default-set-type*
  "The default type of test, which is one of:

      * `:cross-validation`: run a N fold cross validation (default)
      * `:train-test`: train the classifier and then evaluate"
  :cross-validation)

(defn- by-set-type-instances [set-type]
  (case set-type
    :cross-validation (exc/cross-fold-instances)
    :train-test (:train-test (exc/train-test-instances))
    :test (:test (exc/train-test-instances))
    :train (:train (exc/train-test-instances))
    true (throw (ex-info "Unknon set type"
                         {:set-type *default-set-type*}))))

(defn print-model-config
  "Pretty print the model configuation set with [[with-model-conf]]."
  []
  (pprint (model-config)))

(defn analysis-file
  ([]
   (analysis-file "%s-data.arff"))
  ([file-format]
   (let [model-conf (model-config)
         file-name (format file-format (:name model-conf))]
     (io/file (cl/analysis-report-resource) file-name))))

(defn read-arff
  "Read the ARFF file configured with [[with-model-conf]].  If **file** is
  given, use that file instead of getting it
  from [[zensols.model.classifier/analysis-report-resource]]."
  ([]
   (read-arff (analysis-file)))
  ([file]
   (cl/read-arff file)))

(defn write-arff
  "Write the ARFF file configured with [[with-model-conf]].  If **file** is
  given, use that file instead of getting it
  from [[zensols.model.classifier/analysis-report-resource]]."
  ([]
   (write-arff (analysis-file)))
  ([file]
   (binding [cl/*arff-file* file]
     (cl/write-arff (by-set-type-instances :train-test)))
   file))

(defn- feature-matrix
  "Generate a matrix of features as configured in a model with
  [[with-model-conf]].

  For the non-zero-arg form, see [[with-model-conf]]."
  ([]
   (let [{:keys [feature-metas-fn display-feature-metas-fn
                 class-feature-meta-fn create-feature-sets-fn]} (model-config)
         display-feature-metas-fn (or display-feature-metas-fn feature-metas-fn)]
     (feature-matrix (display-feature-metas-fn)
                     (class-feature-meta-fn)
                     (create-feature-sets-fn))))
  ([feature-metas class-feature-meta feature-sets]
   (let [keys (map first (concat (list class-feature-meta) feature-metas))]
     (->> feature-sets
          (map (fn [tok]
                 (map (fn [tkey]
                        (get tok tkey))
                      keys)))
          (hash-map :column-names (map name keys) :data)))))

(defn display-features
  "Display features as configured in a model with [[with-model-conf]].

  For the non-zero-arg form, see [[with-model-conf]]."
  []
  (let [{:keys [column-names data]} (feature-matrix)]
    (dr/display-results data :column-names column-names)))

(defn features-file
  "Return the default file used to create the features output file
  with [[write-features]]."
  []
  (analysis-file "%s-features.csv"))

(defn write-features
  "Write features as configured in a model with [[with-model-conf]] to a CSV
  spreadsheet file.

  See [[features-file]] for the default file

  For the non-zero-arg form, see [[with-model-conf]]."
  ([]
   (write-features (features-file)))
  ([file]<commit_msg>test/train keys for display features<commit_after>(ns ^{:doc
      "A *client* entry point library to help with evaluating machine learning
models.  This library not only wraps the Weka library but also provides
additional functionality like a two pass cross
validation (see [[*two-pass-config*]])."
      :author "<NAME>"}
    zensols.model.eval-classifier
  (:use [clojure.java.io :as io :only [input-stream output-stream file]])
  (:use [clojure.pprint :only [pprint]])
  (:require [clojure.tools.logging :as log]
            [clojure.string :as str]
            [clojure.stacktrace :refer (print-stack-trace)]
            [clojure.data.csv :as csv])
  (:require [zensols.tabres.display-results :as dr])
  (:require [zensols.model.execute-classifier :refer (model-config) :as exc]
            [zensols.model.weka :as weka]
            [zensols.model.classifier :as cl]))

(def ^:dynamic *cross-fold-count*
  "The default number of folds to use during cross fold
  validation (see [[cmpile-results]])."
  10)

(def ^:dynamic *default-set-type*
  "The default type of test, which is one of:

      * `:cross-validation`: run a N fold cross validation (default)
      * `:train-test`: train the classifier and then evaluate"
  :cross-validation)

(defn- by-set-type-instances [set-type]
  (case set-type
    :cross-validation (exc/cross-fold-instances)
    :train-test (:train-test (exc/train-test-instances))
    :test (:test (exc/train-test-instances))
    :train (:train (exc/train-test-instances))
    true (throw (ex-info "Unknon set type"
                         {:set-type *default-set-type*}))))

(defn print-model-config
  "Pretty print the model configuation set with [[with-model-conf]]."
  []
  (pprint (model-config)))

(defn analysis-file
  ([]
   (analysis-file "%s-data.arff"))
  ([file-format]
   (let [model-conf (model-config)
         file-name (format file-format (:name model-conf))]
     (io/file (cl/analysis-report-resource) file-name))))

(defn read-arff
  "Read the ARFF file configured with [[with-model-conf]].  If **file** is
  given, use that file instead of getting it
  from [[zensols.model.classifier/analysis-report-resource]]."
  ([]
   (read-arff (analysis-file)))
  ([file]
   (cl/read-arff file)))

(defn write-arff
  "Write the ARFF file configured with [[with-model-conf]].  If **file** is
  given, use that file instead of getting it
  from [[zensols.model.classifier/analysis-report-resource]]."
  ([]
   (write-arff (analysis-file)))
  ([file]
   (binding [cl/*arff-file* file]
     (cl/write-arff (by-set-type-instances :train-test)))
   file))

(defn- feature-matrix
  "Generate a matrix of features as configured in a model with
  [[with-model-conf]].

  For the non-zero-arg form, see [[with-model-conf]]."
  [& {:keys [max] :as adb-keys
      :or {max Integer/MAX_VALUE}}]
  (let [{:keys [feature-metas-fn display-feature-metas-fn
                class-feature-meta-fn create-feature-sets-fn]} (model-config)
        display-feature-metas-fn (or display-feature-metas-fn feature-metas-fn)
        feature-metas (display-feature-metas-fn)
        class-feature-meta (class-feature-meta-fn)
        adb-keys (if adb-keys
                   (->> (dissoc adb-keys :max)
                        (apply into [])))
        feature-sets (->> adb-keys
                          (apply create-feature-sets-fn)
                          (take max))]
    (let [keys (map first (concat (list class-feature-meta) feature-metas))]
      (->> feature-sets
           (map (fn [tok]
                  (map (fn [tkey]
                         (get tok tkey))
                       keys)))
           (hash-map :column-names (map name keys) :data)))))

(defn display-features
  "Display features as configured in a model with [[with-model-conf]].

  For the non-zero-arg form, see [[with-model-conf]]."
  [& adb-keys]
  (let [{:keys [column-names data]} (apply feature-matrix adb-keys)]
    (dr/display-results data :column-names column-names)))

(defn features-file
  "Return the default file used to create the features output file
  with [[write-features]]."
  []
  (analysis-file "%s-features.csv"))

(defn write-features
  "Write features as configured in a model with [[with-model-conf]] to a CSV
  spreadsheet file.

  See [[features-file]] for the default file

  For the non-zero-arg form, see [[with-model-conf]]."
  ([]
   (write-features (features-file)))
  ([file]
{'fraction_non_alphanumeric': 0.14925722306702538, 'fraction_numerical': 0.0004678909814013335, 'mean_word_length': 2.8670284938941655, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1451269', 'n_tokens_mistral': 2821, 'n_tokens_neox': 2742, 'n_words': 768}