<commit_before>(ns analyze-data.evaluate-model
  (:require [clojure.java.io :as io]
            [analyze-data.csv-to-map :refer [csv-to-map]]
            [analyze-data.find-knn :refer [find-knn]]
            [analyze-data.serialize :refer [read-object]]))

(defn get-predictions
  "Make a prediction for each row of test-data using given model.

  tf-idf-data: map produced by corpus-to-tf-idf-data
  test-data: sequence of maps with 'document-label' and 'document-text' keys

  Return a sequence of the form:
  [[test-label [predicted-label distance]]
    ...]"
  [tf-idf-data test-data]
  (let [test-corpus (map #(get % "document-text") test-data)
        test-labels (map #(get % "document-label") test-data)
        predictions (map (partial find-knn tf-idf-data) test-corpus)]
    (map #(vector %1 (first %2)) test-labels predictions)))

(defn- correct-prediction?
  "Given a prediction of the form [test-label [predicted-label distance]],
  return (= test-label predicted-label)."
  [prediction]
  (let [[test-label [predicted-label distance]] prediction]
    (= test-label predicted-label)))

(defn evaluate-model
  "Make a prediction for each row of test-data using given model and return (#
  correct predictions / # total predictions).

  tf-idf-data: map produced by corpus-to-tf-idf-data
  test-data: sequence of maps with 'document-label' and 'document-text' keys"
  [tf-idf-data test-data]
  (let [predictions (get-predictions tf-idf-data test-data)
        correct-predictions (filter correct-prediction? predictions)]
    (float (/ (count correct-predictions) (count predictions)))))

(defn evaluate-model-file
  "Like evaluate-model, but take file path arguments for both model and test
  data.

  tf-idf-data-file: path to a file containing serialized tf-idf-data
  test-file: path to a csv with 'document-label' and 'document-text' headers"
  [tf-idf-data-file test-file]
  (with-open [in (io/reader test-file)]
    (let [model (read-object tf-idf-data-file)
          test-data (csv-to-map in)]
      (evaluate-model model test-data))))<commit_msg>Update evaluate-model to use predict multimethod and return a confusion matrix instead of percent correct.<commit_after>(ns analyze-data.evaluate-model
  (:require [clojure.java.io :as io]
            [analyze-data.csv-to-map :refer [csv-to-map]]
            [analyze-data.predict :refer [predict]]
            [analyze-data.serialize :refer [read-object]]))

(defn get-predictions
  "Make a prediction for each row of test-corpus using given model.

  model: map produced by analyze-data.naive-bayes.train/train
  test-corpus: sequence of maps with 'document-label' and 'document-text' keys
  options: passed through to analyze-data/predict

  Return a sequence of the form:
  [[label predicted-label]
    ...]"
  [model test-corpus & options]
  (let [document-texts (map #(get % "document-text") test-corpus)
        labels (map #(get % "document-label") test-corpus)
        predicted-labels (map #(apply predict model % options)
                              document-texts)]
    (map vector labels predicted-labels)))

(defn- empty-confusion-matrix
  "Create confusion matrix with all 0 values.
  For example, if classes = [:a :b], returns
  {:a {:a 0
       :b 0}
   :b {:a 0
       :b 0}}"
  [classes]
  (let [entries (zipmap classes (repeat 0))]
    (zipmap classes (repeat entries))))

(defn evaluate-model
  "Make a prediction for each row of test-corpus using given model and return
  a confusion matrix.

  model: model produced by train-model
  test-corpus: sequence of maps with 'document-label' and 'document-text'
               keys
  options: passed through to analyze-data/predict

  Return a confusion matrix as a nested map. For example, if the labels of
  the model are :a and :b, returns
  {:a {:a <number of documents classified as :a that were predicted :a>
       :b <number of documents classified as :a that were predicted :b>}
   :b {:a <number of documents classified as :b that were predicted :a>
       :b <number of documents classified as :b that were predicted :b>}}

  Note that if any counts are 0 then that entry will not exist."
  [model test-corpus & options]
  (let [predictions (apply get-predictions model test-corpus options)
        classes (-> model :dataset :labels)
        inc-prediction-count (fn [m [label predicted-label]]
                               (update-in m [label predicted-label] inc))]
    (reduce inc-prediction-count
            (empty-confusion-matrix classes)
            predictions)))

(defn evaluate-model-file
  "Like evaluate-model, but take file path arguments for both model and test
  data.

  model-file: path to a file containing serialized model
  test-file: path to a csv with 'document-label' and 'document-text' headers"
  [model-file test-file]
  (with-open [in (io/reader test-file)]
    (let [model (read-object model-file)
          test-corpus (csv-to-map in)]
      (evaluate-model model test-corpus))))
{'fraction_non_alphanumeric': 0.11695906432748537, 'fraction_numerical': 0.0018148820326678765, 'mean_word_length': 3.116182572614108, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1788213', 'n_tokens_mistral': 1535, 'n_tokens_neox': 1503, 'n_words': 522}