<commit_before>(ns cortex.loss-test
  (:require [cortex.stream-augment :as stream-augment]
            [cortex.loss :as loss]
            [cortex.keyword-fn :as keyword-fn]
            [clojure.core.matrix :as m]
            [clojure.test :refer :all]))



(deftest labes->indexes
  (let [labels [[1 0 0 0]
                [0 1 0 0]
                [0 0 1 0]
                [0 0 0 1]]
        augment-fn (get (stream-augment/labels->indexes-augmentation :stream)
                        :augmentation)]
    (is (= [0 1 2 3]
           (keyword-fn/call-keyword-fn augment-fn labels)))))


(deftest labes->inverse-counts
  (let [labels [[1 0 0 0 0]
                [0 0 1 0 0]
                [1 0 0 0 0]
                [0 0 0 1 0]]
        augment-fn (get (stream-augment/labels->inverse-counts-augmentation :stream)
                        :augmentation)]
    (is (= [0.5 1.0 0.5 1.0]
           (keyword-fn/call-keyword-fn augment-fn labels)))))


(deftest mse-loss-not-nan
  (let [mse-loss (loss/mse-loss)
        buffer-map {:output 0.0
                    :labels 0.2}
        loss-value (loss/loss mse-loss buffer-map)]
    (println "TODO: Un-comment this assertion after upgrading core.matrix > 0.57.0.")
    #_(is (not (Double/isNaN loss-value)))))<commit_msg>[cortex/loss-test] Un-comment out this test. (#140)<commit_after>(ns cortex.loss-test
  (:require [cortex.stream-augment :as stream-augment]
            [cortex.loss :as loss]
            [cortex.keyword-fn :as keyword-fn]
            [clojure.core.matrix :as m]
            [clojure.test :refer :all]))



(deftest labes->indexes
  (let [labels [[1 0 0 0]
                [0 1 0 0]
                [0 0 1 0]
                [0 0 0 1]]
        augment-fn (get (stream-augment/labels->indexes-augmentation :stream)
                        :augmentation)]
    (is (= [0 1 2 3]
           (keyword-fn/call-keyword-fn augment-fn labels)))))


(deftest labes->inverse-counts
  (let [labels [[1 0 0 0 0]
                [0 0 1 0 0]
                [1 0 0 0 0]
                [0 0 0 1 0]]
        augment-fn (get (stream-augment/labels->inverse-counts-augmentation :stream)
                        :augmentation)]
    (is (= [0.5 1.0 0.5 1.0]
           (keyword-fn/call-keyword-fn augment-fn labels)))))


(deftest mse-loss-not-nan
  (let [mse-loss (loss/mse-loss)
        buffer-map {:output 0.0
                    :labels 0.2}
        loss-value (loss/loss mse-loss buffer-map)]
    (is (not (Double/isNaN loss-value)))))
{'fraction_non_alphanumeric': 0.14107717041800644, 'fraction_numerical': 0.044614147909967844, 'mean_word_length': 1.650692225772098, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1183757', 'n_tokens_mistral': 967, 'n_tokens_neox': 835, 'n_words': 225}