<commit_before>(ns cortex.scrabble-test
  (:require
    [cortex.core :refer [calc output forward backward input-gradient parameters gradient parameter-count]]
    #?(:cljs
        [cljs.test :refer-macros [deftest is are testing]]
        :clj
        [clojure.test :refer [deftest is are testing]])
    [cortex.optimise :as opt]
    [clojure.core.matrix :as mat]
    #?(:cljs [thi.ng.ndarray.core :as nd])
    [cortex.network :as net]
    [cortex.core :as core]
    [cortex.layers :as layers]))

(def scrabble-values {\a 1 \b 3 \c 3 \d 2 \e 1 \f 4 \g 2 \h 4 \i 1 \j 8 \k 5
                      \l 1 \m 3 \n 1 \o 1 \p 3 \q 10 \r 1 \s 1 \t 1 \u 1 \v 4
                      \w 4 \x 8 \y 4 \z 10})

#?(:cljs (enable-console-print!))

(defn- parse-int2 [string]
  #?(:clj (Integer/parseInt string 2))
  :cljs (js/parseInt string 2))

(defn- charval [character]
  (.indexOf "abcdefghijklmnopqrstuvwxyz" #?(:clj (str character) :cljs character)))

#?(:clj (do

(defn num->bit-array [input & {:keys [bits]
                               :or {bits 4}}]
 (->> (clojure.pprint/cl-format nil (str "~" bits ",'0',B") input)
      (mapv #(new Double (str %)))))

(defn char->bit-array [character]
  (let [offset (charval character)
        begin (take offset (repeatedly #(new Double 0.0)))
        end (take (- 26 offset 1) (repeatedly #(new Double 0.0)))]
  (into [] (concat begin [1.0] end)))))

:cljs (do

(defn num->bit-array [input & {:keys [bits]
                               :or {bits 4}}]
 (->> (cljs.pprint/cl-format nil (str "~" bits ",'0',B") input)
      (mapv #(js/parseFloat (str %)))))

(defn char->bit-array [character]
  (let [offset (charval character)
        begin (take offset (repeatedly #(js/parseFloat 0.0)))
        end (take (- 26 offset 1) (repeatedly #(js/parseFloat 0.0)))]
  (into [] (concat begin [1.0] end))))))

(defn bit-array->num [input]
  (-> (map int input)
      (clojure.string/join)
      (parse-int2)))

(defn classify [network character]
  (->> (net/run network [(char->bit-array character)])
       (first)
       (map #(if (> % 0.5) 1 0))
       (clojure.string/join)
       (parse-int2)))

(deftest scrabble-test []
  (let [training-data (into [] (for [k (keys scrabble-values)] (char->bit-array k)))
        training-labels (into [] (for [v (vals scrabble-values)] (num->bit-array v)))
        input-width (last (mat/shape training-data))
        output-width (last (mat/shape training-labels))
        hidden-layer-size 4
        n-epochs 200
        batch-size 1
        loss-fn (opt/mse-loss)
        network (core/stack-module [(layers/linear-layer input-width hidden-layer-size)
                         (layers/logistic [hidden-layer-size])
                         (layers/linear-layer hidden-layer-size output-width)])
        optimizer (opt/sgd-optimiser (core/parameter-count network))
        trained-network (net/train network optimizer loss-fn training-data training-labels batch-size n-epochs)]

      (is (= (classify trained-network \j) 8))
      (is (= (classify trained-network \k) 5))))
 
<commit_msg>[test/scrabble] Fix preprocessor directive.<commit_after>(ns cortex.scrabble-test
  (:require
    [cortex.core :refer [calc output forward backward input-gradient parameters gradient parameter-count]]
    #?(:cljs
        [cljs.test :refer-macros [deftest is are testing]]
        :clj
        [clojure.test :refer [deftest is are testing]])
    [cortex.optimise :as opt]
    [clojure.core.matrix :as mat]
    #?(:cljs [thi.ng.ndarray.core :as nd])
    [cortex.network :as net]
    [cortex.core :as core]
    [cortex.layers :as layers]))

(def scrabble-values {\a 1 \b 3 \c 3 \d 2 \e 1 \f 4 \g 2 \h 4 \i 1 \j 8 \k 5
                      \l 1 \m 3 \n 1 \o 1 \p 3 \q 10 \r 1 \s 1 \t 1 \u 1 \v 4
                      \w 4 \x 8 \y 4 \z 10})

#?(:cljs (enable-console-print!))

(defn- parse-int2 [string]
  #?(:clj (Integer/parseInt string 2)
     :cljs (js/parseInt string 2)))

(defn- charval [character]
  (.indexOf "abcdefghijklmnopqrstuvwxyz" #?(:clj (str character) :cljs character)))

#?(:clj (do

(defn num->bit-array [input & {:keys [bits]
                               :or {bits 4}}]
 (->> (clojure.pprint/cl-format nil (str "~" bits ",'0',B") input)
      (mapv #(new Double (str %)))))

(defn char->bit-array [character]
  (let [offset (charval character)
        begin (take offset (repeatedly #(new Double 0.0)))
        end (take (- 26 offset 1) (repeatedly #(new Double 0.0)))]
  (into [] (concat begin [1.0] end)))))

:cljs (do

(defn num->bit-array [input & {:keys [bits]
                               :or {bits 4}}]
 (->> (cljs.pprint/cl-format nil (str "~" bits ",'0',B") input)
      (mapv #(js/parseFloat (str %)))))

(defn char->bit-array [character]
  (let [offset (charval character)
        begin (take offset (repeatedly #(js/parseFloat 0.0)))
        end (take (- 26 offset 1) (repeatedly #(js/parseFloat 0.0)))]
  (into [] (concat begin [1.0] end))))))

(defn bit-array->num [input]
  (-> (map int input)
      (clojure.string/join)
      (parse-int2)))

(defn classify [network character]
  (->> (net/run network [(char->bit-array character)])
       (first)
       (map #(if (> % 0.5) 1 0))
       (clojure.string/join)
       (parse-int2)))

(deftest scrabble-test []
  (let [training-data (into [] (for [k (keys scrabble-values)] (char->bit-array k)))
        training-labels (into [] (for [v (vals scrabble-values)] (num->bit-array v)))
        input-width (last (mat/shape training-data))
        output-width (last (mat/shape training-labels))
        hidden-layer-size 4
        n-epochs 200
        batch-size 1
        loss-fn (opt/mse-loss)
        network (core/stack-module [(layers/linear-layer input-width hidden-layer-size)
                         (layers/logistic [hidden-layer-size])
                         (layers/linear-layer hidden-layer-size output-width)])
        optimizer (opt/sgd-optimiser (core/parameter-count network))
        trained-network (net/train network optimizer loss-fn training-data training-labels batch-size n-epochs)]

      (is (= (classify trained-network \j) 8))
      (is (= (classify trained-network \k) 5))))
 

{'fraction_non_alphanumeric': 0.16875304828483173, 'fraction_numerical': 0.021459925215412127, 'mean_word_length': 2.3932708218422505, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3721190', 'n_tokens_mistral': 2316, 'n_tokens_neox': 2168, 'n_words': 619}