<commit_before>;; word_stats.clj
;; <NAME>, <EMAIL>
;;
;; Problem Source: iJava (http://ijava.cs.umass.edu/)
;;
;; Given a file with <= 100 characters, print the number of words containing X
;; characters for X from 1 to the length of the longest word, in the format:
;;   words of length 1: 12
;;   words of length 2: 3
;;   ...
;; At the end of the output, should print a line that gives the number of
;; sentences, and line that gives the average number of words per sentence,
;; using the form:
;;  number of sentences: 4
;;  average sentence length: 7.452423455
;; A word is any string of consecutive non-whitespace characters (including
;; sentence terminators). Every input will contain at least one sentence
;; terminator (period, exclamation point, or question mark). The average number
;; of words should simply count the number of words in the file and divide by
;; the number of sentence terminator characters.
;;
;; input stack has the working file on top of stack, full file next

(ns clojush.problems.software.word-stats
  (:use clojush.pushgp.pushgp
        [clojush pushstate interpreter random util globals]
        clojush.instructions.tag
        clojure.math.numeric-tower)
    (:require [clojure.string :as string]))

; Define new instructions
(define-registered
  file_readchar
  ^{:stack-types [:char]}
  (fn [state]
    (let [file (top-item :input state)
          first-char (first file)
          inp-result (push-item (apply str (rest file))
                                :input
                                (pop-item :input state))]
      (if (= file "")
        state
        (push-item first-char
                   :char
                   inp-result)))))

(define-registered
  file_readline
  ^{:stack-types [:string]}
  (fn [state]
    (let [file (top-item :input state)
          index (inc (.indexOf file "\n"))
          has-no-newline (= 0 index)
          inp-result (push-item (if has-no-newline
                                  ""
                                  (subs file index))
                                :input
                                (pop-item :input state))]
      (if (= file "")
        state
        (if has-no-newline
          (push-item file :string inp-result)
          (push-item (subs file 0 index)
                     :string
                     inp-result))))))

(define-registered
  file_EOF
  ^{:stack-types [:boolean]}
  (fn [state]
    (let [file (top-item :input state)
          result (empty? file)]
      (push-item result :boolean state))))

(define-registered
  file_begin
  ^{:stack-types [:input]}
  (fn [state]
    (push-item (stack-ref :input 1 state)
               :input
               (pop-item :input state))))

; Atom generators
(def word-stats-atom-generators
  (concat (list
            \. \? \! \space \tab \newline
            []
            "words of length "
            ": "
            "number of sentences: "
            "average sentence length: "
            ;;; end constants
            (fn [] (- (lrand-int 201) 100)) ;Integer ERC
            ;;; end ERCs
            (tag-instruction-erc [:string :char :integer :boolean :exec] 1000)
            (tagged-instruction-erc 1000)
            ;;; end tag ERCs
            'file_readline
            'file_readchar
            'file_EOF
            'file_begin
            ;;; end input instructions
            )
          (registered-for-stacks [:string :vector_string :char :integer :vector_integer :float :vector_float :boolean :exec :print])))


;; Define test cases
(defn word-stats-input
  "Makes a Word Stats input of length len."
  [len]
  (let [candidate (apply str
                         (repeatedly len
                                     (fn []
                                       (let [rand-cond (lrand)
                                             tab-prob (+ 0.01 (* 0.02 (lrand))) ;prob of tab is between 0.01 and 0.03
                                             newline-prob (+ 0.02 (* 0.05 (lrand))) ;prob of newline is between 0.02 and 0.07
                                             space-prob (+ 0.05 (* 0.3 (lrand))) ;prob of space is between 0.05 and 0.35
                                             terminator-prob (+ 0.01 (* 0.19 (lrand))) ;prob of sentence terminator is between 0.01 and 0.20
                                             ]
                                         (cond
                                           (< rand-cond tab-prob) \tab
                                           (< rand-cond (+ tab-prob newline-prob)) \newline
                                           (< rand-cond (+ tab-prob newline-prob space-prob)) \space
                                           (< rand-cond (+ tab-prob newline-prob space-prob terminator-prob)) (lrand-nth [\. \! \?])
                                           :else (lrand-nth (map char (range 33 127))))))))]
    (if (some #{\. \! \?} candidate)
      candidate
      (apply str (assoc (vec candidate)
                        (lrand-int (count candidate))
                        (lrand-nth [\. \! \?]))))))

;; A list of data domains for the problem. Each domain is a vector containing
;; a "set" of inputs and two integers representing how many cases from the set
;; should be used as training and testing cases respectively. Each "set" of
;; inputs is either a list or a function that, when called, will create a
;; random element of the set.
(def word-stats-data-domains
  [[(list ".", "!", "?", "\t.", "\n!", " ?", ".#", "A.\n", "! \n", "?\t\n", "\n?\n",
          ".!?.!?", ".txt", "!RACECAR!", "www.google.com"
          "Pirate basketball? Envelope WARS!"
          ".hello there wo.RLD"
          "out. at. the. plate."
          "nap time on planets!"
          "supercalifragilisticexpialidocious?"
          (apply str (concat (repeat 99 \newline) [\.]))
          (apply str (concat (repeat 99 \=) [\?]))
          (apply str (concat [\!] (repeat 99 \space)))
          (apply str (concat [\.] (repeat 99 \h)))
          (apply str (concat (repeat 99 \tab) [\?]))
          (apply str (concat (repeat 99 \@) [\!]))
          (apply str (repeat 100 \.))
          (apply str (repeat 100 \!))
          (apply str (repeat 100 \?))
          (apply str (take 100 (cycle (list \. \newline))))
          (apply str (take 100 (cycle (list \? \newline \newline))))
          (apply str (take 100 (cycle (list \! \D \newline))))
          (apply str (take 100 (cycle (list \! \space))))
          (apply str (take 100 (cycle (list \. \tab))))
          (apply str (take 100 (cycle (list \? \newline \y \space))))
          (apply str (take 100 (cycle (list \5 \!))))) 36 0] ;; "Special" inputs covering most base cases
   [(fn [] (word-stats-input (inc (lrand-int 100)))) 164 2000] ;; Random inputs
   ])

;;Can make Word Stats test data like this:
;(test-and-train-data-from-domains word-stats-data-domains)

; Helper function for error function
(defn word-stats-test-cases
  "Takes a sequence of inputs and gives IO test cases of the form
   [input output]."
  [inputs]
  (map (fn [in]
         (let [words (filter not-empty (string/split in #"\s+"))
               word-lengths (map count words)
               num-sentences (count (filter #(some #{%} ".?!") in))]
           (vector in
                   (vector (str (apply str (for [n (range 1 (inc (apply max word-lengths)))]
                                             (format "words of length %d: %d\n" n (count (filter #(= % n) word-lengths)))))
                                (format "number of sentences: %d\n" num-sentences)
                                (format "average sentence length: %s" (pr-str (round-to-n-decimal-places (* 1.0 (/ (count words) num-sentences)) 10))))
                           num-sentences
                           (float (/ (count words) num-sentences))))))
       inputs))

; Define error function. For now, each run uses different random inputs
(defn word-stats-error-function
  "Returns the error function for the Word Stats problem. Takes as
   input Word Stats data domains."
  [data-domains]
  (let [[train-cases test-cases] (map #(sort-by (comp count first) %)
                                      (map word-stats-test-cases
                                           (test-and-train-data-from-domains data-domains)))]
    (when false ;; Change to false to not print test cases
      (doseq [[i case] (map vector (range) train-cases)]
        (println (format "Train Case: %3d | Input/Output: %s" i (str case))))
      (doseq [[i case] (map vector (range) test-cases)]
        (println (format "Test Case: %3d | Input/Output: %s" i (str case)))))
    (fn the-actual-word-stats-error-function
      ([program]
        (the-actual-word-stats-error-function program :train))
      ([program data-cases] ;; data-cases should be :train or :test
        (the-actual-word-stats-error-function program data-cases false))
      ([program data-cases print-outputs]
        (let [behavior (atom '())
              errors (flatten<commit_msg>Changed printing of test cases to true for Word Stats<commit_after>;; word_stats.clj
;; <NAME>, <EMAIL>
;;
;; Problem Source: iJava (http://ijava.cs.umass.edu/)
;;
;; Given a file with <= 100 characters, print the number of words containing X
;; characters for X from 1 to the length of the longest word, in the format:
;;   words of length 1: 12
;;   words of length 2: 3
;;   ...
;; At the end of the output, should print a line that gives the number of
;; sentences, and line that gives the average number of words per sentence,
;; using the form:
;;  number of sentences: 4
;;  average sentence length: 7.452423455
;; A word is any string of consecutive non-whitespace characters (including
;; sentence terminators). Every input will contain at least one sentence
;; terminator (period, exclamation point, or question mark). The average number
;; of words should simply count the number of words in the file and divide by
;; the number of sentence terminator characters.
;;
;; input stack has the working file on top of stack, full file next

(ns clojush.problems.software.word-stats
  (:use clojush.pushgp.pushgp
        [clojush pushstate interpreter random util globals]
        clojush.instructions.tag
        clojure.math.numeric-tower)
    (:require [clojure.string :as string]))

; Define new instructions
(define-registered
  file_readchar
  ^{:stack-types [:char]}
  (fn [state]
    (let [file (top-item :input state)
          first-char (first file)
          inp-result (push-item (apply str (rest file))
                                :input
                                (pop-item :input state))]
      (if (= file "")
        state
        (push-item first-char
                   :char
                   inp-result)))))

(define-registered
  file_readline
  ^{:stack-types [:string]}
  (fn [state]
    (let [file (top-item :input state)
          index (inc (.indexOf file "\n"))
          has-no-newline (= 0 index)
          inp-result (push-item (if has-no-newline
                                  ""
                                  (subs file index))
                                :input
                                (pop-item :input state))]
      (if (= file "")
        state
        (if has-no-newline
          (push-item file :string inp-result)
          (push-item (subs file 0 index)
                     :string
                     inp-result))))))

(define-registered
  file_EOF
  ^{:stack-types [:boolean]}
  (fn [state]
    (let [file (top-item :input state)
          result (empty? file)]
      (push-item result :boolean state))))

(define-registered
  file_begin
  ^{:stack-types [:input]}
  (fn [state]
    (push-item (stack-ref :input 1 state)
               :input
               (pop-item :input state))))

; Atom generators
(def word-stats-atom-generators
  (concat (list
            \. \? \! \space \tab \newline
            []
            "words of length "
            ": "
            "number of sentences: "
            "average sentence length: "
            ;;; end constants
            (fn [] (- (lrand-int 201) 100)) ;Integer ERC
            ;;; end ERCs
            (tag-instruction-erc [:string :char :integer :boolean :exec] 1000)
            (tagged-instruction-erc 1000)
            ;;; end tag ERCs
            'file_readline
            'file_readchar
            'file_EOF
            'file_begin
            ;;; end input instructions
            )
          (registered-for-stacks [:string :vector_string :char :integer :vector_integer :float :vector_float :boolean :exec :print])))


;; Define test cases
(defn word-stats-input
  "Makes a Word Stats input of length len."
  [len]
  (let [candidate (apply str
                         (repeatedly len
                                     (fn []
                                       (let [rand-cond (lrand)
                                             tab-prob (+ 0.01 (* 0.02 (lrand))) ;prob of tab is between 0.01 and 0.03
                                             newline-prob (+ 0.02 (* 0.05 (lrand))) ;prob of newline is between 0.02 and 0.07
                                             space-prob (+ 0.05 (* 0.3 (lrand))) ;prob of space is between 0.05 and 0.35
                                             terminator-prob (+ 0.01 (* 0.19 (lrand))) ;prob of sentence terminator is between 0.01 and 0.20
                                             ]
                                         (cond
                                           (< rand-cond tab-prob) \tab
                                           (< rand-cond (+ tab-prob newline-prob)) \newline
                                           (< rand-cond (+ tab-prob newline-prob space-prob)) \space
                                           (< rand-cond (+ tab-prob newline-prob space-prob terminator-prob)) (lrand-nth [\. \! \?])
                                           :else (lrand-nth (map char (range 33 127))))))))]
    (if (some #{\. \! \?} candidate)
      candidate
      (apply str (assoc (vec candidate)
                        (lrand-int (count candidate))
                        (lrand-nth [\. \! \?]))))))

;; A list of data domains for the problem. Each domain is a vector containing
;; a "set" of inputs and two integers representing how many cases from the set
;; should be used as training and testing cases respectively. Each "set" of
;; inputs is either a list or a function that, when called, will create a
;; random element of the set.
(def word-stats-data-domains
  [[(list ".", "!", "?", "\t.", "\n!", " ?", ".#", "A.\n", "! \n", "?\t\n", "\n?\n",
          ".!?.!?", ".txt", "!RACECAR!", "www.google.com"
          "Pirate basketball? Envelope WARS!"
          ".hello there wo.RLD"
          "out. at. the. plate."
          "nap time on planets!"
          "supercalifragilisticexpialidocious?"
          (apply str (concat (repeat 99 \newline) [\.]))
          (apply str (concat (repeat 99 \=) [\?]))
          (apply str (concat [\!] (repeat 99 \space)))
          (apply str (concat [\.] (repeat 99 \h)))
          (apply str (concat (repeat 99 \tab) [\?]))
          (apply str (concat (repeat 99 \@) [\!]))
          (apply str (repeat 100 \.))
          (apply str (repeat 100 \!))
          (apply str (repeat 100 \?))
          (apply str (take 100 (cycle (list \. \newline))))
          (apply str (take 100 (cycle (list \? \newline \newline))))
          (apply str (take 100 (cycle (list \! \D \newline))))
          (apply str (take 100 (cycle (list \! \space))))
          (apply str (take 100 (cycle (list \. \tab))))
          (apply str (take 100 (cycle (list \? \newline \y \space))))
          (apply str (take 100 (cycle (list \5 \!))))) 36 0] ;; "Special" inputs covering most base cases
   [(fn [] (word-stats-input (inc (lrand-int 100)))) 164 2000] ;; Random inputs
   ])

;;Can make Word Stats test data like this:
;(test-and-train-data-from-domains word-stats-data-domains)

; Helper function for error function
(defn word-stats-test-cases
  "Takes a sequence of inputs and gives IO test cases of the form
   [input output]."
  [inputs]
  (map (fn [in]
         (let [words (filter not-empty (string/split in #"\s+"))
               word-lengths (map count words)
               num-sentences (count (filter #(some #{%} ".?!") in))]
           (vector in
                   (vector (str (apply str (for [n (range 1 (inc (apply max word-lengths)))]
                                             (format "words of length %d: %d\n" n (count (filter #(= % n) word-lengths)))))
                                (format "number of sentences: %d\n" num-sentences)
                                (format "average sentence length: %s" (pr-str (round-to-n-decimal-places (* 1.0 (/ (count words) num-sentences)) 10))))
                           num-sentences
                           (float (/ (count words) num-sentences))))))
       inputs))

; Define error function. For now, each run uses different random inputs
(defn word-stats-error-function
  "Returns the error function for the Word Stats problem. Takes as
   input Word Stats data domains."
  [data-domains]
  (let [[train-cases test-cases] (map #(sort-by (comp count first) %)
                                      (map word-stats-test-cases
                                           (test-and-train-data-from-domains data-domains)))]
    (when true ;; Change to false to not print test cases
      (doseq [[i case] (map vector (range) train-cases)]
        (println (format "Train Case: %3d | Input/Output: %s" i (str case))))
      (doseq [[i case] (map vector (range) test-cases)]
        (println (format "Test Case: %3d | Input/Output: %s" i (str case)))))
    (fn the-actual-word-stats-error-function
      ([program]
        (the-actual-word-stats-error-function program :train))
      ([program data-cases] ;; data-cases should be :train or :test
        (the-actual-word-stats-error-function program data-cases false))
      ([program data-cases print-outputs]
        (let [behavior (atom '())
              errors (flatten
{'fraction_non_alphanumeric': 0.13051908481524568, 'fraction_numerical': 0.016841172234225252, 'mean_word_length': 1.7015863513917988, 'pattern_counts': {'":': 2, '<': 17, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 66, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5890762', 'n_tokens_mistral': 5363, 'n_tokens_neox': 4933, 'n_words': 1734}