<commit_before>(ns babel.italiano.benchmark
  (:refer-clojure :exclude [get-in])
  (:require [babel.italiano.grammar :refer [small medium np-grammar]]
            [babel.italiano.lexicon :refer [lexicon]]
            [babel.italiano.morphology :as morph :refer [analyze-regular fo replace-patterns]]
            [babel.italiano.morphology.nouns :as nouns]
            [babel.italiano.morphology.verbs :as verbs]
            [babel.italiano.workbook :refer [analyze generate parse]]
            [babel.parse :as parse]
            #?(:cljs [cljs.test :refer-macros [deftest is]])
            #?(:clj [clojure.tools.logging :as log])
            #?(:cljs [babel.logjs :as log])
            [clojure.string :as string]
            [dag_unify.core :refer [get-in strip-refs]]))

(defn exception [error-string]
  #?(:clj
     (throw (Exception. error-string)))
  #?(:cljs
     (throw (js/Error. error-string))))

(defn run-benchmark [times]
  (count (take (Integer/parseInt times)
               (repeatedly #(let [debug (println "starting generation..")
                                  expr (time (generate {:comp {:synsem {:agr {:person :3rd}}}
                                                        :synsem {:cat :verb}}))]
                              (println (str "generated: " (fo expr)))
                              (println (str "starting parsing.."))
                              (let [parsed (time (take 1 (parse (fo expr))))]
                                (if (empty? parsed)
                                  (throw (exception (str "could not parse: " (fo expr) " with semantics:"
                                                         (strip-refs (get-in expr [:synsem :sem]))))))
                                                         
                                (println (str "parsed: " (fo (first parsed))))
                                (println "")))))))
(defn -main [times]
  (run-benchmark times))


<commit_msg>use 10 as default benchmark run number<commit_after>(ns babel.italiano.benchmark
  (:refer-clojure :exclude [get-in])
  (:require [babel.engine :refer [generate-all]]
            [babel.italiano.grammar :refer [small medium np-grammar]]
            [babel.italiano.lexicon :refer [lexicon]]
            [babel.italiano.morphology :as morph :refer [analyze-regular fo replace-patterns]]
            [babel.italiano.morphology.nouns :as nouns]
            [babel.italiano.morphology.verbs :as verbs]
            [babel.italiano.workbook :refer [analyze generate parse]]
            [babel.parse :as parse]
            #?(:cljs [cljs.test :refer-macros [deftest is]])
            #?(:clj [clojure.tools.logging :as log])
            #?(:cljs [babel.logjs :as log])
            [clojure.string :as string]
            [dag_unify.core :refer [get-in strip-refs]]))

(defn exception [error-string]
  #?(:clj
     (throw (Exception. error-string)))
  #?(:cljs
     (throw (js/Error. error-string))))

(defn run-benchmark
  ([]
   (run-benchmark 10))

  ([times]
   (count (take (Integer/parseInt times)
                (repeatedly #(let [debug (println "starting generation..")
                                   expr (time (generate {:comp {:synsem {:agr {:person :3rd}}}
                                                         :synsem {:cat :verb}}))]
                               (println (str "generated: " (fo expr)))
                               (println (str "starting parsing.."))
                               (let [parsed (time (take 1 (parse (fo expr))))]
                                 (if (empty? parsed)
                                   (throw (exception (str "could not parse: " (fo expr) " with semantics:"
                                                          (strip-refs (get-in expr [:synsem :sem]))))))
                                 
                                 (println (str "parsed: " (fo (first parsed))))
                                 (println ""))))))))
(defn -main [times]
  (run-benchmark times))



{'fraction_non_alphanumeric': 0.13677354709418837, 'fraction_numerical': 0.002004008016032064, 'mean_word_length': 1.4987484355444305, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1765056', 'n_tokens_mistral': 1149, 'n_tokens_neox': 1098, 'n_words': 285}