<commit_before>(ns babel.italiano.workbook
  (:refer-clojure :exclude [get-in merge resolve find parents])
  (:require
   [babel.engine :as engine]
   [babel.forest :refer [lightning-bolt]]
   [babel.italiano.grammar :refer [medium small np-grammar]]
   [babel.italiano.lexicon :refer :all]
   [babel.italiano.morphology :as morph :refer [fo]]
   [babel.italiano.writer :refer [expression]]
   [babel.html :as html]
   [babel.korma :as korma]
   [babel.over :as over]
   [babel.parse :as parse]
   [babel.pos :as pos]
   [babel.reader :as reader]
   [babel.writer :as writer :refer [reload]]
   [clojail.core :refer [sandbox]]
   [clojail.testers :refer :all]
   [clojure.core :exclude [get-in]]
   [clojure.core :as core] ;; This allows us to use core's get-in by doing "(core/get-in ..)"
   [clojure.set :as set]
   [clojure.string :as string]
   [clojure.tools.logging :as log]
   [compojure.core :as compojure :refer [GET PUT POST DELETE ANY]]
   [dag_unify.core :refer [fail-path-between get-in merge remove-false
                           remove-matching-keys strip-refs unify]]
   [hiccup.core :refer [html]]
))

(defn generate
  ([spec]
   (engine/generate spec medium))
  ([spec model]
   (engine/generate spec model)))

(defn parse
  ([string]
   (map #(conj {:surface (fo %)}
               %)
        (parse/parse string
                     (:lexicon medium)
                     (:lookup medium)
                     (:grammar medium))))
  ([string model]
   (map #(conj {:surface (fo %)}
               %)
        (parse/parse string
                     (:lexicon model)
                     (:lookup model)
                     (:grammar model)))))

(defn expr [id]
  (reader/id2expression (Integer. id)))

;; this def is needed to avoid initialization errors when evaluating within the workbook
;; e.g.: evaluating things like:
;;(generate {:synsem {:subcat '()
;;                                          :infl :imperfect
;;                                          :sem {:subj {:pred :I} :pred :be}}}
;;                                fr/small)

(def foo (expression {:synsem {:cat :verb}}))
;(def foo (lightning-bolt nil nil nil))
(def foo2 (expression {:synsem {:sem {:pred :have-fun}}}))

(def rules (:grammar-map medium))

;; TODO: do morphological analysis
;; do find non-infinitives (e.g. find 'parler' given 'parle')
;; and then apply conjugated parts to lexeme
;; i.e. if input is 'parle', return
;; list of lexemes; for each, [:synsem :agr :person] will be
;; 1st, 2nd, or 3rd, and for all, number will be singular.
(defn lookup
  ([string]
   ((:lookup medium) string))
  ([string model]
   ((:lookup model) string)))

;; TODO: deprecate lookup; use analyze instead.
(defn analyze
  ([surface-form]
   (morph/analyze surface-form (:lexicon medium)))
  ([surface-form model]
   (morph/analyze surface-form (:lexicon model))))

(defn over
  ([arg1]
   (over/over (vals (:grammar-map medium)) (lookup arg1)))
  ([grammar arg1]
   (over/over grammar (lookup arg1)))
  ([grammar arg1 arg2]
   (cond (string? arg1)
         (over grammar (lookup arg1)
               arg2)

         (string? arg2)
         (over grammar arg1 (lookup arg2))

         true
         (over/over grammar arg1 arg2))))

;(def fooexpr (expr 1))

;(def foo2 (lookup "je"))
;(def foo3 (lookup "me"))

(def workbook-sandbox-it
  (sandbox
   (conj
    clojail.testers/secure-tester-without-def
    (blacklist-nses '[
                      clojure.main
                      java
                      javax<commit_msg>- (lookup)<commit_after>(ns babel.italiano.workbook
  (:refer-clojure :exclude [get-in merge resolve find parents])
  (:require
   [babel.engine :as engine]
   [babel.forest :refer [lightning-bolt]]
   [babel.italiano.grammar :refer [medium small np-grammar]]
   [babel.italiano.lexicon :refer :all]
   [babel.italiano.morphology :as morph :refer [fo]]
   [babel.italiano.writer :refer [expression]]
   [babel.html :as html]
   [babel.korma :as korma]
   [babel.over :as over]
   [babel.parse :as parse]
   [babel.pos :as pos]
   [babel.reader :as reader]
   [babel.writer :as writer :refer [reload]]
   [clojail.core :refer [sandbox]]
   [clojail.testers :refer :all]
   [clojure.core :exclude [get-in]]
   [clojure.core :as core] ;; This allows us to use core's get-in by doing "(core/get-in ..)"
   [clojure.set :as set]
   [clojure.string :as string]
   [clojure.tools.logging :as log]
   [compojure.core :as compojure :refer [GET PUT POST DELETE ANY]]
   [dag_unify.core :refer [fail-path-between get-in merge remove-false
                           remove-matching-keys strip-refs unify]]
   [hiccup.core :refer [html]]
))

(defn generate
  ([spec]
   (engine/generate spec medium))
  ([spec model]
   (engine/generate spec model)))

(defn parse
  ([string]
   (map #(conj {:surface (fo %)}
               %)
        (parse/parse string
                     (:lexicon medium)
                     (:lookup medium)
                     (:grammar medium))))
  ([string model]
   (map #(conj {:surface (fo %)}
               %)
        (parse/parse string
                     (:lexicon model)
                     (:lookup model)
                     (:grammar model)))))

(defn expr [id]
  (reader/id2expression (Integer. id)))

;; this def is needed to avoid initialization errors when evaluating within the workbook
;; e.g.: evaluating things like:
;;(generate {:synsem {:subcat '()
;;                                          :infl :imperfect
;;                                          :sem {:subj {:pred :I} :pred :be}}}
;;                                fr/small)

(def foo (expression {:synsem {:cat :verb}}))
;(def foo (lightning-bolt nil nil nil))
(def foo2 (expression {:synsem {:sem {:pred :have-fun}}}))

(def rules (:grammar-map medium))

(defn analyze
  ([surface-form]
   (morph/analyze surface-form (:lookup medium)))
  ([surface-form model]
   (morph/analyze surface-form (:lookup medium))))

(defn over
  ([arg1]
   (over/over (vals (:grammar-map medium)) (analyze arg1 medium)))
  ([grammar arg1]
   (over/over grammar (analyze arg1)))
  ([grammar arg1 arg2]
   (cond (string? arg1)
         (over grammar (analyze arg1)
               arg2)

         (string? arg2)
         (over grammar arg1 (analyze arg2))

         true
         (over/over grammar arg1 arg2))))

;(def fooexpr (expr 1))

;(def foo2 (lookup "je"))
;(def foo3 (lookup "me"))

(def workbook-sandbox-it
  (sandbox
   (conj
    clojail.testers/secure-tester-without-def
    (blacklist-nses '[
                      clojure.main
                      java
                      javax
{'fraction_non_alphanumeric': 0.14155666818388712, 'fraction_numerical': 0.006220603853739948, 'mean_word_length': 2.302605210420842, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3287384', 'n_tokens_mistral': 2255, 'n_tokens_neox': 2108, 'n_words': 618}