<commit_before>(ns babel.espanol.writer
  (:refer-clojure :exclude [get-in]))

(require '[babel.engine :as engine :refer [generate]])
(require '[babel.enrich :refer [enrich]])
(require '[babel.espanol :refer [small]])
(require '[babel.espanol.grammar :refer [parse]])
(require '[babel.espanol.lexicon :refer [lexicon]])
(require '[babel.espanol.morphology :as morph])
(require '[babel.espanol.pos :refer :all])
(require '[babel.lexiconfn :refer (compile-lex map-function-on-map-vals unify)])
(require '[babel.log :refer [log4j!]])
(require '[babel.parse :as parse])
(require '[babel.ug :refer :all])
(require '[babel.writer :as writer :refer [process reload write-lexicon]])
(require '[clojure.string :as string])
(require '[clojure.tools.logging :as log])
(require '[dag_unify.core :refer (fail? get-in strip-refs)])
(require '[dag_unify.core :as unify])

;; for debugging:
(require '[babel.espanol.morphology.verbs :as esverbs])

(defn expression [& [spec]]
  (let [spec (if spec spec :top)]
    (engine/expression (small) spec)))

(defn fo [expression]
  (morph/fo expression))

;; for debugging: use map: in-order rather than interleaved by multiple workers.
;; TODO: using pmap fails: cannot generate sentences; figure out why.
(def use-map-fn map)

(defn root-verbs [lexicon]
  (let [lexemes (vals lexicon)]
    (zipmap
     (keys lexicon)
     (use-map-fn (fn [lexeme-set]
                   (filter (fn [lexeme]
                             (and
                              ;; how to only generate for a single infinitive (for testing/development):
                              ;;                           (= (get-in lexeme [:espanol :espanol]) "abandonar")
                                  (= (get-in lexeme [:synsem :cat]) :verb)
                                  (= (get-in lexeme [:synsem :infl]) :top)
                                  (not (= :top (get-in lexeme [:synsem :sem :pred] :top)))))
                           lexeme-set))
                 lexemes))))

(defn todos [ & [count lexeme]]
  (let [count (if count (Integer. count) 10)
        lexemes (if lexeme (list (get lexicon lexeme))
                    (vals lexicon))

        ;; for debugging: use map: in-order rather than interleaved by multiple workers.
        ;; TODO: using pmap fails: cannot generate sentences; figure out why.
        use-map-fn map

        root-verbs (root-verbs lexicon)

        root-verb-array
        (reduce concat
                (use-map-fn (fn [key]
                              (get root-verbs key))
                            (if lexeme
                              [lexeme]
                              (sort (keys root-verbs)))))]
    (write-lexicon "es" lexicon)
    (log/info (str "done writing lexicon."))
    (log/info (str "generating examples with this many verbs:"
                   (.size root-verb-array)))

    (.size
     (-> root-verb-array
         (use-map-fn
          (fn [verb]
            (log/debug (str "verb: " (strip-refs verb)))
            (let [root-form (get-in verb [:espanol :espanol])]
              (log/info (str "generating with verb: '" root-form "'"))
              (doall (use-map-fn
                      (fn [tense]
                        (let [debug (log/debug (str "{:root {:espanol {:espanol " root-form))
                              spec (unify/unify {:root {:espanol {:espanol root-form}}}
                                                tense)]
                          (doall
                           (use-map-fn
                            (fn [gender]
                              (let [spec (unify/unify spec
                                                      {:comp {:synsem {:agr gender}}})]
                                  (log/debug (str "generating from gender: " gender " with spec: " spec))
                                  (doall
                                   (use-map-fn
                                    (fn [person]
                                      (let [debug (log/debug (str "generating from spec(pre-person):" spec))
                                            unify-with {:comp {:synsem {:agr {:person person}}}}
                                            unified (unify/unifyc spec unify-with)
                                            debug (if (fail? unified)
                                                    (do
                                                      (log/warn (str "spec=fail:tried to generate with spec(pre-person):" spec))
                                                      (log/warn (str "spec=fail:tried to generate with unify-with      : " unify-with))))
                                            spec
                                            (if (fail? unified)
                                              (let [message (str "could not unify " spec " with person specification:" unify-with ".")]
                                                (cond
                                                  (and (= (get-in spec [:comp :synsem :agr :person])
                                                          :3rd)
                                                       (= (get-in spec [:comp :synsem :pronoun])
                                                          false))
                                                  (do<commit_msg>use threading macro (->>) to improve readability<commit_after>(ns babel.espanol.writer
  (:refer-clojure :exclude [get-in]))

(require '[babel.engine :as engine :refer [generate]])
(require '[babel.enrich :refer [enrich]])
(require '[babel.espanol :refer [small]])
(require '[babel.espanol.grammar :refer [parse]])
(require '[babel.espanol.lexicon :refer [lexicon]])
(require '[babel.espanol.morphology :as morph])
(require '[babel.espanol.pos :refer :all])
(require '[babel.lexiconfn :refer (compile-lex map-function-on-map-vals unify)])
(require '[babel.log :refer [log4j!]])
(require '[babel.parse :as parse])
(require '[babel.ug :refer :all])
(require '[babel.writer :as writer :refer [process reload write-lexicon]])
(require '[clojure.string :as string])
(require '[clojure.tools.logging :as log])
(require '[dag_unify.core :refer (fail? get-in strip-refs)])
(require '[dag_unify.core :as unify])

;; for debugging:
(require '[babel.espanol.morphology.verbs :as esverbs])

(defn expression [& [spec]]
  (let [spec (if spec spec :top)]
    (engine/expression (small) spec)))

(defn fo [expression]
  (morph/fo expression))

;; for debugging: use map: in-order rather than interleaved by multiple workers.
;; TODO: using pmap fails: cannot generate sentences; figure out why.
(def use-map-fn map)

(defn root-verbs [lexicon]
  (let [lexemes (vals lexicon)]
    (zipmap
     (keys lexicon)
     (use-map-fn (fn [lexeme-set]
                   (filter (fn [lexeme]
                             (and
                              ;; how to only generate for a single infinitive (for testing/development):
                              ;;                           (= (get-in lexeme [:espanol :espanol]) "abandonar")
                                  (= (get-in lexeme [:synsem :cat]) :verb)
                                  (= (get-in lexeme [:synsem :infl]) :top)
                                  (not (= :top (get-in lexeme [:synsem :sem :pred] :top)))))
                           lexeme-set))
                 lexemes))))

(defn todos [ & [count lexeme]]
  (let [count (if count (Integer. count) 10)
        lexemes (if lexeme (list (get lexicon lexeme))
                    (vals lexicon))

        ;; for debugging: use map: in-order rather than interleaved by multiple workers.
        ;; TODO: using pmap fails: cannot generate sentences; figure out why.
        use-map-fn map

        root-verbs (root-verbs lexicon)

        root-verb-array
        (reduce concat
                (use-map-fn (fn [key]
                              (get root-verbs key))
                            (if lexeme
                              [lexeme]
                              (sort (keys root-verbs)))))]
    (write-lexicon "es" lexicon)
    (log/info (str "done writing lexicon."))
    (log/info (str "generating examples with this many verbs:"
                   (.size root-verb-array)))

    (.size
     (->> root-verb-array
          (use-map-fn
          (fn [verb]
            (log/debug (str "verb: " (strip-refs verb)))
            (let [root-form (get-in verb [:espanol :espanol])]
              (log/info (str "generating with verb: '" root-form "'"))
              (doall (use-map-fn
                      (fn [tense]
                        (let [debug (log/debug (str "{:root {:espanol {:espanol " root-form))
                              spec (unify/unifyc {:root {:espanol {:espanol root-form}}}
                                                tense)]
                          (doall
                           (use-map-fn
                            (fn [gender]
                              (let [spec (unify/unifyc spec
                                                      {:comp {:synsem {:agr gender}}})]
                                  (log/debug (str "generating from gender: " gender " with spec: " spec))
                                  (doall
                                   (use-map-fn
                                    (fn [person]
                                      (let [debug (log/debug (str "generating from spec(pre-person):" spec))
                                            unify-with {:comp {:synsem {:agr {:person person}}}}
                                            unified (unify/unifyc spec unify-with)
                                            debug (if (fail? unified)
                                                    (do
                                                      (log/warn (str "spec=fail:tried to generate with spec(pre-person):" spec))
                                                      (log/warn (str "spec=fail:tried to generate with unify-with      : " unify-with))))
                                            spec
                                            (if (fail? unified)
                                              (let [message (str "could not unify " spec " with person specification:" unify-with ".")]
                                                (cond
                                                  (and (= (get-in spec [:comp :synsem :agr :person])
                                                          :3rd)
                                                       (= (get-in spec [:comp :synsem :pronoun])
                                                          false))
                                                  (do
{'fraction_non_alphanumeric': 0.11918413384716608, 'fraction_numerical': 0.0007519503712754958, 'mean_word_length': 1.2226864424482975, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 92, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1049991', 'n_tokens_mistral': 2852, 'n_tokens_neox': 2711, 'n_words': 794}