<commit_before>(ns babel.italiano.writer
  (:refer-clojure :exclude [get-in]))

(require '[babel.generate :as generate])
(require '[babel.italiano.grammar :as grammar :refer [create-model-for-spec]])
(require '[babel.italiano.morphology :as morph])
(require '[babel.korma :refer [init-db]])
(require '[babel.writer :as writer :refer [process write-lexicon]])
(require '[clojure.string :refer [join]])
(require '[clojure.tools.logging :as log])
(require '[dag_unify.core :refer (fail? get-in strip-refs unify)])

(def model (grammar/medium))

(defn expression [spec]
  (generate/generate model spec))

(defn fo [spec]
  (morph/fo spec))

(defn rewrite-lexicon []
  (let [lexicon (:lexicon model)]
    (write-lexicon "it" lexicon)))

(declare generate-one-verb)

(defn tutti [ & [count lexeme no-older-than]]
  (let [count (cond (nil? count) 10
                    (= "all" count) 10
                    true (Integer. count))
        ;; subset of the lexicon: only verbs which are infinitives and that can be roots:
        ;; (i.e. those that have a specific (non- :top) value for [:synsem :sem :pred])
        lexicon (or (:lexicon (:generate model))
                    (:lexicon model))
        lexemes
        (cond (nil? lexeme) (vals lexicon)
              (= "all" lexeme) (vals lexicon)
              true (list (get lexicon lexeme)))
        root-verbs 
        (zipmap
         (keys lexicon)
         (map (fn [lexeme-set]
                (filter (fn [lexeme]
                          (and
                           (= (get-in lexeme [:synsem :cat]) :verb)
                           (= (get-in lexeme [:synsem :infl]) :top)
                           (= (get-in lexeme [:synsem :aux] false) false)
                           (not (= :top (get-in lexeme [:synsem :sem :pred] :top)))
                           (or (= :unspec (get-in lexeme [:synsem :sem :obj]))
                               (= true (get-in lexeme [:synsem :sem :reflexive])))))
                        lexeme-set))
              lexemes))

        tutti
        (reduce concat
                (map (fn [key]
                       (get root-verbs key))
                     (sort (keys root-verbs))))]

    ;; TODO: lexicon-writing should be moved out of (defn tutti []).
    (init-db)
    (write-lexicon "it" lexicon)
    (log/info (str "done writing lexicon."))
    (log/info (str "generating examples with this many verbs:"
                   (.size tutti)))
    (.size (pmap (fn [verb]
                   (log/debug (str "tutti: verb: " (strip-refs verb)))
                   (let [root-form (get-in verb [:italiano :italiano])]
                     (generate-one-verb (unify
                                         {:synsem {:subcat []}}
                                         {:modified false}
                                         {:synsem {:sem
                                                   (get-in verb [:synsem :sem] :top)}}
                                         {:root {:italiano {:italiano root-form}}})
                                        count no-older-than)))
                 tutti))))

(defn generate-one-verb [spec & [count no-older-than]]
  (log/debug (str "generate-one-verb with spec:" spec "; count=" count))
  (writer/generate-from-spec
   (let [model (create-model-for-spec spec)]
     (log/debug (str "created custom model from spec: " spec "; new model: " (keys model)))
     model)
   (strip-refs spec)

   (cond
     (= (get-in spec [:root :italiano :italiano])
        "chiamarsi")
     ;; for this verb, generate only present<commit_msg>don't generate any sentences with [:sem :subj :city]=true<commit_after>(ns babel.italiano.writer
  (:refer-clojure :exclude [get-in]))

(require '[babel.generate :as generate])
(require '[babel.italiano.grammar :as grammar :refer [create-model-for-spec]])
(require '[babel.italiano.morphology :as morph])
(require '[babel.korma :refer [init-db]])
(require '[babel.writer :as writer :refer [process write-lexicon]])
(require '[clojure.string :refer [join]])
(require '[clojure.tools.logging :as log])
(require '[dag_unify.core :refer (fail? get-in strip-refs unify)])

(def model (grammar/medium))

(defn expression [spec]
  (generate/generate model spec))

(defn fo [spec]
  (morph/fo spec))

(defn rewrite-lexicon []
  (let [lexicon (:lexicon model)]
    (write-lexicon "it" lexicon)))

(declare generate-one-verb)

(defn tutti [ & [count lexeme no-older-than]]
  (let [count (cond (nil? count) 10
                    (= "all" count) 10
                    true (Integer. count))
        ;; subset of the lexicon: only verbs which are infinitives and that can be roots:
        ;; (i.e. those that have a specific (non- :top) value for [:synsem :sem :pred])
        lexicon (or (:lexicon (:generate model))
                    (:lexicon model))
        lexemes
        (cond (nil? lexeme) (vals lexicon)
              (= "all" lexeme) (vals lexicon)
              true (list (get lexicon lexeme)))
        root-verbs 
        (zipmap
         (keys lexicon)
         (map (fn [lexeme-set]
                (filter (fn [lexeme]
                          (and
                           (= (get-in lexeme [:synsem :cat]) :verb)
                           (= (get-in lexeme [:synsem :infl]) :top)
                           (= (get-in lexeme [:synsem :aux] false) false)
                           (not (= :top (get-in lexeme [:synsem :sem :pred] :top)))
                           (or (= :unspec (get-in lexeme [:synsem :sem :obj]))
                               (= true (get-in lexeme [:synsem :sem :reflexive])))))
                        lexeme-set))
              lexemes))

        tutti
        (reduce concat
                (map (fn [key]
                       (get root-verbs key))
                     (sort (keys root-verbs))))]

    ;; TODO: lexicon-writing should be moved out of (defn tutti []).
    (init-db)
    (write-lexicon "it" lexicon)
    (log/info (str "done writing lexicon."))
    (log/info (str "generating examples with this many verbs:"
                   (.size tutti)))
    (.size (pmap (fn [verb]
                   (log/debug (str "tutti: verb: " (strip-refs verb)))
                   (let [root-form (get-in verb [:italiano :italiano])]
                     (generate-one-verb (unify
                                         {:synsem {:subcat []}}
                                         {:modified false}
                                         {:synsem {:sem
                                                   (get-in verb [:synsem :sem] :top)}}
                                         {:root {:italiano {:italiano root-form}}}
                                         {:synsem {:sem {:subj {:city false}}}})
                                        count no-older-than)))
                 tutti))))

(defn generate-one-verb [spec & [count no-older-than]]
  (log/debug (str "generate-one-verb with spec:" spec "; count=" count))
  (writer/generate-from-spec
   (let [model (create-model-for-spec spec)]
     (log/debug (str "created custom model from spec: " spec "; new model: " (keys model)))
     model)
   (strip-refs spec)

   (cond
     (= (get-in spec [:root :italiano :italiano])
        "chiamarsi")
     ;; for this verb, generate only present
{'fraction_non_alphanumeric': 0.14039680352714246, 'fraction_numerical': 0.0011022320198401765, 'mean_word_length': 1.6609237536656891, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 26, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '494322', 'n_tokens_mistral': 2240, 'n_tokens_neox': 2107, 'n_words': 627}