<commit_before>(ns babel.italiano.lexicon
  (:refer-clojure :exclude [get-in merge])
  (:require
   [babel.lexiconfn :refer [compile-lex map-function-on-map-vals unify]]
   [babel.italiano.morphology :as morph]
   [babel.italiano.pos :refer :all]
   [dag-unify.core :refer [fail? get-in merge]]))

(def lexicon-source
  {"Antonia"
   {:synsem {:sem {:pred :antonia
                   :human true}
             :agr {:number :sing
                   :person :3rd
                   :gender :fem}
             :propernoun true}}
   "Antonio"
   {:synsem {:agr {:number :sing
                   :person :3rd
                   :gender :masc}
             :sem {:pred :antonio
                   :human true}
             :propernoun true}}
   "a"
   [(let [location (ref {:place true})]
      {:synsem {:cat :prep
                :sem {:pred :a
                      :obj location
                      :comparative false}
                :subcat {:1 {:cat :noun
                             :subcat '()
                             :sem location}
                         :2 '()}}})

       {:synsem {:cat :prep
             :sem {:pred :in}
             :subcat {:1 {:cat :noun
                          :sem {:city true}}}}}

       ;; e.g. "a ridere": tu hai fatto bene a ridere (you did well to laugh)"
       (let [complement-semantics (ref {:pred :a
                                        :mod {:pred :a}})]
         {:synsem {:cat :prep
                   :sem complement-semantics
                   :subcat {:1 {:cat :verb
                                :sem complement-semantics
                                :subcat {:1 :top
                                         :2 '()}}
                            :2 '()}}})]

   "abbandonare" {:synsem {:cat :verb
                           :sem {:pred :abandon}}}

   "abbassare"  {:synsem {:cat :verb
                          :sem {:pred :lower}}}
   "abbracciare"
    {:synsem {:cat :verb
              :essere false
              :sem {:pred :abbracciare
                    :activity false
                    :discrete false
                    :subj {:human true}
                    :obj {:animate true}}}}

   "accettare" {:synsem {:cat :verb
                          :sem {:pred :accept}}}

   "accompagnare" {:synsem {:cat :verb
                            :sem {:pred :accompany}}}

   "acqua"
   (unify (:agreement noun)
          (:drinkable noun)
          (:feminine noun)
          {:synsem {:sem {:artifact false
                          :animate false
                          :pred :acqua}}})

   "addormentarsi" (let [subject-semantics (ref {:animate true})
                   subject-agr (ref :top)]
               {:synsem {:cat :verb
                         :essere true
                         :sem {:pred :fall-asleep
                               :reflexive true
                               :subj subject-semantics
                               :obj subject-semantics}
                         :subcat {:1 {:agr subject-agr
                                      :sem subject-semantics}
                                  :2 {:agr subject-agr
                                      :pronoun true
                                      :reflexive true
                                      :sem subject-semantics}}}})
   
   "affolato"
   [;; comparative
    (let [is-place (ref {:place true}) ;; only places can be crowded.
          than-this (ref {:pred :di
                          :mod is-place})]
      (unify adjective
              comparative
              {:synsem {:sem {:pred :affolato
                              :arg1 is-place
                              :arg2 is-place}
                        :subcat {:1 {:cat :noun
                                     :sem is-place}
                                 :2 {:cat :prep
                                     :sem than-this}}}}))
    ;; non-comparative
    (unify adjective
            {:synsem {:cat :adjective
                      :sem {:pred :affolato
                            :comparative false
                            :place true}}})

    ] ;; only places can be crowded.

   "aggiungere" {:synsem {:cat :verb
                          :sem {:pred :add}}}
   "aiutare"
    {:synsem {:essere false
              :sem {:pred :aiutare
                    :activity true
                    :obj {:human true}}}}
   "alto"
   [;; non-comparative:
    (let [subject-sem (ref {:human true}) ;; only humans can be tall.
          subject-agr (ref :top)] 
      (unify adjective
             non-comparative-adjective
             {:synsem {:cat :adjective
                       :sem {:pred :alto
                             :comparative false
                             :arg1 subject-sem
                             :human true}
                       :subcat {:1 {:cat :noun
                                    :agr subject-agr
                                    :sem subject-sem}
                                :2 '()}}}))
    ;; comparative:
    (let [complement-complement-sem (ref {:human true}) ;; only humans can be tall.
          complement-sem (ref {:pred :di
                               :mod complement-complement-sem})
          subject-sem (ref {:human true})] ;; only humans can be tall.
      (unify adjective
             comparative
             {:synsem {:sem {:pred :alto
                             :arg1 subject-sem
                             :arg2 complement-complement-sem}
                       :subcat {:1 {:cat :noun
                                    :sem subject-sem}
                                :2 {:cat :prep
                                    :sem complement-sem}}}}))]

   "alzarsi" (let [subject-semantics (ref {:animate true})
                   subject-agr (ref :top)]
               {:synsem {:cat :verb
                         :essere true
                         :sem {:pred :get-up
                               :reflexive true
                               :subj subject-semantics
                               :obj subject-semantics}
                         :subcat {:1 {:agr subject-agr
                                      :sem subject-semantics}
                                  :2 {:agr subject-agr
                                      :pronoun true
                                      :reflexive true
                                      :sem subject-semantics}}}})
   "amare"
    {:synsem {:cat :verb
              :essere false
              :sem {:pred :amare
                    :activity false
                    :discrete false
                    :subj {:human true}}}}
   "amico"
   (unify agreement-noun
          common-noun
          countable-noun
          masculine-noun
          {:synsem {:sem {:pred :amico
                          :human true
                          :child false}}})
   "ammirare" {:synsem {:cat :verb
                        :sem {:pred :admire}}}
   "andare"
    {:italiano {:italiano "andare"
                :essere true
                :drop-e true
                :future-stem "andr"
                :present {:1sing "vado"
                          :2sing "vai"
                          :3sing "va"
                          :1plur "andiamo"
                          :2plur "andate"
                          :3plur "vanno"}}
     :synsem {:cat :verb
              :essere true
              :sem {:subj {:animate true}
                    :pred :go}}}
    
;   (map (fn [each]
;          (unify
;           each
;           ;; common part of all andare lexemes:
;           {:italiano {:italiano "andare"
;                       :essere true
;                       :drop-e true
;                       :present {:1sing "vado"
;                                 :2sing "vai"
;                                 :3sing "va"
;                                 :1plur "andiamo"
;                                 :2plur "andate"
;                                 :3plur "vanno"}}
;            :synsem {:essere true
;                     :sem {:subj {:animate true}
;                           :activity false ;; because "I was going when (something happened) .." sounds weird.
;                           :pred :andare
;                           :discrete false
;                           :motion false}}}))
;        (list
;          ;; "andare"-intransitive
;          

;           {:synsem {:sem {:location '()}}})))

;          (unify
;           transitive
;           ;; "andare" that takes a prepositional phrase
;           (let [place-sem (ref {:place true
;                                 :pred :a})]
;             {:synsem {:sem {:location place-sem}
;                       :subcat {:2 {:sem place-sem
;                                    :subcat '()
;                                    :cat :prep}}}
;              :note "andare-pp"}))))


   "annunciare"  {:synsem {:cat :verb
                           :sem {:pred :announce}}}

   "appoggiare"  {:synsem {:cat :verb
                           :sem {:pred :support}}}

   "apprendere"  {:synsem {:cat :verb
                           :sem {:pred :learn}}}

   "approfittare"  {:synsem {:cat :verb
                             :sem {:pred :take-advantage-of}}}
   
   "approvare" {:synsem {:cat :verb
                         :sem {:pred :approve}}}

   "arrabbiarsi" (let [subject-semantics (ref {:animate true})
                   subject-agr (ref :top)]
               {:synsem {:cat :verb
                         :essere true
                         :sem {:pred :get-angry
                               :reflexive true
                               :subj subject-semantics
                               :obj subject-semantics}
                         :subcat {:1 {:agr subject-agr
                                      :sem subject-semantics}
                                  :2 {:agr subject-agr
                                      :pronoun true
                                      :reflexive true
                                      :sem subject-semantics}}}})
   
   "ascoltare"  {:synsem {:cat :verb
                          :sem {:pred :listen-to}}}

   "aspettare"  {:synsem {:cat :verb
                          :sem {:pred :wait-for}}}

   "assicurare" [ {:synsem {:cat :verb
                            :sem {:pred :assure}}}
                  {:synsem {:cat :verb
                            :sem {:pred :insure}}}]

   "aumentare"   {:synsem {:cat :verb
                           :sem {:pred :increase}}}
   "avere"
   (let [avere-common {:synsem {:essere false
                                :cat :verb
                                :sem {:reflexive false}}
                       :italiano {:italiano "avere"
                                  :future-stem "avr"
                                  :drop-e true
                                  :present {:1sing "ho"
                                            :2sing "hai"
                                            :3sing "ha"
                                            :1plur "abbiamo"
                                            :2plur "avete"
                                            :3plur "hanno"}}}]
     [(unify ;; 1. "avere": to possess something buyable
       transitive
       avere-common
       {:note "avere (possess)"
        :synsem {:sem {:pred :avere
                       :activity false
                       :discrete false
                       :subj {:human true}
                       :obj {:buyable true}}}})


      ;; 2. avere: unspecified object
      (unify
       avere-common
       verb-subjective
       intransitive-unspecified-obj
       {:note "avere (possess): unspecified object"
        :synsem {:sem {:pred :avere
                       :activity false
                       :discrete false
                       :subj {:human true}}}})

      ;; 3. "avere" that takes a transitive verb: e.g. "io l'ho vista (i saw her)"
      (let [agr-of-obj-of-main-verb (ref :top)]
        (unify
         verb-aux
         verb-subjective
         avere-common
        {:note "avere(aux): takes trans"
         :synsem {:infl :present
                  :subcat {:2 {:agr agr-of-obj-of-main-verb
                               :reflexive false
                               :subcat {:2 {:agr agr-of-obj-of-main-verb
                                            :pronoun true}}
                               :essere false}}}}))

      ;; 4. "avere" that takes an intransitive verb or a transitive verb within a VP
      ;;    with the object (e.g. "io ho dormito (i slept)" or "io ho [mangiato la pizza] (i ate the pizza)"
      ;; "avere": auxiliary-verb: takes 2 args:<commit_msg>pred from avere to have<commit_after>(ns babel.italiano.lexicon
  (:refer-clojure :exclude [get-in merge])
  (:require
   [babel.lexiconfn :refer [compile-lex map-function-on-map-vals unify]]
   [babel.italiano.morphology :as morph]
   [babel.italiano.pos :refer :all]
   [dag-unify.core :refer [fail? get-in merge]]))

(def lexicon-source
  {"Antonia"
   {:synsem {:sem {:pred :antonia
                   :human true}
             :agr {:number :sing
                   :person :3rd
                   :gender :fem}
             :propernoun true}}
   "Antonio"
   {:synsem {:agr {:number :sing
                   :person :3rd
                   :gender :masc}
             :sem {:pred :antonio
                   :human true}
             :propernoun true}}
   "a"
   [(let [location (ref {:place true})]
      {:synsem {:cat :prep
                :sem {:pred :a
                      :obj location
                      :comparative false}
                :subcat {:1 {:cat :noun
                             :subcat '()
                             :sem location}
                         :2 '()}}})

       {:synsem {:cat :prep
             :sem {:pred :in}
             :subcat {:1 {:cat :noun
                          :sem {:city true}}}}}

       ;; e.g. "a ridere": tu hai fatto bene a ridere (you did well to laugh)"
       (let [complement-semantics (ref {:pred :a
                                        :mod {:pred :a}})]
         {:synsem {:cat :prep
                   :sem complement-semantics
                   :subcat {:1 {:cat :verb
                                :sem complement-semantics
                                :subcat {:1 :top
                                         :2 '()}}
                            :2 '()}}})]

   "abbandonare" {:synsem {:cat :verb
                           :sem {:pred :abandon}}}

   "abbassare"  {:synsem {:cat :verb
                          :sem {:pred :lower}}}
   "abbracciare"
    {:synsem {:cat :verb
              :essere false
              :sem {:pred :abbracciare
                    :activity false
                    :discrete false
                    :subj {:human true}
                    :obj {:animate true}}}}

   "accettare" {:synsem {:cat :verb
                          :sem {:pred :accept}}}

   "accompagnare" {:synsem {:cat :verb
                            :sem {:pred :accompany}}}

   "acqua"
   (unify (:agreement noun)
          (:drinkable noun)
          (:feminine noun)
          {:synsem {:sem {:artifact false
                          :animate false
                          :pred :acqua}}})

   "addormentarsi" (let [subject-semantics (ref {:animate true})
                   subject-agr (ref :top)]
               {:synsem {:cat :verb
                         :essere true
                         :sem {:pred :fall-asleep
                               :reflexive true
                               :subj subject-semantics
                               :obj subject-semantics}
                         :subcat {:1 {:agr subject-agr
                                      :sem subject-semantics}
                                  :2 {:agr subject-agr
                                      :pronoun true
                                      :reflexive true
                                      :sem subject-semantics}}}})
   
   "affolato"
   [;; comparative
    (let [is-place (ref {:place true}) ;; only places can be crowded.
          than-this (ref {:pred :di
                          :mod is-place})]
      (unify adjective
              comparative
              {:synsem {:sem {:pred :affolato
                              :arg1 is-place
                              :arg2 is-place}
                        :subcat {:1 {:cat :noun
                                     :sem is-place}
                                 :2 {:cat :prep
                                     :sem than-this}}}}))
    ;; non-comparative
    (unify adjective
            {:synsem {:cat :adjective
                      :sem {:pred :affolato
                            :comparative false
                            :place true}}})

    ] ;; only places can be crowded.

   "aggiungere" {:synsem {:cat :verb
                          :sem {:pred :add}}}
   "aiutare"
    {:synsem {:essere false
              :sem {:pred :aiutare
                    :activity true
                    :obj {:human true}}}}
   "alto"
   [;; non-comparative:
    (let [subject-sem (ref {:human true}) ;; only humans can be tall.
          subject-agr (ref :top)] 
      (unify adjective
             non-comparative-adjective
             {:synsem {:cat :adjective
                       :sem {:pred :alto
                             :comparative false
                             :arg1 subject-sem
                             :human true}
                       :subcat {:1 {:cat :noun
                                    :agr subject-agr
                                    :sem subject-sem}
                                :2 '()}}}))
    ;; comparative:
    (let [complement-complement-sem (ref {:human true}) ;; only humans can be tall.
          complement-sem (ref {:pred :di
                               :mod complement-complement-sem})
          subject-sem (ref {:human true})] ;; only humans can be tall.
      (unify adjective
             comparative
             {:synsem {:sem {:pred :alto
                             :arg1 subject-sem
                             :arg2 complement-complement-sem}
                       :subcat {:1 {:cat :noun
                                    :sem subject-sem}
                                :2 {:cat :prep
                                    :sem complement-sem}}}}))]

   "alzarsi" (let [subject-semantics (ref {:animate true})
                   subject-agr (ref :top)]
               {:synsem {:cat :verb
                         :essere true
                         :sem {:pred :get-up
                               :reflexive true
                               :subj subject-semantics
                               :obj subject-semantics}
                         :subcat {:1 {:agr subject-agr
                                      :sem subject-semantics}
                                  :2 {:agr subject-agr
                                      :pronoun true
                                      :reflexive true
                                      :sem subject-semantics}}}})
   "amare"
    {:synsem {:cat :verb
              :essere false
              :sem {:pred :amare
                    :activity false
                    :discrete false
                    :subj {:human true}}}}
   "amico"
   (unify agreement-noun
          common-noun
          countable-noun
          masculine-noun
          {:synsem {:sem {:pred :amico
                          :human true
                          :child false}}})
   "ammirare" {:synsem {:cat :verb
                        :sem {:pred :admire}}}
   "andare"
    {:italiano {:italiano "andare"
                :essere true
                :drop-e true
                :future-stem "andr"
                :present {:1sing "vado"
                          :2sing "vai"
                          :3sing "va"
                          :1plur "andiamo"
                          :2plur "andate"
                          :3plur "vanno"}}
     :synsem {:cat :verb
              :essere true
              :sem {:subj {:animate true}
                    :pred :go}}}
    
;   (map (fn [each]
;          (unify
;           each
;           ;; common part of all andare lexemes:
;           {:italiano {:italiano "andare"
;                       :essere true
;                       :drop-e true
;                       :present {:1sing "vado"
;                                 :2sing "vai"
;                                 :3sing "va"
;                                 :1plur "andiamo"
;                                 :2plur "andate"
;                                 :3plur "vanno"}}
;            :synsem {:essere true
;                     :sem {:subj {:animate true}
;                           :activity false ;; because "I was going when (something happened) .." sounds weird.
;                           :pred :andare
;                           :discrete false
;                           :motion false}}}))
;        (list
;          ;; "andare"-intransitive
;          

;           {:synsem {:sem {:location '()}}})))

;          (unify
;           transitive
;           ;; "andare" that takes a prepositional phrase
;           (let [place-sem (ref {:place true
;                                 :pred :a})]
;             {:synsem {:sem {:location place-sem}
;                       :subcat {:2 {:sem place-sem
;                                    :subcat '()
;                                    :cat :prep}}}
;              :note "andare-pp"}))))


   "annunciare"  {:synsem {:cat :verb
                           :sem {:pred :announce}}}

   "appoggiare"  {:synsem {:cat :verb
                           :sem {:pred :support}}}

   "apprendere"  {:synsem {:cat :verb
                           :sem {:pred :learn}}}

   "approfittare"  {:synsem {:cat :verb
                             :sem {:pred :take-advantage-of}}}
   
   "approvare" {:synsem {:cat :verb
                         :sem {:pred :approve}}}

   "arrabbiarsi" (let [subject-semantics (ref {:animate true})
                   subject-agr (ref :top)]
               {:synsem {:cat :verb
                         :essere true
                         :sem {:pred :get-angry
                               :reflexive true
                               :subj subject-semantics
                               :obj subject-semantics}
                         :subcat {:1 {:agr subject-agr
                                      :sem subject-semantics}
                                  :2 {:agr subject-agr
                                      :pronoun true
                                      :reflexive true
                                      :sem subject-semantics}}}})
   
   "ascoltare"  {:synsem {:cat :verb
                          :sem {:pred :listen-to}}}

   "aspettare"  {:synsem {:cat :verb
                          :sem {:pred :wait-for}}}

   "assicurare" [ {:synsem {:cat :verb
                            :sem {:pred :assure}}}
                  {:synsem {:cat :verb
                            :sem {:pred :insure}}}]

   "aumentare"   {:synsem {:cat :verb
                           :sem {:pred :increase}}}
   "avere"
   (let [avere-common {:synsem {:essere false
                                :cat :verb
                                :sem {:reflexive false}}
                       :italiano {:italiano "avere"
                                  :future-stem "avr"
                                  :drop-e true
                                  :present {:1sing "ho"
                                            :2sing "hai"
                                            :3sing "ha"
                                            :1plur "abbiamo"
                                            :2plur "avete"
                                            :3plur "hanno"}}}]
     [(unify ;; 1. "avere": to possess something buyable
       transitive
       avere-common
       {:note "avere (possess)"
        :synsem {:sem {:pred :have
                       :activity false
                       :discrete false
                       :subj {:human true}
                       :obj {:buyable true}}}})


      ;; 2. avere: unspecified object
      (unify
       avere-common
       verb-subjective
       intransitive-unspecified-obj
       {:note "avere (possess): unspecified object"
        :synsem {:sem {:pred :have
                       :activity false
                       :discrete false
                       :subj {:human true}}}})

      ;; 3. "avere" that takes a transitive verb: e.g. "io l'ho vista (i saw her)"
      (let [agr-of-obj-of-main-verb (ref :top)]
        (unify
         verb-aux
         verb-subjective
         avere-common
        {:note "avere(aux): takes trans"
         :synsem {:infl :present
                  :subcat {:2 {:agr agr-of-obj-of-main-verb
                               :reflexive false
                               :subcat {:2 {:agr agr-of-obj-of-main-verb
                                            :pronoun true}}
                               :essere false}}}}))

      ;; 4. "avere" that takes an intransitive verb or a transitive verb within a VP
      ;;    with the object (e.g. "io ho dormito (i slept)" or "io ho [mangiato la pizza] (i ate the pizza)"
      ;; "avere": auxiliary-verb: takes 2 args:
{'fraction_non_alphanumeric': 0.10866129287181919, 'fraction_numerical': 0.004109372530425162, 'mean_word_length': 0.914303002798578, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 128, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2989399', 'n_tokens_mistral': 7008, 'n_tokens_neox': 6331, 'n_words': 1729}