<commit_before>(ns babel.english.lexicon
  (:require
   [babel.lexiconfn :refer (compile-lex map-function-on-map-vals unify)]
   [babel.english.morphology :as morph]
   [babel.english.pos :refer :all]))

(def lexicon-source
  {
   "Antonia"
   {:synsem {:sem {:pred :antonia
                   :human true}
             :agr {:number :sing
                   :person :3rd
                   :gender :fem}
             :propernoun true}}

   "Antonio"
   {:synsem {:agr {:number :sing
                   :person :3rd
                   :gender :masc}
             :sem {:pred :antonio
                   :human true}
             :propernoun true}}

   "a"
   {:synsem {:cat :det
             :def :indef
             :mass false
             :number :sing}}

   "abandon" {:synsem {:cat :verb
                       :sem {:pred :abandon}}}

   "accept"  {:synsem {:cat :verb
                       :sem {:pred :accept}}}
   
   "accompany" {:synsem {:cat :verb
                         :sem {:pred :accompany}}
                :english {:past "accompanied"}}
   
   "add" {:synsem {:cat :verb
                   :sem {:pred :add}}}

   "admire" {:synsem {:cat :verb
                      :sem {:pred :admire}}}

   "announce" {:synsem {:cat :verb
                        :sem {:pred :announce}}}

   "answer" {:synsem {:cat :verb
                      :sem {:pred :answer
                            :subj {:human true}}}}

   "approve" {:synsem {:cat :verb
                :sem {:pred :approve}}}

   "ask" {:synsem {:cat :verb
                   :sem {:pred :chiedere
                         :subj {:human true}}}}

   "assure" {:synsem {:cat :verb
                      :sem {:pred :assure}}}

   "attend" {:synsem {:cat :verb
                      :sem {:pred :attend}}}

   "avoid" {:synsem {:cat :verb
                     :sem {:pred :avoid}}}

   "bag" {:synsem {:cat :noun
                   :sem {:pred :bag
                         :place false}}}

   "base" {:synsem {:cat :verb
                    :sem {:pred :support}}}
   
   "be"
   {:synsem {:cat :verb
             :sem {:pred :be}}
    :english {:present {:1sing "am"
                        :2sing "are"
                        :3sing "is"
                        :1plur "are"
                        :2plur "are"
                        :3plur "are"}
              :past {:1sing "was"
                     :2sing "were"
                     :3sing "was"
                     :1plur "were"
                     :2plur "were"
                     :3plur "were"}}}
   
   "be missed" {:synsem {:cat :verb
                         :sem {:pred :mancare}}
                :english {:participle "be missed"
                          :future "be missed"
                          :conditional "be missed"
                          :past {:english "was missed"
                                 :2sing "were missed"
                                 :2plur "were missed"
                                 :3plur "were missed"}
                          :present {:1sing "am missed"
                                    :2sing "are missed"
                                    :3sing "is missed"
                                    :1plur "are missed"
                                    :2plur "are missed"
                                    :3plur "are missed"}}}

   ;; TODO: for imperfect, generates things like 'he was be missinging'.
   "be missing" {:english {:imperfect {:1sing "was missing"
                                       :2sing "were missing"
                                       :3sing "was missing"
                                       :1plur "were missing"
                                       :2plur "were missing"
                                       :3plur "were missing"}
                           :present {:1sing "am missing"
                                     :2sing "are missing"
                                     :3sing "is missing"
                                     :1plur "are missing"
                                     :2plur "are missing"
                                     :3plur "are missing"}
                           :past {:1sing "was missing"
                                  :2sing "were missing"
                                  :3sing "was missing"
                                  :1plur "were missing"
                                  :2plur "were missing"
                                  :3plur "were missing"}}
                 :synsem {:cat :verb
                          :sem {:pred :to-be-missing}}}

   "begin" {:synsem {:cat :verb
                     :sem {:pred :begin}}
            :english {:past "began"}}

   "believe" {:synsem {:cat :verb
                       :sem {:pred :believe}}}
   
   "bicycle" {:synsem {:cat :noun
                       :sem {:pred :bicycle
                             :place false}}}

   "black"
   (unify adjective
          {:synsem {:cat :adjective
                    :sem {:pred :nero
                          :comparative false
                          :physical-object true
                          :human false}}})

   "book"
   (unify agreement-noun
          common-noun
          countable-noun
          {:synsem {:sem {:pred :libro
                          :legible true
                          :speakable false
                          :mass false
                          :buyable true
                          :place false
                          :consumable false
                          :artifact true}}})

   "break" {:synsem {:cat :verb
                     :sem {:pred :break }}
            :english {:past "break"}}

   "buy" {:synsem {:cat :verb
                   :sem {:pred :comprare
                         :subj {:human true}
                         :obj {:buyable true}}}
          :english {:past "bought"
                    :present {:3sing "buys"}}}

   "call" {:synsem {:cat :verb
                     :sem {:pred :call}}}

   "car" {:synsem {:cat :noun
                   :sem {:pred :car<commit_msg>corrected past of break from break to broke<commit_after>(ns babel.english.lexicon
  (:require
   [babel.lexiconfn :refer (compile-lex map-function-on-map-vals unify)]
   [babel.english.morphology :as morph]
   [babel.english.pos :refer :all]))

(def lexicon-source
  {
   "Antonia"
   {:synsem {:sem {:pred :antonia
                   :human true}
             :agr {:number :sing
                   :person :3rd
                   :gender :fem}
             :propernoun true}}

   "Antonio"
   {:synsem {:agr {:number :sing
                   :person :3rd
                   :gender :masc}
             :sem {:pred :antonio
                   :human true}
             :propernoun true}}

   "a"
   {:synsem {:cat :det
             :def :indef
             :mass false
             :number :sing}}

   "abandon" {:synsem {:cat :verb
                       :sem {:pred :abandon}}}

   "accept"  {:synsem {:cat :verb
                       :sem {:pred :accept}}}
   
   "accompany" {:synsem {:cat :verb
                         :sem {:pred :accompany}}
                :english {:past "accompanied"}}
   
   "add" {:synsem {:cat :verb
                   :sem {:pred :add}}}

   "admire" {:synsem {:cat :verb
                      :sem {:pred :admire}}}

   "announce" {:synsem {:cat :verb
                        :sem {:pred :announce}}}

   "answer" {:synsem {:cat :verb
                      :sem {:pred :answer
                            :subj {:human true}}}}

   "approve" {:synsem {:cat :verb
                :sem {:pred :approve}}}

   "ask" {:synsem {:cat :verb
                   :sem {:pred :chiedere
                         :subj {:human true}}}}

   "assure" {:synsem {:cat :verb
                      :sem {:pred :assure}}}

   "attend" {:synsem {:cat :verb
                      :sem {:pred :attend}}}

   "avoid" {:synsem {:cat :verb
                     :sem {:pred :avoid}}}

   "bag" {:synsem {:cat :noun
                   :sem {:pred :bag
                         :place false}}}

   "base" {:synsem {:cat :verb
                    :sem {:pred :support}}}
   
   "be"
   {:synsem {:cat :verb
             :sem {:pred :be}}
    :english {:present {:1sing "am"
                        :2sing "are"
                        :3sing "is"
                        :1plur "are"
                        :2plur "are"
                        :3plur "are"}
              :past {:1sing "was"
                     :2sing "were"
                     :3sing "was"
                     :1plur "were"
                     :2plur "were"
                     :3plur "were"}}}
   
   "be missed" {:synsem {:cat :verb
                         :sem {:pred :mancare}}
                :english {:participle "be missed"
                          :future "be missed"
                          :conditional "be missed"
                          :past {:english "was missed"
                                 :2sing "were missed"
                                 :2plur "were missed"
                                 :3plur "were missed"}
                          :present {:1sing "am missed"
                                    :2sing "are missed"
                                    :3sing "is missed"
                                    :1plur "are missed"
                                    :2plur "are missed"
                                    :3plur "are missed"}}}

   ;; TODO: for imperfect, generates things like 'he was be missinging'.
   "be missing" {:english {:imperfect {:1sing "was missing"
                                       :2sing "were missing"
                                       :3sing "was missing"
                                       :1plur "were missing"
                                       :2plur "were missing"
                                       :3plur "were missing"}
                           :present {:1sing "am missing"
                                     :2sing "are missing"
                                     :3sing "is missing"
                                     :1plur "are missing"
                                     :2plur "are missing"
                                     :3plur "are missing"}
                           :past {:1sing "was missing"
                                  :2sing "were missing"
                                  :3sing "was missing"
                                  :1plur "were missing"
                                  :2plur "were missing"
                                  :3plur "were missing"}}
                 :synsem {:cat :verb
                          :sem {:pred :to-be-missing}}}

   "begin" {:synsem {:cat :verb
                     :sem {:pred :begin}}
            :english {:past "began"}}

   "believe" {:synsem {:cat :verb
                       :sem {:pred :believe}}}
   
   "bicycle" {:synsem {:cat :noun
                       :sem {:pred :bicycle
                             :place false}}}

   "black"
   (unify adjective
          {:synsem {:cat :adjective
                    :sem {:pred :nero
                          :comparative false
                          :physical-object true
                          :human false}}})

   "book"
   (unify agreement-noun
          common-noun
          countable-noun
          {:synsem {:sem {:pred :libro
                          :legible true
                          :speakable false
                          :mass false
                          :buyable true
                          :place false
                          :consumable false
                          :artifact true}}})

   "break" {:synsem {:cat :verb
                     :sem {:pred :break }}
            :english {:past "broke"}}

   "buy" {:synsem {:cat :verb
                   :sem {:pred :comprare
                         :subj {:human true}
                         :obj {:buyable true}}}
          :english {:past "bought"
                    :present {:3sing "buys"}}}

   "call" {:synsem {:cat :verb
                     :sem {:pred :call}}}

   "car" {:synsem {:cat :noun
                   :sem {:pred :car
{'fraction_non_alphanumeric': 0.11385073638921066, 'fraction_numerical': 0.006950190302829721, 'mean_word_length': 0.9191806922832645, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 46, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2242618', 'n_tokens_mistral': 3453, 'n_tokens_neox': 3014, 'n_words': 874}