<commit_before>(ns babel.latin
  (:refer-clojure :exclude [get-in])
  (:require [babel.latin.morphology :as morph]
            [babel.lexiconfn :refer [default listify map-function-on-map-vals
                                     verb-pred-defaults]]
            [babel.encyclopedia :as encyc]
            [clojure.java.io :refer [resource]]
            [clojure.repl :refer [doc]]
            [dag_unify.core :refer [fail? get-in unifyc]]))

(defn edn2lexicon [resource]
  (-> (read-string (slurp resource)) ;; read .edn file into a Clojure map.
      listify
      (map-function-on-map-vals
       (fn [lexicon-string lexemes]
         (map (fn [lexeme]
                (merge lexeme
                       {:root lexicon-string}))
              lexemes)))
      (default
       {:phrasal false})))

(def lexicon
  (-> (edn2lexicon (resource "babel/latin/lexicon.edn"))
      (verb-pred-defaults encyc/verb-pred-defaults)))
<commit_msg>add missing {:obj :unspec} as lexical compilation rule<commit_after>(ns babel.latin
  (:refer-clojure :exclude [get-in])
  (:require [babel.latin.morphology :as morph]
            [babel.lexiconfn :refer [default listify map-function-on-map-vals
                                     verb-pred-defaults]]
            [babel.encyclopedia :as encyc]
            [clojure.java.io :refer [resource]]
            [clojure.repl :refer [doc]]
            [dag_unify.core :refer [fail? get-in unifyc]]))

(defn edn2lexicon [resource]
  (-> (read-string (slurp resource)) ;; read .edn file into a Clojure map.
      listify
      (map-function-on-map-vals
       (fn [lexicon-string lexemes]
         (map (fn [lexeme]
                (merge lexeme
                       {:root lexicon-string}))
              lexemes)))
      (default
       {:phrasal false})))

(def lexicon
  (-> (edn2lexicon (resource "babel/latin/lexicon.edn"))


      (default ;; intransitive verbs' :obj is :unspec.
       {:synsem {:cat :verb
                 :subcat {:1 {:top :top}
                          :2 '()}
                 :sem {:obj :unspec}}})

      (verb-pred-defaults encyc/verb-pred-defaults)))

{'fraction_non_alphanumeric': 0.1359773371104816, 'fraction_numerical': 0.0028328611898017, 'mean_word_length': 1.9107142857142858, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '618466', 'n_tokens_mistral': 727, 'n_tokens_neox': 695, 'n_words': 154}