<commit_before>(ns hyphen-keeper.export
  (:require [clojure.core.async :as async]
            [clojure.java
             [io :as io]
             [shell :refer [sh]]]
            [clojure.string :as string]
            [clojure.tools.logging :as log]
            [hyphen-keeper
             [db :as db]
             [hyphenate :as hyphenate]]
            [org.tobereplaced.nio.file :as nio])
  (:import java.nio.file.StandardCopyOption))

(def ^:private substrings-program
  "Program to prepare the hyphenation dics. Expected to be installed
  on the system."
  "/usr/share/libhyphen/substrings.pl")

(def dictionaries {0 ["/tmp/whitelist_de_DE_OLDSPELL.txt"
                      "/usr/share/hyphen/generated/hyph_de_DE_OLDSPELL.dic"
                      "dicts/hyph_de_DE_OLDSPELL.dic"]
                   1 ["/tmp/whitelist_de.txt"
                      "/usr/share/hyphen/generated/hyph_de_DE.dic"
                      "dicts/hyph_de_DE.dic"]})

(defn- prepare-for-libhyphen
  "Prepare a hyphenation string for consumption by libhyphen"
  [s]
  (-> s
   (string/replace #"(.)" "$18") ; place "8" between each char
   (string/replace #"^(.*)$" ".$1.") ; suround with .
   (string/replace "8-8" "9") ; give hyphens more weight
   (string/replace "8." "."))) ; drop the last 8

(defn- get-hyphenations [spelling]
  (->>
   ;; get words for given spelling from db
   (db/read-words spelling)
   ;; filter the ones that aren't hyphenated correctly
   (remove
    (fn [{:keys [word hyphenation]}]
      (= (hyphenate/hyphenate word spelling) hyphenation)))
   (map :hyphenation)
   (remove string/blank?) ; drop empty ones
   (filter #(string/includes? % "-")) ; drop hyphenations w/o a hyphen
   (map string/lower-case) ; make sure it's lowercase
   sort
   (map prepare-for-libhyphen)))


(defn- write-file [words file-name original-dict-name]
  (with-open [w (io/writer file-name :encoding "ISO-8859-1")]<commit_msg>Export using the base hyphenation dictionaries<commit_after>(ns hyphen-keeper.export
  (:require [clojure.core.async :as async]
            [clojure.java
             [io :as io]
             [shell :refer [sh]]]
            [clojure.string :as string]
            [clojure.tools.logging :as log]
            [hyphen-keeper
             [db :as db]
             [hyphenate :as hyphenate]]
            [org.tobereplaced.nio.file :as nio])
  (:import java.nio.file.StandardCopyOption))

(def ^:private substrings-program
  "Program to prepare the hyphenation dics. Expected to be installed
  on the system."
  "/usr/share/libhyphen/substrings.pl")

(def dictionaries {0 ["/tmp/whitelist_de_DE_OLDSPELL.txt"
                      "/usr/share/hyphen/generated/hyph_de_DE_OLDSPELL.dic"
                      "dicts/hyph_de_DE_OLDSPELL.dic"]
                   1 ["/tmp/whitelist_de.txt"
                      "/usr/share/hyphen/generated/hyph_de_DE.dic"
                      "dicts/hyph_de_DE.dic"]})

(defn- prepare-for-libhyphen
  "Prepare a hyphenation string for consumption by libhyphen"
  [s]
  (-> s
   (string/replace #"(.)" "$18") ; place "8" between each char
   (string/replace #"^(.*)$" ".$1.") ; suround with .
   (string/replace "8-8" "9") ; give hyphens more weight
   (string/replace "8." "."))) ; drop the last 8

(defn- get-hyphenations [spelling]
  (->>
   ;; get words for given spelling from db
   (db/read-words spelling)
   ;; filter the ones that aren't hyphenated correctly
   (remove
    (fn [{:keys [word hyphenation]}]
      (= (hyphenate/hyphenate-against-base word spelling) hyphenation)))
   (map :hyphenation)
   (remove string/blank?) ; drop empty ones
   (filter #(string/includes? % "-")) ; drop hyphenations w/o a hyphen
   (map string/lower-case) ; make sure it's lowercase
   sort
   (map prepare-for-libhyphen)))


(defn- write-file [words file-name original-dict-name]
  (with-open [w (io/writer file-name :encoding "ISO-8859-1")]
{'fraction_non_alphanumeric': 0.13896037056098817, 'fraction_numerical': 0.008234688625836336, 'mean_word_length': 2.4767441860465116, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3607025', 'n_tokens_mistral': 1389, 'n_tokens_neox': 1314, 'n_words': 316}