<commit_before>;;;; -*- Mode: LISP; Syntax: COMMON-LISP; Package: CL-CUNEIFORM; Base: 10 -*-
;;;; parser.lisp

;;;; Copyright (c) 2014 "the Phoeron" <NAME> <//thephoeron.com>
;;;; See LICENSE for additional information.

(in-package :cl-cuneiform)

;; EOF
<commit_msg>Add TOKENIZE-CUNEIFORM and ASCII->LATIN parser functions<commit_after>;;;; -*- Mode: LISP; Syntax: COMMON-LISP; Package: CL-CUNEIFORM; Base: 10 -*-
;;;; parser.lisp

;;;; Copyright (c) 2014 "the Phoeron" <NAME> <//thephoeron.com>
;;;; See LICENSE for additional information.

(in-package :cl-cuneiform)

(defun tokenize-cuneiform (str)
  "Return a 2d list of cuneiform signs parsed from a string of cuneiform text."
  (let ((wordlist (split-sequence #\Space str)))
    (loop for word in wordlist
          collect (split-sequence #\- word))))

(defun ascii->latin (str)
  "Returns an upcase extended Latin unicode string with the appropriate transformations from ASCII-representation of Cuneiform signs."
  (let ((latin-string (format nil "~:@(~A~)")))
    (setf latin-string (replace-all latin-string "x" "×" :test #'char=))
    (setf latin-string (replace-all latin-string "SH" "Š"))
    (setf latin-string (replace-all latin-string "NG" "Ŋ"))
    (setf latin-string (replace-all latin-string "H" "Ḫ"))
    (setf latin-string (replace-all latin-string "0" "₀"))
    (setf latin-string (replace-all latin-string "1" "₁"))
    (setf latin-string (replace-all latin-string "2" "₂"))
    (setf latin-string (replace-all latin-string "3" "₃"))
    (setf latin-string (replace-all latin-string "4" "₄"))
    (setf latin-string (replace-all latin-string "5" "₅"))
    (setf latin-string (replace-all latin-string "6" "₆"))
    (setf latin-string (replace-all latin-string "7" "₇"))
    (setf latin-string (replace-all latin-string "8" "₈"))
    (setf latin-string (replace-all latin-string "9" "₉"))
    (setf latin-string (replace-all latin-string "_" "."))
    (setf latin-string (replace-all latin-string "×₃" "×3"))
    (setf latin-string (replace-all latin-string "×₄" "×4"))
    latin-string))

;; EOF

{'fraction_non_alphanumeric': 0.17576050217286335, 'fraction_numerical': 0.012071463061323033, 'mean_word_length': 3.909952606635071, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1616682', 'n_tokens_mistral': 804, 'n_tokens_neox': 763, 'n_words': 212}