<commit_before>#|
exec /usr/local/bin/sbcl --noinform --non-interactive --load "$0" "$@"
|#

;;
;; Script for formatting draft proposed American National Standard (dpANS) on Common Lisp
;; so that each section has its own contents in place
;;
;; Usage: sbcl dpans-formatter.lisp DPANS_INDEX_HTML
;;
;;        DPANS_DIRECTORY is a top directory of Dpans.
;;

(in-package :cl-user)

(eval-when (:compile-toplevel :load-toplevel :execute)
  (ql:quickload :alexandria :silent t)
  (ql:quickload :cl-fad :silent t)
  (ql:quickload :cl-ppcre :silent t))

(defpackage dpans
  (:use cl)
  (:import-from :alexandria
                :iota
                :read-file-into-string
                :write-string-into-file)
  (:import-from :cl-ppcre
                :scan
                :register-groups-bind
                :regex-replace
                :regex-replace-all)
  (:import-from :cl-fad
                :pathname-as-directory
                :pathname-directory-pathname
                :merge-pathnames-as-file
                :list-directory))

(in-package :dpans)

(defparameter *src-path* nil)   ; Source directory including original dpANS HTML documents
(defparameter *dst-path* nil)   ; Formatted dpANS HTML documents are saved in this directory
(defparameter *index-name* nil) ; Top HTML filename (e.g. index.html)

(defun scan-href (tag)
  "Pick up an href value of <A> tag."
  (register-groups-bind (tag)
      ("href=\"([^\"#]*)#.*\"" tag)
    tag))

(defun iterate-on (tag menu scanner)
  "Iterate on each tag"
  (labels ((rec (acc &key (start 0))
             (multiple-value-bind (begin end)
                 (scan tag menu :start start)
               (if (and begin end)
                   (rec (cons (subseq menu begin (length menu)) acc)
                        :start end)
                   (mapcar scanner (reverse acc))))))
    (rec nil)))

(defun body-content (src)
  "Return the contents of <body> in src"
  (register-groups-bind (content)
      ("(?s)<div class=\"node\">.*?</div>(.*)</body>" src)
    content))

(defun link-content (link)
  "Return the contents of a given link"
  (let* ((path (merge-pathnames-as-file *src-path* link)))
    (body-content (read-file-into-string path))))

(defun scan-menu-list (src)
  "Search a <ul> tag which is a menu list except for a dictionary list."
  (multiple-value-bind (begin end)
      (scan "(?s)<ul class=\"menu\">.*?</ul>" src)
    (let ((dict (scan "(?m)^<p>Dictionary$" src)))
      (if (or (null dict)
              (and dict (< begin dict)))
          (values begin end)))))

(defun replace-menu (src)
  "Replace menu part in src with its contents in place recursively."
  (multiple-value-bind (begin end)
      (scan-menu-list src)
    (if (and begin end)
        (let ((menu (subseq src begin end))
              (head (subseq src 0 begin))
              (tail (subseq src end (length src))))
          (apply #'concatenate 
                 (append (list 'string head "<hr>")
                         (mapcar #'replace-menu
                                 (mapcar #'link-content (Iterate-on "<li>" menu #'scan-href)))
                         (list tail))))
        ;; a leaf document
        (concatenate 'string src "<hr>"))))

(defun file-size (path)
  "Return file size in byte"
  (with-open-file (in path)
    (file-length in)))
<commit_msg>Remove unused imported symbol and fix upper case to lower case of a function name.<commit_after>#|
exec /usr/local/bin/sbcl --noinform --non-interactive --load "$0" "$@"
|#

;;
;; Script for formatting draft proposed American National Standard (dpANS) on Common Lisp
;; so that each section has its own contents in place
;;
;; Usage: sbcl dpans-formatter.lisp DPANS_INDEX_HTML
;;
;;        DPANS_DIRECTORY is a top directory of Dpans.
;;

(in-package :cl-user)

(eval-when (:compile-toplevel :load-toplevel :execute)
  (ql:quickload :alexandria :silent t)
  (ql:quickload :cl-fad :silent t)
  (ql:quickload :cl-ppcre :silent t))

(defpackage dpans
  (:use cl)
  (:import-from :alexandria
                :read-file-into-string
                :write-string-into-file)
  (:import-from :cl-ppcre
                :scan
                :register-groups-bind
                :regex-replace
                :regex-replace-all)
  (:import-from :cl-fad
                :pathname-as-directory
                :pathname-directory-pathname
                :merge-pathnames-as-file
                :list-directory))

(in-package :dpans)

(defparameter *src-path* nil)   ; Source directory including original dpANS HTML documents
(defparameter *dst-path* nil)   ; Formatted dpANS HTML documents are saved in this directory
(defparameter *index-name* nil) ; Top HTML filename (e.g. index.html)

(defun scan-href (tag)
  "Pick up an href value of <A> tag."
  (register-groups-bind (tag)
      ("href=\"([^\"#]*)#.*\"" tag)
    tag))

(defun iterate-on (tag menu scanner)
  "Iterate on each tag"
  (labels ((rec (acc &key (start 0))
             (multiple-value-bind (begin end)
                 (scan tag menu :start start)
               (if (and begin end)
                   (rec (cons (subseq menu begin (length menu)) acc)
                        :start end)
                   (mapcar scanner (reverse acc))))))
    (rec nil)))

(defun body-content (src)
  "Return the contents of <body> in src"
  (register-groups-bind (content)
      ("(?s)<div class=\"node\">.*?</div>(.*)</body>" src)
    content))

(defun link-content (link)
  "Return the contents of a given link"
  (let* ((path (merge-pathnames-as-file *src-path* link)))
    (body-content (read-file-into-string path))))

(defun scan-menu-list (src)
  "Search a <ul> tag which is a menu list except for a dictionary list."
  (multiple-value-bind (begin end)
      (scan "(?s)<ul class=\"menu\">.*?</ul>" src)
    (let ((dict (scan "(?m)^<p>Dictionary$" src)))
      (if (or (null dict)
              (and dict (< begin dict)))
          (values begin end)))))

(defun replace-menu (src)
  "Replace menu part in src with its contents in place recursively."
  (multiple-value-bind (begin end)
      (scan-menu-list src)
    (if (and begin end)
        (let ((menu (subseq src begin end))
              (head (subseq src 0 begin))
              (tail (subseq src end (length src))))
          (apply #'concatenate 
                 (append (list 'string head "<hr>")
                         (mapcar #'replace-menu
                                 (mapcar #'link-content (iterate-on "<li>" menu #'scan-href)))
                         (list tail))))
        ;; a leaf document
        (concatenate 'string src "<hr>"))))

(defun file-size (path)
  "Return file size in byte"
  (with-open-file (in path)
    (file-length in)))

{'fraction_non_alphanumeric': 0.14330497089117777, 'fraction_numerical': 0.0008956560680698612, 'mean_word_length': 2.2414126753749395, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1736440', 'n_tokens_mistral': 2161, 'n_tokens_neox': 2071, 'n_words': 655}