<commit_before>#| link : http://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html |#
(defpackage :lem-posix-shell-mode
  (:use :cl :lem :lem.language-mode)
  (:export :*posix-shell-mode-hook*))
(in-package :lem-posix-shell-mode)

(defparameter *reserved-words*
  '("case" "do" "done" "elif" "else"
    "esac" "fi" "for" "if" "in"
    "then" "until" "while"
    "function" "select"))

(defparameter *shell-variables*
  '("ENV" "HOME" "IFS" "LANG" "LC_ALL"
    "LC_COLLATE" "LC_CTYPE" "LC_MESSAGES" "LINENO" "NLSPATH"
    "PATH" "PPID" "PS1" "PS2" "PS4" "PWD"))

(defparameter *special-built-in-utilities*
  '("break" "colon" "continue" "dot" "eval"
    "exec" "exit" "export" "readonly" "return"
    "set" "shift" "times" "trap" "unset"))

(defun tokens (boundary strings)
  (let ((alternation
         `(:alternation ,@(sort (copy-list strings) #'> :key #'length))))
    (if boundary
        `(:sequence ,boundary ,alternation ,boundary)
        alternation)))

(defun make-tm-string-region (sepalator)
  (make-tm-region `(:sequence ,sepalator)
                  `(:sequence ,sepalator)
                  :name 'syntax-string-attribute
                  :patterns (make-tm-patterns (make-tm-match "\\\\."))))

(defun parameter-expression ()
  (let ((special-parameters "[*@#?\\-$!0]")
        (name "([a-zA-Z_][0-9a-zA-Z_]*)"))
    (format nil "\\$((~a)|(~a)|(\\{~a\\}))" special-parameters name name)))

(defun make-tmlanguage-posix-shell ()
  (let* ((patterns (make-tm-patterns
                    (make-tm-region "#" "$" :name 'syntax-comment-attribute)
                    (make-tm-string-region "\"")
                    (make-tm-string-region "'")
                    (make-tm-match (tokens :word-boundary
                                           *reserved-words*)
                                   :name 'syntax-keyword-attribute)
                    (make-tm-match (parameter-expression)
                                   :name 'syntax-variable-attribute)
                    (make-tm-match (tokens :word-boundary
                                           *shell-variables*)
                                   :name 'syntax-variable-attribute)<commit_msg>modify posix shell mode with language mode tools<commit_after>#| link : http://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html |#
(defpackage :lem-posix-shell-mode
  (:use :cl :lem :lem.language-mode :lem.language-mode-tools)
  (:export :*posix-shell-mode-hook*))
(in-package :lem-posix-shell-mode)

(defparameter *reserved-words*
  '("case" "do" "done" "elif" "else"
    "esac" "fi" "for" "if" "in"
    "then" "until" "while"
    "function" "select"))

(defparameter *shell-variables*
  '("ENV" "HOME" "IFS" "LANG" "LC_ALL"
    "LC_COLLATE" "LC_CTYPE" "LC_MESSAGES" "LINENO" "NLSPATH"
    "PATH" "PPID" "PS1" "PS2" "PS4" "PWD"))

(defparameter *special-built-in-utilities*
  '("break" "colon" "continue" "dot" "eval"
    "exec" "exit" "export" "readonly" "return"
    "set" "shift" "times" "trap" "unset"))

(defun tokens (boundary strings)
  (let ((alternation
         `(:alternation ,@(sort (copy-list strings) #'> :key #'length))))
    (if boundary
        `(:sequence ,boundary ,alternation ,boundary)
        alternation)))

(defun parameter-expression ()
  (let ((special-parameters "[*@#?\\-$!0]")
        (name "([a-zA-Z_][0-9a-zA-Z_]*)"))
    (format nil "\\$((~a)|(~a)|(\\{~a\\}))" special-parameters name name)))

(defun make-tmlanguage-posix-shell ()
  (let* ((patterns (make-tm-patterns
                    (make-tm-region "#" "$" :name 'syntax-comment-attribute)
                    (make-tm-string-region "\"")
                    (make-tm-string-region "'")
                    (make-tm-match (tokens :word-boundary
                                           *reserved-words*)
                                   :name 'syntax-keyword-attribute)
                    (make-tm-match (parameter-expression)
                                   :name 'syntax-variable-attribute)
                    (make-tm-match (tokens :word-boundary
                                           *shell-variables*)
                                   :name 'syntax-variable-attribute)
{'fraction_non_alphanumeric': 0.180622009569378, 'fraction_numerical': 0.00909090909090909, 'mean_word_length': 2.1698256254738437, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2130358', 'n_tokens_mistral': 1432, 'n_tokens_neox': 1353, 'n_words': 266}