<commit_before>(defpackage :lem-rust-mode
  (:use :cl :lem :lem.language-mode)
  (:export :rust-mode
           :*rust-mode-hook*))
(in-package :lem-rust-mode)

(defvar *rust-mode-hook* '())

(defun tokens (boundary strings)
  (let ((alternation
          `(:alternation ,@(sort (copy-list strings) #'> :key #'length))))
    (if boundary
        `(:sequence ,boundary ,alternation ,boundary)
        alternation)))

(defun make-tmlanguage-rust ()
  (let* ((patterns (make-tm-patterns
                    (make-tm-region "//" "$" :name 'syntax-comment-attribute)
                    (make-tm-region '(:sequence "/*")
                                    '(:sequence "*/")
                                    :name 'syntax-comment-attribute)
                    (make-tm-region '(:sequence "#[")
                                    '(:sequence "]")
                                    :name 'syntax-builtin-attribute)
                    (make-tm-region '(:sequence "\"")
                                    '(:sequence "\"")
                                    :name 'syntax-string-attribute
                                    :patterns (make-tm-patterns
                                               (make-tm-match "\\\\.")))
                    (make-tm-match "'\\\\?.'"
                                   :name 'syntax-string-attribute)
                    (make-tm-match (tokens :word-boundary
                                           '("as"
                                             "box" "break"
                                             "const" "continue" "crate"
                                             "do" "dyn"
                                             "else" "enum" "extern"
                                             "false" "fn" "for"
                                             "if" "impl" "in"
                                             "let" "loop"
                                             "match" "mod" "move" "mut"
                                             "priv" "pub"
                                             "ref" "return"
                                             "self" "static" "struct" "super"
                                             "true" "trait" "type"
                                             "use"
                                             "virtual"
                                             "where" "while"
                                             "yield"))
                                   :name 'syntax-keyword-attribute)
                    (make-tm-match
                     (tokens :word-boundary
                             '("u8" "i8"
                               "u16" "i16"
                               "u32" "i32"
                               "u64" "i64"
                               "u128" "i128"

                               "f32" "f64"
                               "isize" "usize"
                               "bool"
                               "str" "char"))
                     :name 'syntax-type-attribute)
                    (make-tm-match "\\b[A-Z][a-zA-Z_0-9]*\\b"
                                   :name 'syntax-type-attribute)
                    (make-tm-match (tokens :word-boundary
                                           '("const" "extern" "register" "restrict"
                                             "static" "volatile" "inline"))
                                   :name 'syntax-builtin-attribute)
                    (make-tm-match (tokens :word-boundary
                                           '("NULL" "true" "false" "TRUE" "FALSE"))
                                   :name 'syntax-constant-attribute)
                    (make-tm-match (tokens nil '("+" "++" "+=" "-" "--" "-=" "*" "*=" "/" "/=" "%" "%=" 
                                                 "<" "<=" ">" ">=" "!=" "==" 
                                                 "!" "&&" "||"
                                                 "<<" "<<=" ">>" ">>=" "~" "&" "&=" "|" "|=" "^" "^=" <commit_msg>[rust-mode] syntax for function name.<commit_after>(defpackage :lem-rust-mode
  (:use :cl :lem :lem.language-mode)
  (:export :rust-mode
           :*rust-mode-hook*))
(in-package :lem-rust-mode)

(defvar *rust-mode-hook* '())

(defun tokens (boundary strings)
  (let ((alternation
          `(:alternation ,@(sort (copy-list strings) #'> :key #'length))))
    (if boundary
        `(:sequence ,boundary ,alternation ,boundary)
        alternation)))

(defun make-tmlanguage-rust ()
  (let* ((patterns (make-tm-patterns
                    (make-tm-region "//" "$" :name 'syntax-comment-attribute)
                    (make-tm-region '(:sequence "/*")
                                    '(:sequence "*/")
                                    :name 'syntax-comment-attribute)
                    (make-tm-region '(:sequence "#[")
                                    '(:sequence "]")
                                    :name 'syntax-builtin-attribute)
                    (make-tm-region '(:sequence "\"")
                                    '(:sequence "\"")
                                    :name 'syntax-string-attribute
                                    :patterns (make-tm-patterns
                                               (make-tm-match "\\\\.")))
                    (make-tm-match "'\\\\?.'"
                                   :name 'syntax-string-attribute)
                    (make-tm-match (tokens :word-boundary
                                           '("as"
                                             "box" "break"
                                             "const" "continue" "crate"
                                             "do" "dyn"
                                             "else" "enum" "extern"
                                             "false" "fn" "for"
                                             "if" "impl" "in"
                                             "let" "loop"
                                             "match" "mod" "move" "mut"
                                             "priv" "pub"
                                             "ref" "return"
                                             "self" "static" "struct" "super"
                                             "true" "trait" "type"
                                             "use"
                                             "virtual"
                                             "where" "while"
                                             "yield"))
                                   :name 'syntax-keyword-attribute)
                    (make-tm-match `(:sequence
                                     (:register "fn")
                                     (:greedy-repetition 1 nil :whitespace-char-class)
                                     ,(ppcre:parse-string "([^\\t <>()-]+)")
                                     :word-boundary)
                                   :captures (vector nil
                                                     (make-tm-name 'syntax-keyword-attribute)
                                                     (make-tm-name 'syntax-function-name-attribute)))
                    (make-tm-match
                     (tokens :word-boundary
                             '("u8" "i8"
                               "u16" "i16"
                               "u32" "i32"
                               "u64" "i64"
                               "u128" "i128"

                               "f32" "f64"
                               "isize" "usize"
                               "bool"
                               "str" "char"))
                     :name 'syntax-type-attribute)
                    (make-tm-match "\\b[A-Z][a-zA-Z_0-9]*\\b"
                                   :name 'syntax-type-attribute)
                    (make-tm-match (tokens :word-boundary
                                           '("const" "extern" "register" "restrict"
                                             "static" "volatile" "inline"))
                                   :name 'syntax-builtin-attribute)
                    (make-tm-match (tokens :word-boundary
                                           '("NULL" "true" "false" "TRUE" "FALSE"))
                                   :name 'syntax-constant-attribute)
                    (make-tm-match (tokens nil '("+" "++" "+=" "-" "--" "-=" "*" "*=" "/" "/=" "%" "%=" 
                                                 "<" "<=" ">" ">=" "!=" "==" 
                                                 "!" "&&" "||"
                                                 "<<" "<<=" ">>" ">>=" "~" "&" "&=" "|" "|=" "^" "^=" 
{'fraction_non_alphanumeric': 0.13538889532861945, 'fraction_numerical': 0.006143502955836328, 'mean_word_length': 0.6695046439628483, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 129, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3488960', 'n_tokens_mistral': 1971, 'n_tokens_neox': 1815, 'n_words': 334}