<commit_before>;;; -*- Gerbil -*-
;;; (C) vyzo
;;; std parser regular language parsing with derivatives
package: std/parser

(export
  @nul @nul? @eps @eps?
  @char @charset @negset @dot
  @alt @cat @rep* @rep+ @maybe
  deriv delta recognizes?
  )

;; regular expressions for lexical tokens
(defstruct rlang ())
(defstruct (nul rlang) ()          final: #t) ; null [atom]
(defstruct (eps rlang) ()          final: #t) ; epsilon [atom]
(defstruct (char rlang) (c)        final: #t) ; character c
(defstruct (charset rlang) (chars) final: #t) ; [...]        -- character set
(defstruct (negset rlang) (chars)  final: #t) ; [^...]       -- negated charset
(defstruct (alt rlang) (L1 L2)     final: #t) ; L1 | L2      -- alternation
(defstruct (cat rlang) (L1 L2)     final: #t) ; L1 . L2      -- concatenation
(defstruct (rep rlang) (L)         final: #t) ; L*           -- Kleene star

(def @nul (make-nul))
(def @eps (make-eps))

(def (@nul? L)
  (eq? @nul L))
(def (@eps? L)
  (eq? @eps L))

;; constructors
(def (@char c)
  (make-char c))

(def (@charset chars)
  (if (null? chars)
    @nul
    (make-charset (list->hash-table-eq (map (cut cons <> #t) chars)))))

(def (@negset chars)
  (make-negset (list->hash-table-eq (map (cut cons <> #t) chars))))

(def @dot
  (@negset []))

(def* @alt
  ((L1 L2)
   (cond
    ((@nul? L1) L2)
    ((@nul? L2) L1)
    (else
     (make-alt L1 L2))))
  ((L1 L2 . rest)
   (@alt L1 (apply @alt L2 rest))))

(def* @cat
  ((L1 L2)
   (if (or (@nul? L1) (@nul? L2))
     @nul
     (make-cat L1 L2)))
  ((L1 L2 . rest)
   (@cat L1 (apply @cat L2 rest))))

(def (@rep* L)
  (if (@nul? L)<commit_msg>std/parser/rlang: allow @cat and @alt to accept single argument<commit_after>;;; -*- Gerbil -*-
;;; (C) vyzo
;;; std parser regular language parsing with derivatives
package: std/parser

(export
  @nul @nul? @eps @eps?
  @char @charset @negset @dot
  @alt @cat @rep* @rep+ @maybe
  deriv delta recognizes?
  )

;; regular expressions for lexical tokens
(defstruct rlang ())
(defstruct (nul rlang) ()          final: #t) ; null [atom]
(defstruct (eps rlang) ()          final: #t) ; epsilon [atom]
(defstruct (char rlang) (c)        final: #t) ; character c
(defstruct (charset rlang) (chars) final: #t) ; [...]        -- character set
(defstruct (negset rlang) (chars)  final: #t) ; [^...]       -- negated charset
(defstruct (alt rlang) (L1 L2)     final: #t) ; L1 | L2      -- alternation
(defstruct (cat rlang) (L1 L2)     final: #t) ; L1 . L2      -- concatenation
(defstruct (rep rlang) (L)         final: #t) ; L*           -- Kleene star

(def @nul (make-nul))
(def @eps (make-eps))

(def (@nul? L)
  (eq? @nul L))
(def (@eps? L)
  (eq? @eps L))

;; constructors
(def (@char c)
  (make-char c))

(def (@charset chars)
  (if (null? chars)
    @nul
    (make-charset (list->hash-table-eq (map (cut cons <> #t) chars)))))

(def (@negset chars)
  (make-negset (list->hash-table-eq (map (cut cons <> #t) chars))))

(def @dot
  (@negset []))

(def* @alt
  ((L) L)
  ((L1 L2)
   (cond
    ((@nul? L1) L2)
    ((@nul? L2) L1)
    (else
     (make-alt L1 L2))))
  ((L1 L2 . rest)
   (@alt L1 (apply @alt L2 rest))))

(def* @cat
  ((L) L)
  ((L1 L2)
   (if (or (@nul? L1) (@nul? L2))
     @nul
     (make-cat L1 L2)))
  ((L1 L2 . rest)
   (@cat L1 (apply @cat L2 rest))))

(def (@rep* L)
  (if (@nul? L)
{'fraction_non_alphanumeric': 0.2015015015015015, 'fraction_numerical': 0.018018018018018018, 'mean_word_length': 2.6970033296337403, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4477051', 'n_tokens_mistral': 1395, 'n_tokens_neox': 1357, 'n_words': 427}