<commit_before>;;;; command.lisp
;;;;
;;;; This file is part of the cl-closure-template library, released under Lisp-LGPL.
;;;; See file COPYING for details.
;;;;
;;;; Author: <NAME> <<EMAIL>>

(in-package #:closure-template.parser)

;;; simple text

(defun simple-text-char-p (character)
  (not (member character '(#\{ #\} #\Space #\Tab #\Return #\Newline))))

(define-rule simple-text (+ (simple-text-char-p character))
  (:text t))

;;;; commment

(define-rule simple-comment (and  "//" (*  (and (! #\Newline) character)) whitespace)
  (:lambda (list)
    (make-instance 'comment
                   :text (text list))))

(define-rule multiline-comment (and  "/*" (*  (and (! "*/") character)) "*/" (? whitespace))
  (:lambda (list)
    (make-instance 'comment
                   :text (text list)
                   :multiline t)))

(define-rule comment (or simple-comment multiline-comment))

(defclass comment ()
  ((text :initarg :text :reader comment-text)
   (multiline :initarg :multiline :initform nil :reader comment-multiline-p)))

;;;; substition

(defvar *substitiotn-map* (make-hash-table))

(defmacro define-substitions (name str value)
  `(progn
     (setf (gethash ',name *substitiotn-map*)
           ,(if (characterp  value)
                (string value)
                value))
     (define-rule ,name (and (? whitespace) ,str (? whitespace))
       (:constant ',name))))

(define-substitions space-tag "{sp}" #\Space)
(define-substitions emptry-string "{nil}" "")
(define-substitions carriage-return "{\\r}" #\Return)
(define-substitions line-feed "{\\n}" #\Newline)
(define-substitions tag "{\\t}" #\Tab)
(define-substitions left-brace "{lb}" #\{)
(define-substitions right-brace "{rb}" #\})

(define-rule substition (or space-tag emptry-string carriage-return line-feed tag left-brace right-brace))

;;;; literag command

(define-rule literal (and "{literal}" (*  (and (! "{/literal}") character)) "{/literal}")
  (:destructure (start any end)
    (declare (ignore start end))
    (make-instance 'literal
                   :content (text any))))

(defclass literal ()
  ((content :initarg :content :reader literal-content)))

;;;; print

(defmacro define-print-directive (name value)
  `(define-rule ,name (and (? whitespace) "|" (? whitespace) ,value (? whitespace))
     (:constant '(,(lispify-name value) t))))

(define-print-directive no-autoescape-d "noAutoescape")
(define-print-directive id-d "id")
(define-print-directive escape-html-d "escapeHtml")
(define-print-directive escape-uri-d "escapeUri")
(define-print-directive escape-js-d "escapeJs")

(define-rule insert-word-breaks-d (and (? whitespace) "|" (? whitespace) "insertWordBreaks:" decimal-integer (? whitespace))
  (:destructure (w1 d w2 i value w3)
    (declare (ignore w1 d w2 i w3))
    (list :insert-word-breaks value)))
  
(define-rule print-directive (or no-autoescape-d id-d escape-html-d escape-uri-d escape-js-d insert-word-breaks-d))

(define-rule print-command (and (or "{print " "{")  expression (* print-directive) "}")
  (:destructure (start expr directives end)
    (declare (ignore start end))
    (make-instance 'print-command
                   :expr expr
                   :directives (iter (for (key value) in directives)
                                     (collect key)
                                     (collect value)))))
(defclass print-command ()
  ((expr :initarg :expr :reader print-expression)
   (directives :initarg :directives :reader print-directives)))

(defmacro wrap-user-print-directive (d-symbol-cl-templ d-symbol-orig)
  `(define-rule ,d-symbol-cl-templ (and (? whitespace) "|" (? whitespace) ,d-symbol-orig (? whitespace))
     (:destructure (w1 d w2 d-params w3)
                   (declare (ignore w1 d w2 w3))
                   (list ',d-symbol-cl-templ d-params))))

(defmacro register-print-directive (d-symbol d-handler) ;; TODO user will have to specify the result list transformation according to backend type
  (alexandria:with-gensyms (rl-expr)
    (let ((local-d-symbol (intern (gensym (symbol-name d-symbol)))))
      `(progn
         (wrap-user-print-directive ,local-d-symbol ,d-symbol)
         (with-closure-template-rules
           (let ((,rl-expr (rule-expression (find-rule 'print-directive))))
             (change-rule 'print-directive (append ,rl-expr (list ',local-d-symbol)))))
         (setf (gethash ',local-d-symbol *user-print-directives*) ,d-handler)))))

;;; witch

(define-rule with-variable (and whitespace  simple-name "=\"" expression #\")
  (:destructure (w var e expr q)
    (declare (ignore w e q))
    (list (make-instance 'var
                         :name (lispify-name var)
                         :jsname var)
          expr)))
                   
(define-rule with (and "{with" (+ with-variable) (? whitespace) "}" (? code-block) "{/with}")
  (:destructure (start vars w rb code end)
    (declare (ignore start w rb end))
    (make-instance 'with
                   :vars vars
                   :body code)))

(defclass with ()
  ((vars :initarg :vars :reader with-vars)<commit_msg>Syntax fix in print directive registering<commit_after>;;;; command.lisp
;;;;
;;;; This file is part of the cl-closure-template library, released under Lisp-LGPL.
;;;; See file COPYING for details.
;;;;
;;;; Author: <NAME> <<EMAIL>>

(in-package #:closure-template.parser)

;;; simple text

(defun simple-text-char-p (character)
  (not (member character '(#\{ #\} #\Space #\Tab #\Return #\Newline))))

(define-rule simple-text (+ (simple-text-char-p character))
  (:text t))

;;;; commment

(define-rule simple-comment (and  "//" (*  (and (! #\Newline) character)) whitespace)
  (:lambda (list)
    (make-instance 'comment
                   :text (text list))))

(define-rule multiline-comment (and  "/*" (*  (and (! "*/") character)) "*/" (? whitespace))
  (:lambda (list)
    (make-instance 'comment
                   :text (text list)
                   :multiline t)))

(define-rule comment (or simple-comment multiline-comment))

(defclass comment ()
  ((text :initarg :text :reader comment-text)
   (multiline :initarg :multiline :initform nil :reader comment-multiline-p)))

;;;; substition

(defvar *substitiotn-map* (make-hash-table))

(defmacro define-substitions (name str value)
  `(progn
     (setf (gethash ',name *substitiotn-map*)
           ,(if (characterp  value)
                (string value)
                value))
     (define-rule ,name (and (? whitespace) ,str (? whitespace))
       (:constant ',name))))

(define-substitions space-tag "{sp}" #\Space)
(define-substitions emptry-string "{nil}" "")
(define-substitions carriage-return "{\\r}" #\Return)
(define-substitions line-feed "{\\n}" #\Newline)
(define-substitions tag "{\\t}" #\Tab)
(define-substitions left-brace "{lb}" #\{)
(define-substitions right-brace "{rb}" #\})

(define-rule substition (or space-tag emptry-string carriage-return line-feed tag left-brace right-brace))

;;;; literag command

(define-rule literal (and "{literal}" (*  (and (! "{/literal}") character)) "{/literal}")
  (:destructure (start any end)
    (declare (ignore start end))
    (make-instance 'literal
                   :content (text any))))

(defclass literal ()
  ((content :initarg :content :reader literal-content)))

;;;; print

(defmacro define-print-directive (name value)
  `(define-rule ,name (and (? whitespace) "|" (? whitespace) ,value (? whitespace))
     (:constant '(,(lispify-name value) t))))

(define-print-directive no-autoescape-d "noAutoescape")
(define-print-directive id-d "id")
(define-print-directive escape-html-d "escapeHtml")
(define-print-directive escape-uri-d "escapeUri")
(define-print-directive escape-js-d "escapeJs")

(define-rule insert-word-breaks-d (and (? whitespace) "|" (? whitespace) "insertWordBreaks:" decimal-integer (? whitespace))
  (:destructure (w1 d w2 i value w3)
    (declare (ignore w1 d w2 i w3))
    (list :insert-word-breaks value)))
  
(define-rule print-directive (or no-autoescape-d id-d escape-html-d escape-uri-d escape-js-d insert-word-breaks-d))

(define-rule print-command (and (or "{print " "{")  expression (* print-directive) "}")
  (:destructure (start expr directives end)
    (declare (ignore start end))
    (make-instance 'print-command
                   :expr expr
                   :directives (iter (for (key value) in directives)
                                     (collect key)
                                     (collect value)))))
(defclass print-command ()
  ((expr :initarg :expr :reader print-expression)
   (directives :initarg :directives :reader print-directives)))

(defmacro wrap-user-print-directive (d-symbol-cl-templ d-symbol-orig)
  `(define-rule ,d-symbol-cl-templ (and (? whitespace) "|" (? whitespace) ,d-symbol-orig (? whitespace))
     (:destructure (w1 d w2 d-params w3)
                   (declare (ignore w1 d w2 w3))
                   (list ',d-symbol-cl-templ d-params))))

(defmacro register-print-directive (d-symbol d-handler) ;; TODO user will have to specify the result list transformation according to backend type
  (alexandria:with-gensyms (rl-expr)
    (let ((local-d-symbol (intern (symbol-name (gensym (symbol-name d-symbol))))))
      `(progn
         (wrap-user-print-directive ,local-d-symbol ,d-symbol)
         (with-closure-template-rules
           (let ((,rl-expr (rule-expression (find-rule 'print-directive))))
             (change-rule 'print-directive (append ,rl-expr (list ',local-d-symbol)))))
         (setf (gethash ',local-d-symbol *user-print-directives*) ,d-handler)))))

;;; witch

(define-rule with-variable (and whitespace  simple-name "=\"" expression #\")
  (:destructure (w var e expr q)
    (declare (ignore w e q))
    (list (make-instance 'var
                         :name (lispify-name var)
                         :jsname var)
          expr)))
                   
(define-rule with (and "{with" (+ with-variable) (? whitespace) "}" (? code-block) "{/with}")
  (:destructure (start vars w rb code end)
    (declare (ignore start w rb end))
    (make-instance 'with
                   :vars vars
                   :body code)))

(defclass with ()
  ((vars :initarg :vars :reader with-vars)
{'fraction_non_alphanumeric': 0.16588096641131408, 'fraction_numerical': 0.002357100766057749, 'mean_word_length': 2.956099456099456, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '736687', 'n_tokens_mistral': 3350, 'n_tokens_neox': 3234, 'n_words': 855}