<commit_before>(in-package #:djula)

(defun get-closing-delimiter (type)
  (ecase type
    (:comment "#}")
    (:unparsed-variable "}}")
    (:unparsed-translation-variable "_}")
    (:unparsed-tag "%}")))

(defun split-template-string (string start)
  (let (({ (position #\{ string :start start :test 'char=)))
    (if (null {)
	(if (< start (length string))
	    (list `(:string ,(subseq string start))))
	(if (> { start)
	    (cons (list :string (subseq string start {))
		  (split-template-string string {))
	    (let* ((next (char string (1+ {)))
		   (type (case next
			   (#\# :comment)
			   (#\{ :unparsed-variable)
			   (#\_ :unparsed-translation-variable)
			   (#\% :unparsed-tag)
			   (otherwise :not-special))))
	      (ecase type
		((:comment :unparsed-variable :unparsed-translation-variable :unparsed-tag)
		 (let ((end (search (get-closing-delimiter type)
				    string
				    :start2 (1+ {))))
		   (if (null end)
		       (list (list :string (subseq string start)))
		       (cons (list type (subseq string (+ 2 {) end))
			     (split-template-string string (+ 2 end))))))
		(:not-special (cons `(:string "{") (split-template-string string (1+ start))))))))))

(defun parse-template-string (string)
  (split-template-string string 0))<commit_msg>Verbatim syntax<commit_after>(in-package #:djula)

(defun get-closing-delimiter (type)
  (ecase type
    (:comment "#}")
    (:unparsed-variable "}}")
    (:unparsed-translation-variable "_}")
    (:unparsed-tag "%}")
    (:verbatim "$}")))

(defun split-template-string (string start)
  (let (({ (position #\{ string :start start :test 'char=)))
    (if (null {)
	(if (< start (length string))
	    (list `(:string ,(subseq string start))))
	(if (> { start)
	    (cons (list :string (subseq string start {))
		  (split-template-string string {))
	    (let* ((next (char string (1+ {)))
		   (type (case next
			   (#\# :comment)
			   (#\{ :unparsed-variable)
			   (#\_ :unparsed-translation-variable)
			   (#\% :unparsed-tag)
			   (#\$ :verbatim)
			   (otherwise :not-special))))
	      (ecase type
		((:comment 
		  :unparsed-variable 
		  :unparsed-translation-variable 
		  :unparsed-tag
		  :verbatim)
		 (let ((end (search (get-closing-delimiter type)
				    string
				    :start2 (1+ {))))
		   (if (null end)
		       (list (list :string (subseq string start)))
		       (cons (list type (subseq string (+ 2 {) end))
			     (split-template-string string (+ 2 end))))))
		(:not-special (cons `(:string "{") (split-template-string string (1+ start))))))))))

(def-token-compiler :verbatim (string)
  (lambda (stream)
    (write-string string stream)))

(defun parse-template-string (string)
  (split-template-string string 0))
{'fraction_non_alphanumeric': 0.19174041297935104, 'fraction_numerical': 0.005162241887905605, 'mean_word_length': 3.0553064275037367, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4373309', 'n_tokens_mistral': 1017, 'n_tokens_neox': 894, 'n_words': 219}