<commit_before>#lang racket/base

;                      /\ \__                /\ \
;  _ __    __      ____\ \ ,_\    __      ___\ \ \___      __
; /\`'__\/'__`\   /',__\\ \ \/  /'__`\   /'___\ \  _ `\  /'__`\
; \ \ \//\ \L\.\_/\__, `\\ \ \_/\ \L\.\_/\ \__/\ \ \ \ \/\  __/
;  \ \_\\ \__/.\_\/\____/ \ \__\ \__/.\_\ \____\\ \_\ \_\ \____\
;   \/_/ \/__/\/_/\/___/   \/__/\/__/\/_/\/____/ \/_/\/_/\/____/
; Mustache template engine for Racket

; Set Delimiter tags are used to change the tag delimiters for all
; content following the tag in the current compilation unit.
;
; The tag's content MUST be any two non-whitespace sequences
; (separated by whitespace) EXCEPT an equals sign ('=') followed by
; the current closing delimiter.
;
; Set Delimiter tags SHOULD be treated as standalone when appropriate.

(require rackunit
         rackunit/text-ui
         "../scanner.rkt"
         "rastache-test-case.rkt")

(define delimiters-tests
  (test-suite
   "Delimiters tests"

   (rast-t-case "Pair Behavior"
                #hash{(text . "Hey!")}
                "{{=<% %>=}}(<%text%>)"
                "(Hey!)"
                (list (token 'static "" null)
                      (token 'static "(" null)
                      (token 'etag 'text null)
                      (token 'static ")" null))
                "The equals sign (used on both sides) should permit delimiter changes.")

   (rast-t-case "Special Characters"
                #hash{(text . "It worked!")}
                "({{=[ ]=}}[text])"
                "(It worked!)"
                (list (token 'static "(" null)
                      (token 'etag 'text null)
                      (token 'static ")" null))
                "Characters with special meaning regexen should be valid delimiters.")

#;
   (rast-t-case "Sections"
                #hash{(section . #t)
                      (data . "I got interpolated.")}
                "[
                 {{#section}}
                   {{data}}
                   |data|
                 {{/section}}

                 {{= | | =}}
                 |#section|
                   {{data}}
                   |data|
                 |/section|
                 ]"
                "[
                   I got interpolated.
                   |data|

                   {{data}}
                   I got interpolated.
                 ]"
                "Delimiters set outside sections should persist.")

#;
   (rast-t-case "Inverted Sections"
                #hash{(section . #f)
                      (data . "I got interpolated.")}
                "[
                 {{^section}}
                   {{data}}
                   |data|
                 {{/section}}

                 {{= | | =}}
                 |^section|
                   {{data}}
                   |data|
                 |/section|
                 ]"
                "[
                   I got interpolated.
                   |data|

                   {{data}}
                   I got interpolated.
                 ]"
                "Delimiters set outside inverted sections should persist.")
<commit_msg>Add tocken-mock for the section part of delimiters test<commit_after>#lang racket/base

;                      /\ \__                /\ \
;  _ __    __      ____\ \ ,_\    __      ___\ \ \___      __
; /\`'__\/'__`\   /',__\\ \ \/  /'__`\   /'___\ \  _ `\  /'__`\
; \ \ \//\ \L\.\_/\__, `\\ \ \_/\ \L\.\_/\ \__/\ \ \ \ \/\  __/
;  \ \_\\ \__/.\_\/\____/ \ \__\ \__/.\_\ \____\\ \_\ \_\ \____\
;   \/_/ \/__/\/_/\/___/   \/__/\/__/\/_/\/____/ \/_/\/_/\/____/
; Mustache template engine for Racket

; Set Delimiter tags are used to change the tag delimiters for all
; content following the tag in the current compilation unit.
;
; The tag's content MUST be any two non-whitespace sequences
; (separated by whitespace) EXCEPT an equals sign ('=') followed by
; the current closing delimiter.
;
; Set Delimiter tags SHOULD be treated as standalone when appropriate.

(require rackunit
         rackunit/text-ui
         "../scanner.rkt"
         "rastache-test-case.rkt")

(define delimiters-tests
  (test-suite
   "Delimiters tests"

   (rast-t-case "Pair Behavior"
                #hash{(text . "Hey!")}
                "{{=<% %>=}}(<%text%>)"
                "(Hey!)"
                (list (token 'static "" null)
                      (token 'static "(" null)
                      (token 'etag 'text null)
                      (token 'static ")" null))
                "The equals sign (used on both sides) should permit delimiter changes.")

   (rast-t-case "Special Characters"
                #hash{(text . "It worked!")}
                "({{=[ ]=}}[text])"
                "(It worked!)"
                (list (token 'static "(" null)
                      (token 'etag 'text null)
                      (token 'static ")" null))
                "Characters with special meaning regexen should be valid delimiters.")

   (rast-t-case "Sections"
                #hash{(section . #t)
                      (data . "I got interpolated.")}
                "[
                 {{#section}}
                   {{data}}
                   |data|
                 {{/section}}

                 {{=| |=}}
                 |#section|
                   {{data}}
                   |data|
                 |/section|
                 ]"
                "[
                   I got interpolated.
                   |data|

                   {{data}}
                   I got interpolated.
                 ]"
                (list
                 (token 'static "[\n" null)
                 (token 'section
                        'section
                        (list
                         (token 'static "                   " null)
                         (token 'etag 'data null)
                         (token 'static "\n" null)
                         (token 'static "                   |data|\n" null)))
                 (token 'static "\n" null)
                 (token 'section
                        'section
                        (list
                         (token 'static "                   {{data}}\n" null)
                         (token 'static "                   " null)
                         (token 'etag 'data null)
                         (token 'static "\n" null)))
                 (token 'static "                 ]" null))
                "Delimiters set outside sections should persist.")

   (rast-t-case "Inverted Sections"
                #hash{(section . #f)
                      (data . "I got interpolated.")}
                "[
                 {{^section}}
                   {{data}}
                   |data|
                 {{/section}}

                 {{=| |=}}
                 |^section|
                   {{data}}
                   |data|
                 |/section|
                 ]"
                "[
                   I got interpolated.
                   |data|

                   {{data}}
                   I got interpolated.
                 ]"
                (list
                 (token 'static "[\n" null)
                 (token 'inverted-section
                        'section
                        (list
                         (token 'static "                   " null)
                         (token 'etag 'data null)
                         (token 'static "\n" null)
                         (token 'static "                   |data|\n" null)))
                 (token 'static "\n" null)
                 (token 'inverted-section
                        'section
                        (list
                         (token 'static "                   {{data}}\n" null)
                         (token 'static "                   " null)
                         (token 'etag 'data null)
                         (token 'static "\n" null)))
                 (token 'static "                 ]" null))
                "Delimiters set outside inverted sections should persist.")

{'fraction_non_alphanumeric': 0.13750313361744798, 'fraction_numerical': 0.0, 'mean_word_length': 0.8756464503996239, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2445463', 'n_tokens_mistral': 2198, 'n_tokens_neox': 1980, 'n_words': 498}