<commit_before>#+TITLE: Bug Tracker
#+AUTHOR: Rl3<NAME> ⚓

* Bug tracker
** Scanner
*** Standalone tag
    Standalone tag should be removed from the template:
    - [[file:comments.rkt::47]]
      #+BEGIN_SRC racket
      (rast-t-case "Standalone"
               #hash()
               "Begin.
                {{! Comment Block! }}
                End."
               "Begin.
                End."
               (list (token 'static "Begin.\n" null)
                     (token 'static "                 End."  null))
               "All standalone comment lines should be removed.")
      #+END_SRC
    - [[file:comments.rkt::58]]
      #+BEGIN_SRC racket
      (rast-t-case "Indented Standalone"
                  #hash()
                  "Begin.
                     {{! Comment Block! }}
                   End."
                  "Begin.
                   End."
                  (list (token 'static "Begin.\n" null)
                        (token 'static "                 End."  null))
                  "All standalone comment lines should be removed.")
      #+END_SRC
    - [[file:comments.rkt::102]]
      #+BEGIN_SRC racket
      (rast-t-case "Multiline Standalone"
                  #hash()
                  "Begin.
                   {{!
                   Something's going on here...
                   }}
                   End."
                  "Begin.
                   End."
                  (list (token 'static "Begin.\n" null)
                        (token 'static "                 End."  null))
                  "All standalone comment lines should be removed.")
      #+END_SRC

    Tag is not treated as standalone if a non blank char happens
    before :
    - [[file:comments.rkt::128]]
      #+BEGIN_SRC racket
      (rast-t-case "Indented Inline"
                #hash()
                "  12 {{! 34 }}\n"
                "  12 \n"
                (list (token 'static "  12 " null)
                      (token 'static "\n" null))
                "Inline comments should not strip whitespace")
      #+END_SRC
    - [[file:sections.rkt::492]]
      #+BEGIN_SRC racket
      (rast-t-case "Standalone Without Newline"
                #hash{( boolean . #t )}
                "#{{#boolean}}\n/\n  {{/boolean}}"
                ; Template should be considered as
                ; "#{{#boolean}}↩
                ;  /↩
                ;  {{/boolean}}
                "#\n/\n"
                "Standalone tags should not require a newline to follow them.")
      #+END_SRC
    - [[file:sections.rkt::480]]
      #+BEGIN_SRC racket
      (rast-t-case "Standalone Without Previous Line"
                #hash{( boolean . #t )}
                "  {{#boolean}}\n#{{/boolean}}\n/"
                "#\n/"
                ; Template should be considered as:
                ; "  {{#boolean}}
                ;  #{{/boolean}}↩
                ;  /"
                (list (token 'static "" null)
                      (token 'section 'boolean
                             (list (token 'static "#" null)))
                      (token 'static "\n/" null))
                "Standalone tags should not require a newline to precede them.")
      #+END_SRC
    - [[file:inverted.rkt::335]]
      #+BEGIN_SRC racket
      (rast-t-case "Standalone Without Newline"
                #hash{( boolean . #f )}
                "^{{^boolean}}\n/\n  {{/boolean}}"
                "^\n/\n"
                ; Should be considered as:
                ; "^{{^boolean}}↩
                ;  /↩
                ;    {{/boolean}}"
                (list (token 'static "^" null)
                      (token 'inverted-section
                             'boolean
                             (list (token 'static "\n/\n" null)))
                      (token 'static "" null))
                "Standalone tags should not require a newline to follow them.")
      #+END_SRC
    - [[file:inverted.rkt::320]]
      #+BEGIN_SRC racket
      (rast-t-case "Standalone Without Previous Line"
                #hash{( boolean . #f )}
                "  {{^boolean}}\n^{{/boolean}}\n/"
                "^\n/"
                ; Should be considered as:
                ; "  {{^boolean}}
                ; ^{{/boolean}}↩
                ; /"
                (list (token 'static "" null)
                      (token 'inverted-section
                             'boolean
                             (list (token 'static "^" null)
                                   (token 'static "\n" null)))
                      (token 'static "/" null))
                "Standalone tags should not require a newline to precede them.")
      #+END_SRC

    Standalone interpolation tag should not be removed from the
    template:
    - [[file:interpolation.rkt::309]]
      #+BEGIN_SRC racket
      (rast-t-case "Interpolation - Standalone"
                #hash{(string . "---")}
                "  {{string}}\n"
                "  ---\n"
                (list (token 'static "  " null)
                      (token 'etag 'string null)
                      (token 'static "\n" null))
                "Standalone interpolation should not alter surrounding whitespace.")
      #+END_SRC
    - [[file:interpolation.rkt::318]]
      #+BEGIN_SRC racket
      (rast-t-case "Triple Mustache - Standalone"
                #hash{(string . "---")}
                "  {{{string}}}\n"
                "  ---\n"
                (list (token 'static "  " null)
                      (token 'utag 'string null)
                      (token 'static "\n" null))
                "Standalone interpolation should not alter surrounding whitespace.")
      #+END_SRC
    - [[file:interpolation.rkt::327]]
      #+BEGIN_SRC racket
      (rast-t-case "Ampersand - Standalone"
                #hash{(string . "---")}
                "  {{&string}}\n"
                "  ---\n"
                (list (token 'static "  " null)
                      (token 'utag 'string null)
                      (token 'static "\n" null))
                "Standalone interpolation should not alter surrounding whitespace.")
      #+END_SRC
<commit_msg>Discussion for the scan of standalone tag<commit_after>#+AUTHOR: Rl3x Fish ⚓

* Bug tracker
** Scanner
*** Standalone tag
Standalone tag should be removed from the template:
- [[file:comments.rkt::47]]
  #+BEGIN_SRC racket
  (rast-t-case "Standalone"
           #hash()
           "Begin.
            {{! Comment Block! }}
            End."
           "Begin.
            End."
           (list (token 'static "Begin.\n" null)
                 (token 'static "                 End."  null))
           "All standalone comment lines should be removed.")
  #+END_SRC
- [[file:comments.rkt::58]]
  #+BEGIN_SRC racket
  (rast-t-case "Indented Standalone"
              #hash()
              "Begin.
                 {{! Comment Block! }}
               End."
              "Begin.
               End."
              (list (token 'static "Begin.\n" null)
                    (token 'static "                 End."  null))
              "All standalone comment lines should be removed.")
  #+END_SRC
- [[file:comments.rkt::102]]
  #+BEGIN_SRC racket
  (rast-t-case "Multiline Standalone"
              #hash()
              "Begin.
               {{!
               Something's going on here...
               }}
               End."
              "Begin.
               End."
              (list (token 'static "Begin.\n" null)
                    (token 'static "                 End."  null))
              "All standalone comment lines should be removed.")
  #+END_SRC

Tag is not treated as standalone if a non blank char happens
around it:
- [[file:comments.rkt::128]]
  #+BEGIN_SRC racket
  (rast-t-case "Indented Inline"
            #hash()
            "  12 {{! 34 }}\n"
            "  12 \n"
            (list (token 'static "  12 " null)
                  (token 'static "\n" null))
            "Inline comments should not strip whitespace")
  #+END_SRC
- [[file:sections.rkt::492]]
  #+BEGIN_SRC racket
  (rast-t-case "Standalone Without Newline"
            #hash{( boolean . #t )}
            "#{{#boolean}}\n/\n  {{/boolean}}"
            ; Template should be considered as
            ; "#{{#boolean}}↩
            ;  /↩
            ;  {{/boolean}}
            "#\n/\n"
            "Standalone tags should not require a newline to follow them.")
  #+END_SRC
- [[file:sections.rkt::480]]
  #+BEGIN_SRC racket
  (rast-t-case "Standalone Without Previous Line"
            #hash{( boolean . #t )}
            "  {{#boolean}}\n#{{/boolean}}\n/"
            "#\n/"
            ; Template should be considered as:
            ; "  {{#boolean}}
            ;  #{{/boolean}}↩
            ;  /"
            (list (token 'static "" null)
                  (token 'section 'boolean
                         (list (token 'static "#" null)))
                  (token 'static "\n/" null))
            "Standalone tags should not require a newline to precede them.")
  #+END_SRC
- [[file:inverted.rkt::335]]
  #+BEGIN_SRC racket
  (rast-t-case "Standalone Without Newline"
            #hash{( boolean . #f )}
            "^{{^boolean}}\n/\n  {{/boolean}}"
            "^\n/\n"
            ; Should be considered as:
            ; "^{{^boolean}}↩
            ;  /↩
            ;    {{/boolean}}"
            (list (token 'static "^" null)
                  (token 'inverted-section
                         'boolean
                         (list (token 'static "\n/\n" null)))
                  (token 'static "" null))
            "Standalone tags should not require a newline to follow them.")
  #+END_SRC
- [[file:inverted.rkt::320]]
  #+BEGIN_SRC racket
  (rast-t-case "Standalone Without Previous Line"
            #hash{( boolean . #f )}
            "  {{^boolean}}\n^{{/boolean}}\n/"
            "^\n/"
            ; Should be considered as:
            ; "  {{^boolean}}
            ; ^{{/boolean}}↩
            ; /"
            (list (token 'static "" null)
                  (token 'inverted-section
                         'boolean
                         (list (token 'static "^" null)
                               (token 'static "\n" null)))
                  (token 'static "/" null))
            "Standalone tags should not require a newline to precede them.")
  #+END_SRC

Standalone interpolation tag should not be removed from the
template:
- [[file:interpolation.rkt::309]]
  #+BEGIN_SRC racket
  (rast-t-case "Interpolation - Standalone"
            #hash{(string . "---")}
            "  {{string}}\n"
            "  ---\n"
            (list (token 'static "  " null)
                  (token 'etag 'string null)
                  (token 'static "\n" null))
            "Standalone interpolation should not alter surrounding whitespace.")
  #+END_SRC
- [[file:interpolation.rkt::318]]
  #+BEGIN_SRC racket
  (rast-t-case "Triple Mustache - Standalone"
            #hash{(string . "---")}
            "  {{{string}}}\n"
            "  ---\n"
            (list (token 'static "  " null)
                  (token 'utag 'string null)
                  (token 'static "\n" null))
            "Standalone interpolation should not alter surrounding whitespace.")
  #+END_SRC
- [[file:interpolation.rkt::327]]
  #+BEGIN_SRC racket
  (rast-t-case "Ampersand - Standalone"
            #hash{(string . "---")}
            "  {{&string}}\n"
            "  ---\n"
            (list (token 'static "  " null)
                  (token 'utag 'string null)
                  (token 'static "\n" null))
            "Standalone interpolation should not alter surrounding whitespace.")
  #+END_SRC

Solution is while reading a line, test if the line is a standalone
one:
#+BEGIN_SRC racket
;; Match plain standalone tag, e.g., "  {{! 88 mph }}  "
#px"^\\s*\\{\\{(!|#|\\^|/|>|=)\\s*.*?\\}\\}?\\s*"
;; Match partial standalone tag, e.g., "  {{! 88 mph "
#px"^\\s*\\{\\{(!|#|\\^|/|>|=)\\s*[^(\\}\\})]*"
;; Match plain and partial standalone tag
#px"^\\s*\\{\\{(!|#|\\^|/|>|=)\\s*(.*?\\}\\}\\s*|[^(\\}\\})]*)"

;; Regexp pattern for standalone tag.
(define standalone-pattern
  (pregexp
     (string-append "^\\s*"
                    "\\{\\{"
                    "(!|#|\\^|/|>|=)"
                    "\\s*"
                    "("
                      ".*?\\}\\}\\s*"
                    "|"
                      "[^(\\}\\})]*"
                    ")")))

(define (is-standalone? line)
  (regexp-match-exact? standalone-pattern line))

(require rackunit)
(check-true (is-standalone? "{{! Comment Block! }}"))
(check-true (is-standalone? "{{#boolean}}"))
(check-true (is-standalone? "{{/boolean}}"))
(check-true (is-standalone? "  {{! Comment Block! }}"))
(check-true (is-standalone? "  {{#boolean}}"))
(check-true (is-standalone? "  {{/boolean}}"))
(check-true (is-standalone? "{{!"))

(check-false (is-standalone? "/"))
(check-false (is-standalone? "  {{! 12 }}34"))
(check-false (is-standalone? "  12 {{! 34 }}"))
(check-false (is-standalone? "#{{#boolean}}"))
(check-false (is-standalone? "#{{/boolean}}"))
(check-false (is-standalone? "^{{^boolean}}"))
(check-false (is-standalone? "^{{/boolean}}"))
(check-false (is-standalone? "  #{{#boolean}}"))
(check-false (is-standalone? "  #{{/boolean}}"))
(check-false (is-standalone? "  ^{{^boolean}}"))
(check-false (is-standalone? "  ^{{/boolean}}"))
(check-false (is-standalone? "  {{#boolean}}#"))
(check-false (is-standalone? "  {{/boolean}}#"))
(check-false (is-standalone? "  {{^boolean}}^"))
(check-false (is-standalone? "  {{/boolean}}^"))
(check-false (is-standalone? "{{string}}"))
(check-false (is-standalone? "{{{string}}}"))
(check-false (is-standalone? "{{&string}}"))
(check-false (is-standalone? "  {{string}}"))
(check-false (is-standalone? "  {{{string}}}"))
(check-false (is-standalone? "  {{&string}}"))
#+END_SRC

{'fraction_non_alphanumeric': 0.16430800493504608, 'fraction_numerical': 0.006676827055664417, 'mean_word_length': 1.417119803543238, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2187494', 'n_tokens_mistral': 4220, 'n_tokens_neox': 3963, 'n_words': 993}