<commit_before>(define (display-token token)
"
  This is used as a place holder.
"
  (if (lexical-token? token)
    (format #t
      "lexical-category = ~a, lexical-source = ~a, lexical-value = ~a\n"  (lexical-token-category token)
      (lexical-token-source token)
      (lexical-token-value token)
    )
    (begin
      ;(display "passing on \n")
      token)
  )
)

(define (get-source-location port column)
"
  This returns a record of the source location. It doesn't correct for
  whitespaces that may get trimmed off.
"
  (make-source-location
    (port-filename port)
    (port-line port)
    column
    (false-if-exception (ftell port))
    #f
  )
)

(define (show-location loc)
  (format #f "line = ~a & column = ~a"
    (source-location-line loc)
    (source-location-column loc)
  )
)

(define (tokeniz str location)
  (define current-match '())
  (define (has-match? pattern str)
    (let ((match
            (string-match
              (string-append "^[ \t]*" pattern "[ \t]*") str)))
      (if match
        (begin (set! current-match match) #t)
        #f
      )))

  (define (result:suffix token location value)
    ; Returns a lexical-token
    (cons
      (make-lexical-token token location value)
      (match:suffix current-match)))

  (define (command-pair)
    ; Return command string.
    (let ((match (string-match "~" (match:substring current-match))))
      ; The suffix is the command and the prefix is the argument
      (cons (match:suffix match) (string-trim-both (match:prefix match)))
    ))

  ; NOTE
  ; 1. The matching must be done starting from the most specific to the
  ;    the broadest regex patterns.
  ; 2. #f is given as a token value for those cases which don't have any
  ;    semantic values.
  (cond
    ((has-match? "\\(" str) (result:suffix 'LPAREN location #f))
    ((has-match? "\\)" str) (result:suffix 'RPAREN location #f))
    ; Chatscript declarations
    ((has-match? "concept:" str) (result:suffix 'CONCEPT location #f))
    ; The token value for 'CR is empty-string so as to handle multiline
    ; rules.
    ((has-match? "\r" str) (result:suffix 'CR location ""))
    ; FIXME: This is not really newline.
    ((string=? "" str) (cons (make-lexical-token 'NEWLINE location #f) ""))
    ((has-match? "urge:" str) (result:suffix 'URGE location #f))
    ((has-match? "ordered-goal:" str) (result:suffix 'ORD-GOAL location #f))
    ((has-match? "goal:" str) (result:suffix 'GOAL location #f))
    ((has-match? "#goal:" str) (result:suffix 'RGOAL location #f))
    ((has-match? "parallel-rules:" str)
      (result:suffix 'PARALLEL-RULES location #f))
    ((has-match? "link-concept:" str)
      (result:suffix 'LINK-CONCEPT location #f))
    ((has-match? "#link-concept:" str)
      (result:suffix 'RLINK-CONCEPT location #f))
    ((has-match? "#!" str) ; This should be checked always before #
      ; TODO Add tester function for this
      (cons (make-lexical-token 'SAMPLE_INPUT location #f) ""))
    ((has-match? "*#" str)
      (cons (make-lexical-token 'COMMENT location #f) ""))
    ; Reactive rules (aka Responders)
    ; Starts with "r:"; "s:", "?:" and "u:" are for backward compatibility
    ((has-match? "[rs?u]:" str)
      (result:suffix 'REACTIVE-RULES location
        (car (string->list (substring (string-trim-both
          (match:substring current-match)) 0 1)))))
    ((has-match? "[a-q]:" str)
      (result:suffix 'REJOINDERS location<commit_msg>Change the rejoinder label as well<commit_after>(define (display-token token)
"
  This is used as a place holder.
"
  (if (lexical-token? token)
    (format #t
      "lexical-category = ~a, lexical-source = ~a, lexical-value = ~a\n"  (lexical-token-category token)
      (lexical-token-source token)
      (lexical-token-value token)
    )
    (begin
      ;(display "passing on \n")
      token)
  )
)

(define (get-source-location port column)
"
  This returns a record of the source location. It doesn't correct for
  whitespaces that may get trimmed off.
"
  (make-source-location
    (port-filename port)
    (port-line port)
    column
    (false-if-exception (ftell port))
    #f
  )
)

(define (show-location loc)
  (format #f "line = ~a & column = ~a"
    (source-location-line loc)
    (source-location-column loc)
  )
)

(define (tokeniz str location)
  (define current-match '())
  (define (has-match? pattern str)
    (let ((match
            (string-match
              (string-append "^[ \t]*" pattern "[ \t]*") str)))
      (if match
        (begin (set! current-match match) #t)
        #f
      )))

  (define (result:suffix token location value)
    ; Returns a lexical-token
    (cons
      (make-lexical-token token location value)
      (match:suffix current-match)))

  (define (command-pair)
    ; Return command string.
    (let ((match (string-match "~" (match:substring current-match))))
      ; The suffix is the command and the prefix is the argument
      (cons (match:suffix match) (string-trim-both (match:prefix match)))
    ))

  ; NOTE
  ; 1. The matching must be done starting from the most specific to the
  ;    the broadest regex patterns.
  ; 2. #f is given as a token value for those cases which don't have any
  ;    semantic values.
  (cond
    ((has-match? "\\(" str) (result:suffix 'LPAREN location #f))
    ((has-match? "\\)" str) (result:suffix 'RPAREN location #f))
    ; Chatscript declarations
    ((has-match? "concept:" str) (result:suffix 'CONCEPT location #f))
    ; The token value for 'CR is empty-string so as to handle multiline
    ; rules.
    ((has-match? "\r" str) (result:suffix 'CR location ""))
    ; FIXME: This is not really newline.
    ((string=? "" str) (cons (make-lexical-token 'NEWLINE location #f) ""))
    ((has-match? "urge:" str) (result:suffix 'URGE location #f))
    ((has-match? "ordered-goal:" str) (result:suffix 'ORD-GOAL location #f))
    ((has-match? "goal:" str) (result:suffix 'GOAL location #f))
    ((has-match? "#goal:" str) (result:suffix 'RGOAL location #f))
    ((has-match? "parallel-rules:" str)
      (result:suffix 'PARALLEL-RULES location #f))
    ((has-match? "link-concept:" str)
      (result:suffix 'LINK-CONCEPT location #f))
    ((has-match? "#link-concept:" str)
      (result:suffix 'RLINK-CONCEPT location #f))
    ((has-match? "#!" str) ; This should be checked always before #
      ; TODO Add tester function for this
      (cons (make-lexical-token 'SAMPLE_INPUT location #f) ""))
    ((has-match? "*#" str)
      (cons (make-lexical-token 'COMMENT location #f) ""))
    ; Reactive rules (aka Responders)
    ; Starts with "r:"; "s:", "?:" and "u:" are for backward compatibility
    ((has-match? "[rs?u]:" str)
      (result:suffix 'REACTIVE-RULES location
        (car (string->list (substring (string-trim-both
          (match:substring current-match)) 0 1)))))
    ; Rejoinders
    ; Starts with "j" follow by a number to denote the level
    ((has-match? "j[0-9]+:" str)
      (result:suffix 'REJOINDERS location
        (car (string->list (substring (string-trim-both
          (match:substring current-match)) 0 1)))))
    ; Rejoinders again, for backward compatibility, but limit the support
    ; up to the letter "e" instead of "q" as in ChatScript
    ((has-match? "[a-e]:" str)
      (result:suffix 'REJOINDERS location
{'fraction_non_alphanumeric': 0.1505982670884335, 'fraction_numerical': 0.001650391968092422, 'mean_word_length': 2.96943231441048, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6058603', 'n_tokens_mistral': 2433, 'n_tokens_neox': 2349, 'n_words': 738}