<commit_before>#lang racket/base
(require parser-tools/lex
         (prefix-in : parser-tools/lex-sre)
         "../private/lex.rkt")

(provide get-syntax-token)

(define (syn-val lex a b c d)
  (values lex a b (position-offset c) (position-offset d)))

(define (colorize-string my-start-pos)
  (define lxr
    (lexer
     [(:~ #\" #\\) (lxr input-port)]
     [(:: #\\ #\\) (lxr input-port)]
     [(:: #\\ #\newline) (lxr input-port)]
     [(:: #\\ #\") (lxr input-port)]
     [(eof) (syn-val "" 'error #f my-start-pos end-pos)]
     [#\" (syn-val "" 'string #f my-start-pos end-pos)]))
  lxr)

(define get-syntax-token
  (lexer
   [(:+ whitespace)
    (syn-val lexeme 'whitespace #f start-pos end-pos)]
   [comment-re
    (syn-val lexeme 'comment #f start-pos end-pos)]
   [variable-re
    (syn-val lexeme 'symbol #f start-pos end-pos)]
   [identifier-re
    (syn-val lexeme 'identifier #f start-pos end-pos)]
   [(:or #\) #\() (syn-val lexeme 'parenthesis #f start-pos end-pos)]
   [(:or "!=" #\= #\? #\~ #\. #\, ":-") (syn-val lexeme 'parenthesis #f start-pos end-pos)]
   [(eof) (syn-val lexeme 'eof #f start-pos end-pos)]
   [#\" ((colorize-string start-pos) input-port)]
   [any-char (syn-val lexeme 'error #f start-pos end-pos)]))<commit_msg>fix datalog colorer for specials<commit_after>#lang racket/base
(require parser-tools/lex
         (prefix-in : parser-tools/lex-sre)
         "../private/lex.rkt")

(provide get-syntax-token)

(define (syn-val lex a b c d)
  (values lex a b (position-offset c) (position-offset d)))

(define (colorize-string my-start-pos)
  (define lxr
    (lexer
     [(:~ #\" #\\) (lxr input-port)]
     [(:: #\\ #\\) (lxr input-port)]
     [(:: #\\ #\newline) (lxr input-port)]
     [(:: #\\ #\") (lxr input-port)]
     [(eof) (syn-val "" 'error #f my-start-pos end-pos)]
     [#\" (syn-val "" 'string #f my-start-pos end-pos)]))
  lxr)

(define get-syntax-token
  (lexer
   [(:+ whitespace)
    (syn-val lexeme 'whitespace #f start-pos end-pos)]
   [comment-re
    (syn-val lexeme 'comment #f start-pos end-pos)]
   [variable-re
    (syn-val lexeme 'symbol #f start-pos end-pos)]
   [identifier-re
    (syn-val lexeme 'identifier #f start-pos end-pos)]
   [(:or #\) #\() (syn-val lexeme 'parenthesis #f start-pos end-pos)]
   [(:or "!=" #\= #\? #\~ #\. #\, ":-") (syn-val lexeme 'parenthesis #f start-pos end-pos)]
   [(eof) (syn-val lexeme 'eof #f start-pos end-pos)]
   [#\" ((colorize-string start-pos) input-port)]
   [any-char (syn-val lexeme 'error #f start-pos end-pos)]
   [(special) (syn-val lexeme 'error #f start-pos end-pos)]
   [(special-comment) (syn-val lexeme 'error #f start-pos end-pos)]))
{'fraction_non_alphanumeric': 0.2158790170132325, 'fraction_numerical': 0.0, 'mean_word_length': 2.78, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1667644', 'n_tokens_mistral': 1093, 'n_tokens_neox': 1036, 'n_words': 243}