<commit_before>(module lexer
   (library utf)
   (export *JS-grammar*
	   *care-future-reserved*
	   *Reg-exp-grammar*
	   (reserved-word?::bool symbol::symbol)
	   (token-type token)
	   (token-val token)
	   (token-pos token)))

(define (the-coord input-port)
   (list 'at (input-port-name input-port) (input-port-position input-port)))

(define *care-future-reserved* #t)

(define *keyword-list*
   '("as"
     "break"
     "case"
     "catch"
     "class"
     "const"
     "continue"
     "default"
     "delete"
     "do"
     "else"
     "extends"
     "false"
     "finally"
     "for"
     "function"
     "if"
     "import"
     "in"
     "instanceof"
     "is"
     "namespace"
     "new"
     "null"
     "package"
     "private"
     "public"
     "return"
     "super"
     "switch"
     "this"
     "throw"
     "true"
     "try"
     "typeof"
     "use"
     "var"
     "void"
     "while"
     "with"))

(define *future-reserved-list*
   '("abstract"
     "debugger"
     "enum"
     "export"
     "goto"
     "implements"
     "interface"
     "native"
     "protected"
     "synchronized"
     "throws"
     "transient"
     "volatile"))

(for-each (lambda (word)
	     (putprop! (string->symbol word) 'reserved #t))
	  *keyword-list*)
(for-each (lambda (word)
	     (putprop! (string->symbol word) 'future-reserved #t))
	  *future-reserved-list*)

(define (reserved-word? symbol)
   (or (getprop symbol 'reserved)
       (and *care-future-reserved*
	    (getprop symbol 'future-reserved))))

(define-macro (token type value)
   `(econs ,type ,value (the-coord input-port)))
(define (token-type token) (car token))
(define (token-val token) (cdr token))
(define (token-pos token) (and (epair? token) (cer token)))

(define (unescape-unicode-escape-sequences str)
   (define (surrogate-1st? n) (=fx (bit-and n #xFC00) #xD800))
   (define (surrogate-2nd? n) (=fx (bit-and n #xFC00) #xFC00))
   (define (surrogate? n) (or (surrogate-1st? n) (surrogate-2nd? n)))

   (let ((i (string-contains str "\\u")))
      (if (not i)
	  str
	  (let ((first-part (substring str 0 i))
		(u-part (string->integer (substring str (+fx i 2) (+fx i 6))
					 16))
		(last-part (substring str (+fx i 6) (string-length str))))
	     (cond
		((not (surrogate? u-part))
		 (let* ((buffer (make-string 4))
			(len (utf8-string-set! buffer 0 u-part)))
		    (unescape-unicode-escape-sequences
		     (string-append first-part
				    (string-shrink! buffer len)
				    last-part))))
		((and (surrogate-1st? u-part)
		      (>=fx (string-length str) (+fx i 12))
		      (char=? #\\ (string-ref str (+fx i 6)))
		      (char=? #\u (string-ref str (+fx i 7)))
		      (let* ((sub-str (substring str (+fx i 8) (+fx i 12)))
			     (u-part2 (string->integer sub-str 16)))
			 (and (surrogate-2nd? u-part2)
			      u-part2)))
		 =>
		 (lambda (u-part2)
		    (let ((ucs2-buffer (make-ucs2-string 2))
			  (u1 (unchecked-integer->ucs2 u-part))
			  (u2 (unchecked-integer->ucs2 u-part2)))
		       (ucs2-string-set! ucs2-buffer 0 u1)
		       (ucs2-string-set! ucs2-buffer 1 u2)
		       (unescape-unicode-escape-sequences
			(string-append first-part
				       (utf16-string->utf8-string ucs2-buffer)
				       last-part)))))
		(else #f))))))
   
(define *JS-grammar*
   (utf8-regular-grammar
	 ((source-char (out))
	  (blank_no_lt  (or (in #x9 #xB #xC #x20 #xA0) Zs))
	  (lt           (in #a013 #\Newline #x2028 #x2029))
	  (not-lt-no-single-quote (out #a013 #\Newline #x2028 #x2029 #\'))
	  (not-lt-no-double-quote (out #a013 #\Newline #x2028 #x2029 #\"))
	  (blank        (or blank_no_lt lt))
	  (nonzero-digit   (in ("19")))
	  (unicode-letter (or Lu Ll Lt Lm Lo Nl))
	  (unicode-combining-mark (or Mn Mc))
	  (unicode-digit Nd)
	  (unicode-connector-punctuation Pc)
	  (hex-digit (or (in ("09")) (in ("af")) (in ("AF"))))
	  (unicode-escape-sequence (: #\u hex-digit hex-digit
					  hex-digit hex-digit))
	  (id_start (or unicode-letter #\$ #\_
			(: #\\ unicode-escape-sequence)))
	  (id_part (or id_start unicode-combining-mark unicode-digit
		       unicode-connector-punctuation
		       (: #\\ unicode-escape-sequence))))


      ((+ blank_no_lt)
       (ignore))

      ((: (* blank_no_lt) lt (* blank))
       (token 'NEW_LINE #\newline))
      
      ;; LineComment
      ((:"//" (* source-char))
       (ignore))

      ;; multi-line comment on one line
      ((: "/*" (* (or (out #\*) (: (+ #\*) (out #\/ #\*)))) (+ #\*) "/")
       (ignore))

      ;; multi-line comment with LineTerminators (several lines)
      ((: "/*"
	  (* (or lt
		 (out #\*)
		 (: (+ #\*) (out #\/ #\*))))
	  (+ #\*) "/")
       (token 'NEW_LINE #\newline))

      ;; unfinished multi-line comment (added for repl)
      ((eof (: "/*"
	       (* (or lt
		      (out #\*)
		      (: (+ #\*) (out #\/ #\*))))))<commit_msg>fixed stupid bug that made //-comments go till the end of file.<commit_after>(module lexer
   (library utf)
   (export *JS-grammar*
	   *care-future-reserved*
	   *Reg-exp-grammar*
	   (reserved-word?::bool symbol::symbol)
	   (token-type token)
	   (token-val token)
	   (token-pos token)))

(define (the-coord input-port)
   (list 'at (input-port-name input-port) (input-port-position input-port)))

(define *care-future-reserved* #t)

(define *keyword-list*
   '("as"
     "break"
     "case"
     "catch"
     "class"
     "const"
     "continue"
     "default"
     "delete"
     "do"
     "else"
     "extends"
     "false"
     "finally"
     "for"
     "function"
     "if"
     "import"
     "in"
     "instanceof"
     "is"
     "namespace"
     "new"
     "null"
     "package"
     "private"
     "public"
     "return"
     "super"
     "switch"
     "this"
     "throw"
     "true"
     "try"
     "typeof"
     "use"
     "var"
     "void"
     "while"
     "with"))

(define *future-reserved-list*
   '("abstract"
     "debugger"
     "enum"
     "export"
     "goto"
     "implements"
     "interface"
     "native"
     "protected"
     "synchronized"
     "throws"
     "transient"
     "volatile"))

(for-each (lambda (word)
	     (putprop! (string->symbol word) 'reserved #t))
	  *keyword-list*)
(for-each (lambda (word)
	     (putprop! (string->symbol word) 'future-reserved #t))
	  *future-reserved-list*)

(define (reserved-word? symbol)
   (or (getprop symbol 'reserved)
       (and *care-future-reserved*
	    (getprop symbol 'future-reserved))))

(define-macro (token type value)
   `(econs ,type ,value (the-coord input-port)))
(define (token-type token) (car token))
(define (token-val token) (cdr token))
(define (token-pos token) (and (epair? token) (cer token)))

(define (unescape-unicode-escape-sequences str)
   (define (surrogate-1st? n) (=fx (bit-and n #xFC00) #xD800))
   (define (surrogate-2nd? n) (=fx (bit-and n #xFC00) #xFC00))
   (define (surrogate? n) (or (surrogate-1st? n) (surrogate-2nd? n)))

   (let ((i (string-contains str "\\u")))
      (if (not i)
	  str
	  (let ((first-part (substring str 0 i))
		(u-part (string->integer (substring str (+fx i 2) (+fx i 6))
					 16))
		(last-part (substring str (+fx i 6) (string-length str))))
	     (cond
		((not (surrogate? u-part))
		 (let* ((buffer (make-string 4))
			(len (utf8-string-set! buffer 0 u-part)))
		    (unescape-unicode-escape-sequences
		     (string-append first-part
				    (string-shrink! buffer len)
				    last-part))))
		((and (surrogate-1st? u-part)
		      (>=fx (string-length str) (+fx i 12))
		      (char=? #\\ (string-ref str (+fx i 6)))
		      (char=? #\u (string-ref str (+fx i 7)))
		      (let* ((sub-str (substring str (+fx i 8) (+fx i 12)))
			     (u-part2 (string->integer sub-str 16)))
			 (and (surrogate-2nd? u-part2)
			      u-part2)))
		 =>
		 (lambda (u-part2)
		    (let ((ucs2-buffer (make-ucs2-string 2))
			  (u1 (unchecked-integer->ucs2 u-part))
			  (u2 (unchecked-integer->ucs2 u-part2)))
		       (ucs2-string-set! ucs2-buffer 0 u1)
		       (ucs2-string-set! ucs2-buffer 1 u2)
		       (unescape-unicode-escape-sequences
			(string-append first-part
				       (utf16-string->utf8-string ucs2-buffer)
				       last-part)))))
		(else #f))))))
   
(define *JS-grammar*
   (utf8-regular-grammar
	 ((source-char (out))
	  (blank_no_lt  (or (in #x9 #xB #xC #x20 #xA0) Zs))
	  (lt           (in  #a013 #\Newline #x2028 #x2029))
	  (not-lt       (out #a013 #\Newline #x2028 #x2029))
	  (not-lt-no-single-quote (out #a013 #\Newline #x2028 #x2029 #\'))
	  (not-lt-no-double-quote (out #a013 #\Newline #x2028 #x2029 #\"))
	  (blank        (or blank_no_lt lt))
	  (nonzero-digit   (in ("19")))
	  (unicode-letter (or Lu Ll Lt Lm Lo Nl))
	  (unicode-combining-mark (or Mn Mc))
	  (unicode-digit Nd)
	  (unicode-connector-punctuation Pc)
	  (hex-digit (or (in ("09")) (in ("af")) (in ("AF"))))
	  (unicode-escape-sequence (: #\u hex-digit hex-digit
					  hex-digit hex-digit))
	  (id_start (or unicode-letter #\$ #\_
			(: #\\ unicode-escape-sequence)))
	  (id_part (or id_start unicode-combining-mark unicode-digit
		       unicode-connector-punctuation
		       (: #\\ unicode-escape-sequence))))


      ((+ blank_no_lt)
       (ignore))

      ((: (* blank_no_lt) lt (* blank))
       (token 'NEW_LINE #\newline))
      
      ;; LineComment
      ((:"//" (* not-lt))
       (ignore))

      ;; multi-line comment on one line
      ((: "/*" (* (or (out #\*) (: (+ #\*) (out #\/ #\*)))) (+ #\*) "/")
       (ignore))

      ;; multi-line comment with LineTerminators (several lines)
      ((: "/*"
	  (* (or lt
		 (out #\*)
		 (: (+ #\*) (out #\/ #\*))))
	  (+ #\*) "/")
       (token 'NEW_LINE #\newline))

      ;; unfinished multi-line comment (added for repl)
      ((eof (: "/*"
	       (* (or lt
		      (out #\*)
		      (: (+ #\*) (out #\/ #\*))))))
{'fraction_non_alphanumeric': 0.1927437641723356, 'fraction_numerical': 0.021541950113378686, 'mean_word_length': 2.316131237183869, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1133747', 'n_tokens_mistral': 4069, 'n_tokens_neox': 3571, 'n_words': 847}