<commit_before>(include "./runtime")
(import [list list? count empty? first second third rest
         cons rest] "./list")
(import [odd? dictionary merge] "./runtime")
(import [symbol? symbol keyword? keyword quote syntax-quote
         unquote unquote-splicing meta with-meta name deref] "./ast")

(declare nil)

(defn PushbackReader
  "StringPushbackReader"
  [source index buffer]
  (set! this.source source)
  (set! this.index-atom index)
  (set! this.buffer-atom buffer)
  this)

(defn push-back-reader
  "Creates a StringPushbackReader from a given string"
  [source]
  (new PushbackReader source 0 ""))
(defn next-char
  "Returns next char from the Reader without reading it.
  nil if the end of stream has being reached."
  [reader]
  (if (empty? reader.buffer-atom)
    (aget reader.source reader.index-atom)
    (aget reader.buffer-atom 0)))

(defn read-char
  "Returns the next char from the Reader, nil if the end
  of stream has been reached"
  [reader]
  (if (empty? reader.buffer-atom)
    (let [index reader.index-atom]
      (set! reader.index-atom (+ index 1))
      (aget reader.source index))
    (let [buffer reader.buffer-atom]
      (set! reader.buffer-atom (.substr buffer 1))
      (aget buffer 0))))


(defn unread
  "Push back a single character on to the stream"
  [reader ch]
  (if ch (set! reader.buffer-atom (.concat ch reader.buffer-atom))))


;; Predicates

(defn- ^boolean breaking-whitespace?
 "Checks if a string is all breaking whitespace."
 [ch]
 (>= (.index-of "\t\n\r " ch) 0))

(defn- ^boolean whitespace?
  "Checks whether a given character is whitespace"
  [ch]
  (or (breaking-whitespace? ch) (identical? "," ch)))

(defn- ^boolean numeric?
 "Checks whether a given character is numeric"
 [ch]
 (>= (.index-of "01234567890" ch) 0))

(defn- ^boolean comment-prefix?
  "Checks whether the character begins a comment."
  [ch]
  (identical? ";" ch))


(defn- ^boolean number-literal?
  "Checks whether the reader is at the start of a number literal"
  [reader initch]
  (or (numeric? initch)
      (and (or (identical? \+ initch)
               (identical? \- initch))
           (numeric? (let [next-ch (read-char reader)]
                       (unread reader next-ch)
                       next-ch)))))



;; read helpers
<commit_msg>Make use of next-char instead of read-char where possible.<commit_after>(include "./runtime")
(import [list list? count empty? first second third rest
         cons rest] "./list")
(import [odd? dictionary merge] "./runtime")
(import [symbol? symbol keyword? keyword quote syntax-quote
         unquote unquote-splicing meta with-meta name deref] "./ast")

(declare nil)

(defn PushbackReader
  "StringPushbackReader"
  [source index buffer]
  (set! this.source source)
  (set! this.index-atom index)
  (set! this.buffer-atom buffer)
  this)

(defn push-back-reader
  "Creates a StringPushbackReader from a given string"
  [source]
  (new PushbackReader source 0 ""))
(defn next-char
  "Returns next char from the Reader without reading it.
  nil if the end of stream has being reached."
  [reader]
  (if (empty? reader.buffer-atom)
    (aget reader.source reader.index-atom)
    (aget reader.buffer-atom 0)))

(defn read-char
  "Returns the next char from the Reader, nil if the end
  of stream has been reached"
  [reader]
  (if (empty? reader.buffer-atom)
    (let [index reader.index-atom]
      (set! reader.index-atom (+ index 1))
      (aget reader.source index))
    (let [buffer reader.buffer-atom]
      (set! reader.buffer-atom (.substr buffer 1))
      (aget buffer 0))))


(defn unread
  "Push back a single character on to the stream"
  [reader ch]
  (if ch (set! reader.buffer-atom (.concat ch reader.buffer-atom))))


;; Predicates

(defn- ^boolean breaking-whitespace?
 "Checks if a string is all breaking whitespace."
 [ch]
 (>= (.index-of "\t\n\r " ch) 0))

(defn- ^boolean whitespace?
  "Checks whether a given character is whitespace"
  [ch]
  (or (breaking-whitespace? ch) (identical? "," ch)))

(defn- ^boolean numeric?
 "Checks whether a given character is numeric"
 [ch]
 (>= (.index-of "01234567890" ch) 0))

(defn- ^boolean comment-prefix?
  "Checks whether the character begins a comment."
  [ch]
  (identical? ";" ch))


(defn- ^boolean number-literal?
  "Checks whether the reader is at the start of a number literal"
  [reader initch]
  (or (numeric? initch)
      (and (or (identical? \+ initch)
               (identical? \- initch))
           (numeric? (next-char reader)))))



;; read helpers

{'fraction_non_alphanumeric': 0.1295187402971834, 'fraction_numerical': 0.007984031936127744, 'mean_word_length': 3.2268041237113403, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1299729', 'n_tokens_mistral': 1513, 'n_tokens_neox': 1446, 'n_words': 514}