<commit_before>;; Copyright (c) Cognitect, Inc.
;; All rights reserved.

(ns transit.read
  (:refer-clojure :exclude [read])
  (:require [transit.write :as w]
            [clojure.edn :as edn])
  (:import [com.fasterxml.jackson.core
            JsonFactory JsonParser JsonToken JsonParseException]
           org.msgpack.MessagePack
           [org.msgpack.unpacker Unpacker MessagePackUnpacker]
           [org.msgpack.type Value MapValue ArrayValue RawValue ValueType]
           [org.apache.commons.codec.binary Base64]
           [java.io InputStream EOFException]))

(set! *warn-on-reflection* true)

(defn read-int-str
  [s]
  (let [o (edn/read-string s)]
    (when (number? o) o)))

(def default-decoders {"'" identity
                       ":" #(keyword %)
                       "?" #(Boolean/valueOf ^boolean (= "t" %))
                       "b" #(Base64/decodeBase64 ^bytes %)
                       "_" (fn [_] nil)
                       "c" #(.charAt ^String % 0)
                       "i" #(try
                              (Long/parseLong %)
                              (catch NumberFormatException _ (read-int-str %)))
                       "d" #(Double. ^String %)
                       "f" #(java.math.BigDecimal. ^String %)
                       "t" #(if (string? %)
                              (clojure.instant/read-instant-date %)
                              (java.util.Date. ^long %))
                       "u" #(if (string? %)
                              (java.util.UUID/fromString %)
                              (java.util.UUID. (first %) (second %)))
                       "r" #(java.net.URI. %)
                       "$" #(symbol %)
                       "set" #(reduce (fn [s v] (conj s v)) #{} %)
                       "list" #(reverse (into '() %))
                       "cmap" #(reduce (fn [m v] (assoc m (nth v 0) (nth v 1)))
                                       {} (partition 2 %))
                       "ratio" #(/ (first %) (second %))})

(def ^:private ^:dynamic *decoders* default-decoders)

(defn default-default-decoder
  [^String tag rep]
  (if (and (= (.length tag) 1) (string? rep))
    (str "`~" tag rep)
    (w/tagged-value tag rep)))

(def ^:private ^:dynamic *default* default-default-decoder)

(defn decode
  [tag rep]
  ;;(prn "decode" tag rep)
  (if-let [decoder (*decoders* tag)]
    (decoder rep)
    (*default* tag rep)))

(defprotocol ReadCache
  (cache-read [cache str as-map-key]))

(defn parse-str
  [s]
  ;;(prn "parse-str before" s)
  (let [res (if (and (string? s)
                     (> (.length ^String s) 1)
                     (= "~" (subs ^String s 0 1))) ;; ESC
              (case (subs s 1 2)
                "~" (subs s 1) ;; ESC
                "^" (subs s 1) ;; SUB
                "`" (subs s 1) ;; RESERVED
                "#" s ;; TAG
                (decode (subs s 1 2) (subs s 2)))
              s)]
    ;;(prn "parse-str after" res)
    res))

(defn parse-tagged-map
  [^java.util.Map m]
  (let [entries (.entrySet m)
        iter (.iterator entries)
        entry (when (.hasNext iter) (.next iter))
        key (when entry (.getKey ^java.util.Map$Entry entry))]
    (if (and entry (string? key) (> (.length ^String key) 1) (= "#" (subs key 1 2)))
      (decode (subs key 2) (.getValue ^java.util.Map$Entry entry))
      m)))

(defn cache-code?
  [^String s]
  (= w/SUB (subs s 0 1)))

(defn code->idx
  [^String s]
  (- (byte ^Character (.charAt s 1)) w/BASE_CHAR_IDX))

(deftype ReadCacheImpl [^:unsynchronized-mutable idx cache]
  ReadCache
  (cache-read [_ str as-map-key]
    ;;(prn "cache read before" idx str)
    (let [res (if (and str (not (zero? (.length ^String str))))
                (if (w/cacheable? str as-map-key)
                  (do 
                    (when (= idx w/MAX_CACHE_ENTRIES)
                      (set! idx 0))
                    (let [o (parse-str str)]
                      (aset ^objects cache idx o)
                      (set! idx (inc idx))
                      o))
                  (if (cache-code? str)
                    (aget ^objects cache (code->idx str))
                    str))
                str)]
      ;;(prn "cache read after" idx res)
      res)))

(defn read-cache [] (ReadCacheImpl. 0 (make-array Object w/MAX_CACHE_ENTRIES)))

(defprotocol Parser
  (unmarshal [p cache])
  (parse-val [p as-map-key cache])
  (parse-map [p as-map-key cache])
  (parse-array [p as-map-key cache]))

(extend-protocol Parser
  JsonParser
  (unmarshal [^JsonParser jp cache]
    (when (.nextToken jp) (parse-val jp false cache)))

  (parse-val [^JsonParser jp as-map-key cache]
    ;;(prn "parse-val" (.getCurrentToken jp))<commit_msg>Fix bug trying to read (.charAt s 1) on 1-char strings<commit_after>;; Copyright (c) Cognitect, Inc.
;; All rights reserved.

(ns transit.read
  (:refer-clojure :exclude [read])
  (:require [transit.write :as w]
            [clojure.edn :as edn])
  (:import [com.fasterxml.jackson.core
            JsonFactory JsonParser JsonToken JsonParseException]
           org.msgpack.MessagePack
           [org.msgpack.unpacker Unpacker MessagePackUnpacker]
           [org.msgpack.type Value MapValue ArrayValue RawValue ValueType]
           [org.apache.commons.codec.binary Base64]
           [java.io InputStream EOFException]))

(set! *warn-on-reflection* true)

(defn read-int-str
  [s]
  (let [o (edn/read-string s)]
    (when (number? o) o)))

(def default-decoders {"'" identity
                       ":" #(keyword %)
                       "?" #(Boolean/valueOf ^boolean (= "t" %))
                       "b" #(Base64/decodeBase64 ^bytes %)
                       "_" (fn [_] nil)
                       "c" #(.charAt ^String % 0)
                       "i" #(try
                              (Long/parseLong %)
                              (catch NumberFormatException _ (read-int-str %)))
                       "d" #(Double. ^String %)
                       "f" #(java.math.BigDecimal. ^String %)
                       "t" #(if (string? %)
                              (clojure.instant/read-instant-date %)
                              (java.util.Date. ^long %))
                       "u" #(if (string? %)
                              (java.util.UUID/fromString %)
                              (java.util.UUID. (first %) (second %)))
                       "r" #(java.net.URI. %)
                       "$" #(symbol %)
                       "set" #(reduce (fn [s v] (conj s v)) #{} %)
                       "list" #(reverse (into '() %))
                       "cmap" #(reduce (fn [m v] (assoc m (nth v 0) (nth v 1)))
                                       {} (partition 2 %))
                       "ratio" #(/ (first %) (second %))})

(def ^:private ^:dynamic *decoders* default-decoders)

(defn default-default-decoder
  [^String tag rep]
  (if (and (= (.length tag) 1) (string? rep))
    (str "`~" tag rep)
    (w/tagged-value tag rep)))

(def ^:private ^:dynamic *default* default-default-decoder)

(defn decode
  [tag rep]
  ;;(prn "decode" tag rep)
  (if-let [decoder (*decoders* tag)]
    (decoder rep)
    (*default* tag rep)))

(defprotocol ReadCache
  (cache-read [cache str as-map-key]))

(defn parse-str
  [s]
  ;;(prn "parse-str before" s)
  (let [res (if (and (string? s)
                     (> (.length ^String s) 1)
                     (= "~" (subs ^String s 0 1))) ;; ESC
              (case (subs s 1 2)
                "~" (subs s 1) ;; ESC
                "^" (subs s 1) ;; SUB
                "`" (subs s 1) ;; RESERVED
                "#" s ;; TAG
                (decode (subs s 1 2) (subs s 2)))
              s)]
    ;;(prn "parse-str after" res)
    res))

(defn parse-tagged-map
  [^java.util.Map m]
  (let [entries (.entrySet m)
        iter (.iterator entries)
        entry (when (.hasNext iter) (.next iter))
        key (when entry (.getKey ^java.util.Map$Entry entry))]
    (if (and entry (string? key) (> (.length ^String key) 1) (= "#" (subs key 1 2)))
      (decode (subs key 2) (.getValue ^java.util.Map$Entry entry))
      m)))

(defn cache-code?
  [^String s]
  (= w/SUB (subs s 0 1)))

(defn code->idx
  [^String s]
  (- (byte ^Character (.charAt s 1)) w/BASE_CHAR_IDX))

(deftype ReadCacheImpl [^:unsynchronized-mutable idx cache]
  ReadCache
  (cache-read [_ str as-map-key]
    ;;(prn "cache read before" idx str)
    (let [res (if (and str (> (.length ^String str) 1))
                (if (w/cacheable? str as-map-key)
                  (do 
                    (when (= idx w/MAX_CACHE_ENTRIES)
                      (set! idx 0))
                    (let [o (parse-str str)]
                      (aset ^objects cache idx o)
                      (set! idx (inc idx))
                      o))
                  (if (cache-code? str)
                    (aget ^objects cache (code->idx str))
                    str))
                str)]
      ;;(prn "cache read after" idx res)
      res)))

(defn read-cache [] (ReadCacheImpl. 0 (make-array Object w/MAX_CACHE_ENTRIES)))

(defprotocol Parser
  (unmarshal [p cache])
  (parse-val [p as-map-key cache])
  (parse-map [p as-map-key cache])
  (parse-array [p as-map-key cache]))

(extend-protocol Parser
  JsonParser
  (unmarshal [^JsonParser jp cache]
    (when (.nextToken jp) (parse-val jp false cache)))

  (parse-val [^JsonParser jp as-map-key cache]
    ;;(prn "parse-val" (.getCurrentToken jp))
{'fraction_non_alphanumeric': 0.15366606363733107, 'fraction_numerical': 0.00691710120251144, 'mean_word_length': 1.5769125308472718, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1638821', 'n_tokens_mistral': 3245, 'n_tokens_neox': 2965, 'n_words': 832}