<commit_before>(ns caesium.byte-bufs
  "Byte buffer utilities, like conversions and length."
  (:require [byte-streams :as bs])
  (:import (java.nio ByteBuffer)))

(defn ^ByteBuffer ->indirect-byte-buf
  "Convert buffer to a [[ByteBuffer]].

  If the input is not a byte buffer already, will be converted to an
  indirect byte buffer. For example, a byte array will just be
  wrapped. If the input is already a byte buffer, return it
  unmodified, *even if it was a direct byte buffer*. (This might seem
  confusing, but is generally what callers of this API want.)"
  [x]
  (bs/convert x ByteBuffer {:direct? false}))

(defn ^ByteBuffer ->direct-byte-buf
  "Convert buffer to a [[ByteBuffer]].

  If the input is not a byte buffer already, will be converted to a
  direct byte buffer. For example, a byte array will be copied into a
  newly allocated direct byte buffer. If the input is already a byte
  buffer, return it unmodified, *even if it was an indirect byte
  buffer*. (This might seem confusing, but is generally what callers
  of this API want.)"
  [x]
  (bs/convert x ByteBuffer {:direct? true}))

(defprotocol BufLen
  (^Long buflen [this]))

(extend-protocol BufLen
  (Class/forName "[B")
  (buflen [this] (long (alength ^bytes this)))

  ByteBuffer
  (buflen [this] (long (.remaining ^ByteBuffer this))))<commit_msg>Style fix<commit_after>(ns caesium.byte-bufs
  "Byte buffer utilities, like conversions and length."
  (:require [byte-streams :as bs])
  (:import (java.nio ByteBuffer)))

(defn ^ByteBuffer ->indirect-byte-buf
  "Convert buffer to a [[ByteBuffer]].

  If the input is not a byte buffer already, will be converted to an
  indirect byte buffer. For example, a byte array will just be
  wrapped. If the input is already a byte buffer, return it
  unmodified, *even if it was a direct byte buffer*. (This might seem
  confusing, but is generally what callers of this API want.)"
  [x]
  (bs/convert x ByteBuffer {:direct? false}))

(defn ^ByteBuffer ->direct-byte-buf
  "Convert buffer to a [[ByteBuffer]].

  If the input is not a byte buffer already, will be converted to a
  direct byte buffer. For example, a byte array will be copied into a
  newly allocated direct byte buffer. If the input is already a byte
  buffer, return it unmodified, *even if it was an indirect byte
  buffer*. (This might seem confusing, but is generally what callers
  of this API want.)"
  [x]
  (bs/convert x ByteBuffer {:direct? true}))

(defprotocol BufLen
  (^Long buflen [this]))

(extend-protocol BufLen
  (Class/forName "[B")
  (buflen [this]
    (long (alength ^bytes this)))

  ByteBuffer
  (buflen [this]
    (long (.remaining ^ByteBuffer this))))
{'fraction_non_alphanumeric': 0.09820089955022489, 'fraction_numerical': 0.0, 'mean_word_length': 3.8527272727272726, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4408402', 'n_tokens_mistral': 818, 'n_tokens_neox': 797, 'n_words': 376}