<commit_before>(ns multihash.core
  "Core multihash type definition and helper methods."
  (:refer-clojure :exclude [hash])
  (:require
    [clojure.string :as str])
  (:import
    java.security.MessageDigest))


;; ## Hash Function Algorithms

(def ^:const algorithms
  "Map of content hashing algorithms to system names and byte encodings."
  {:sha1     {:code 0x11, :length 20, :system "SHA-1"}
   :sha2-256 {:code 0x12, :length 32, :system "SHA-256"}
   :sha2-512 {:code 0x13, :length 64, :system "SHA-512"}
   :sha3     {:code 0x14, :length 64}
   :blake2b  {:code 0x40, :length 64}
   :blake2s  {:code 0x41, :length 32}})


(defn app-code?
  "True if the given code number is assigned to the application-specfic range."
  [code]
  (< 0 code 0x10))


(defn get-algorithm
  "Looks up an algorithm by keyword name or code number. Returns `nil` if the
  value does not map to any valid algorithm."
  [value]
  (cond
    (keyword? value)
    (when-let [data (get algorithms value)]
      (assoc data :name value))

    (not (integer? value))
    nil

    (app-code? value)
    {:code value, :name (keyword (str "app-" value))}

    :else
    (some #(when (= value (:code (val %)))
             (assoc (val %) :name (key %)))
          algorithms)))



;; ## Utilities

(defn- zero-pad
  "Pads a string with leading zeroes up to the given width."
  [width value]
  (let [string (str value)]
    (if (<= width (count string))
      string
      (-> width
          (- (count string))
          (repeat "0")
          str/join
          (str string)))))


(defn- bytes->hex
  "Converts a byte array into a lowercase hexadecimal string."
  [^bytes value]
  (let [width (* 2 (count value))
        hex (-> (BigInteger. 1 value)
                (.toString 16)
                str/lower-case)]
    (zero-pad width hex)))


(defn- hex->bytes
  "Parses a hexadecimal string into a byte array. Ensures that the resulting
  array is zero-padded to match the hex string length."
  ^bytes
  [^String value]
  (when (odd? (count value))
    (throw (IllegalArgumentException.
             (str "Input string '" value "' is not valid hex: number of "
                  "characters (" (count value) ") is odd"))))
  (let [width (/ (count value) 2)
        data (.toByteArray (BigInteger. value 16))]
    (if (= width (count data))
      data
      (let [data' (byte-array width)]
        (System/arraycopy data 0 data' 0 (count data))
        data'))))



;; ## Multihash Type

;; Multihash identifiers have two properties:
;;
;; - `:code` is a numeric code for an algorithm entry in `algorithms` or an
;;   application-specific algorithm code.
;; - `:digest` is a string holding the hex-encoded algorithm output.
(deftype Multihash
  [^long code ^String digest _meta]

  Object

  (toString [this]
    (str "hash:" (name (:name (get-algorithm code))) \: digest))

  (equals [this that]
    (cond
      (identical? this that) true
      (instance? Multihash that)
        (and (= code   (.code   ^Multihash that))
             (= digest (.digest ^Multihash that)))
      :else false))

  (hashCode [this]
    (hash-combine code digest))
<commit_msg>Allow nil values in hex conversions.<commit_after>(ns multihash.core
  "Core multihash type definition and helper methods."
  (:refer-clojure :exclude [hash])
  (:require
    [clojure.string :as str])
  (:import
    java.security.MessageDigest))


;; ## Hash Function Algorithms

(def ^:const algorithms
  "Map of content hashing algorithms to system names and byte encodings."
  {:sha1     {:code 0x11, :length 20, :system "SHA-1"}
   :sha2-256 {:code 0x12, :length 32, :system "SHA-256"}
   :sha2-512 {:code 0x13, :length 64, :system "SHA-512"}
   :sha3     {:code 0x14, :length 64}
   :blake2b  {:code 0x40, :length 64}
   :blake2s  {:code 0x41, :length 32}})


(defn app-code?
  "True if the given code number is assigned to the application-specfic range."
  [code]
  (< 0 code 0x10))


(defn get-algorithm
  "Looks up an algorithm by keyword name or code number. Returns `nil` if the
  value does not map to any valid algorithm."
  [value]
  (cond
    (keyword? value)
    (when-let [data (get algorithms value)]
      (assoc data :name value))

    (not (integer? value))
    nil

    (app-code? value)
    {:code value, :name (keyword (str "app-" value))}

    :else
    (some #(when (= value (:code (val %)))
             (assoc (val %) :name (key %)))
          algorithms)))



;; ## Utilities

(defn- zero-pad
  "Pads a string with leading zeroes up to the given width."
  [width value]
  (let [string (str value)]
    (if (<= width (count string))
      string
      (-> width
          (- (count string))
          (repeat "0")
          str/join
          (str string)))))


(defn- bytes->hex
  "Converts a byte array into a lowercase hexadecimal string."
  [^bytes value]
  (when (and value (pos? (alength value)))
    (let [width (* 2 (alength value))
          hex (-> (BigInteger. 1 value)
                  (.toString 16)
                  str/lower-case)]
      (zero-pad width hex))))


(defn- hex->bytes
  "Parses a hexadecimal string into a byte array. Ensures that the resulting
  array is zero-padded to match the hex string length."
  ^bytes
  [^String value]
  (when (and value (not (empty? value)))
    (when (odd? (count value))
      (throw (IllegalArgumentException.
               (str "Input string '" value "' is not valid hex: number of "
                    "characters (" (count value) ") is odd"))))
    (let [width (/ (count value) 2)
          data (.toByteArray (BigInteger. value 16))]
      (if (= width (alength data))
        data
        (let [data' (byte-array width)]
          (System/arraycopy data 0 data' (- width (alength data)) (alength data))
          data')))))



;; ## Multihash Type

;; Multihash identifiers have two properties:
;;
;; - `:code` is a numeric code for an algorithm entry in `algorithms` or an
;;   application-specific algorithm code.
;; - `:digest` is a string holding the hex-encoded algorithm output.
(deftype Multihash
  [^long code ^String digest _meta]

  Object

  (toString [this]
    (str "hash:" (name (:name (get-algorithm code))) \: digest))

  (equals [this that]
    (cond
      (identical? this that) true
      (instance? Multihash that)
        (and (= code   (.code   ^Multihash that))
             (= digest (.digest ^Multihash that)))
      :else false))

  (hashCode [this]
    (hash-combine code digest))

{'fraction_non_alphanumeric': 0.1357986326911125, 'fraction_numerical': 0.01942200124300808, 'mean_word_length': 2.6021264689423615, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2449750', 'n_tokens_mistral': 2210, 'n_tokens_neox': 2023, 'n_words': 741}