<commit_before>(ns hbase-region-inspector.hbase
  (:require [clojure.string :as str]
            [hbase-region-inspector.util :as util]
            [hbase-region-inspector.hbase.impl :as hbase-impl])
  (:import org.apache.hadoop.hbase.client.HBaseAdmin
           org.apache.hadoop.hbase.HBaseConfiguration
           org.apache.hadoop.hbase.util.Bytes
           org.apache.hadoop.security.UserGroupInformation))

(defn server-load->map
  "Transforms ServerLoad object into clojure map"
  [load]
  (assoc
    {:max-heap-mb        (.getMaxHeapMB load)
     :used-heap-mb       (.getUsedHeapMB load)
     :regions            (.getNumberOfRegions load)
     :requests-rate      (.getNumberOfRequests load)
     :store-files        (.getStorefiles load)
     :store-file-size-mb (.getStorefileSizeInMB load)}
    :store-uncompressed-size-mb
    (->> load
         str
         (re-find #"storefileUncompressedSizeMB=([0-9]+)")
         last
         Integer/parseInt)))

(defn- collect-server-info
  "Collects server statistics"
  [cluster-status]
  (let [server-names (.getServers cluster-status)
        server-loads (map #(.getLoad cluster-status %) server-names)]
    (into
      {}
      (map (fn [[name load]]
             (let [name (str name)]
               [name
                (assoc (server-load->map load) :name name)]))
           (zipmap server-names server-loads)))))

(defn collect-region-info
  ([admin]
   (hbase-impl/collect-region-info admin (.getClusterStatus admin)))
  ([admin cluster-status]
   (hbase-impl/collect-region-info admin cluster-status)))

(defn collect-info
  "Collects server and region statistics"
  [admin]
  (let [cluster-status (.getClusterStatus admin)]
    {:servers (doall (collect-server-info cluster-status))
     :regions (doall (collect-region-info admin cluster-status))}))

(def bytes-comp Bytes/BYTES_COMPARATOR)

(defn byte-buffer->str
  "Returns the string representation of a bytes array"<commit_msg>ServerLoad.toString may not contain storefileUncompressedSizeMB<commit_after>(ns hbase-region-inspector.hbase
  (:require [clojure.string :as str]
            [hbase-region-inspector.util :as util]
            [hbase-region-inspector.hbase.impl :as hbase-impl])
  (:import org.apache.hadoop.hbase.client.HBaseAdmin
           org.apache.hadoop.hbase.HBaseConfiguration
           org.apache.hadoop.hbase.util.Bytes
           org.apache.hadoop.security.UserGroupInformation))

(defn server-load->map
  "Transforms ServerLoad object into clojure map"
  [load]
  (assoc
    {:max-heap-mb        (.getMaxHeapMB load)
     :used-heap-mb       (.getUsedHeapMB load)
     :regions            (.getNumberOfRegions load)
     :requests-rate      (.getNumberOfRequests load)
     :store-files        (.getStorefiles load)
     :store-file-size-mb (.getStorefileSizeInMB load)}
    :store-uncompressed-size-mb
    (some->> load
             str
             (re-find #"storefileUncompressedSizeMB=([0-9]+)")
             last
             Integer/parseInt)))

(defn- collect-server-info
  "Collects server statistics"
  [cluster-status]
  (let [server-names (.getServers cluster-status)
        server-loads (map #(.getLoad cluster-status %) server-names)]
    (into
      {}
      (map (fn [[name load]]
             (let [name (str name)]
               [name
                (assoc (server-load->map load) :name name)]))
           (zipmap server-names server-loads)))))

(defn collect-region-info
  ([admin]
   (hbase-impl/collect-region-info admin (.getClusterStatus admin)))
  ([admin cluster-status]
   (hbase-impl/collect-region-info admin cluster-status)))

(defn collect-info
  "Collects server and region statistics"
  [admin]
  (let [cluster-status (.getClusterStatus admin)]
    {:servers (doall (collect-server-info cluster-status))
     :regions (doall (collect-region-info admin cluster-status))}))

(def bytes-comp Bytes/BYTES_COMPARATOR)

(defn byte-buffer->str
  "Returns the string representation of a bytes array"
{'fraction_non_alphanumeric': 0.1269482151835093, 'fraction_numerical': 0.001005530417295123, 'mean_word_length': 2.7326454033771106, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '400644', 'n_tokens_mistral': 1271, 'n_tokens_neox': 1240, 'n_words': 266}