<commit_before>(ns clojurewerkz.statistiker.clustering.kmeans
  (:import [org.apache.commons.math3.ml.clustering KMeansPlusPlusClusterer]
           [clojurewerkz.statistiker DoublePointWithMeta]))

(defn double-point
  [nums]
  (DoublePointWithMeta. (meta nums) (double-array nums)))

(defn- ^KMeansPlusPlusClusterer clusterer
  ([k max-iter]
     (KMeansPlusPlusClusterer. k max-iter))
  ([k max-iter distance-measure]
     (KMeansPlusPlusClusterer. k max-iter distance-measure)))

(defn cluster
  [initial k max-iter]
  (let [clusterer (clusterer k max-iter)]
    (->> initial
         (map double-point)
         (.cluster clusterer)
         (map #(hash-map :center (vec (.getPoint (.getCenter %)))
                         :points (map (fn [a] (with-meta (vec (.getPoint a))
                                               (.getMetadata a)))
                                      (.getPoints %)))))))

(defn cluster-by
  "Clusters the hashmaps by fields. Fields should be given as vector. Field values
   should be numerical.

   Resulting hashmap will be returned with :cluster-id field that identifies the cluster"
  [data fields]
  (let [vectors  (map (fn [point]
                        (with-meta
                          (into [] (for [field fields] (get point field)))
                          point))
                      data)
        clusters (map :points (cluster vectors 5 200))]
    (->> clusters
         (map vector (iterate inc (int 0)))
         (map (fn [[cluster points]]
                (map #(assoc (meta %) :cluster-id cluster) points)))
         ;; mapcat identity?
         flatten)))<commit_msg>Add number of clusters to cluster-by<commit_after>(ns clojurewerkz.statistiker.clustering.kmeans
  (:import [org.apache.commons.math3.ml.clustering KMeansPlusPlusClusterer]
           [clojurewerkz.statistiker DoublePointWithMeta]))

(defn double-point
  [nums]
  (DoublePointWithMeta. (meta nums) (double-array nums)))

(defn- ^KMeansPlusPlusClusterer clusterer
  ([k max-iter]
     (KMeansPlusPlusClusterer. k max-iter))
  ([k max-iter distance-measure]
     (KMeansPlusPlusClusterer. k max-iter distance-measure)))

(defn cluster
  [initial k max-iter]
  (let [clusterer (clusterer k max-iter)]
    (->> initial
         (map double-point)
         (.cluster clusterer)
         (map #(hash-map :center (vec (.getPoint (.getCenter %)))
                         :points (map (fn [a] (with-meta (vec (.getPoint a))
                                               (.getMetadata a)))
                                      (.getPoints %)))))))

(defn cluster-by
  "Clusters the hashmaps by fields. Fields should be given as vector. Field values
   should be numerical.

   Resulting hashmap will be returned with :cluster-id field that identifies the cluster"
  ([data fields clusters]
     (cluster-by data fields clusters 100))
  ([data fields clusters iterations]
     (let [vectors  (map (fn [point]
                           (with-meta
                             (into [] (for [field fields] (get point field)))
                             point))
                         data)
           clusters (map :points (cluster vectors clusters iterations))]
       (->> clusters
            (map vector (iterate inc (int 0)))
            (map (fn [[cluster points]]
                   (map #(assoc (meta %) :cluster-id cluster) points)))
            ;; mapcat identity?
            flatten))))
{'fraction_non_alphanumeric': 0.11458637825197311, 'fraction_numerical': 0.003215434083601286, 'mean_word_length': 2.0043898156277438, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6739493', 'n_tokens_mistral': 1007, 'n_tokens_neox': 982, 'n_words': 286}