<commit_before>(ns refactor-nrepl.artifacts
  (:require [clj-http.client :as http]
            [clojure.edn :as edn]
            [clojure.java.io :as io]
            [clojure.tools.nrepl.middleware :refer [set-descriptor!]]
            [clojure.tools.nrepl.misc :refer [response-for]]
            [clojure.tools.nrepl.transport :as transport]))

(def artifacts (atom {} :meta {:last-modified nil}))
(def millis-per-day (* 24 60 60 1000))

(defn get-artifacts-from-clojars! []
  (let [rdr (io/reader (java.net.URL. "https://clojars.org/repo/all-jars.clj"))
        add-lib (fn [line]
                  ;; each line looks like [some/lib "0.1"]
                  (let [lib (edn/read-string line)]
                    (swap! artifacts update-in [(str (first lib))] conj (second lib))))]
    (dorun (map add-lib (line-seq rdr)))))

(defn- stale-cache? []
  (or (empty? @artifacts)
      (neg? (- millis-per-day
               (- (-> artifacts meta :last-modified .getTime)
                  (-> (java.util.Date.) .getTime))))))

(defn- get-all-clj-artifacts! []
  "All the artifacts under org.clojure in mvn central"
  (let [search-url "http://search.maven.org/solrsearch/select?q=g:%22org.clojure%22+AND+p:%22jar%22&rows=2000&wt=json"
        search-result (http/get search-url {:as :json})]
    (map :a (-> search-result :body :response :docs))))

(defn- get-versions! [artifact]
  (->> (http/get (str "http://search.maven.org/solrsearch/select?"
                      "q=g:%22org.clojure%22+AND+a:%22"
                      artifact
                      "%22&core=gav&rows=200&wt=json")
                 {:as :json})
       :body
       :response
       :docs
       (map :v)
       doall))

(defn- collate-artifact-and-versions [artifact]
  (->> artifact
       get-versions!
       (vector (str "org.clojure/" artifact))))

(defn- add-artifact [[artifact versions]]
  (swap! artifacts update-in [artifact] (constantly versions)))

(defn- add-artifacts [artifacts]
  (->> artifacts
       (partition-all 2)
       (map #(future (->> %
                         (map collate-artifact-and-versions)
                         (map add-artifact)
                         dorun)))))

(defn- get-artifacts-from-mvn-central! []
  (-> (get-all-clj-artifacts!)
      (add-artifacts)))

(defn- update-artifact-cache! []
  (let [mvn-central-futures (get-artifacts-from-mvn-central!)
        clojars-future (future (get-artifacts-from-clojars!))]
    (-> (map deref mvn-central-futures) dorun)
    @clojars-future)
  (alter-meta! artifacts update-in [:last-modified]
               (constantly (java.util.Date.))))

(defn- artifacts-list [{:keys [transport force] :as msg}]
  (when (or (= force "true") (stale-cache?))
    (update-artifact-cache!))
  (let [names (->> @artifacts
                   keys
                   (interpose " ")
                   (apply str))]
    (transport/send transport (response-for msg :value names)))
  (transport/send transport (response-for msg :status :done)))

(defn- artifact-versions [{:keys [transport artifact] :as msg}]
  (let [versions (->> artifact (@artifacts) (interpose " ") (apply str))]
    (transport/send transport (response-for msg :value versions))
    (transport/send transport (response-for msg :status :done))))

(defn wrap-artifacts
  [handler]
  (fn [{:keys [op] :as msg}]
    (cond (= op "artifact-list") (artifacts-list msg)
          (= op "artifact-versions") (artifact-versions msg)
          :else
          (handler msg))))

(set-descriptor!
 #'wrap-artifacts<commit_msg>Minor refactoring to make testing easier<commit_after>(ns refactor-nrepl.artifacts
  (:require [clj-http.client :as http]
            [clojure.edn :as edn]
            [clojure.java.io :as io]
            [clojure.tools.nrepl.middleware :refer [set-descriptor!]]
            [clojure.tools.nrepl.misc :refer [response-for]]
            [clojure.tools.nrepl.transport :as transport]))

(def artifacts (atom {} :meta {:last-modified nil}))
(def millis-per-day (* 24 60 60 1000))

(defn get-artifacts-from-clojars! []
  "Returns a vector of [[some/lib \"0.1\"]...]."
  (->> "https://clojars.org/repo/all-jars.clj"
       java.net.URL.
       io/reader
       line-seq
       (map edn/read-string)))

(defn add-artifacts-from-clojars! []
  (->> (get-artifacts-from-clojars!)
       (map #(swap! artifacts update-in [(str (first %))] conj (second %)))
       dorun))

(defn- stale-cache? []
  (or (empty? @artifacts)
      (neg? (- millis-per-day
               (- (-> artifacts meta :last-modified .getTime)
                  (-> (java.util.Date.) .getTime))))))

(defn- get-all-clj-artifacts! []
  "All the artifacts under org.clojure in mvn central"
  (let [search-url "http://search.maven.org/solrsearch/select?q=g:%22org.clojure%22+AND+p:%22jar%22&rows=2000&wt=json"
        search-result (http/get search-url {:as :json})]
    (map :a (-> search-result :body :response :docs))))

(defn- get-versions! [artifact]
  "Gets all the versions from an artifact belonging to the org.clojure."
  (->> (http/get (str "http://search.maven.org/solrsearch/select?"
                      "q=g:%22org.clojure%22+AND+a:%22"
                      artifact
                      "%22&core=gav&rows=200&wt=json")
                 {:as :json})
       :body
       :response
       :docs
       (map :v)
       doall))

(defn- collate-artifact-and-versions [artifact]
  (->> artifact
       get-versions!
       (vector (str "org.clojure/" artifact))))

(defn- add-artifact [[artifact versions]]
  (swap! artifacts update-in [artifact] (constantly versions)))

(defn- add-artifacts [artifacts]
  (->> artifacts
       (partition-all 2)
       (map #(future (->> %
                          (map collate-artifact-and-versions)
                          (map add-artifact)
                          dorun)))))

(defn- get-artifacts-from-mvn-central! []
  (-> (get-all-clj-artifacts!)
      (add-artifacts)))

(defn- update-artifact-cache! []
  (let [mvn-central-futures (get-artifacts-from-mvn-central!)
        clojars-future (future (add-artifacts-from-clojars!))]
    (-> (map deref mvn-central-futures) dorun)
    @clojars-future)
  (alter-meta! artifacts update-in [:last-modified]
               (constantly (java.util.Date.))))

(defn- artifacts-list [{:keys [transport force] :as msg}]
  (when (or (= force "true") (stale-cache?))
    (update-artifact-cache!))
  (let [names (->> @artifacts
                   keys
                   (interpose " ")
                   (apply str))]
    (transport/send transport (response-for msg :value names)))
  (transport/send transport (response-for msg :status :done)))

(defn- artifact-versions [{:keys [transport artifact] :as msg}]
  (let [versions (->> artifact (@artifacts) (interpose " ") (apply str))]
    (transport/send transport (response-for msg :value versions))
    (transport/send transport (response-for msg :status :done))))

(defn wrap-artifacts
  [handler]
  (fn [{:keys [op] :as msg}]
    (cond (= op "artifact-list") (artifacts-list msg)
          (= op "artifact-versions") (artifact-versions msg)
          :else
          (handler msg))))

(set-descriptor!
 #'wrap-artifacts
{'fraction_non_alphanumeric': 0.16974531206269242, 'fraction_numerical': 0.010075566750629723, 'mean_word_length': 2.41145584725537, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 41, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4101366', 'n_tokens_mistral': 2546, 'n_tokens_neox': 2391, 'n_words': 519}