<commit_before>(ns clojars.dev.setup
  "Tools to setup a dev db."
  (:require [clojars.config :refer [config]]
            [clojars.db :as db]
            [clojars.db.sql :as sql]
            [clojars.file-utils :as fu]
            [clojars.search :as search]
            [clojure.java.io :as io]
            [clojure.string :as str])
  (:import [org.apache.maven.artifact.repository.metadata Metadata Versioning]
           [org.apache.maven.artifact.repository.metadata.io.xpp3
            MetadataXpp3Reader
            MetadataXpp3Writer]))

(defn reset-db! [db]
  (sql/clear-jars! {} {:connection db})
  (sql/clear-groups! {} {:connection db})
  (sql/clear-users! {} {:connection db}))

(defn add-test-users
  "Adds n test users of the form test0/test0."
  [db n]
  (mapv #(let [name (str "test" %)]
           (if (db/find-user db name)
             (println "User" name "already exists")
             (do
               (printf "Adding user %s/%s\n" name name)
               (db/add-user db (str name "@example.com") name name)))
           name)
    (range n)))

(defn write-metadata [md file]
  (with-open [w (io/writer file)]
    (.write (MetadataXpp3Writer.) w md)))

(defn read-metadata [file]
  (.read (MetadataXpp3Reader.) (io/input-stream file)))

(defn create-metadata [md-file group-id artifact-id version]
  (write-metadata
    (doto (Metadata.)
      (.setGroupId group-id)
      (.setArtifactId artifact-id)
      (.setVersioning
        (doto (Versioning.)
          (.addVersion version))))
    md-file))

(defn update-metadata
  ([dir group-id artifact-id version]
   (let [md-file (io/file dir "maven-metadata.xml")]
     (if (.exists md-file)
       (update-metadata md-file version)
       (create-metadata md-file group-id artifact-id version))))
  ([md-file version]
   (-> md-file
     read-metadata
     (doto (-> .getVersioning (.addVersion version)))
     (write-metadata md-file))))

; TODO: If there are more places that require this, move to a utility location.
(defn get-path
  "Ensures that / is used as separator when regex depends on it."
  [file]
  (str/replace (.getPath file) (java.io.File/separator) "/"))

(defn import-repo
  "Builds a dev db from the contents of the repo."
  [db repo stats-dir users]
  (let [group-artifact-pattern (re-pattern (str repo "/(.*)/([^/]*)$"))
        stats-file (io/file stats-dir "all.edn")]
    (let [d (io/file stats-dir)]
      (when-not (.exists d)
        (.mkdirs d)))
    (->>
      (for [version-dir (file-seq (io/file repo))
            :when (and (.isDirectory version-dir)
                    (re-find #"^[0-9]\." (.getName version-dir)))
            :let [parent (.getParentFile version-dir)
                  [_ group-path artifact-id] (re-find group-artifact-pattern (get-path parent))
                  version (.getName version-dir)
                  group-id (str/lower-case (fu/path->group group-path))
                  user (or (first (db/group-adminnames db group-id)) (rand-nth users))]]
        (when-not (db/find-jar db group-id artifact-id version)
          (printf "Importing %s/%s %s (user: %s)\n" group-id artifact-id version user)
          (db/add-group db user group-id)
          (db/verify-group! db user group-id)
          (db/add-jar db
                      user {:group group-id
                            :name artifact-id
                            :version version
                            :description (format "Description for %s/%s" group-id artifact-id)
                            :homepage (format "http://example.com/%s/%s" group-id artifact-id)
                            :authors ["Foo" "Bar" "Basil"]})
          (update-metadata parent group-id artifact-id version)
          [group-id artifact-id version (rand-int 1000)]))
      (remove nil?)
      (reduce (fn [accum [g a v dl]]
                (assoc-in accum [[g a] v] dl))
        {})
      pr-str
      (spit stats-file))
    (println "Wrote download stats to" (.getAbsolutePath stats-file))))

(defn setup-dev-environment []
  (let [{:keys [repo stats-dir db]} (config)]
    (println "NOTE: this will clear the contents of" db
      "and import all of the projects in" repo "into the db.\n")
    (print "Are you sure you want to continue? [y/N] ")
    (flush)
    (when-not (= "y" (.toLowerCase (read-line)))
      (println "Aborting.")
      (System/exit 1))
    (println "==> Clearing the" db "db...")
    (reset-db! db)
    (println "==> Creating 10 test users...")
    (let [test-users (add-test-users db 10)]<commit_msg>Expand dev setup to include scm info in test jars<commit_after>(ns clojars.dev.setup
  "Tools to setup a dev db."
  (:require [clojars.config :refer [config]]
            [clojars.db :as db]
            [clojars.db.sql :as sql]
            [clojars.file-utils :as fu]
            [clojars.search :as search]
            [clojure.java.io :as io]
            [clojure.string :as str])
  (:import [org.apache.maven.artifact.repository.metadata Metadata Versioning]
           [org.apache.maven.artifact.repository.metadata.io.xpp3
            MetadataXpp3Reader
            MetadataXpp3Writer]))

(defn reset-db! [db]
  (sql/clear-jars! {} {:connection db})
  (sql/clear-groups! {} {:connection db})
  (sql/clear-users! {} {:connection db}))

(defn add-test-users
  "Adds n test users of the form test0/test0."
  [db n]
  (mapv #(let [name (str "test" %)]
           (if (db/find-user db name)
             (println "User" name "already exists")
             (do
               (printf "Adding user %s/%s\n" name name)
               (db/add-user db (str name <EMAIL>") name name)))
           name)
    (range n)))

(defn write-metadata [md file]
  (with-open [w (io/writer file)]
    (.write (MetadataXpp3Writer.) w md)))

(defn read-metadata [file]
  (.read (MetadataXpp3Reader.) (io/input-stream file)))

(defn create-metadata [md-file group-id artifact-id version]
  (write-metadata
    (doto (Metadata.)
      (.setGroupId group-id)
      (.setArtifactId artifact-id)
      (.setVersioning
        (doto (Versioning.)
          (.addVersion version))))
    md-file))

(defn update-metadata
  ([dir group-id artifact-id version]
   (let [md-file (io/file dir "maven-metadata.xml")]
     (if (.exists md-file)
       (update-metadata md-file version)
       (create-metadata md-file group-id artifact-id version))))
  ([md-file version]
   (-> md-file
     read-metadata
     (doto (-> .getVersioning (.addVersion version)))
     (write-metadata md-file))))

; TODO: If there are more places that require this, move to a utility location.
(defn get-path
  "Ensures that / is used as separator when regex depends on it."
  [file]
  (str/replace (.getPath file) (java.io.File/separator) "/"))

(defn import-repo
  "Builds a dev db from the contents of the repo."
  [db repo stats-dir users]
  (let [group-artifact-pattern (re-pattern (str repo "/(.*)/([^/]*)$"))
        stats-file (io/file stats-dir "all.edn")]
    (let [d (io/file stats-dir)]
      (when-not (.exists d)
        (.mkdirs d)))
    (->>
      (for [version-dir (file-seq (io/file repo))
            :when (and (.isDirectory version-dir)
                    (re-find #"^[0-9]\." (.getName version-dir)))
            :let [parent (.getParentFile version-dir)
                  [_ group-path artifact-id] (re-find group-artifact-pattern (get-path parent))
                  version (.getName version-dir)
                  group-id (str/lower-case (fu/path->group group-path))
                  user (or (first (db/group-adminnames db group-id)) (rand-nth users))]]
        (when-not (db/find-jar db group-id artifact-id version)
          (printf "Importing %s/%s %s (user: %s)\n" group-id artifact-id version user)
          (db/add-group db user group-id)
          (db/verify-group! db user group-id)
          (db/add-jar db
                      user {:group group-id
                            :name artifact-id
                            :version version
                            :description (format "Description for %s/%s" group-id artifact-id)
                            :homepage (format "http://example.com/%s/%s" group-id artifact-id)
                            :authors ["Foo" "Bar" "Basil"]
                            :scm {:tag "abcde1"
                                  :url (format "https://%s/%s/%s"
                                               (rand-nth ["github.com" "gitlab.com" "other.org"])
                                               group-id artifact-id)}})
          (update-metadata parent group-id artifact-id version)
          [group-id artifact-id version (rand-int 1000)]))
      (remove nil?)
      (reduce (fn [accum [g a v dl]]
                (assoc-in accum [[g a] v] dl))
        {})
      pr-str
      (spit stats-file))
    (println "Wrote download stats to" (.getAbsolutePath stats-file))))

(defn setup-dev-environment []
  (let [{:keys [repo stats-dir db]} (config)]
    (println "NOTE: this will clear the contents of" db
      "and import all of the projects in" repo "into the db.\n")
    (print "Are you sure you want to continue? [y/N] ")
    (flush)
    (when-not (= "y" (.toLowerCase (read-line)))
      (println "Aborting.")
      (System/exit 1))
    (println "==> Clearing the" db "db...")
    (reset-db! db)
    (println "==> Creating 10 test users...")
    (let [test-users (add-test-users db 10)]
{'fraction_non_alphanumeric': 0.14887700534759357, 'fraction_numerical': 0.003957219251336898, 'mean_word_length': 2.0193735873425895, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 18, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '208646', 'n_tokens_mistral': 3067, 'n_tokens_neox': 2910, 'n_words': 846}