<commit_before>(ns joplin.core
  (:require [clj-time.core :as t]
            [clj-time.format :as f]
            [clojure.java.io :as io]
            [clojure.set]
            [clojure.string]
            [ragtime.core]
            [ragtime.main]))

;; ==========================================================================
;; methods implemented by migrator/seed targets

(defmulti migrate-db
  "Migrate target database described by a joplin database map."
  (fn [target & args] (get-in target [:db :type])))

(defmulti rollback-db
  "Rollback target database described by a joplin database map N steps.
N is an optional argument and if present should be the first in args."
  (fn [target & args] (get-in target [:db :type])))

(defmulti seed-db
  "Migrate target database described by a joplin database map."
  (fn [target & args] (get-in target [:db :type])))

(defmulti reset-db
  "Reset (re-migrate and seed) target database described by a joplin database map."
  (fn [target & args] (get-in target [:db :type])))

(defmulti create-migration
  "Create migrations file(s) for target database described by a joplin database map.
The first argument must be the name of the migration to create"
  (fn [target & args] (get-in target [:db :type])))

;; ==========================================================================
;; Helpers

(def verbose-migration @#'ragtime.main/verbose-migration)

(defn load-var
  "Load a var specified by a string"
  [v]
  (try
    (@#'ragtime.main/load-var v)
    (catch Exception e
      (println (format "Function '%s' not found" v))
      (println (.getMessage e)))))

(defn get-full-migrator-id [id]
  (str (f/unparse (f/formatter "YYYYMMddHHmmss") (t/now)) "-" id))

(defn- get-migration-ns [path]
  (when path
    (let [ns (->> (clojure.string/split path #"/")
                  rest
                  (interpose ".")
                  (apply str))
          folder (io/file path)]
      (->> (.listFiles folder)
           (map #(.getName %))
           (map #(re-matches #"(.*)(\.clj)$" %))
           (keep second)
           (map #(clojure.string/replace % "_" "-"))
           sort
           (mapv #(vector % (symbol (str ns "." %))))))))

(defn get-migrations
  "Get all seq of ragtime migrators given a path (will scan the filesystem)"
  [path]
  (for [[id ns] (get-migration-ns path)]
    (do
      (require ns)
      (verbose-migration
       {:id id
        :up (load-var (str ns "/up"))
        :down (load-var (str ns "/down"))}))))

(defn do-migrate
  "Perform migration on a database"
  [migrations db]
  (println "Migrating" db)
  (ragtime.core/migrate-all db migrations))

(defn do-rollback
  "Perform rollback on a database"
  [migrations db n-str]
  (println "Rolling back" db)
  (doseq [m migrations]
    (ragtime.core/remember-migration m))
  (ragtime.core/rollback-last db (or (when n-str (Integer/parseInt n-str))
                                     1)))

(defn do-seed-fn
  "Run a seeder function with migration check"
  [migrations db target args]
  (println "Seeding" db)
  (when-let [seed-fn (and (:seed target) (load-var (:seed target)))]
    (let [migrations (->> migrations (map :id) set)
          applied-migrations (set (ragtime.core/applied-migration-ids db))]

      (cond
       (not= (count migrations) (count applied-migrations))
       (do
         (println "There are" (- (count migrations) (count applied-migrations)) "pending migration(s)")
         (println (clojure.set/difference migrations applied-migrations)))

       seed-fn
       (do
         (println "Appying seed function" (:seed target))
         (apply seed-fn target args))

       :else
       (println "Skipping" (:seed target))))))

(defn do-reset
  "Perform a reset on a database"
  [db target args]

  ;; Roll back all
  (ragtime.core/rollback-last db Integer/MAX_VALUE)

  ;; Migrate
  (apply migrate-db target args)

  ;; Seed
  (apply seed-db target args))

(defn do-create-migration
  "Create a scaffold migrator file"
  [target id ns]
  (when (:migrator target)
    (let [migration-id (get-full-migrator-id id)
          path (str (:migrator target) "/"
                    (clojure.string/replace migration-id "-" "_")
                    ".clj")]
      (println "creating" path)
      (try
        (spit path (format "(ns %s
  (:use [%s]))

(defn up [db]
  ;; TODO - up migration code here
  )

(defn down [db]
  ;; TODO - down migration goes here
  )
" (apply str (interpose "."
                        (concat
                         (-> (:migrator target) (clojure.string/split #"/") rest)
                         [migration-id]))) ns))
        (catch Exception e
          (println "Error creating file" path))))))<commit_msg>look for migrations in jars on the classpath<commit_after>(ns joplin.core
  (:require [clj-time.core :as t]
            [clj-time.format :as f]
            [clojure.java.classpath :as classpath]
            [clojure.java.io :as io]
            [clojure.set]
            [clojure.string :as string]
            [ragtime.core]
            [ragtime.main]))


;; ==========================================================================
;; methods implemented by migrator/seed targets

(defmulti migrate-db
  "Migrate target database described by a joplin database map."
  (fn [target & args] (get-in target [:db :type])))

(defmulti rollback-db
  "Rollback target database described by a joplin database map N steps.
N is an optional argument and if present should be the first in args."
  (fn [target & args] (get-in target [:db :type])))

(defmulti seed-db
  "Migrate target database described by a joplin database map."
  (fn [target & args] (get-in target [:db :type])))

(defmulti reset-db
  "Reset (re-migrate and seed) target database described by a joplin database map."
  (fn [target & args] (get-in target [:db :type])))

(defmulti create-migration
  "Create migrations file(s) for target database described by a joplin database map.
The first argument must be the name of the migration to create"
  (fn [target & args] (get-in target [:db :type])))

;; ==========================================================================
;; Helpers

(def verbose-migration @#'ragtime.main/verbose-migration)

(defn load-var
  "Load a var specified by a string"
  [v]
  (try
    (@#'ragtime.main/load-var v)
    (catch Exception e
      (println (format "Function '%s' not found" v))
      (println (.getMessage e)))))

(defn get-full-migrator-id [id]
  (str (f/unparse (f/formatter "YYYYMMddHHmmss") (t/now)) "-" id))

(defn- get-migration-files [path]
  (let [folder (io/file path)
        classpath-folder (->> (string/split path "/")
                              rest (interpose "/") (apply str))]
    (if (.isDirectory folder)
      ;; If it's a folder just read the file from there
      (map #(.getName %) (.listFiles folder))

      ;; Try finding this path in a jar on the classpath
      (->> (classpath/classpath-jarfiles)
           (mapcat classpath/filenames-in-jar)
           (filter #(.startsWith % classpath-folder))))))

(defn- get-migration-namespaces [path]
  (when path
    (let [ns (->> (string/split path #"/")
                  rest
                  (interpose ".")
                  (apply str))]
      (->> (get-migration-files path)
           (map #(re-matches #"(.*)(\.clj)$" %))
           (keep second)
           (map #(string/replace % "_" "-"))
           sort
           (mapv #(vector % (symbol (str ns "." %))))))))

(defn get-migrations
  "Get all seq of ragtime migrators given a path (will scan the filesystem)"
  [path]
  (for [[id ns] (get-migration-namespaces path)]
    (do
      (require ns)
      (verbose-migration
       {:id id
        :up (load-var (str ns "/up"))
        :down (load-var (str ns "/down"))}))))

(defn do-migrate
  "Perform migration on a database"
  [migrations db]
  (println "Migrating" db)
  (ragtime.core/migrate-all db migrations))

(defn do-rollback
  "Perform rollback on a database"
  [migrations db n-str]
  (println "Rolling back" db)
  (doseq [m migrations]
    (ragtime.core/remember-migration m))
  (ragtime.core/rollback-last db (or (when n-str (Integer/parseInt n-str))
                                     1)))

(defn do-seed-fn
  "Run a seeder function with migration check"
  [migrations db target args]
  (println "Seeding" db)
  (when-let [seed-fn (and (:seed target) (load-var (:seed target)))]
    (let [migrations (->> migrations (map :id) set)
          applied-migrations (set (ragtime.core/applied-migration-ids db))]

      (cond
       (not= (count migrations) (count applied-migrations))
       (do
         (println "There are" (- (count migrations) (count applied-migrations)) "pending migration(s)")
         (println (clojure.set/difference migrations applied-migrations)))

       seed-fn
       (do
         (println "Appying seed function" (:seed target))
         (apply seed-fn target args))

       :else
       (println "Skipping" (:seed target))))))

(defn do-reset
  "Perform a reset on a database"
  [db target args]

  ;; Roll back all
  (ragtime.core/rollback-last db Integer/MAX_VALUE)

  ;; Migrate
  (apply migrate-db target args)

  ;; Seed
  (apply seed-db target args))

(defn do-create-migration
  "Create a scaffold migrator file"
  [target id ns]
  (when (:migrator target)
    (let [migration-id (get-full-migrator-id id)
          path (str (:migrator target) "/"
                    (string/replace migration-id "-" "_")
                    ".clj")]
      (println "creating" path)
      (try
        (spit path (format "(ns %s
  (:use [%s]))

(defn up [db]
  ;; TODO - up migration code here
  )

(defn down [db]
  ;; TODO - down migration goes here
  )
" (apply str (interpose "."
                        (concat
                         (-> (:migrator target) (string/split #"/") rest)
                         [migration-id]))) ns))
        (catch Exception e
          (println "Error creating file" path))))))
{'fraction_non_alphanumeric': 0.16661635979474795, 'fraction_numerical': 0.00020122748767481639, 'mean_word_length': 2.5086480762442642, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6589625', 'n_tokens_mistral': 3225, 'n_tokens_neox': 3145, 'n_words': 979}