<commit_before>(ns cmr.metadata-db.services.jobs
  (:require [clojurewerkz.quartzite.scheduler :as qs]
            [clojurewerkz.quartzite.triggers :as t]
            [clojurewerkz.quartzite.jobs :as jobs]
            [clojurewerkz.quartzite.stateful :refer [def-stateful-job]]
            [clojurewerkz.quartzite.schedule.calendar-interval :refer [schedule with-interval-in-seconds]]
            [clojurewerkz.quartzite.conversion :as c]
            [cmr.metadata-db.db-holder :as db-holder]
            [cmr.common.log :as log :refer (debug info warn error)]
            [cmr.metadata-db.data.providers :as provider-db]
            [cmr.metadata-db.data.oracle.providers]
            [cmr.metadata-db.services.concept-service :as srv]))

(def job-key "jobs.expired.1")

(def EXPIRED_CONCEPT_CLEANUP_INTERVAL
  "The number of seconds between jobs run to cleanup expired granules and collections"
  (* 3600 5))

(defn configure-quartz-system-properties
  [db]
  (let [{{:keys [subprotocol subname user password]} :spec} db]
    (System/setProperty
      "org.quartz.dataSource.myDS.URL"
      (str "jdbc:" subprotocol ":" subname))
    (System/setProperty "org.quartz.dataSource.myDS.user" user)
    (System/setProperty "org.quartz.dataSource.myDS.password" password)))


(def-stateful-job ExpiredConceptCleanupJob
  [ctx]
  (info "Executing expired concepts cleanup.")
  (try
    (let [db (db-holder/get-db)]
      (doseq [provider (provider-db/get-providers db)]
        (srv/delete-expired-concepts db provider :collection)
        (srv/delete-expired-concepts db provider :granule)))
    (catch Throwable e
      (error e "ExpiredConceptCleanupJob caught Exception.")))
  (info "Finished expired concepts cleanup."))

(defn start
  [db]
  (info "Starting expired concepts cleanup job")
  (configure-quartz-system-properties db)
  (qs/initialize)
  (qs/start)
  ;; We delete exiting job and recreate it
  (when (qs/get-job job-key)
    (qs/delete-job (jobs/key job-key)))
  (let [job (jobs/build
              (jobs/of-type ExpiredConceptCleanupJob)
              (jobs/with-identity (jobs/key job-key)))
        trigger (t/build
                  (t/with-identity (t/key "triggers.1"))
                  (t/start-now)
                  (t/with-schedule (schedule
                                     (with-interval-in-seconds EXPIRED_CONCEPT_CLEANUP_INTERVAL))))]<commit_msg>Updating job scheduling to slightly delay the start of the jobs.<commit_after>(ns cmr.metadata-db.services.jobs
  (:require [clojurewerkz.quartzite.scheduler :as qs]
            [clojurewerkz.quartzite.triggers :as t]
            [clojurewerkz.quartzite.jobs :as jobs]
            [clojurewerkz.quartzite.stateful :refer [def-stateful-job]]
            [clojurewerkz.quartzite.schedule.calendar-interval :refer [schedule with-interval-in-seconds]]
            [clojurewerkz.quartzite.conversion :as c]
            [cmr.metadata-db.db-holder :as db-holder]
            [cmr.common.log :as log :refer (debug info warn error)]
            [cmr.metadata-db.data.providers :as provider-db]
            [cmr.metadata-db.data.oracle.providers]
            [cmr.metadata-db.services.concept-service :as srv]
            [clj-time.core :as ct]))

(def job-key "jobs.expired.1")

(def EXPIRED_CONCEPT_CLEANUP_INTERVAL
  "The number of seconds between jobs run to cleanup expired granules and collections"
  (* 3600 5))

(defn configure-quartz-system-properties
  [db]
  (let [{{:keys [subprotocol subname user password]} :spec} db]
    (System/setProperty
      "org.quartz.dataSource.myDS.URL"
      (str "jdbc:" subprotocol ":" subname))
    (System/setProperty "org.quartz.dataSource.myDS.user" user)
    (System/setProperty "org.quartz.dataSource.myDS.password" password)))


(def-stateful-job ExpiredConceptCleanupJob
  [ctx]
  (info "Executing expired concepts cleanup.")
  (try
    (let [db (db-holder/get-db)]
      (doseq [provider (provider-db/get-providers db)]
        (srv/delete-expired-concepts db provider :collection)
        (srv/delete-expired-concepts db provider :granule)))
    (catch Throwable e
      (error e "ExpiredConceptCleanupJob caught Exception.")))
  (info "Finished expired concepts cleanup."))

(defn start
  [db]
  (info "Starting expired concepts cleanup job")
  (configure-quartz-system-properties db)
  (qs/initialize)
  (qs/start)
  ;; We delete exiting job and recreate it
  (when (qs/get-job job-key)
    (qs/delete-job (jobs/key job-key)))
  (let [job (jobs/build
              (jobs/of-type ExpiredConceptCleanupJob)
              (jobs/with-identity (jobs/key job-key)))
        trigger (t/build
                  (t/with-identity (t/key "triggers.1"))
                  (t/start-at (-> 5 ct/seconds ct/from-now))
                  (t/with-schedule (schedule
                                     (with-interval-in-seconds EXPIRED_CONCEPT_CLEANUP_INTERVAL))))]
{'fraction_non_alphanumeric': 0.12415349887133183, 'fraction_numerical': 0.003078185922429715, 'mean_word_length': 2.75211701308699, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1543282', 'n_tokens_mistral': 1668, 'n_tokens_neox': 1588, 'n_words': 343}