<commit_before>(ns chronograph.data
  (:require [chronograph.db :as d])
  (:require [clj-http.client :as client])
  (:require [crossref.util.config :refer [config]]
            [crossref.util.date :as crdate]
            [crossref.util.doi :as crdoi])
  (:require [korma.core :as k])
  (:require [clj-time.core :as t]
            [clj-time.periodic :as time-period])
  (:require [clj-time.coerce :as coerce]
            [clj-time.format :refer [parse formatter unparse]])
  (:require [robert.bruce :refer [try-try-again]]))

(def works-endpoint "http://api.crossref.org/v1/works")
(def api-page-size 1000)
(def yyyy-mm-dd (formatter (t/default-time-zone) "yyyy-MM-dd" "yyyy-MM-dd"))

(defn get-type-id-by-name [type-name]
  (:id (first (k/select d/types (k/where (= :ident type-name))))))

(defn get-source-id-by-name [source-name]
  (:id (first (k/select d/sources (k/where (= :ident source-name))))))

(defn get-doi-id [doi create?]
  (try
  (let [retrieved (first (k/select d/doi (k/where (= :doi doi))))]
    (if retrieved
      (:id retrieved)
      (when create? (let [inserted (k/insert d/doi (k/values {:doi doi}))]
        (:GENERATED_KEY inserted)))))
  ; Return nil on error
  (catch Exception e (prn "EXCEPTION" e))))

(defn insert-event [doi type-id source-id date overwrite cnt arg1 arg2 arg3]
  (let [doi-id (get-doi-id doi true)]
    (when overwrite (k/delete d/events (k/where {:doi doi-id
                                                :event (coerce/to-sql-time date)
                                                :source source-id
                                                :type type-id})))
    (try
    (k/insert d/events (k/values {:doi doi-id
                                  :type type-id
                                  :source source-id
                                  :event (coerce/to-sql-time date)
                                  :inserted (coerce/to-sql-time (t/now))
                                  :count (or cnt 1)
                                  :arg1 arg1 :arg2 arg2 :arg3 arg3}))
    (catch Exception e (prn "EXCEPTION" e)))))

(defn insert-event-timeline
  "Insert parts of an DOI's event timeline. This will merge the existing data.
  This should be used to update large quantities of data per DOI, source, type.
  merge-fn is used to replace duplicates. It should accept [old, new] and return new. E.g.  #(max %1 %2)"
  [doi type-id source-id data merge-fn]
  (let [doi-id (get-doi-id doi true)
        initial-row (first (k/select d/event-timelines
                                     (k/where (and
                                                (= :doi doi-id)
                                                (= :type type-id)
                                                (= :source source-id)))))
        initial-row-data (or (:timeline initial-row) {})
        merged-data (merge-with merge-fn initial-row-data data)]
    (if initial-row
      (k/update d/event-timelines
                (k/where {:doi doi-id
                          :type type-id
                          :source source-id})
                (k/set-fields {:timeline merged-data}))
      
      (k/insert d/event-timelines
                (k/values {:doi doi-id
                           :type type-id
                           :source source-id
                           :inserted (coerce/to-sql-time (t/now))
                           :timeline merged-data})))))

(defn insert-domain-timeline
  "Insert parts of a Referrer Domain's event timeline. This will merge the existing data.
  This should be used to update large quantities of data per Domain.
  merge-fn is used to replace duplicates. It should accept [old, new] and return new. E.g.  #(max %1 %2)"
  [host domain type-id source-id data merge-fn]
  (let [initial-row (first (k/select d/referrer-domain-timelines
                                     (k/where (and
                                                (= :domain domain)
                                                (= :host host)
                                                (= :type type-id)
                                                (= :source source-id)))))
        initial-row-data (or (:timeline initial-row) {})
        merged-data (merge-with merge-fn initial-row-data data)]
    (if initial-row
      (k/update d/referrer-domain-timelines
                (k/where {:domain domain
                          :host host
                          :type type-id
                          :source source-id})
                (k/set-fields {:timeline merged-data}))
      
      (k/insert d/referrer-domain-timelines
                (k/values {:domain domain
                           :host host
                           :type type-id
                           :source source-id
                           :inserted (coerce/to-sql-time (t/now))
                           :timeline merged-data})))))

(defn insert-subdomain-timeline
  "Insert parts of a Referrer Subdomain's event timeline. This will merge the existing data.
  This should be used to update large quantities of data per Domain.
  merge-fn is used to replace duplicates. It should accept [old, new] and return new. E.g.  #(max %1 %2)"
  [host domain type-id source-id data merge-fn]
  (let [initial-row (first (k/select d/referrer-subdomain-timelines
                                     (k/where (and
                                                (= :domain domain)
                                                (= :host host)
                                                (= :type type-id)
                                                (= :source source-id)))))
        initial-row-data (or (:timeline initial-row) {})
        merged-data (merge-with merge-fn initial-row-data data)]
    (if initial-row
      (k/update d/referrer-subdomain-timelines
                (k/where {:domain domain
                          :host host
                          :type type-id
                          :source source-id})
                (k/set-fields {:timeline merged-data}))
      
      (k/insert d/referrer-subdomain-timelines
                (k/values {:domain domain
                           :host host
                           :type type-id
                           :source source-id
                           :inserted (coerce/to-sql-time (t/now))
                           :timeline merged-data})))))

(defn sort-timeline-values
  "For a hashmap timeline, return as sorted list"
  [timeline]
  (into (sorted-map) (sort-by first t/before? (seq timeline))))

(defn get-doi-timelines
  "Get all timelines for a DOI"
  [doi]
  (when-let [timelines (k/select
    d/event-timelines
    (k/where {:doi (get-doi-id doi false)})
    (k/with d/types))]
    (map (fn [timeline]
           (assoc timeline :timeline (sort-timeline-values (:timeline timeline))))
         timelines)))

(defn get-domain-timelines
  "Get all timelines for a domain"
  [domain]
  (when-let [timelines (k/select
    d/referrer-domain-timelines
    (k/where {:host domain})
    (k/with d/types))]
    (map (fn [timeline]
           (assoc timeline :timeline (sort-timeline-values (:timeline timeline))))
         timelines)))

(defn get-subdomain-timelines
  "Get all timelines for a subdomain"
  [subdomain]
  (when-let [timelines (k/select
    d/referrer-domain-timelines
    (k/where {:host subdomain})
    (k/with d/types))]
    (map (fn [timeline]
           (assoc timeline :timeline (sort-timeline-values (:timeline timeline))))
         timelines)))

(defn insert-domain-event [host domain type-id source-id date overwrite cnt]
    (when overwrite (k/delete d/referrer-domain-events (k/where {:domain domain
                                                                :event (coerce/to-sql-time date)
                                                                :source source-id
                                                                :type type-id})))
    (try
    (k/insert d/referrer-domain-events (k/values {:host host
                                                  :domain domain
                                                  :type type-id
                                                  :source source-id
                                                  :event (coerce/to-sql-time date)
                                                  :inserted (coerce/to-sql-time (t/now))
                                                  :count (or cnt 1)}))
    (catch Exception e (prn "EXCEPTION" e))))

(defn insert-subdomain-event [host domain type-id source-id date overwrite cnt]
    (when overwrite (k/delete d/referrer-domain-events (k/where {:domain domain
                                                                 :host host
                                                                :event (coerce/to-sql-time date)
                                                                :source source-id
                                                                :type type-id})))
    (try
    (k/insert d/referrer-subdomain-events (k/values { :host host
                                                      :domain domain
                                                      :type type-id
                                                      :source source-id
                                                      :event (coerce/to-sql-time date)
                                                      :inserted (coerce/to-sql-time (t/now))
                                                      :count (or cnt 1)}))
    (catch Exception e (prn "EXCEPTION" e))))

(defn get-last-run-date
  "Last date that the DOI import was run."
  []
  (or (-> (k/select d/state (k/where (= "last-run" :name))) first :theDate coerce/from-sql-date)
      (when-let [date (:start-date config)] (apply t/date-time date))
      (t/now)))

(defn set-last-run-date!
  [date]
  (k/delete d/state (k/where (= :name "last-run")))
  (k/insert d/state (k/values {:name "last-run" :theDate (coerce/to-sql-date date)})))

(defn get-dois-updated-since [date]
  (let [base-q (if date 
          {:filter (str "from-update-date:" (unparse yyyy-mm-dd date))}
          {})
        results (client/get (str works-endpoint \? (client/generate-query-string (assoc base-q :rows 0))) {:as :json})
        num-results (-> results :body :message :total-results)
        ; Extra page just in case.
        num-pages (+ (quot num-results api-page-size) 2)
        page-queries (map #(str works-endpoint \? (client/generate-query-string (assoc base-q
                                                                                  :rows api-page-size
                                                                                  :offset (* api-page-size %)))) (range num-pages))
        
        issued-type-id (get-type-id-by-name "issued")
                  updated-type-id (get-type-id-by-name "updated")
                  metadata-source-id (get-source-id-by-name "CrossRefMetadata")]
    
      (doseq [page-query page-queries]
        (prn "Fetching" page-query)
        (let [response (client/get page-query {:as :json})
              items (-> response :body :message :items)]
          (prn "Fetched")
          (doseq [item items]
            (let [the-doi (:DOI item)
                  ; list of date parts in CrossRef date format
                  issued-input (-> item :issued :date-parts first)
                  ; CrossRef date native format<commit_msg>Made accesses to CrossRef API more robust to deal with network issues.<commit_after>(ns chronograph.data
  (:require [chronograph.db :as d])
  (:require [clj-http.client :as client])
  (:require [crossref.util.config :refer [config]]
            [crossref.util.date :as crdate]
            [crossref.util.doi :as crdoi])
  (:require [korma.core :as k])
  (:require [clj-time.core :as t]
            [clj-time.periodic :as time-period])
  (:require [clj-time.coerce :as coerce]
            [clj-time.format :refer [parse formatter unparse]])
  (:require [robert.bruce :refer [try-try-again]]))

(def works-endpoint "http://api.crossref.org/v1/works")
(def api-page-size 1000)
(def yyyy-mm-dd (formatter (t/default-time-zone) "yyyy-MM-dd" "yyyy-MM-dd"))

(defn get-type-id-by-name [type-name]
  (:id (first (k/select d/types (k/where (= :ident type-name))))))

(defn get-source-id-by-name [source-name]
  (:id (first (k/select d/sources (k/where (= :ident source-name))))))

(defn get-doi-id [doi create?]
  (try
  (let [retrieved (first (k/select d/doi (k/where (= :doi doi))))]
    (if retrieved
      (:id retrieved)
      (when create? (let [inserted (k/insert d/doi (k/values {:doi doi}))]
        (:GENERATED_KEY inserted)))))
  ; Return nil on error
  (catch Exception e (prn "EXCEPTION" e))))

(defn insert-event [doi type-id source-id date overwrite cnt arg1 arg2 arg3]
  (let [doi-id (get-doi-id doi true)]
    (when overwrite (k/delete d/events (k/where {:doi doi-id
                                                :event (coerce/to-sql-time date)
                                                :source source-id
                                                :type type-id})))
    (try
    (k/insert d/events (k/values {:doi doi-id
                                  :type type-id
                                  :source source-id
                                  :event (coerce/to-sql-time date)
                                  :inserted (coerce/to-sql-time (t/now))
                                  :count (or cnt 1)
                                  :arg1 arg1 :arg2 arg2 :arg3 arg3}))
    (catch Exception e (prn "EXCEPTION" e)))))

(defn insert-event-timeline
  "Insert parts of an DOI's event timeline. This will merge the existing data.
  This should be used to update large quantities of data per DOI, source, type.
  merge-fn is used to replace duplicates. It should accept [old, new] and return new. E.g.  #(max %1 %2)"
  [doi type-id source-id data merge-fn]
  (let [doi-id (get-doi-id doi true)
        initial-row (first (k/select d/event-timelines
                                     (k/where (and
                                                (= :doi doi-id)
                                                (= :type type-id)
                                                (= :source source-id)))))
        initial-row-data (or (:timeline initial-row) {})
        merged-data (merge-with merge-fn initial-row-data data)]
    (if initial-row
      (k/update d/event-timelines
                (k/where {:doi doi-id
                          :type type-id
                          :source source-id})
                (k/set-fields {:timeline merged-data}))
      
      (k/insert d/event-timelines
                (k/values {:doi doi-id
                           :type type-id
                           :source source-id
                           :inserted (coerce/to-sql-time (t/now))
                           :timeline merged-data})))))

(defn insert-domain-timeline
  "Insert parts of a Referrer Domain's event timeline. This will merge the existing data.
  This should be used to update large quantities of data per Domain.
  merge-fn is used to replace duplicates. It should accept [old, new] and return new. E.g.  #(max %1 %2)"
  [host domain type-id source-id data merge-fn]
  (let [initial-row (first (k/select d/referrer-domain-timelines
                                     (k/where (and
                                                (= :domain domain)
                                                (= :host host)
                                                (= :type type-id)
                                                (= :source source-id)))))
        initial-row-data (or (:timeline initial-row) {})
        merged-data (merge-with merge-fn initial-row-data data)]
    (if initial-row
      (k/update d/referrer-domain-timelines
                (k/where {:domain domain
                          :host host
                          :type type-id
                          :source source-id})
                (k/set-fields {:timeline merged-data}))
      
      (k/insert d/referrer-domain-timelines
                (k/values {:domain domain
                           :host host
                           :type type-id
                           :source source-id
                           :inserted (coerce/to-sql-time (t/now))
                           :timeline merged-data})))))

(defn insert-subdomain-timeline
  "Insert parts of a Referrer Subdomain's event timeline. This will merge the existing data.
  This should be used to update large quantities of data per Domain.
  merge-fn is used to replace duplicates. It should accept [old, new] and return new. E.g.  #(max %1 %2)"
  [host domain type-id source-id data merge-fn]
  (let [initial-row (first (k/select d/referrer-subdomain-timelines
                                     (k/where (and
                                                (= :domain domain)
                                                (= :host host)
                                                (= :type type-id)
                                                (= :source source-id)))))
        initial-row-data (or (:timeline initial-row) {})
        merged-data (merge-with merge-fn initial-row-data data)]
    (if initial-row
      (k/update d/referrer-subdomain-timelines
                (k/where {:domain domain
                          :host host
                          :type type-id
                          :source source-id})
                (k/set-fields {:timeline merged-data}))
      
      (k/insert d/referrer-subdomain-timelines
                (k/values {:domain domain
                           :host host
                           :type type-id
                           :source source-id
                           :inserted (coerce/to-sql-time (t/now))
                           :timeline merged-data})))))

(defn sort-timeline-values
  "For a hashmap timeline, return as sorted list"
  [timeline]
  (into (sorted-map) (sort-by first t/before? (seq timeline))))

(defn get-doi-timelines
  "Get all timelines for a DOI"
  [doi]
  (when-let [timelines (k/select
    d/event-timelines
    (k/where {:doi (get-doi-id doi false)})
    (k/with d/types))]
    (map (fn [timeline]
           (assoc timeline :timeline (sort-timeline-values (:timeline timeline))))
         timelines)))

(defn get-domain-timelines
  "Get all timelines for a domain"
  [domain]
  (when-let [timelines (k/select
    d/referrer-domain-timelines
    (k/where {:host domain})
    (k/with d/types))]
    (map (fn [timeline]
           (assoc timeline :timeline (sort-timeline-values (:timeline timeline))))
         timelines)))

(defn get-subdomain-timelines
  "Get all timelines for a subdomain"
  [subdomain]
  (when-let [timelines (k/select
    d/referrer-domain-timelines
    (k/where {:host subdomain})
    (k/with d/types))]
    (map (fn [timeline]
           (assoc timeline :timeline (sort-timeline-values (:timeline timeline))))
         timelines)))

(defn insert-domain-event [host domain type-id source-id date overwrite cnt]
    (when overwrite (k/delete d/referrer-domain-events (k/where {:domain domain
                                                                :event (coerce/to-sql-time date)
                                                                :source source-id
                                                                :type type-id})))
    (try
    (k/insert d/referrer-domain-events (k/values {:host host
                                                  :domain domain
                                                  :type type-id
                                                  :source source-id
                                                  :event (coerce/to-sql-time date)
                                                  :inserted (coerce/to-sql-time (t/now))
                                                  :count (or cnt 1)}))
    (catch Exception e (prn "EXCEPTION" e))))

(defn insert-subdomain-event [host domain type-id source-id date overwrite cnt]
    (when overwrite (k/delete d/referrer-domain-events (k/where {:domain domain
                                                                 :host host
                                                                :event (coerce/to-sql-time date)
                                                                :source source-id
                                                                :type type-id})))
    (try
    (k/insert d/referrer-subdomain-events (k/values { :host host
                                                      :domain domain
                                                      :type type-id
                                                      :source source-id
                                                      :event (coerce/to-sql-time date)
                                                      :inserted (coerce/to-sql-time (t/now))
                                                      :count (or cnt 1)}))
    (catch Exception e (prn "EXCEPTION" e))))

(defn get-last-run-date
  "Last date that the DOI import was run."
  []
  (or (-> (k/select d/state (k/where (= "last-run" :name))) first :theDate coerce/from-sql-date)
      (when-let [date (:start-date config)] (apply t/date-time date))
      (t/now)))

(defn set-last-run-date!
  [date]
  (k/delete d/state (k/where (= :name "last-run")))
  (k/insert d/state (k/values {:name "last-run" :theDate (coerce/to-sql-date date)})))

(defn get-dois-updated-since [date]
  (let [base-q (if date 
          {:filter (str "from-update-date:" (unparse yyyy-mm-dd date))}
          {})
        results (try-try-again #(client/get (str works-endpoint \? (client/generate-query-string (assoc base-q :rows 0))) {:as :json}))
        num-results (-> results :body :message :total-results)
        ; Extra page just in case.
        num-pages (+ (quot num-results api-page-size) 2)
        page-queries (map #(str works-endpoint \? (client/generate-query-string (assoc base-q
                                                                                  :rows api-page-size
                                                                                  :offset (* api-page-size %)))) (range num-pages))
        
        issued-type-id (get-type-id-by-name "issued")
                  updated-type-id (get-type-id-by-name "updated")
                  metadata-source-id (get-source-id-by-name "CrossRefMetadata")]
    
      (doseq [page-query page-queries]
        (prn "Fetching" page-query)
        (let [response (try-try-again #(client/get page-query {:as :json}))
              items (-> response :body :message :items)]
          (prn "Fetched")
          (doseq [item items]
            (let [the-doi (:DOI item)
                  ; list of date parts in CrossRef date format
                  issued-input (-> item :issued :date-parts first)
                  ; CrossRef date native format
{'fraction_non_alphanumeric': 0.11466829502333493, 'fraction_numerical': 0.0021808348235704627, 'mean_word_length': 1.2843479127229251, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 162, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5491244', 'n_tokens_mistral': 6385, 'n_tokens_neox': 6063, 'n_words': 1718}