<commit_before>(ns showkr.data
  (:import [goog.net Jsonp]))

(def URL "https://api.flickr.com/services/rest/")
(def OPTS {:api_key "1606ff0ad63a3b5efeaa89443fe80704"
           :format "json"})

(defonce world
  (atom {:opts {:target nil
                :path nil}
         :data {:form {}
                :sets {}
                :users {}}}))

(defn- flickr-error [payload]
  (js/console.error "Error fetching data with parameters:" payload))

(defn flickr-call [payload callback]
  (let [q (Jsonp. URL "jsoncallback")]
    (.send q (clj->js (merge OPTS payload)) callback flickr-error)))

(defn flickr-fetch [path attr payload & [cb]]
  (when-not (get-in @world path)
    (swap! world assoc-in path ^{:state :waiting} {})
    (flickr-call payload
      (fn [data]
        (let [data (js->clj data :keywordize-keys true)]
          (swap! world assoc-in path
            (condp = (:stat data)
              "ok"   (with-meta (attr data) {:state :fetched})
              "fail" (with-meta data {:state :failed})))
          (when cb (cb)))))))

(defn fetched? [data]
  (= :fetched (:state (meta data))))<commit_msg>make data fetcher react on oldness<commit_after>(ns showkr.data
  (:import [goog.net Jsonp]))

(def URL "https://api.flickr.com/services/rest/")
(def OPTS {:api_key "1606ff0ad63a3b5efeaa89443fe80704"
           :format "json"})

(def ^:dynamic *old-threshold* (* 1000 60 60 2)) ;; 2 hours

(defonce world
  (atom {:opts {:target nil
                :path nil}
         :data {:form {}
                :sets {}
                :users {}}}))

(defn- flickr-error [payload]
  (js/console.error "Error fetching data with parameters:" payload))

(defn flickr-call [payload callback]
  (let [q (Jsonp. URL "jsoncallback")]
    (.send q (clj->js (merge OPTS payload)) callback flickr-error)))

(defn old? [data]
  (> (- (.getTime (js/Date.))
       (or (-> data meta :date) 0))
    *old-threshold*))

(defn flickr-fetch [path attr payload & [cb]]
  (let [data (get-in @world path)]
    (when (empty? data)
      (swap! world assoc-in path
        ^{:state :waiting :date (.getDate (js/Date.))} {}))
    (when (old? data)
      (flickr-call payload
        (fn [data]
          (let [data (js->clj data :keywordize-keys true)]
            (swap! world assoc-in path
              (condp = (:stat data)
                "ok"
                (with-meta (attr data)
                  {:state :fetched
                   :date (.getTime (js/Date.))})

                "fail"
                (if (>= (:code data) 100) ;; not user input fault
                  nil
                  (with-meta data
                    {:state :failed}))))
            (when cb (cb))))))))

(defn fetched? [data]
  (= :fetched (:state (meta data))))
{'fraction_non_alphanumeric': 0.16386247256766642, 'fraction_numerical': 0.019019751280175568, 'mean_word_length': 2.1436781609195403, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 10, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5133335', 'n_tokens_mistral': 980, 'n_tokens_neox': 898, 'n_words': 260}