<commit_before>(ns layzee.use-cases.lazy-web
  (:gen-class)
  (:require [clojure.test :refer :all]
            [layzee.adapters.twitter.search :refer :all :as twitter]
            [layzee.adapters.settings :refer :all :as settings]
            [clj-http.util :as util]
            [clj-time.core :as t]
            [clj-time.format :as f]
            [clojure.core.memoize :as memo]))

(defn- clean[text]
  (.replace text "\n" ""))

(defn- layout[t width]
  (clojure.pprint/cl-format nil (format "~%sA" width) t))

(defn- earl[t]
  ;; https://twitter.com/marick/status/633460947193978880
  (layout (format "https://twitter.com/%s/status/%s" (util/url-encode (-> t :user :id_str)) (-> t :id)) 65))

(def local-date-format
     (f/formatter "dd-MM-yy HH:mm"))

(def twitter-date-format
     (f/formatter "EEE MMM dd HH:mm:ss Z yyyy"))

(defn- local[date] ;; https://github.com/clj-time/clj-time ;; Tue Aug 18 02:14:51 +0000 2015
  (f/unparse local-date-format (t/to-time-zone (f/parse twitter-date-format date) (t/time-zone-for-id "Pacific/Auckland"))))

(defn- view[t]
  (format "[%s] %s %s %s" (local (:created_at t)) (earl t) (layout (-> t :user :name) 30) (-> t :text clean)))

(def ^{:private true} no-retweets
     #(nil? (:retweeted_status %1)))

(defn- result[adapters how-many] (apply (:search-adapter-fn adapters) [{:count how-many :filter no-retweets}]))

(def ^{:private true} cached-result
  (let [ttl-in-seconds (* 60 5)]
    (memo/ttl result :ttl/threshold (* 1000 ttl-in-seconds)))) ;; http://clojure.github.io/core.memoize/

(defn run[adapters]
  (let [how-many 100]
    (let [result (apply cached-result [adapters how-many])]
      (println (format "Searched for <%s> <#lazyweb> mentions and found <%s> results (filtered)" how-many (count result)))

      (doseq [tweet result]
        (println (view tweet))))))
<commit_msg>(Caching has no effect in CLI of course...)<commit_after>(ns layzee.use-cases.lazy-web
  (:gen-class)
  (:require [clojure.test :refer :all]
            [layzee.adapters.twitter.search :refer :all :as twitter]
            [layzee.adapters.settings :refer :all :as settings]
            [clj-http.util :as util]
            [clj-time.core :as t]
            [clj-time.format :as f]
            [clojure.core.memoize :as memo]))

(defn- clean[text]
  (.replace text "\n" ""))

(defn- layout[t width]
  (clojure.pprint/cl-format nil (format "~%sA" width) t))

(defn- earl[t]
  ;; https://twitter.com/marick/status/633460947193978880
  (layout (format "https://twitter.com/%s/status/%s" (util/url-encode (-> t :user :id_str)) (-> t :id)) 65))

(def local-date-format
     (f/formatter "dd-MM-yy HH:mm"))

(def twitter-date-format
     (f/formatter "EEE MMM dd HH:mm:ss Z yyyy"))

(defn- local[date] ;; https://github.com/clj-time/clj-time ;; Tue Aug 18 02:14:51 +0000 2015
  (f/unparse local-date-format (t/to-time-zone (f/parse twitter-date-format date) (t/time-zone-for-id "Pacific/Auckland"))))

(defn- view[t]
  (format "[%s] %s %s %s" (local (:created_at t)) (earl t) (layout (-> t :user :name) 30) (-> t :text clean)))

(def ^{:private true} no-retweets
     #(nil? (:retweeted_status %1)))

(defn- result[adapters how-many]
  {
   :timestamp (t/now)
   :result (apply (:search-adapter-fn adapters) [{:count how-many :filter no-retweets}])})

(def ^{:private true} cached-result
  (let [ttl-in-seconds (* 60 5)]
    (memo/ttl result :ttl/threshold (* 1000 ttl-in-seconds)))) ;; http://clojure.github.io/core.memoize/

(defn run[adapters]
  (let [how-many 100]
    (let [cached (apply cached-result [adapters how-many])]
      (let [{timestamp :timestamp result :result} cached]
        (println (format "Searched for <%s> <#lazyweb> mentions and found <%s> results (filtered) -- [at: %s]" how-many (count result) (str timestamp)))
        
        (doseq [tweet result]
          (println (view tweet)))))))

{'fraction_non_alphanumeric': 0.18904180732277331, 'fraction_numerical': 0.02544793560114256, 'mean_word_length': 3.2098360655737705, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 17, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6939644', 'n_tokens_mistral': 1603, 'n_tokens_neox': 1465, 'n_words': 360}