<commit_before>(ns polismath.server
  (:require [polismath.export :as export]
            [ring.component.jetty :refer [jetty-server]]
            [ring.util.response :as response]
            [ring.middleware.params :as ring.params]
            [ring.middleware.keyword-params :as ring.keyword-params]
            [clojure.core.async :as async :refer [chan >!! <!! >! <! go]]
            [com.stuartsierra.component :as component]
            [plumbing.core :as pc]
            [polismath.env :as env]
            [polismath.db :as db]
            [monger.core :as mg]
            [monger.collection :as mc]
            [amazonica.core :as aws]
            [amazonica.aws.s3 :as s3]
            [amazonica.aws.s3transfer :as s3transfer]
            [clj-time.core :as time]
            [bidi.bidi :as bidi]
            [bidi.ring]))


;; First we'll just set up some basic helpers and settings/variables we'll need.
;; We'll really want to move these to the configuration component when that exists. XXX

(def tmp-dir "/tmp/")
(defn full-path [filename] (str tmp-dir filename))

(def app-url-base "https://localhost:3000")
;(def app-url-base "https://polismath-aux.heroku.com")
(defn full-url
  [& path]
  (apply str app-url-base "/" path))


;; A ping handler will just be for debugging purposes

(defn ping-handler
  "Simple ping handler. Returns response with edn representation of the request as the body.
  For testing purposes."
  [request]
  {:status  202
   :headers {"Content-Type" "text/plain" "Location" "chhhdkjdf"}
   :params {:stuff "the x y"}
   :body (with-out-str (clojure.pprint/pprint request))})


;; The filename is actually pretty iportant.
;; It should be unique between different expors, as is used as the identifying key for the aws buckets, mongo
;; tracking and as part of the API requests.

(defn generate-filename
  "Generates a filename based on request-params"
  [{:keys [zid zinvite at-date format] :as request-params}]
  (let [zinvite (or zinvite (export/get-zinvite-from-zid zid))
        last-updated (or at-date (System/currentTimeMillis))
        ext (case format :excel "xlsx" :csv "zip")
        path (str "/tmp/polis-export-" zinvite "-" last-updated "." ext)]
    path))


;; The following is really just a bunch of parameter parsing stuff.
;; Tihs could all possibly be interwoven with the config component as well.

(defn ->int
  "Try to parse as an integer; return nil if not possible."
  [x]
  (try (Integer/parseInt x)
       (catch Exception e nil)))

(def parsers {:zid ->int
              :zinvite ->int})

(def allowed-params #{:filename :zid :zinvite :at-date :format})

(defn parsed-params
  "Parses the params for a request, occording to parsers."
  [params]
  (reduce
    (fn [m [k v]] (if (allowed-params k)
                    (assoc m k ((parsers k) v))
                    m))
    {}
    params))


;; We use mongo to persists the status of our exports, and data needed for downloading from AWS.
;; Here, we're setting up basic mongo read and write helpers.

(defn update-export-status
  [filename document]
  (mc/insert (db/mongo-db (:env/env :mongolab-uri))
             (db/mongo-collection-name "exports")
             {:filename filename}
             (db/format-for-mongo identity (assoc document
                                                  :filename filename
                                                  :lastupdate (System/currentTimeMillis)))
             {:upsert true}))

(defn get-export-status
  [filename]
  (mc/find-one (db/mongo-db (:env/env :mongolab-uri))
               (db/mongo-collection-name "exports")
               {:filename filename}))

(defn notify-mongo
  [filename status params]
  (update-export-status filename {:status status :params params}))


;; We use AWS to store conversation exports whcih took a long time to compute.
;; These exports are set to expire automatically.

;; move to component; uncomment on commit XXX
;(def aws-cred {:access-key (env/env :aws-access-key)
               ;:secret-key (env/env :aws-secret-key)})

(defn full-aws-path
  [filename]
  (str (:math-env env/env) "/" filename))

(defn upload-to-aws
  [aws-cred filename]
  (s3/put-object aws-cred
                 :bucket-name "polis-datadump"
                 :key (full-aws-path filename)
                 :file (full-path filename)))

(defn handle-completion!
  [aws-cred filename params]
  (upload-to-aws aws-cred filename)
  (notify-mongo filename "complete" params))

(defn handle-timedout-datadump!
  [aws-cred filename compute-chan params]
  ;; First let mongo know we're working on it
  (notify-mongo filename "processing" params)
  ;; Start a go routine, which upon completion of the computation loads the data to
  ;; aws and updates the mongo
  (go (when-let [late-result (<! compute-chan)]
        (handle-completion! aws-cred filename params))))

(defn generate-aws-url!
  "Generate a presigned url from amazon for the given filename. Optionally set an expiration in hours (defaulting to the number
  sourced from env variable :download-link-expiration."
  ([aws-cred filename expiration]
   ;; XXX more env/env stuff
   (let [expiration (-> expiration time/hours time/from-now)]
     (s3/generate-presigned-url aws-cred "polis-datadump" (full-aws-path filename) expiration)))
  ([aws-cred filename]
   (let [expiration (->int (:download-link-expiration env/env))]
     (generate-aws-url! aws-cred filename expiration))))


;; What follows is the guts of our responding and computing.
;; The outline of what this looks like is as follows:

;; get export params
;;   * notify-mongo (:started)
;;   * start computing
;;   * complete:
;;       * go:
;;            * upload-aws
;;            * notify-mongo (:complete)
;;       * response 200
;;   * timeout
;;       * notify-mongo (:processing)
;;       * response 202
;;       * go:
;;            * complete:
;;                 * upload-aws
;;                 * notify-mongo (:complete)
;; get export filename
;;   * not ready
;;       * 404 not there
;;   * ready
;;       * proxy aws file
;; get status filename
;;   * check status
;;   * done:
;;       * respond 201
;;       * link for download
;;   * pending
;;       * send link for download.


;; General helpers..

(def respond-404
  {:status 404
   :headers {"Content-Type" "text/plain"}
   :body "Unknown, incomplete or expired conversation export"})


;; Requests for exported files in aws.
;; <commit_msg>Few extra overview notes<commit_after>(ns polismath.server
  (:require [polismath.export :as export]
            [ring.component.jetty :refer [jetty-server]]
            [ring.util.response :as response]
            [ring.middleware.params :as ring.params]
            [ring.middleware.keyword-params :as ring.keyword-params]
            [clojure.core.async :as async :refer [chan >!! <!! >! <! go]]
            [com.stuartsierra.component :as component]
            [plumbing.core :as pc]
            [polismath.env :as env]
            [polismath.db :as db]
            [monger.core :as mg]
            [monger.collection :as mc]
            [amazonica.core :as aws]
            [amazonica.aws.s3 :as s3]
            [amazonica.aws.s3transfer :as s3transfer]
            [clj-time.core :as time]
            [bidi.bidi :as bidi]
            [bidi.ring]))


;; First we'll just set up some basic helpers and settings/variables we'll need.
;; We'll really want to move these to the configuration component when that exists. XXX

(def tmp-dir "/tmp/")
(defn full-path [filename] (str tmp-dir filename))

(def app-url-base "https://localhost:3000")
;(def app-url-base "https://polismath-aux.heroku.com")
(defn full-url
  [& path]
  (apply str app-url-base "/" path))


;; A ping handler will just be for debugging purposes

(defn ping-handler
  "Simple ping handler. Returns response with edn representation of the request as the body.
  For testing purposes."
  [request]
  {:status  202
   :headers {"Content-Type" "text/plain" "Location" "chhhdkjdf"}
   :params {:stuff "the x y"}
   :body (with-out-str (clojure.pprint/pprint request))})


;; The filename is actually pretty iportant.
;; It should be unique between different expors, as is used as the identifying key for the aws buckets, mongo
;; tracking and as part of the API requests.

(defn generate-filename
  "Generates a filename based on request-params"
  [{:keys [zid zinvite at-date format] :as request-params}]
  (let [zinvite (or zinvite (export/get-zinvite-from-zid zid))
        last-updated (or at-date (System/currentTimeMillis))
        ext (case format :excel "xlsx" :csv "zip")
        path (str "/tmp/polis-export-" zinvite "-" last-updated "." ext)]
    path))


;; The following is really just a bunch of parameter parsing stuff.
;; Tihs could all possibly be interwoven with the config component as well.

(defn ->int
  "Try to parse as an integer; return nil if not possible."
  [x]
  (try (Integer/parseInt x)
       (catch Exception e nil)))

(def parsers {:zid ->int
              :zinvite ->int})

(def allowed-params #{:filename :zid :zinvite :at-date :format})

(defn parsed-params
  "Parses the params for a request, occording to parsers."
  [params]
  (reduce
    (fn [m [k v]] (if (allowed-params k)
                    (assoc m k ((parsers k) v))
                    m))
    {}
    params))


;; We use mongo to persists the status of our exports, and data needed for downloading from AWS.
;; Here, we're setting up basic mongo read and write helpers.

(defn update-export-status
  [filename document]
  (mc/insert (db/mongo-db (:env/env :mongolab-uri))
             (db/mongo-collection-name "exports")
             {:filename filename}
             (db/format-for-mongo identity (assoc document
                                                  :filename filename
                                                  :lastupdate (System/currentTimeMillis)))
             {:upsert true}))

(defn get-export-status
  [filename]
  (mc/find-one (db/mongo-db (:env/env :mongolab-uri))
               (db/mongo-collection-name "exports")
               {:filename filename}))

(defn notify-mongo
  [filename status params]
  (update-export-status filename {:status status :params params}))


;; We use AWS to store conversation exports whcih took a long time to compute.
;; These exports are set to expire automatically.

;; move to component; uncomment on commit XXX
;(def aws-cred {:access-key (env/env :aws-access-key)
               ;:secret-key (env/env :aws-secret-key)})

(defn full-aws-path
  [filename]
  (str (:math-env env/env) "/" filename))

(defn upload-to-aws
  [aws-cred filename]
  (s3/put-object aws-cred
                 :bucket-name "polis-datadump"
                 :key (full-aws-path filename)
                 :file (full-path filename)))

(defn handle-completion!
  [aws-cred filename params]
  (upload-to-aws aws-cred filename)
  (notify-mongo filename "complete" params))

(defn handle-timedout-datadump!
  [aws-cred filename compute-chan params]
  ;; First let mongo know we're working on it
  (notify-mongo filename "processing" params)
  ;; Start a go routine, which upon completion of the computation loads the data to
  ;; aws and updates the mongo
  (go (when-let [late-result (<! compute-chan)]
        (handle-completion! aws-cred filename params))))

(defn generate-aws-url!
  "Generate a presigned url from amazon for the given filename. Optionally set an expiration in hours (defaulting to the number
  sourced from env variable :download-link-expiration."
  ([aws-cred filename expiration]
   ;; XXX more env/env stuff
   (let [expiration (-> expiration time/hours time/from-now)]
     (s3/generate-presigned-url aws-cred "polis-datadump" (full-aws-path filename) expiration)))
  ([aws-cred filename]
   (let [expiration (->int (:download-link-expiration env/env))]
     (generate-aws-url! aws-cred filename expiration))))


;; What follows is the guts of our responding and computing.

;; We try to respond as soon as possible to the request.
;; If the computation takes too long, we respond with a 202 and a Location pointing to a status url.
;; When that url is pinged, it responds a 200 saying to check back later if it's not done.
;; If the computation is complete, it responds with a 201, and a url at which the results can be downloaded.

;; A more detailed outline of what this looks like in code is as follows:

;; get export params
;;   * notify-mongo (:started)
;;   * start computing
;;   * complete:
;;       * go:
;;            * upload-aws
;;            * notify-mongo (:complete)
;;       * response 200
;;   * timeout
;;       * notify-mongo (:processing)
;;       * response 202
;;       * go:
;;            * complete:
;;                 * upload-aws
;;                 * notify-mongo (:complete)
;; get export filename
;;   * not ready
;;       * 404 not there
;;   * ready
;;       * proxy aws file
;; get status filename
;;   * check status
;;   * done:
;;       * respond 201
;;       * link for download
;;   * pending
;;       * send link for download.


;; General helpers..

(def respond-404
  {:status 404
   :headers {"Content-Type" "text/plain"}
   :body "Unknown, incomplete or expired conversation export"})


;; Requests for exported files in aws.
{'fraction_non_alphanumeric': 0.12056308181336563, 'fraction_numerical': 0.005373495799591311, 'mean_word_length': 2.5946681175190425, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 17, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 19, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4117628', 'n_tokens_mistral': 4113, 'n_tokens_neox': 3951, 'n_words': 1361}