<commit_before>(ns pc.http.routes
  (:require [cemerick.url :as url]
            [cheshire.core :as json]
            [clojure.java.io :as io]
            [clojure.set :as set]
            [crypto.equality :as crypto]
            [datomic.api :as d]
            [defpage.core :as defpage :refer (defpage)]
            [hiccup.page]
            [pc.analytics :as analytics]
            [pc.assets]
            [pc.auth :as auth]
            [pc.auth.google :as google-auth]
            [pc.convert :as convert]
            [pc.datomic :as pcd]
            [pc.http.doc :as doc-http]
            [pc.http.handlers.custom-domain :as custom-domain]
            [pc.http.lb :as lb]
            [pc.http.sente :as sente]
            [pc.http.urls :as urls]
            [pc.models.access-request :as access-request-model]
            [pc.models.chat-bot :as chat-bot-model]
            [pc.models.cust :as cust-model]
            [pc.models.doc :as doc-model]
            [pc.models.invoice :as invoice-model]
            [pc.models.issue :as issue-model]
            [pc.models.layer :as layer-model]
            [pc.models.team :as team-model]
            [pc.profile :as profile]
            [pc.render :as render]
            [pc.util.md5 :as md5]
            [pc.views.content :as content]
            [pc.views.invoice :as invoice-view]
            [pc.views.team :as team-view]
            [ring.middleware.anti-forgery :as csrf]
            [ring.util.response :refer (redirect)])
  (:import [java.io ByteArrayOutputStream]
           [java.util UUID]))

(defn common-view-data [req]
  (merge
   {:CSRFToken csrf/*anti-forgery-token*
    :google-client-id (google-auth/google-client-id)
    :sente-id (-> req :session :sente-id)
    :hostname (profile/hostname)}
   (when-let [cust (-> req :auth :cust)]
     {:cust (cust-model/read-api cust)
      :admin? (contains? cust-model/admin-emails (:cust/email cust))})
   (when-let [team (-> req :team)]
     {:team (team-model/public-read-api team)})
   (when-let [subdomain (-> req :subdomain)]
     {:subdomain subdomain})))

(defpage root "/" [req]
  (if (:subdomain req)
    (let [db (pcd/default-db)]
      (cond (not (auth/logged-in? req))
            (team-view/login-interstitial req)


            (and (auth/logged-in? req)
                 (not (:team req)))
            {:status 200
             :body (team-view/request-domain req)}

            (and (auth/logged-in? req)
                 (:team req)
                 (not (auth/has-team-permission? db (:team req) (:auth req) :admin))
                 (seq (access-request-model/find-by-team-and-cust (pcd/default-db) (:team req) (get-in req [:auth :cust]))))
            {:status 200
             :body (team-view/requested-access req)}

            (and (auth/logged-in? req)
                 (:team req)
                 (not (auth/has-team-permission? db (:team req) (:auth req) :admin)))
            {:status 200
             :body (team-view/request-access req)}

            (and (auth/logged-in? req)
                 (:team req)
                 (auth/has-team-permission? db (:team req) (:auth req) :admin))
            (let [doc (doc-model/create-team-doc!
                       (:team req)
                       (merge {:document/chat-bot (rand-nth chat-bot-model/chat-bots)}
                              (when-let [cust-uuid (get-in req [:cust :cust/uuid])]
                                {:document/creator cust-uuid})))]
              (redirect (str "/document/" (:db/id doc))))))
    (if-let [cust-uuid (get-in req [:auth :cust :cust/uuid])]
      (redirect (str "/document/" (:db/id (doc-model/create-public-doc!
                                           (merge {:document/chat-bot (rand-nth chat-bot-model/chat-bots)}
                                                  (when cust-uuid {:document/creator cust-uuid})
                                                  ;; needs default permissions set
                                                  (when (:team req) {:document/team (:db/id (:team req))}))))))
      (content/app (common-view-data req)))))

(defpage request-team-permission [:post "/request-team-permission"] [req]
  (let [team (:team req)
        cust (get-in req [:auth :cust])]
    (when-not team
      {:throw 400
       :public-message "Sorry, we couldn't find that team."})
    (when-not cust
      {:throw 400
       :public-message "Please log in before requesting access."})
    (do (access-request-model/create-team-request team cust {:transaction/team (:db/id team)
                                                             :cust/uuid (:cust/uuid cust)
                                                             :transaction/broadcast true})
        (redirect "/"))))

(defn doc-resp [doc req & {:keys [view-data]}]
  (let [read-access? (auth/has-document-permission? (pcd/default-db) doc (-> req :auth) :read)]
    (content/app (merge (common-view-data req)
                        {:initial-document-id (:db/id doc)
                         :meta-image (urls/png-from-doc doc)
                         :meta-url (urls/from-doc doc)
                         :meta-title (when read-access? (:document/name doc))
                         :initial-entities (if read-access?
                                             [(doc-model/read-api doc)]
                                             [{:db/id (:db/id doc)}])}
                        view-data))))

(defn parse-doc-id [doc-id-param]
  (Long/parseLong (re-find #"[0-9]+$" doc-id-param)))

(defn handle-doc [req]
  (let [document-id (some-> req :params :document-id parse-doc-id)
        db (pcd/default-db)
        doc (doc-model/find-by-team-and-id db (:team req) document-id)]
    (if doc
      (doc-resp doc req)
      (if-let [redirect-doc (doc-model/find-by-team-and-invalid-id db (:team req) document-id)]
        (redirect (str "/document/" (:db/id redirect-doc)))
        {:status 404
         :body (if-let [doc (doc-model/find-by-id db document-id)]
                 (hiccup.page/html5
                  [:body "This document lives on a different domain: " [:a {:href (urls/from-doc doc)}
                                                                        (urls/from-doc doc)]])
                 "Document not found")}))))

(defpage document [:get "/document/:document-id" {:document-id #"[A-Za-z0-9_-]*-{0,1}[0-9]+"}] [req]
  (handle-doc req))

(defpage document-overlay [:get "/document/:document-id/:overlay" {:document-id #"[A-Za-z0-9_-]*-{0,1}[0-9]+" :overlay #"[\w-]+"}] [req]
  (handle-doc req))

(def private-layers [{:layer/opacity 1.0, :layer/stroke-width 1.0, :layer/end-x 389.5996, :entity/type :layer, :layer/start-y 120.0, :layer/text "Please log in or request access to view it.", :layer/stroke-color "black", :layer/start-x 100.0, :layer/fill "none", :layer/type :layer.type/text, :layer/end-y 100.0} {:layer/opacity 1.0, :layer/stroke-width 1.0, :layer/end-x 373.5547, :entity/type :layer, :layer/start-y 90.0, :layer/text "Sorry, this document is private.", :layer/stroke-color "black", :layer/start-x 100.0, :layer/fill "none", :layer/type :layer.type/text, :layer/end-y 70.0}])

(defn image-cache-headers [db doc]
  (let [last-modified-instant (or (doc-http/last-modified-instant db doc)
                                  (java.util.Date.))]
    {"Cache-Control" "no-cache; private"
     "ETag" (format "\"%s\"" (md5/encode (str last-modified-instant)))
     "Last-Modified" (->> last-modified-instant
                       (clj-time.coerce/from-date)
                       (clj-time.format/unparse (clj-time.format/formatters :rfc822)))}))

(defpage doc-svg "/document/:document-id.svg" [req]
  (let [document-id (-> req :params :document-id parse-doc-id)
        db (pcd/default-db)
        doc (doc-model/find-by-team-and-id db (:team req) document-id)]
    (cond (nil? doc)
          (if-let [doc (doc-model/find-by-team-and-invalid-id db (:team req) document-id)]
            (redirect (str "/document/" (:db/id doc) ".svg"))

            {:status 404
             ;; TODO: Maybe return a "not found" image.
             :body "Document not found."})

          (auth/has-document-permission? db doc (-> req :auth) :read)
          (if (= :head (:request-method req))
            {:status 200
             :headers (merge {"Content-Type" "image/svg+xml"}
                             (image-cache-headers db doc))
             :pc/doc doc
             :body ""}
            (let [as-of (some-> req :params :as-of (Long/parseLong))
                  layer-db (if as-of (d/as-of db as-of) db)
                  layers (layer-model/find-by-document layer-db doc)]
              {:status 200
               :headers (merge {"Content-Type" "image/svg+xml"}
                               (when (-> req :params :dl)
                                 {"Content-Disposition" (format "attachment; filename=\"precursor-document-%s.svg\""
                                                                (:db/id doc))})
                               (image-cache-headers layer-db doc))
               :pc/doc doc
               :body (render/render-layers layers :invert-colors? (-> req :params :printer-friendly (= "false")))}))

          (auth/logged-in? req)
          {:status 200
           :headers {"Content-Type" "image/svg+xml"
                     "Cache-Control" "no-cache; private"}
           :body (render/render-layers private-layers :invert-colors? (-> req :params :printer-friendly (= "false")))}

          :else
          {:status 200
           :headers {"Content-Type" "image/svg+xml"
                     "Cache-Control" "no-cache; private"}
           :body (render/render-layers private-layers :invert-colors? (-> req :params :printer-friendly (= "false")))})))

(defpage doc-png "/document/:document-id.png" [req]
  (let [document-id (-> req :params :document-id parse-doc-id)
        db (pcd/default-db)
        doc (doc-model/find-by-team-and-id db (:team req) document-id)]
    (cond (nil? doc)
          (if-let [redirect-doc (doc-model/find-by-team-and-invalid-id db (:team req) document-id)]
            (redirect (str "/document/" (:db/id redirect-doc) ".png"))

            {:status 404
             ;; TODO: Return a "not found" image.
             :body "Document not found."})

          (auth/has-document-permission? db doc (-> req :auth) :read)
          (if (= :head (:request-method req))
            {:status 200
             :headers (merge {"Content-Type" "image/png"}
                             (image-cache-headers db doc))
             :pc/doc doc
             :body ""}
            (let [as-of (some-> req :params :as-of (Long/parseLong))
                  layer-db (if as-of (d/as-of db as-of) db)
                  layers (layer-model/find-by-document layer-db doc)]
              {:status 200
               :headers (merge {"Content-Type" "image/png"}
                               (when (-> req :params :dl)
                                 {"Content-Disposition" (format "attachment; filename=\"precursor-document-%s.png\""
                                                                (:db/id doc))})
                               (image-cache-headers layer-db doc))<commit_msg>fix broken utf-8 in svgs<commit_after>(ns pc.http.routes
  (:require [cemerick.url :as url]
            [cheshire.core :as json]
            [clojure.java.io :as io]
            [clojure.set :as set]
            [crypto.equality :as crypto]
            [datomic.api :as d]
            [defpage.core :as defpage :refer (defpage)]
            [hiccup.page]
            [pc.analytics :as analytics]
            [pc.assets]
            [pc.auth :as auth]
            [pc.auth.google :as google-auth]
            [pc.convert :as convert]
            [pc.datomic :as pcd]
            [pc.http.doc :as doc-http]
            [pc.http.handlers.custom-domain :as custom-domain]
            [pc.http.lb :as lb]
            [pc.http.sente :as sente]
            [pc.http.urls :as urls]
            [pc.models.access-request :as access-request-model]
            [pc.models.chat-bot :as chat-bot-model]
            [pc.models.cust :as cust-model]
            [pc.models.doc :as doc-model]
            [pc.models.invoice :as invoice-model]
            [pc.models.issue :as issue-model]
            [pc.models.layer :as layer-model]
            [pc.models.team :as team-model]
            [pc.profile :as profile]
            [pc.render :as render]
            [pc.util.md5 :as md5]
            [pc.views.content :as content]
            [pc.views.invoice :as invoice-view]
            [pc.views.team :as team-view]
            [ring.middleware.anti-forgery :as csrf]
            [ring.util.response :refer (redirect)])
  (:import [java.io ByteArrayOutputStream]
           [java.util UUID]))

(defn common-view-data [req]
  (merge
   {:CSRFToken csrf/*anti-forgery-token*
    :google-client-id (google-auth/google-client-id)
    :sente-id (-> req :session :sente-id)
    :hostname (profile/hostname)}
   (when-let [cust (-> req :auth :cust)]
     {:cust (cust-model/read-api cust)
      :admin? (contains? cust-model/admin-emails (:cust/email cust))})
   (when-let [team (-> req :team)]
     {:team (team-model/public-read-api team)})
   (when-let [subdomain (-> req :subdomain)]
     {:subdomain subdomain})))

(defpage root "/" [req]
  (if (:subdomain req)
    (let [db (pcd/default-db)]
      (cond (not (auth/logged-in? req))
            (team-view/login-interstitial req)


            (and (auth/logged-in? req)
                 (not (:team req)))
            {:status 200
             :body (team-view/request-domain req)}

            (and (auth/logged-in? req)
                 (:team req)
                 (not (auth/has-team-permission? db (:team req) (:auth req) :admin))
                 (seq (access-request-model/find-by-team-and-cust (pcd/default-db) (:team req) (get-in req [:auth :cust]))))
            {:status 200
             :body (team-view/requested-access req)}

            (and (auth/logged-in? req)
                 (:team req)
                 (not (auth/has-team-permission? db (:team req) (:auth req) :admin)))
            {:status 200
             :body (team-view/request-access req)}

            (and (auth/logged-in? req)
                 (:team req)
                 (auth/has-team-permission? db (:team req) (:auth req) :admin))
            (let [doc (doc-model/create-team-doc!
                       (:team req)
                       (merge {:document/chat-bot (rand-nth chat-bot-model/chat-bots)}
                              (when-let [cust-uuid (get-in req [:cust :cust/uuid])]
                                {:document/creator cust-uuid})))]
              (redirect (str "/document/" (:db/id doc))))))
    (if-let [cust-uuid (get-in req [:auth :cust :cust/uuid])]
      (redirect (str "/document/" (:db/id (doc-model/create-public-doc!
                                           (merge {:document/chat-bot (rand-nth chat-bot-model/chat-bots)}
                                                  (when cust-uuid {:document/creator cust-uuid})
                                                  ;; needs default permissions set
                                                  (when (:team req) {:document/team (:db/id (:team req))}))))))
      (content/app (common-view-data req)))))

(defpage request-team-permission [:post "/request-team-permission"] [req]
  (let [team (:team req)
        cust (get-in req [:auth :cust])]
    (when-not team
      {:throw 400
       :public-message "Sorry, we couldn't find that team."})
    (when-not cust
      {:throw 400
       :public-message "Please log in before requesting access."})
    (do (access-request-model/create-team-request team cust {:transaction/team (:db/id team)
                                                             :cust/uuid (:cust/uuid cust)
                                                             :transaction/broadcast true})
        (redirect "/"))))

(defn doc-resp [doc req & {:keys [view-data]}]
  (let [read-access? (auth/has-document-permission? (pcd/default-db) doc (-> req :auth) :read)]
    (content/app (merge (common-view-data req)
                        {:initial-document-id (:db/id doc)
                         :meta-image (urls/png-from-doc doc)
                         :meta-url (urls/from-doc doc)
                         :meta-title (when read-access? (:document/name doc))
                         :initial-entities (if read-access?
                                             [(doc-model/read-api doc)]
                                             [{:db/id (:db/id doc)}])}
                        view-data))))

(defn parse-doc-id [doc-id-param]
  (Long/parseLong (re-find #"[0-9]+$" doc-id-param)))

(defn handle-doc [req]
  (let [document-id (some-> req :params :document-id parse-doc-id)
        db (pcd/default-db)
        doc (doc-model/find-by-team-and-id db (:team req) document-id)]
    (if doc
      (doc-resp doc req)
      (if-let [redirect-doc (doc-model/find-by-team-and-invalid-id db (:team req) document-id)]
        (redirect (str "/document/" (:db/id redirect-doc)))
        {:status 404
         :body (if-let [doc (doc-model/find-by-id db document-id)]
                 (hiccup.page/html5
                  [:body "This document lives on a different domain: " [:a {:href (urls/from-doc doc)}
                                                                        (urls/from-doc doc)]])
                 "Document not found")}))))

(defpage document [:get "/document/:document-id" {:document-id #"[A-Za-z0-9_-]*-{0,1}[0-9]+"}] [req]
  (handle-doc req))

(defpage document-overlay [:get "/document/:document-id/:overlay" {:document-id #"[A-Za-z0-9_-]*-{0,1}[0-9]+" :overlay #"[\w-]+"}] [req]
  (handle-doc req))

(def private-layers [{:layer/opacity 1.0, :layer/stroke-width 1.0, :layer/end-x 389.5996, :entity/type :layer, :layer/start-y 120.0, :layer/text "Please log in or request access to view it.", :layer/stroke-color "black", :layer/start-x 100.0, :layer/fill "none", :layer/type :layer.type/text, :layer/end-y 100.0} {:layer/opacity 1.0, :layer/stroke-width 1.0, :layer/end-x 373.5547, :entity/type :layer, :layer/start-y 90.0, :layer/text "Sorry, this document is private.", :layer/stroke-color "black", :layer/start-x 100.0, :layer/fill "none", :layer/type :layer.type/text, :layer/end-y 70.0}])

(defn image-cache-headers [db doc]
  (let [last-modified-instant (or (doc-http/last-modified-instant db doc)
                                  (java.util.Date.))]
    {"Cache-Control" "no-cache; private"
     "ETag" (format "\"%s\"" (md5/encode (str last-modified-instant)))
     "Last-Modified" (->> last-modified-instant
                       (clj-time.coerce/from-date)
                       (clj-time.format/unparse (clj-time.format/formatters :rfc822)))}))

(defpage doc-svg "/document/:document-id.svg" [req]
  (let [document-id (-> req :params :document-id parse-doc-id)
        db (pcd/default-db)
        doc (doc-model/find-by-team-and-id db (:team req) document-id)]
    (cond (nil? doc)
          (if-let [doc (doc-model/find-by-team-and-invalid-id db (:team req) document-id)]
            (redirect (str "/document/" (:db/id doc) ".svg"))

            {:status 404
             ;; TODO: Maybe return a "not found" image.
             :body "Document not found."})

          (auth/has-document-permission? db doc (-> req :auth) :read)
          (if (= :head (:request-method req))
            {:status 200
             :headers (merge {"Content-Type" "image/svg+xml; charset=UTF-8"}
                             (image-cache-headers db doc))
             :pc/doc doc
             :body ""}
            (let [as-of (some-> req :params :as-of (Long/parseLong))
                  layer-db (if as-of (d/as-of db as-of) db)
                  layers (layer-model/find-by-document layer-db doc)]
              {:status 200
               :headers (merge {"Content-Type" "image/svg+xml; charset=UTF-8"}
                               (when (-> req :params :dl)
                                 {"Content-Disposition" (format "attachment; filename=\"precursor-document-%s.svg\""
                                                                (:db/id doc))})
                               (image-cache-headers layer-db doc))
               :pc/doc doc
               :body (render/render-layers layers :invert-colors? (-> req :params :printer-friendly (= "false")))}))

          (auth/logged-in? req)
          {:status 200
           :headers {"Content-Type" "image/svg+xml; charset=UTF-8"
                     "Cache-Control" "no-cache; private"}
           :body (render/render-layers private-layers :invert-colors? (-> req :params :printer-friendly (= "false")))}

          :else
          {:status 200
           :headers {"Content-Type" "image/svg+xml; charset=UTF-8"
                     "Cache-Control" "no-cache; private"}
           :body (render/render-layers private-layers :invert-colors? (-> req :params :printer-friendly (= "false")))})))

(defpage doc-png "/document/:document-id.png" [req]
  (let [document-id (-> req :params :document-id parse-doc-id)
        db (pcd/default-db)
        doc (doc-model/find-by-team-and-id db (:team req) document-id)]
    (cond (nil? doc)
          (if-let [redirect-doc (doc-model/find-by-team-and-invalid-id db (:team req) document-id)]
            (redirect (str "/document/" (:db/id redirect-doc) ".png"))

            {:status 404
             ;; TODO: Return a "not found" image.
             :body "Document not found."})

          (auth/has-document-permission? db doc (-> req :auth) :read)
          (if (= :head (:request-method req))
            {:status 200
             :headers (merge {"Content-Type" "image/png"}
                             (image-cache-headers db doc))
             :pc/doc doc
             :body ""}
            (let [as-of (some-> req :params :as-of (Long/parseLong))
                  layer-db (if as-of (d/as-of db as-of) db)
                  layers (layer-model/find-by-document layer-db doc)]
              {:status 200
               :headers (merge {"Content-Type" "image/png"}
                               (when (-> req :params :dl)
                                 {"Content-Disposition" (format "attachment; filename=\"precursor-document-%s.png\""
                                                                (:db/id doc))})
                               (image-cache-headers layer-db doc))
{'fraction_non_alphanumeric': 0.1579744354874627, 'fraction_numerical': 0.009753707745067475, 'mean_word_length': 1.6397836821067482, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 52, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3791057', 'n_tokens_mistral': 7138, 'n_tokens_neox': 6811, 'n_words': 1623}