<commit_before>(ns
  ^{:author "<NAME>"
    :doc "xena server main."}
  cavm.core
  (:require [clojure.string :as s])
  (:require [cavm.h2 :as h2])
  (:require [cavm.auth :as auth])
  (:require [clojure.java.io :as io])
  (:require [ring.adapter.jetty :refer [run-jetty]])
  (:require [clojure.data.json :as json])
  (:require [me.raynes.fs :as fs])
  (:require [clojure.tools.cli :refer [parse-opts]])
  (:require [cgdata.core :as cgdata])
  (:require [ring.middleware.resource :refer [wrap-resource]])
  (:require [ring.middleware.content-type :refer [wrap-content-type]])
  (:require [ring.middleware.not-modified :refer [wrap-not-modified]])
  (:require [ring.middleware.params :refer [wrap-params]])
  (:require [ring.middleware.multipart-params :refer [wrap-multipart-params]])
  (:require [ring.middleware.multipart-params.byte-array :refer [byte-array-store]])
  (:require [cavm.views.datasets])
  (:require [ring.middleware.gzip :refer [wrap-gzip]])
  (:require [ring.middleware.stacktrace :refer [wrap-stacktrace-web]]) ; XXX only in dev
  (:require [liberator.dev :refer [wrap-trace]])                   ; XXX only in dev
  (:require [ring.middleware.reload :refer [wrap-reload]]) ; XXX only in dev
  (:require [filevents.core :refer [watch]])
  (:require [cavm.readers :as cr])
  (:require [cavm.loader :as cl])
  (:require [cavm.fs-utils :refer [normalized-path]])
  (:require [cavm.cgdata])
  (:require [clj-http.client :as client])
  (:require [clojure.tools.logging :as log :refer [info trace warn error]])
  (:require [taoensso.timbre :as timbre])
  (:require [less.awful.ssl :as ssl])
  (:require [ring.util.response :as response])
  (:require hiccup.page)
  (:import cavm.XenaImport cavm.XenaServer cavm.Splash cavm.CohortCallback)
  (:import org.slf4j.LoggerFactory
           ch.qos.logback.classic.joran.JoranConfigurator
           ch.qos.logback.core.util.StatusPrinter)
  (:require crypto.equality)
  (:require ring.middleware.session)
  (:require crypto.random)
  (:import [com.google.api.client.json.gson GsonFactory])
  (:import [com.google.api.client.http.javanet NetHttpTransport])
  (:import [com.google.api.client.googleapis.auth.oauth2 GoogleIdToken GoogleIdToken$Payload GoogleIdTokenVerifier GoogleIdTokenVerifier$Builder])
  (:gen-class))

(defn- in-data-path [root path]
  (boolean (fs/child-of? (normalized-path root) (normalized-path path))))

(def pom-props
  (doto (java.util.Properties.)
    (.load (io/reader (io/resource "META-INF/maven/cavm/cavm/pom.properties")))))

;
; web services

(def trusted-hosts
  ;#"https://[-_A-Za-z0-9]+.kilokluster.ucsc.edu|https://genome-cancer.ucsc.edu"
  #".*")

(def public-url "https://genome-cancer.ucsc.edu/")
(def public-xena (str public-url "proj/public/xena"))

(defn local-url [port] (str "https://local.xena.ucsc.edu:" (inc port)))
(defn local-xena [port] (str "http://local.xena.ucsc.edu:" port))

(def cohort-query
  '(map :cohort (query
                  {:select [:%distinct.cohort]
                   :from [:dataset]})))

(defn async-post [url post cb]
  (let [x (agent nil)]
    (set-error-handler! x (fn [_ err] (error err "Fetching cohort list") (cb [])))
    (send-off x (fn [_]
                  (cb (:body (client/post url {:body
                                               (str post)
                                               :as :json
                                               :content-type
                                               "text/plain"})))))))

; need to block in an agent & then invoke callback
(defn retrieve-cohorts [port cb]
  (let [x (atom [])
        update (fn [cohorts]
                 (swap! x (fn [all]
                            (let [ret (conj all cohorts)]
                              (when (= 2 (count ret))
                                (cb (filter identity (apply concat ret))))
                              ret))))]
    (async-post (str public-xena "/data/") cohort-query update)
    (async-post (str (local-xena port) "/data/") cohort-query update)))

(defn- wrap-access-control [handler]
  (fn [{:keys [request-method] :as request}]
    (let [response (if (= request-method :options)
                     (response/response "")
                     (handler request))]
      (info "request" request)
      (if-let [origin (re-matches trusted-hosts (get-in request [:headers "origin"] ""))]
        (-> response
            (assoc-in [:headers "Access-Control-Allow-Origin"] origin)
            (assoc-in [:headers "Access-Control-Expose-Headers"] "Location")
            (assoc-in [:headers "Access-Control-Allow-Credentials"] "true")
            (assoc-in [:headers "Access-Control-Allow-Headers"] "Cancer-Browser-Api, X-Redirect-To"))
        response))))

(defn- attr-middleware [app k v]
  (fn [req]
    (app (assoc req k v))))

(defn- log-middleware [handler]
  (fn [{:keys [uri query-string request-method session] :as request}]
    (info "Start:" (.toUpperCase (name request-method)) uri query-string
          (or (:email (:user session)) "(no session)"))
    (try<commit_msg>replace old public xena with the four current xena hubs on AWS<commit_after>(ns
  ^{:author "<NAME>"
    :doc "xena server main."}
  cavm.core
  (:require [clojure.string :as s])
  (:require [cavm.h2 :as h2])
  (:require [cavm.auth :as auth])
  (:require [clojure.java.io :as io])
  (:require [ring.adapter.jetty :refer [run-jetty]])
  (:require [clojure.data.json :as json])
  (:require [me.raynes.fs :as fs])
  (:require [clojure.tools.cli :refer [parse-opts]])
  (:require [cgdata.core :as cgdata])
  (:require [ring.middleware.resource :refer [wrap-resource]])
  (:require [ring.middleware.content-type :refer [wrap-content-type]])
  (:require [ring.middleware.not-modified :refer [wrap-not-modified]])
  (:require [ring.middleware.params :refer [wrap-params]])
  (:require [ring.middleware.multipart-params :refer [wrap-multipart-params]])
  (:require [ring.middleware.multipart-params.byte-array :refer [byte-array-store]])
  (:require [cavm.views.datasets])
  (:require [ring.middleware.gzip :refer [wrap-gzip]])
  (:require [ring.middleware.stacktrace :refer [wrap-stacktrace-web]]) ; XXX only in dev
  (:require [liberator.dev :refer [wrap-trace]])                   ; XXX only in dev
  (:require [ring.middleware.reload :refer [wrap-reload]]) ; XXX only in dev
  (:require [filevents.core :refer [watch]])
  (:require [cavm.readers :as cr])
  (:require [cavm.loader :as cl])
  (:require [cavm.fs-utils :refer [normalized-path]])
  (:require [cavm.cgdata])
  (:require [clj-http.client :as client])
  (:require [clojure.tools.logging :as log :refer [info trace warn error]])
  (:require [taoensso.timbre :as timbre])
  (:require [less.awful.ssl :as ssl])
  (:require [ring.util.response :as response])
  (:require hiccup.page)
  (:import cavm.XenaImport cavm.XenaServer cavm.Splash cavm.CohortCallback)
  (:import org.slf4j.LoggerFactory
           ch.qos.logback.classic.joran.JoranConfigurator
           ch.qos.logback.core.util.StatusPrinter)
  (:require crypto.equality)
  (:require ring.middleware.session)
  (:require crypto.random)
  (:import [com.google.api.client.json.gson GsonFactory])
  (:import [com.google.api.client.http.javanet NetHttpTransport])
  (:import [com.google.api.client.googleapis.auth.oauth2 GoogleIdToken GoogleIdToken$Payload GoogleIdTokenVerifier GoogleIdTokenVerifier$Builder])
  (:gen-class))

(defn- in-data-path [root path]
  (boolean (fs/child-of? (normalized-path root) (normalized-path path))))

(def pom-props
  (doto (java.util.Properties.)
    (.load (io/reader (io/resource "META-INF/maven/cavm/cavm/pom.properties")))))

;
; web services

(def trusted-hosts
  ;#"https://[-_A-Za-z0-9]+.kilokluster.ucsc.edu|https://genome-cancer.ucsc.edu"
  #".*")

(def public-url "https://genome-cancer.ucsc.edu/")
;(def public-xena (str public-url "proj/public/xena"))
(def tcga-xena "https://tcga.xenahubs.net")
(def toil-xena "https://toil.xenahubs.net")
(def icgc-xena "https://icgc.xenahubs.net")
(def ucscPublic-xena "https://ucscpublic.xenahubs.net")
(defn local-url [port] (str "https://local.xena.ucsc.edu:" (inc port)))
(defn local-xena [port] (str "http://local.xena.ucsc.edu:" port))

(def cohort-query
  '(map :cohort (query
                  {:select [:%distinct.cohort]
                   :from [:dataset]})))

(defn async-post [url post cb]
  (let [x (agent nil)]
    (set-error-handler! x (fn [_ err] (error err "Fetching cohort list") (cb [])))
    (send-off x (fn [_]
                  (cb (:body (client/post url {:body
                                               (str post)
                                               :as :json
                                               :content-type
                                               "text/plain"})))))))

; need to block in an agent & then invoke callback
(defn retrieve-cohorts [port cb]
  (let [x (atom [])
        update (fn [cohorts]
                 (swap! x (fn [all]
                            (let [ret (conj all cohorts)]
                              (when (= 5 (count ret))
                                (cb (filter identity (apply concat ret))))
                              ret))))]
    (async-post (str tcga-xena "/data/") cohort-query update)
    (async-post (str icgc-xena "/data/") cohort-query update)
    (async-post (str toil-xena "/data/") cohort-query update)
    (async-post (str ucscPublic-xena "/data/") cohort-query update)
    (async-post (str (local-xena port) "/data/") cohort-query update)))

(defn- wrap-access-control [handler]
  (fn [{:keys [request-method] :as request}]
    (let [response (if (= request-method :options)
                     (response/response "")
                     (handler request))]
      (info "request" request)
      (if-let [origin (re-matches trusted-hosts (get-in request [:headers "origin"] ""))]
        (-> response
            (assoc-in [:headers "Access-Control-Allow-Origin"] origin)
            (assoc-in [:headers "Access-Control-Expose-Headers"] "Location")
            (assoc-in [:headers "Access-Control-Allow-Credentials"] "true")
            (assoc-in [:headers "Access-Control-Allow-Headers"] "Cancer-Browser-Api, X-Redirect-To"))
        response))))

(defn- attr-middleware [app k v]
  (fn [req]
    (app (assoc req k v))))

(defn- log-middleware [handler]
  (fn [{:keys [uri query-string request-method session] :as request}]
    (info "Start:" (.toUpperCase (name request-method)) uri query-string
          (or (:email (:user session)) "(no session)"))
    (try
{'fraction_non_alphanumeric': 0.16601618277010946, 'fraction_numerical': 0.0013326987148976678, 'mean_word_length': 2.760200429491768, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 12, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2049758', 'n_tokens_mistral': 3613, 'n_tokens_neox': 3476, 'n_words': 824}