<commit_before>(ns cavm.views.datasets
  (:use [noir.core])
  (:use [noir.request])
  (:use [clojure.core.matrix])
  (:require [cavm.models.datasets :as d])
  (:require [cavm.query.expression :as expr])
  (:require [cavm.query.functions :as f])
  (:require [cavm.query.sources :as sources])
  (:require [noir.response :as response])
  (:require [ring.util.codec :as codec])
  (:require [clojure.edn :as edn])
  (:require [cheshire.custom :as json])
  (:require [cavm.h2 :as h2])
  (:import mikera.matrixx.Matrix)
  (:import mikera.vectorz.impl.ArraySubVector)
  (:gen-class))

(defpage [:get "/datasets"] []
         (response/json (d/datasets)))

; XXX temporary alias, to interoperate with client
(defpage [:get "/tracks"] []
  (response/json (d/datasets)))

(defpage [:options [":any" :any #".*"]] []
  (response/empty))

(defn genes-decode [genes]
  (map codec/url-decode (clojure.string/split genes #",")))

(defpage [:get ["/datasets/:id" :id #".+"]] {:keys [id genes]}
  (response/json (d/dataset-by-genes id (genes-decode genes))))

(defpage [:get ["/query/:query" :query #".+"]] {:keys [query]}
  (let [{db :db} (ring-request)]
    (response/json (d/query db (edn/read-string query)))))

; pretty-print query
(defpage [:get ["/query-p/:query" :query #".+"]] {:keys [query]}
  (let [{db :db} (ring-request)]
    (response/content-type "application/json; charset=utf-8"
                           (binding [clojure.pprint/*print-right-margin* 200]
                             (with-out-str (json/encode (d/query db (edn/read-string query)) {:pretty true}))))))

(defn samples-for-attr [attr]
  {:select [[:experiments.name "experiment"] :exp_samples.name]
   :from [:exp_samples]
   :left-join [:experiments [:= :exp_samples.experiments_id :experiments.id]]
   :where [:in :experiments.id
           {:select [:experiments.id]
            :modifiers [:distinct]
            :from [:features]
            :left-join [:probes [:= :features.probes_id :probes.id]
                        :experiments [:= :probes.eid :experiments.id]]
            :where [:= :features.longTitle attr]}]})

; XXX This is probably not the fastest way to do
; this.
(defn- boolean-matrix [m pred]
  (map pred (vec m)))

(defn- filter-matches [samples matches]
  (->> (map vector samples matches)
       (filter second)
       (map first)))

(defn attr-filter [attr pred]
  (->> attr
       (samples-for-attr)
       (h2/run-query)
       (group-by :EXPERIMENT)
       (map (fn [[exp samps]]
              {'table exp
               'columns [attr]
               'samples (map :NAME samps)}))
       (map #(assoc % 'data (sources/read-symbols [%])))
       (map #(assoc % 'samples
                    (filter-matches ('samples %)
                                    (boolean-matrix
                                      (first ('data %)) pred))))
       (mapcat #(map (fn [exp samp]
                         {:experiment exp
                          :sample samp}) (repeat ('table %)) ('samples %)))))

(defn filter-attr [samples attr pred]
  (->> samples
       (group-by :EXPERIMENT)
       (map (fn [[exp samps]]
              {'table exp
               'columns [attr]
               'samples (map :NAME samps)}))
       (map #(assoc % 'data (sources/read-symbols [%])))
       (map #(assoc % 'samples
                    (filter-matches ('samples %)
                                    (boolean-matrix
                                      (first ('data %)) pred))))
       (mapcat #(map (fn [exp samp]
                         {:experiment exp
                          :sample samp}) (repeat ('table %)) ('samples %)))))

; Add a json encoder for primitive float arrays, since that's what
; we get back from the expression engine.
(defn encode-array
  "Encode a primitive array to the json generator."
  [arr jg]
  (json/to-json (seq arr) jg))

(json/add-encoder (Class/forName "[F") encode-array)
; Mike suggests extending mikera.arrayz.INDArray to hit all
; core.matrix types. Leaving these for now.
(json/add-encoder mikera.vectorz.impl.ArraySubVector encode-array)
(json/add-encoder mikera.matrixx.Matrix encode-array)

(def functions
  {'fetch #(map vec (sources/read-symbols %))
   'query h2/run-query
   'filter filter-attr})

; XXX The call to read-symbols needs a list of sources, I think.
(defpage [:get ["/data/:expression" :expression #".+"]] {:keys [expression]}
  (h2/with-db (:db (ring-request))
    (response/json (map vec (expr/expression expression f/functions functions)))))<commit_msg>Fix non-array return values.<commit_after>(ns cavm.views.datasets
  (:use [noir.core])
  (:use [noir.request])
  (:use [clojure.core.matrix])
  (:require [cavm.models.datasets :as d])
  (:require [cavm.query.expression :as expr])
  (:require [cavm.query.functions :as f])
  (:require [cavm.query.sources :as sources])
  (:require [noir.response :as response])
  (:require [ring.util.codec :as codec])
  (:require [clojure.edn :as edn])
  (:require [cheshire.custom :as json])
  (:require [cavm.h2 :as h2])
  (:import mikera.matrixx.Matrix)
  (:import mikera.vectorz.impl.ArraySubVector)
  (:gen-class))

(defpage [:get "/datasets"] []
         (response/json (d/datasets)))

; XXX temporary alias, to interoperate with client
(defpage [:get "/tracks"] []
  (response/json (d/datasets)))

(defpage [:options [":any" :any #".*"]] []
  (response/empty))

(defn genes-decode [genes]
  (map codec/url-decode (clojure.string/split genes #",")))

(defpage [:get ["/datasets/:id" :id #".+"]] {:keys [id genes]}
  (response/json (d/dataset-by-genes id (genes-decode genes))))

(defpage [:get ["/query/:query" :query #".+"]] {:keys [query]}
  (let [{db :db} (ring-request)]
    (response/json (d/query db (edn/read-string query)))))

; pretty-print query
(defpage [:get ["/query-p/:query" :query #".+"]] {:keys [query]}
  (let [{db :db} (ring-request)]
    (response/content-type "application/json; charset=utf-8"
                           (binding [clojure.pprint/*print-right-margin* 200]
                             (with-out-str (json/encode (d/query db (edn/read-string query)) {:pretty true}))))))

(defn samples-for-attr [attr]
  {:select [[:experiments.name "experiment"] :exp_samples.name]
   :from [:exp_samples]
   :left-join [:experiments [:= :exp_samples.experiments_id :experiments.id]]
   :where [:in :experiments.id
           {:select [:experiments.id]
            :modifiers [:distinct]
            :from [:features]
            :left-join [:probes [:= :features.probes_id :probes.id]
                        :experiments [:= :probes.eid :experiments.id]]
            :where [:= :features.longTitle attr]}]})

; XXX This is probably not the fastest way to do
; this.
(defn- boolean-matrix [m pred]
  (map pred (vec m)))

(defn- filter-matches [samples matches]
  (->> (map vector samples matches)
       (filter second)
       (map first)))

(defn attr-filter [attr pred]
  (->> attr
       (samples-for-attr)
       (h2/run-query)
       (group-by :EXPERIMENT)
       (map (fn [[exp samps]]
              {'table exp
               'columns [attr]
               'samples (map :NAME samps)}))
       (map #(assoc % 'data (sources/read-symbols [%])))
       (map #(assoc % 'samples
                    (filter-matches ('samples %)
                                    (boolean-matrix
                                      (first ('data %)) pred))))
       (mapcat #(map (fn [exp samp]
                         {:experiment exp
                          :sample samp}) (repeat ('table %)) ('samples %)))))

(defn filter-attr [samples attr pred]
  (->> samples
       (group-by :EXPERIMENT)
       (map (fn [[exp samps]]
              {'table exp
               'columns [attr]
               'samples (map :NAME samps)}))
       (map #(assoc % 'data (sources/read-symbols [%])))
       (map #(assoc % 'samples
                    (filter-matches ('samples %)
                                    (boolean-matrix
                                      (first ('data %)) pred))))
       (mapcat #(map (fn [exp samp]
                         {:experiment exp
                          :sample samp}) (repeat ('table %)) ('samples %)))))

; Add a json encoder for primitive float arrays, since that's what
; we get back from the expression engine.
(defn encode-array
  "Encode a primitive array to the json generator."
  [arr jg]
  (json/to-json (seq arr) jg))

(json/add-encoder (Class/forName "[F") encode-array)
; Mike suggests extending mikera.arrayz.INDArray to hit all
; core.matrix types. Leaving these for now.
(json/add-encoder mikera.vectorz.impl.ArraySubVector encode-array)
(json/add-encoder mikera.matrixx.Matrix encode-array)

(def functions
  {'fetch #(map vec (sources/read-symbols %))
   'query h2/run-query
   'filter filter-attr})

; XXX The call to read-symbols needs a list of sources, I think.
(defpage [:get ["/data/:expression" :expression #".+"]] {:keys [expression]}
  (h2/with-db (:db (ring-request))
    (response/json (expr/expression expression f/functions functions))))
{'fraction_non_alphanumeric': 0.16779642182464719, 'fraction_numerical': 0.002000222246916324, 'mean_word_length': 2.4482758620689653, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '489780', 'n_tokens_mistral': 3008, 'n_tokens_neox': 2891, 'n_words': 760}