<commit_before>(ns cavm.core
  (:require [clojure.string :as s])
  (:use [cavm.h2 :only [create
                        load-exp
                        load-probemap
                        del-exp
                        datasets
                        with-db
                        clean-sources
                        create-db]]) ; XXX use model instead?
  (:require [digest])
  (:require [cavm.test-data :as data])
  (:require [clojure.java.io :as io])
  (:require [clj-time.coerce :refer [from-long]])
  (:require [clj-time.core :refer [now]])
  (:require [noir.server :as server])
  (:require [clojure.data.json :as json])
  (:require [me.raynes.fs :as fs])
  (:require [clojure.tools.cli :refer [cli]])
  (:require [cavm.cgdata :as cgdata])
  (:gen-class))

(defn- tabbed [line]
  (s/split line #"\t"))

(defn- file-time [file]
  (from-long (.lastModified (java.io.File. file))))

(defn- in-data-path [root path]
  (boolean (fs/child-of? root path))) ; XXX not working with ".."??

(defn- filehash [file]
  (digest/sha1 (io/as-file file)))

;
;
;

(defn- file-stats [root file]
  (let  [full-path (str (fs/file root file))
         ts (file-time full-path)
         h (filehash full-path)]
    {:name file :time ts :hash h}))

(defn- genomic-renames [mdata rfile]
  "Convert cgData genomic attrs to db attrs."
  (-> mdata
      (clojure.set/rename-keys
        {":probeMap" "probeMap"
         "PLATFORM" "platform"
         ":dataSubType" "dataSubType"})
      (assoc "name" rfile)))

(defn load-matrix-file [root file]
  (with-open [in (io/reader file)]
    (let [{:keys [rfile meta refs features]} (cgdata/matrix-file file :root root)
          meta (genomic-renames meta rfile)
          files (if features
                  [(refs ":clinicalFeature") rfile]
                  [rfile])
          stats (mapv #(file-stats root %) files)
          data-fn (fn []
                    (cgdata/matrix-data meta features (map tabbed (line-seq in))))]
      (load-exp stats meta data-fn features))))

(defn- loadtest [name ^Integer m ^Integer n]
  (create)
  (let [mi (Integer. m)
        ni (Integer. n)]
  (load-exp name (now) "FIXME" (cgdata/matrix-data nil (data/matrix mi ni)))))


;
; cgData probemap
;<commit_msg>Additional diffs for parallelizing cgdata transforms.<commit_after>(ns cavm.core
  (:require [clojure.string :as s])
  (:use [cavm.h2 :only [create
                        load-exp
                        load-probemap
                        del-exp
                        datasets
                        with-db
                        clean-sources
                        create-db]]) ; XXX use model instead?
  (:require [digest])
  (:require [cavm.test-data :as data])
  (:require [clojure.java.io :as io])
  (:require [clj-time.coerce :refer [from-long]])
  (:require [clj-time.core :refer [now]])
  (:require [noir.server :as server])
  (:require [clojure.data.json :as json])
  (:require [me.raynes.fs :as fs])
  (:require [clojure.tools.cli :refer [cli]])
  (:require [cavm.cgdata :as cgdata])
  (:gen-class))

(defn- file-time [file]
  (from-long (.lastModified (java.io.File. file))))

(defn- in-data-path [root path]
  (boolean (fs/child-of? root path))) ; XXX not working with ".."??

(defn- filehash [file]
  (digest/sha1 (io/as-file file)))

;
;
;

(defn- file-stats [root file]
  (let  [full-path (str (fs/file root file))
         ts (file-time full-path)
         h (filehash full-path)]
    {:name file :time ts :hash h}))

(defn- genomic-renames [mdata rfile]
  "Convert cgData genomic attrs to db attrs."
  (-> mdata
      (clojure.set/rename-keys
        {":probeMap" "probeMap"
         "PLATFORM" "platform"
         ":dataSubType" "dataSubType"})
      (assoc "name" rfile)))

(defn load-matrix-file [root file]
  (with-open [in (io/reader file)]
    (let [{:keys [rfile meta refs features]} (cgdata/matrix-file file :root root)
          meta (genomic-renames meta rfile)
          files (if features
                  [(refs ":clinicalFeature") rfile]
                  [rfile])
          stats (mapv #(file-stats root %) files)
          data-fn (fn []
                    (cgdata/matrix-data meta features (line-seq in)))]
      (load-exp stats meta data-fn features))))

(defn- loadtest [name ^Integer m ^Integer n]
  (create)
  (let [mi (Integer. m)
        ni (Integer. n)]
  (load-exp name (now) "FIXME" (cgdata/matrix-data nil (data/matrix mi ni)))))


;
; cgData probemap
;
{'fraction_non_alphanumeric': 0.15469488853861743, 'fraction_numerical': 0.0009006980409817609, 'mean_word_length': 2.166072701354241, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6228271', 'n_tokens_mistral': 1568, 'n_tokens_neox': 1506, 'n_words': 415}