<commit_before>(ns kixi.hecuba.api.measurements
  (:require
   [bidi.bidi :as bidi]
   [clj-time.coerce :as tc]
   [clj-time.core :as t]
   [clojure.string :as string]
   [clojure.tools.logging :as log]
   [kixi.hecuba.data.misc :as m]
   [kixi.hecuba.data.misc :as misc]
   [kixi.hecuba.data.validate :as v]
   [kixi.hecuba.protocols :as hecuba]
   [kixi.hecuba.queue :as q]
   [kixi.hecuba.security :as sec]
   [kixi.hecuba.storage.dbnew :as dbnew]
   [kixi.hecuba.webutil :as util]
   [kixi.hecuba.webutil :refer (decode-body authorized? uuid stringify-values sha1-regex time-range)]
   [liberator.core :refer (defresource)]
   [liberator.representation :refer (ring-response)]
   [qbits.hayt :as hayt]
   ))

(defn sensor_metadata-for [store sensor_id]
  (let [{:keys [type device_id]} sensor_id]
    (dbnew/with-session [session (:hecuba-session store)]
      (first (dbnew/execute session
                            (hayt/select
                             :sensor_metadata
                             (hayt/where [[= :device_id device_id]
                                          [= :type type]])))))))

(defn- mk-bounds-from [store type device_id]
  (let [sm  (sensor_metadata-for store {:type type :device_id device_id})
        {:keys [lower_ts upper_ts]} sm]
    (vector (atom (or (tc/to-date-time lower_ts) (org.joda.time.DateTime. Long/MAX_VALUE) ))
            (atom (or (tc/to-date-time upper_ts) (org.joda.time.DateTime. 0))))))

(defn resolve-start-end [store type device_id start end]
  (mapv tc/to-date-time
       (if (not (and start end))
         (let [sm (sensor_metadata-for store {:type type :device_id device_id})
               _ (log/info "sm: " sm)
               [lower upper] ((juxt :lower_ts :upper_ts) sm)]
           [(or start lower)
            (or end upper)])
         [start end])))

(defn all-measurements
  "Returns a sequence of all the measurements for a sensor
   matching (type,device_id). The sequence pages to the database in the
   background. The page size is a clj-time Period representing a range
   in the timestamp column. page size defaults to (clj-time/hours 1)"
  ([store sensor_id & [opts]]
     (let [{:keys [type device_id]} sensor_id
           {:keys [page start end] :or {page (t/hours 1)}} opts
           [start end] (resolve-start-end store type device_id start end )
           _ (log/info "s: " start ", e:" end)
           next-start (t/plus start page)]
       (dbnew/with-session [session (:hecuba-session store)]
         (lazy-cat (dbnew/execute session
                                  (hayt/select :measurements
                                               (hayt/where [[= :device_id device_id]
                                                            [= :type type]
                                                            [= :month (m/get-month-partition-key start)]
                                                            [>= :timestamp start]
                                                            [< :timestamp next-start]]))
                                  nil)
                   (when (t/before? next-start end)
                     (all-measurements store sensor_id (merge opts {:start next-start :end end}))))))))

(defn measurements-slice-handle-ok [store store-new ctx]
  (let [request                (:request ctx)
        {:keys [route-params
                query-string]} request
        {:keys [device_id
                reading-type]} route-params
        decoded-params         (util/decode-query-params query-string)
        start-date             (util/to-db-format (string/replace (get decoded-params "startDate") "%20" " "))
        end-date               (util/to-db-format (string/replace (get decoded-params "endDate") "%20" " "))
        measurements           nil ;; TODO
        ]
    {:measurements (->> measurements
                        (map (fn [m]
                               (-> m
                                   util/parse-value
                                   (update-in [:timestamp] util/db-to-iso)
                                   (dissoc :month :metadata :device_id)))))}))





(defn- min-date [dt1 dt2]
  (let [dt1' (tc/to-date-time dt1)
        dt2' (tc/to-date-time dt2)]
    (if (t/before? dt1' dt2') dt1' dt2')))

(defn- max-date [dt1 dt2]
  (let [dt1' (tc/to-date-time dt1)
        dt2' (tc/to-date-time dt2)]
    (if (t/before? dt1' dt2') dt2' dt1')))

(defn- sensor-exists? [session device_id type]
  (first (dbnew/execute session
                  (hayt/select :sensors
                               (hayt/where [[= :device_id device_id]
                                            [= :type type]])))))

(defn index-post! [store store-new queue ctx]
  (let [{:keys [commander querier]} store
        request       (:request ctx)
        route-params  (:route-params request)
        device_id     (:device_id route-params)
        topic         (get-in queue ["measurements"])
        measurements  (:measurements (decode-body request))<commit_msg>Fix measurements slicing for GET.<commit_after>(ns kixi.hecuba.api.measurements
  (:require
   [bidi.bidi :as bidi]
   [clj-time.coerce :as tc]
   [clj-time.core :as t]
   [clojure.string :as string]
   [clojure.tools.logging :as log]
   [kixi.hecuba.data.misc :as m]
   [kixi.hecuba.data.misc :as misc]
   [kixi.hecuba.data.validate :as v]
   [kixi.hecuba.protocols :as hecuba]
   [kixi.hecuba.queue :as q]
   [kixi.hecuba.security :as sec]
   [kixi.hecuba.storage.dbnew :as dbnew]
   [kixi.hecuba.webutil :as util]
   [kixi.hecuba.webutil :refer (decode-body authorized? uuid stringify-values sha1-regex time-range)]
   [liberator.core :refer (defresource)]
   [liberator.representation :refer (ring-response)]
   [qbits.hayt :as hayt]
   ))

(defn sensor_metadata-for [store sensor_id]
  (let [{:keys [type device_id]} sensor_id]
    (dbnew/with-session [session (:hecuba-session store)]
      (first (dbnew/execute session
                            (hayt/select
                             :sensor_metadata
                             (hayt/where [[= :device_id device_id]
                                          [= :type type]])))))))

(defn- mk-bounds-from [store type device_id]
  (let [sm  (sensor_metadata-for store {:type type :device_id device_id})
        {:keys [lower_ts upper_ts]} sm]
    (vector (atom (or (tc/to-date-time lower_ts) (org.joda.time.DateTime. Long/MAX_VALUE) ))
            (atom (or (tc/to-date-time upper_ts) (org.joda.time.DateTime. 0))))))

(defn resolve-start-end [store type device_id start end]
  (mapv tc/to-date-time
       (if (not (and start end))
         (let [sm (sensor_metadata-for store {:type type :device_id device_id})
               _ (log/info "sm: " sm)
               [lower upper] ((juxt :lower_ts :upper_ts) sm)]
           [(or start lower)
            (or end upper)])
         [start end])))

(defn all-measurements
  "Returns a sequence of all the measurements for a sensor
   matching (type,device_id). The sequence pages to the database in the
   background. The page size is a clj-time Period representing a range
   in the timestamp column. page size defaults to (clj-time/hours 1)"
  ([store sensor_id & [opts]]
     (let [{:keys [type device_id]} sensor_id
           {:keys [page start end] :or {page (t/hours 1)}} opts
           [start end] (resolve-start-end store type device_id start end )
           _ (log/info "s: " start ", e:" end)
           next-start (t/plus start page)]
       (dbnew/with-session [session (:hecuba-session store)]
         (lazy-cat (dbnew/execute session
                                  (hayt/select :measurements
                                               (hayt/where [[= :device_id device_id]
                                                            [= :type type]
                                                            [= :month (m/get-month-partition-key start)]
                                                            [>= :timestamp start]
                                                            [< :timestamp next-start]]))
                                  nil)
                   (when (t/before? next-start end)
                     (all-measurements store sensor_id (merge opts {:start next-start :end end}))))))))

(defn retrieve-measurements 
  "Iterate over a sequence of months and concatanate measurements retrieved from the database."
  [querier start-date end-date device-id reading-type]
  (let [range  (time-range start-date end-date (t/months 1))
        months (map #(util/get-month-partition-key (tc/to-date %)) range)
        where  [[= :device_id device-id]
                [= :type reading-type]
                [>= :timestamp (tc/to-date start-date)]
                [<= :timestamp (tc/to-date end-date)]]]
    (mapcat (fn [month] (hecuba/items querier :measurement (conj where [= :month month]))) months)))

(defn measurements-slice-handle-ok [store store-new ctx]
  (let [request                (:request ctx)
        querier                (:querier store)
        {:keys [route-params
                query-string]} request
        {:keys [device_id
                reading-type]} route-params
        decoded-params         (util/decode-query-params query-string)
        start-date             (util/to-db-format (string/replace (get decoded-params "startDate") "%20" " "))
        end-date               (util/to-db-format (string/replace (get decoded-params "endDate") "%20" " "))
        measurements           (retrieve-measurements querier start-date end-date device_id reading-type)
        ]
    {:measurements (->> measurements
                        (map (fn [m]
                               (-> m
                                   util/parse-value
                                   (update-in [:timestamp] util/db-to-iso)
                                   (dissoc :month :metadata :device_id)))))}))

(defn- min-date [dt1 dt2]
  (let [dt1' (tc/to-date-time dt1)
        dt2' (tc/to-date-time dt2)]
    (if (t/before? dt1' dt2') dt1' dt2')))

(defn- max-date [dt1 dt2]
  (let [dt1' (tc/to-date-time dt1)
        dt2' (tc/to-date-time dt2)]
    (if (t/before? dt1' dt2') dt2' dt1')))

(defn- sensor-exists? [session device_id type]
  (first (dbnew/execute session
                  (hayt/select :sensors
                               (hayt/where [[= :device_id device_id]
                                            [= :type type]])))))

(defn index-post! [store store-new queue ctx]
  (let [{:keys [commander querier]} store
        request       (:request ctx)
        route-params  (:route-params request)
        device_id     (:device_id route-params)
        topic         (get-in queue ["measurements"])
        measurements  (:measurements (decode-body request))
{'fraction_non_alphanumeric': 0.12644105615470436, 'fraction_numerical': 0.005299367794719226, 'mean_word_length': 1.7329776422764227, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 38, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2475354', 'n_tokens_mistral': 3345, 'n_tokens_neox': 3252, 'n_words': 846}