<commit_before>(ns cfpb.qu.query
  (:require [clojure.string :as str]
            [monger.query :as q]
            [monger.collection :as coll]
            [cfpb.qu.data :as data]
            [cfpb.qu.query.mongo :as mongo]
            [cfpb.qu.query.select :as select]
            [cfpb.qu.query.validation :as validation]
            [cfpb.qu.util :refer [->int ->num]]
            [lonocloud.synthread :as ->]
            [taoensso.timbre :as log]
            [clj-statsd :as sd]))

(def default-limit 100)
(def default-aggregation-limit 10000)
(def default-offset 0)

(declare parse-params mongo-find mongo-aggregation)

(defrecord Query [select group where orderBy limit offset callback mongo errors slicedef])
(def allowed-clauses #{:$select :$where :$orderBy :$group :$limit :$offset :$callback :$page :$perPage})

;; TODO refactor and move this and other validations into separate namespace
(defn valid? [query]
  (or (not (:errors query))
      (zero? (reduce + (map count (:errors query))))))

(defn params->Query
  "Convert params from a web request plus a dateset definition and a slice name into a Query record."
  [params metadata slice]
  (let [slicedef (get-in metadata [:slices slice])
        {:keys [dimensions clauses]} (parse-params params slicedef)
        {select :$select
         group :$group
         orderBy :$orderBy
         where :$where
         limit :$limit
         offset :$offset
         page :$page
         perPage :$perPage
         callback :$callback} clauses]
    (map->Query {:select select
                 :group group
                 :where where
                 :limit limit
                 :offset offset
                 :page page
                 :perPage perPage
                 :orderBy orderBy
                 :callback callback
                 :dimensions dimensions
                 :metadata metadata
                 :slice slice
                 :slicedef slicedef})))

(defn is-aggregation? [query]
  (:group query false))

(defn- resolve-limit-and-offset [{:keys [limit offset page perPage] :as query}]
  (let [limit (or (->int limit nil)
                  (->int perPage nil)
                  default-limit)
        limit (if (zero? limit)
                default-limit
                limit)
        offset (->int offset nil)
        page (->int page (when (and offset
                                    (zero? (mod offset limit)))
                           (inc (/ offset limit))))]
    (cond
     page (merge query {:offset (-> page
                                    dec
                                    (* limit))
                        :limit limit
                        :page page})
     offset (merge query {:offset offset
                          :limit limit
                          :page page})
     :default (merge query {:offset default-offset
                            :limit limit
                            :page 1}))))

(defn execute
  "Execute the query against the provided collection."
  [dataset collection query]

  (sd/with-timing "qu.queries.execute"
    (let [_ (log/info (str "Raw query: " (into {} (dissoc query :metadata :slicedef))))
          query (-> query
                    validation/validate
                    resolve-limit-and-offset
                    mongo/process)
          _ (log/info (str "Post-process query: " (into {} (dissoc query :metadata :slicedef))))]
      (assoc query :result
             (cond
              (seq (:errors query))
               (sd/increment "qu.queries.invalid")

              (is-aggregation? query)
              (data/get-aggregation dataset collection (mongo-aggregation query))

              :default
              (data/get-find dataset collection (mongo-find query)))))))


(defn- cast-value [value type]
  (case type
    "integer" (->int value)
    "number" (->num value)
    value))

(defn- cast-dimensions
  "Given a slice definition and a set of dimensions from the request,
cast the requested dimensions into the right type for comparison when
querying the database."
  [slice-def dimensions]
  (into {}
        (for [[dimension value] dimensions]
          (vector dimension (cast-value
                             value
                             (get-in slice-def [:types dimension]))))))<commit_msg>Fix execute case when query error is found<commit_after>(ns cfpb.qu.query
  (:require [clojure.string :as str]
            [monger.query :as q]
            [monger.collection :as coll]
            [cfpb.qu.data :as data]
            [cfpb.qu.query.mongo :as mongo]
            [cfpb.qu.query.select :as select]
            [cfpb.qu.query.validation :as validation]
            [cfpb.qu.util :refer [->int ->num]]
            [lonocloud.synthread :as ->]
            [taoensso.timbre :as log]
            [clj-statsd :as sd]))

(def default-limit 100)
(def default-aggregation-limit 10000)
(def default-offset 0)

(declare parse-params mongo-find mongo-aggregation)

(defrecord Query [select group where orderBy limit offset callback mongo errors slicedef])
(def allowed-clauses #{:$select :$where :$orderBy :$group :$limit :$offset :$callback :$page :$perPage})

;; TODO refactor and move this and other validations into separate namespace
(defn valid? [query]
  (or (not (:errors query))
      (zero? (reduce + (map count (:errors query))))))

(defn params->Query
  "Convert params from a web request plus a dateset definition and a slice name into a Query record."
  [params metadata slice]
  (let [slicedef (get-in metadata [:slices slice])
        {:keys [dimensions clauses]} (parse-params params slicedef)
        {select :$select
         group :$group
         orderBy :$orderBy
         where :$where
         limit :$limit
         offset :$offset
         page :$page
         perPage :$perPage
         callback :$callback} clauses]
    (map->Query {:select select
                 :group group
                 :where where
                 :limit limit
                 :offset offset
                 :page page
                 :perPage perPage
                 :orderBy orderBy
                 :callback callback
                 :dimensions dimensions
                 :metadata metadata
                 :slice slice
                 :slicedef slicedef})))

(defn is-aggregation? [query]
  (:group query false))

(defn- resolve-limit-and-offset [{:keys [limit offset page perPage] :as query}]
  (let [limit (or (->int limit nil)
                  (->int perPage nil)
                  default-limit)
        limit (if (zero? limit)
                default-limit
                limit)
        offset (->int offset nil)
        page (->int page (when (and offset
                                    (zero? (mod offset limit)))
                           (inc (/ offset limit))))]
    (cond
     page (merge query {:offset (-> page
                                    dec
                                    (* limit))
                        :limit limit
                        :page page})
     offset (merge query {:offset offset
                          :limit limit
                          :page page})
     :default (merge query {:offset default-offset
                            :limit limit
                            :page 1}))))

(defn execute
  "Execute the query against the provided collection."
  [dataset collection query]

  (sd/with-timing "qu.queries.execute"
    (let [_ (log/info (str "Raw query: " (into {} (dissoc query :metadata :slicedef))))
          query (-> query
                    validation/validate
                    resolve-limit-and-offset
                    mongo/process)
          _ (log/info (str "Post-process query: " (into {} (dissoc query :metadata :slicedef))))]
      (assoc query :result
             (cond
              (seq (:errors query))
               (do
                 (sd/increment "qu.queries.invalid")
                 [])

              (is-aggregation? query)
              (data/get-aggregation dataset collection (mongo-aggregation query))

              :default
              (data/get-find dataset collection (mongo-find query)))))))


(defn- cast-value [value type]
  (case type
    "integer" (->int value)
    "number" (->num value)
    value))

(defn- cast-dimensions
  "Given a slice definition and a set of dimensions from the request,
cast the requested dimensions into the right type for comparison when
querying the database."
  [slice-def dimensions]
  (into {}
        (for [[dimension value] dimensions]
          (vector dimension (cast-value
                             value
                             (get-in slice-def [:types dimension]))))))
{'fraction_non_alphanumeric': 0.10512909575083941, 'fraction_numerical': 0.002315618849137432, 'mean_word_length': 1.6432068543451652, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7182359', 'n_tokens_mistral': 2358, 'n_tokens_neox': 2242, 'n_words': 749}