<commit_before>(ns qu.query.where
  "This namespace parses WHERE clauses into an AST and turns that AST
into a Monger query."
  (:require [clojure.string :as str]
            [clojure.walk :refer [postwalk]]
            [protoflex.parse :as p]
            [qu.query.parser :refer [where-expr]])
  (:import (java.util.regex Pattern)))

(defn parse
  "Parse a valid WHERE expression and return an abstract syntax tree
for use in constructing Mongo queries."
  [clause]
  (p/parse where-expr clause))

(def mongo-operators
  {:AND "$and"
   :OR "$or"
   :IN "$in"
   :< "$lt"
   :<= "$lte"
   :> "$gt"
   :>= "$gte"
   :!= "$ne"})

(defn mongo-not [comparison]
  (let [ident (first (keys comparison))
        operation (first (vals comparison))]

    (cond
     (map? operation)
     (let [operator (first (keys operation))
           value (first (vals operation))]
       (if (= operator "$ne")
         {ident value}
         {ident {"$not" operation}}))

     (= (type operation) Pattern)
     {ident {"$not" operation}}

     :default
     {ident {"$ne" operation}})))

(declare mongo-eval-not)

(defn sql-pattern-to-regex-str
  "Converts a SQL search string, such as 'foo%', into a regular expression string"
  [value]
  (str "^"
       (str/replace value
                    #"[%_]|[^%_]+"
                    (fn [match]
                      (case match
                        "%" ".*"
                        "_" "."
                        (Pattern/quote match))))
       "$"))

(defn like-to-regex
  "Converts a SQL LIKE value into a regular expression."
  [like]
  (re-pattern (sql-pattern-to-regex-str like)))

(defn ilike-to-regex
  "Converts a SQL ILIKE value into a regular expression."
  [ilike]
  (re-pattern
   (str "(?i)"
        (sql-pattern-to-regex-str ilike))))

(defn- has-key?
  "Returns true if map? is a map and has the key key."
  [map key]
  (and (map? map)
       (contains? map key)))

(defn- fix-mongo-maps
  "In Mongo queries, equality or comparison is done with one-element
  maps. mapcat turns these into 2-element vectors, so this fixes them."
  [evaled-operands]
  (postwalk #(if (and (vector? %) (= 2 (count %)))
                   (apply hash-map %)
                   %)
            evaled-operands))

(defn mongo-eval<commit_msg>Ensure only the right vectors in Mongo queries get turned into maps<commit_after>(ns qu.query.where
  "This namespace parses WHERE clauses into an AST and turns that AST
into a Monger query."
  (:require [clojure.string :as str]
            [clojure.walk :refer [postwalk]]
            [protoflex.parse :as p]
            [taoensso.timbre :as log]
            [qu.query.parser :refer [where-expr]])
  (:import (java.util.regex Pattern)))

(defn parse
  "Parse a valid WHERE expression and return an abstract syntax tree
for use in constructing Mongo queries."
  [clause]
  (p/parse where-expr clause))

(def mongo-operators
  {:AND "$and"
   :OR "$or"
   :IN "$in"
   :< "$lt"
   :<= "$lte"
   :> "$gt"
   :>= "$gte"
   :!= "$ne"})

(defn mongo-not [comparison]
  (let [ident (first (keys comparison))
        operation (first (vals comparison))]

    (cond
     (map? operation)
     (let [operator (first (keys operation))
           value (first (vals operation))]
       (if (= operator "$ne")
         {ident value}
         {ident {"$not" operation}}))

     (= (type operation) Pattern)
     {ident {"$not" operation}}

     :default
     {ident {"$ne" operation}})))

(declare mongo-eval-not)

(defn sql-pattern-to-regex-str
  "Converts a SQL search string, such as 'foo%', into a regular expression string"
  [value]
  (str "^"
       (str/replace value
                    #"[%_]|[^%_]+"
                    (fn [match]
                      (case match
                        "%" ".*"
                        "_" "."
                        (Pattern/quote match))))
       "$"))

(defn like-to-regex
  "Converts a SQL LIKE value into a regular expression."
  [like]
  (re-pattern (sql-pattern-to-regex-str like)))

(defn ilike-to-regex
  "Converts a SQL ILIKE value into a regular expression."
  [ilike]
  (re-pattern
   (str "(?i)"
        (sql-pattern-to-regex-str ilike))))

(defn- has-key?
  "Returns true if map? is a map and has the key key."
  [map key]
  (and (map? map)
       (contains? map key)))

(defn- fix-mongo-maps
  "In Mongo queries, equality or comparison is done with one-element
  maps. mapcat turns these into 2-element vectors, so this fixes them."
  [evaled-operands]
  (postwalk #(if (and (vector? %)
                      (= 2 (count %))
                      (keyword? (first %)))
               (apply hash-map %)
               %)
            evaled-operands))

(defn mongo-eval
{'fraction_non_alphanumeric': 0.14502026018340797, 'fraction_numerical': 0.0008530603540200469, 'mean_word_length': 2.137123745819398, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7283434', 'n_tokens_mistral': 1549, 'n_tokens_neox': 1489, 'n_words': 471}