<commit_before>(ns akvo.lumen.postgres.filter
  (:require [akvo.lumen.postgres :as core]
            [clojure.tools.logging :as log]
            [clojure.string :as str])
  (:import [java.sql Timestamp]))

(defn invalid-filter [msg map]
  (throw (ex-info (format "Invalid filter: %s" msg) map)))

(defmulti filter-sql (fn [filter]
                       (:strategy filter)))

(defn- char-range [start end]
  (map char (range (int start) (inc (int end)))))

(def alphanumerics (concat (char-range \a \z)
                           (char-range \A \Z)
                           (take 10 (range))))

(defn random-tag []
  (format "tag_%s" (->> #(rand-nth alphanumerics)
                        repeatedly
                        (take 10)
                        (apply str))))

(defn parse-number [s]
  (try
    (Double/parseDouble s)
    (catch NumberFormatException e
      (invalid-filter "Not a number" {:string s}))))

(defn parse-date [s]
  (try
    (java.sql.Timestamp. (* s 1000))
    (catch NumberFormatException e
      (invalid-filter "Not a timestamp" {:string s}))))

(defn comparison [op column-type column-name value]
  (condp = column-type
    "number" (format "%s %s %s"
                     column-name
                     op
                     (parse-number value))
    "date" (format "%s %s '%s'::timestamptz"
                   column-name
                   op
                   (parse-date value))
    (invalid-filter "isHigher/isLower not supported on this column type" {:type column-type})))

(defmethod filter-sql "isHigher"
  [{:keys [column operation value]}]
  (let [op (if (= operation "keep") ">" "<=")]
    (comparison op (:type column) (:columnName column) value)))

(defmethod filter-sql "isLower"
  [{:keys [column operation value]}]
  (let [op (if (= operation "keep") "<" ">=")]
    (comparison op (:type column) (:columnName column) value)))

(defmethod filter-sql "is"
  [{:keys [column operation value]}]
  (let [op (if (= operation "keep") "=" "<>")]
    (condp = (:type column)
      "number" (format "%1$s IS NULL OR %1$s %2$s %3$s"
                       (column :columnName)
                       op
                       (parse-number value))
      "date" (format "%1$s IS NULL OR %1$s %2$s '%3$s'::timestamptz"
                     (column :columnName)
                     op
                     (parse-date value))
      "text" (format "coalesce(%1$s, '') %2$s '%3$s'"
                     (column :columnName)
                     op
                     (core/escape-string value))
      "option" (format "coalesce(%1$s, '') %2$s '%3$s'"
                       (column :columnName)
                       op
                       (core/escape-string value))
      (invalid-filter "Type not supported" {:type (:type column)}))))

(defmethod filter-sql "contains"
  [{:keys [column operation value]}]
  (let [op (if (= operation "keep") "=" "<>")]
    (condp = (:type column)
      "option" (format "position('%3$s' in %1$s) %2$s 0"
                       (column :columnName)
                       (if (= "=" op) ">" "=")
                       (core/escape-string value))
      (invalid-filter "Type not supported" {:type (:type column)}))))

(defmethod filter-sql "not_contains"
  [{:keys [column operation value]}]
  (let [op (if (= operation "keep") "=" "<>")]
    (condp = (:type column)
      "option" (format "position('%3$s' in %1$s) %2$s 0"
                       (column :columnName)
                       (if (= "=" op) "=" ">")
                       (core/escape-string value))
      (invalid-filter "Type not supported" {:type (:type column)}))))

(defmethod filter-sql "isEmpty"
  [{:keys [column operation]}]
  (if (= (:type column) "text")
    (format "coalesce(%s, '') %s ''"
            (column :columnName)
            (if (= operation "keep") "=" "<>"))
    (format "%s IS %s"
            (column :columnName)
            (if (= operation "keep")
              "NULL"
              "NOT NULL"))))

(defmethod filter-sql :default [filter]
  (invalid-filter "No such filter strategy" {:strategy (:strategy filter)}))

(defn find-column [columns column-name]
  (if-let [column (first (filter #(= (:columnName %) column-name) columns))]
    column
    (invalid-filter "No such column" {:column column-name})))

(defn sql-str [columns filters]
  (if (empty? filters)
    "TRUE"
    (let [filters (->> filters
                       (map #(assoc % :column (find-column columns (:column %)))))]
      (str/join " AND " (map #(format "(%s)" (filter-sql %))
                             filters)))))
<commit_msg>[#2964] Filter isEmpty option column as text type<commit_after>(ns akvo.lumen.postgres.filter
  (:require [akvo.lumen.postgres :as core]
            [clojure.tools.logging :as log]
            [clojure.string :as str])
  (:import [java.sql Timestamp]))

(defn invalid-filter [msg map]
  (throw (ex-info (format "Invalid filter: %s" msg) map)))

(defmulti filter-sql (fn [filter]
                       (:strategy filter)))

(defn- char-range [start end]
  (map char (range (int start) (inc (int end)))))

(def alphanumerics (concat (char-range \a \z)
                           (char-range \A \Z)
                           (take 10 (range))))

(defn random-tag []
  (format "tag_%s" (->> #(rand-nth alphanumerics)
                        repeatedly
                        (take 10)
                        (apply str))))

(defn parse-number [s]
  (try
    (Double/parseDouble s)
    (catch NumberFormatException e
      (invalid-filter "Not a number" {:string s}))))

(defn parse-date [s]
  (try
    (java.sql.Timestamp. (* s 1000))
    (catch NumberFormatException e
      (invalid-filter "Not a timestamp" {:string s}))))

(defn comparison [op column-type column-name value]
  (condp = column-type
    "number" (format "%s %s %s"
                     column-name
                     op
                     (parse-number value))
    "date" (format "%s %s '%s'::timestamptz"
                   column-name
                   op
                   (parse-date value))
    (invalid-filter "isHigher/isLower not supported on this column type" {:type column-type})))

(defmethod filter-sql "isHigher"
  [{:keys [column operation value]}]
  (let [op (if (= operation "keep") ">" "<=")]
    (comparison op (:type column) (:columnName column) value)))

(defmethod filter-sql "isLower"
  [{:keys [column operation value]}]
  (let [op (if (= operation "keep") "<" ">=")]
    (comparison op (:type column) (:columnName column) value)))

(defmethod filter-sql "is"
  [{:keys [column operation value]}]
  (let [op (if (= operation "keep") "=" "<>")]
    (condp = (:type column)
      "number" (format "%1$s IS NULL OR %1$s %2$s %3$s"
                       (column :columnName)
                       op
                       (parse-number value))
      "date" (format "%1$s IS NULL OR %1$s %2$s '%3$s'::timestamptz"
                     (column :columnName)
                     op
                     (parse-date value))
      "text" (format "coalesce(%1$s, '') %2$s '%3$s'"
                     (column :columnName)
                     op
                     (core/escape-string value))
      "option" (format "coalesce(%1$s, '') %2$s '%3$s'"
                       (column :columnName)
                       op
                       (core/escape-string value))
      (invalid-filter "Type not supported" {:type (:type column)}))))

(defmethod filter-sql "contains"
  [{:keys [column operation value]}]
  (let [op (if (= operation "keep") "=" "<>")]
    (condp = (:type column)
      "option" (format "position('%3$s' in %1$s) %2$s 0"
                       (column :columnName)
                       (if (= "=" op) ">" "=")
                       (core/escape-string value))
      (invalid-filter "Type not supported" {:type (:type column)}))))

(defmethod filter-sql "not_contains"
  [{:keys [column operation value]}]
  (let [op (if (= operation "keep") "=" "<>")]
    (condp = (:type column)
      "option" (format "position('%3$s' in %1$s) %2$s 0"
                       (column :columnName)
                       (if (= "=" op) "=" ">")
                       (core/escape-string value))
      (invalid-filter "Type not supported" {:type (:type column)}))))

(defmethod filter-sql "isEmpty"
  [{:keys [column operation]}]
  (if (or (= (:type column) "option") (= (:type column) "text"))
    (format "coalesce(%s, '') %s ''"
            (column :columnName)
            (if (= operation "keep") "=" "<>"))
    (format "%s IS %s"
            (column :columnName)
            (if (= operation "keep")
              "NULL"
              "NOT NULL"))))

(defmethod filter-sql :default [filter]
  (invalid-filter "No such filter strategy" {:strategy (:strategy filter)}))

(defn find-column [columns column-name]
  (if-let [column (first (filter #(= (:columnName %) column-name) columns))]
    column
    (invalid-filter "No such column" {:column column-name})))

(defn sql-str [columns filters]
  (if (empty? filters)
    "TRUE"
    (let [filters (->> filters
                       (map #(assoc % :column (find-column columns (:column %)))))]
      (str/join " AND " (map #(format "(%s)" (filter-sql %))
                             filters)))))

{'fraction_non_alphanumeric': 0.16863424672370844, 'fraction_numerical': 0.00693165818260587, 'mean_word_length': 1.9719987125844867, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '515775', 'n_tokens_mistral': 2963, 'n_tokens_neox': 2801, 'n_words': 805}