<commit_before>(ns akvo.lumen.lib.aggregation.bar
  (:require [akvo.lumen.lib :as lib]
            [akvo.lumen.lib.aggregation.commons :refer (run-query sql-option-bucket-column) :as commons]
            [akvo.lumen.lib.dataset.utils :refer (find-column)]
            [akvo.lumen.postgres.filter :refer (sql-str)]
            [akvo.lumen.util :as util]
            [clojure.java.jdbc :as jdbc]
            [clojure.string :as str]
            [clojure.pprint :refer (pprint)]
            [clojure.tools.logging :as log]))

(defn- sort* [v column]
  (case v
    nil   "x ASC"
    "asc" (format "%s ASC NULLS FIRST" column)
    "dsc" (format "%s DESC NULLS LAST" column)))

(defn- subbucket-sql [table-name bucket-column subbucket-column aggregation filter-sql sort-sql truncate-size]
  (format "
          WITH
          sort_table AS
            (SELECT %1$s AS x, %2$s AS sort_value
             FROM %3$s
             WHERE %4$s
             GROUP BY x
             ORDER BY %5$s
             LIMIT %6$s),
          data_table AS
            (SELECT %1$s as x, %2$s as y, %7$s as s
             FROM %3$s
             WHERE %4$s
             GROUP BY x, s)
          SELECT
            data_table.x AS x,
            data_table.y,
            data_table.s,
            sort_table.sort_value
          FROM
            data_table
          JOIN
            sort_table
          ON
            sort_table.x = data_table.x
          ORDER BY %5$s, data_table.s"
          (sql-option-bucket-column bucket-column)
          aggregation
          table-name
          filter-sql
          sort-sql
          truncate-size
          (sql-option-bucket-column subbucket-column)))

(defn- bucket-sql [table-name bucket-column aggregation filter-sql sort-sql truncate-size]
  (format "SELECT *
           FROM (SELECT %1$s as x, %2$s FROM %3$s WHERE %4$s GROUP BY x)z
           ORDER BY %5$s
           LIMIT %6$s"
          (sql-option-bucket-column bucket-column)
          aggregation
          table-name
          filter-sql
          sort-sql
          truncate-size))

(defn- aggregation* [aggregation-method metric-column bucket-column]
  (let [aggregation-method (if-not metric-column "count" (or aggregation-method "count"))]
    (format
     (case aggregation-method
       nil                         "NULL"
       ("min" "max" "count" "sum") (str aggregation-method  "(%s)")
       "mean"                      "avg(%s)"
       "median"                    "percentile_cont(0.5) WITHIN GROUP (ORDER BY %s)"
       "distinct"                  "COUNT(DISTINCT %s)"
       "q1"                        "percentile_cont(0.25) WITHIN GROUP (ORDER BY %s)"
       "q3"                        "percentile_cont(0.75) WITHIN GROUP (ORDER BY %s)")
     (or (:columnName metric-column)
         (:columnName bucket-column)))))<commit_msg>[#2964] Use count(*) instead of count(col_name)<commit_after>(ns akvo.lumen.lib.aggregation.bar
  (:require [akvo.lumen.lib :as lib]
            [akvo.lumen.lib.aggregation.commons :refer (run-query sql-option-bucket-column) :as commons]
            [akvo.lumen.lib.dataset.utils :refer (find-column)]
            [akvo.lumen.postgres.filter :refer (sql-str)]
            [akvo.lumen.util :as util]
            [clojure.java.jdbc :as jdbc]
            [clojure.string :as str]
            [clojure.pprint :refer (pprint)]
            [clojure.tools.logging :as log]))

(defn- sort* [v column]
  (case v
    nil   "x ASC"
    "asc" (format "%s ASC NULLS FIRST" column)
    "dsc" (format "%s DESC NULLS LAST" column)))

(defn- subbucket-sql [table-name bucket-column subbucket-column aggregation filter-sql sort-sql truncate-size]
  (format "
          WITH
          sort_table AS
            (SELECT %1$s AS x, %2$s AS sort_value
             FROM %3$s
             WHERE %4$s
             GROUP BY x
             ORDER BY %5$s
             LIMIT %6$s),
          data_table AS
            (SELECT %1$s as x, %2$s as y, %7$s as s
             FROM %3$s
             WHERE %4$s
             GROUP BY x, s)
          SELECT
            data_table.x AS x,
            data_table.y,
            data_table.s,
            sort_table.sort_value
          FROM
            data_table
          JOIN
            sort_table
          ON
            sort_table.x = data_table.x
          ORDER BY %5$s, data_table.s"
          (sql-option-bucket-column bucket-column)
          aggregation
          table-name
          filter-sql
          sort-sql
          truncate-size
          (sql-option-bucket-column subbucket-column)))

(defn- bucket-sql [table-name bucket-column aggregation filter-sql sort-sql truncate-size]
  (format "SELECT *
           FROM (SELECT %1$s as x, %2$s FROM %3$s WHERE %4$s GROUP BY x)z
           ORDER BY %5$s
           LIMIT %6$s"
          (sql-option-bucket-column bucket-column)
          aggregation
          table-name
          filter-sql
          sort-sql
          truncate-size))

(defn- aggregation* [aggregation-method metric-column bucket-column]
  (let [aggregation-method (if-not metric-column "count" (or aggregation-method "count"))]
    (format
     (case aggregation-method
       nil                         "NULL"
       ("min" "max" "sum")         (str aggregation-method  "(%s)")
       "count"                     (str aggregation-method  "(*)")
       "mean"                      "avg(%s)"
       "median"                    "percentile_cont(0.5) WITHIN GROUP (ORDER BY %s)"
       "distinct"                  "COUNT(DISTINCT %s)"
       "q1"                        "percentile_cont(0.25) WITHIN GROUP (ORDER BY %s)"
       "q3"                        "percentile_cont(0.75) WITHIN GROUP (ORDER BY %s)")
     (or (:columnName metric-column)
         (:columnName bucket-column)))))
{'fraction_non_alphanumeric': 0.11421628189550426, 'fraction_numerical': 0.010414858531504947, 'mean_word_length': 1.574620196604111, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1076412', 'n_tokens_mistral': 1866, 'n_tokens_neox': 1791, 'n_words': 458}