<commit_before>(ns cmr.common.util
  "Utility functions that might be useful throughout the CMR."
  (:require [cmr.common.log :refer (debug info warn error)]
            [cmr.common.services.errors :as errors]
            [camel-snake-kebab.core :as csk]
            [clojure.set :as set]
            [clojure.string :as str]
            [clojure.walk :as w]
            [clojure.template :as template]
            [clojure.test :as test])
  (:import java.text.DecimalFormat))

(defmacro are2
  "Based on the are macro from clojure.test. Checks multiple assertions with a template expression.
  Wraps each tested expression in a testing block to identify what's being tested.
  See clojure.template/do-template for an explanation of templates.

  Example: (are2 [x y] (= x y)
                \"The most basic case with 1\"
                2 (+ 1 1)
                \"A more complicated test\"
                4 (* 2 2))
  Expands to:
           (do
              (testing \"The most basic case with 1\"
                (is (= 2 (+ 1 1))))
              (testing \"A more complicated test\"
                (is (= 4 (* 2 2)))))

  Note: This breaks some reporting features, such as line numbers."
  [argv expr & args]
  (if (or
        ;; (are2 [] true) is meaningless but ok
        (and (empty? argv) (empty? args))
        ;; Catch wrong number of args
        (and (pos? (count argv))
             (pos? (count args))
             (zero? (mod (count args) (inc (count argv))))))
    (let [testing-var (gensym "testing-msg")
          argv (vec (cons testing-var argv))]
      `(template/do-template ~argv (test/testing ~testing-var (test/is ~expr)) ~@args))
    (throw (IllegalArgumentException.
             "The number of args doesn't match are2's argv or testing doc string may be missing."))))

(defn trunc
  "Returns the given string truncated to n characters."
  [s n]
  (when s
    (subs s 0 (min (count s) n))))

(defn sequence->fn
  [vals]
  "Creates a stateful function that returns individual values from the sequence. It returns the first
  value when called the first time, the second value on the second call and so on until the sequence
  is exhausted of values. Returns nil forever after that.

      user=> (def my-ints (sequence->fn [1 2 3]))
      user=> (my-ints)
      1
      user=> (my-ints)
      2
      user=> (my-ints)
      3
      user=> (my-ints)
      nil"
  (let [vals-atom (atom {:curr-val nil :next-vals (seq vals)})]
    (fn []
      (:curr-val (swap! vals-atom
                        (fn [{:keys [next-vals]}]
                          {:curr-val (first next-vals)
                           :next-vals (rest next-vals)}))))))

(defmacro future-with-logging
  "Creates a future that will log when a task starts and completes or if exceptions occur."
  [taskname & body ]
  `(future
    (info "Starting " ~taskname)
    (try
      (let [result# (do ~@body)]
        (info ~taskname " completed without exception")
        result#)
      (catch Throwable e#
        (error e# "Exception in " ~taskname)
        (throw e#))
      (finally
        (info ~taskname " complete.")))))

(defmacro time-execution
  "Times the execution of the body and returns a tuple of time it took and the results"
  [& body]
  `(let [start# (System/currentTimeMillis)
         result# (do ~@body)]
     [(- (System/currentTimeMillis) start#) result#]))


(defmacro defn-timed
  "Creates a function that logs how long it took to execute the body"
  [fn-name & fn-tail]
  (let [fn-name-str (name fn-name)
        ns-str (str *ns*)
        ;; Extract the doc string from the function if present
        [doc-string fn-tail] (if (string? (first fn-tail))
                               [(first fn-tail) (next fn-tail)]
                               [nil fn-tail])
        ;; Wrap single arity functions in a list
        fn-tail (if (vector? (first fn-tail))
                  (list fn-tail)
                  fn-tail)
        ;; extract other arities defined in the function which will not be timed.
        other-arities (drop-last fn-tail)
        ;; extract the last arity definitions bindings and body
        [timed-arity-bindings & timed-arity-body] (last fn-tail)]
    `(defn ~fn-name
       ~@(when doc-string [doc-string])
       ~@other-arities
       (~timed-arity-bindings
         (let [start# (System/currentTimeMillis)]
           (try<commit_msg>CMR-1359: Adding more comments to defn-timed.<commit_after>(ns cmr.common.util
  "Utility functions that might be useful throughout the CMR."
  (:require [cmr.common.log :refer (debug info warn error)]
            [cmr.common.services.errors :as errors]
            [camel-snake-kebab.core :as csk]
            [clojure.set :as set]
            [clojure.string :as str]
            [clojure.walk :as w]
            [clojure.template :as template]
            [clojure.test :as test])
  (:import java.text.DecimalFormat))

(defmacro are2
  "Based on the are macro from clojure.test. Checks multiple assertions with a template expression.
  Wraps each tested expression in a testing block to identify what's being tested.
  See clojure.template/do-template for an explanation of templates.

  Example: (are2 [x y] (= x y)
                \"The most basic case with 1\"
                2 (+ 1 1)
                \"A more complicated test\"
                4 (* 2 2))
  Expands to:
           (do
              (testing \"The most basic case with 1\"
                (is (= 2 (+ 1 1))))
              (testing \"A more complicated test\"
                (is (= 4 (* 2 2)))))

  Note: This breaks some reporting features, such as line numbers."
  [argv expr & args]
  (if (or
        ;; (are2 [] true) is meaningless but ok
        (and (empty? argv) (empty? args))
        ;; Catch wrong number of args
        (and (pos? (count argv))
             (pos? (count args))
             (zero? (mod (count args) (inc (count argv))))))
    (let [testing-var (gensym "testing-msg")
          argv (vec (cons testing-var argv))]
      `(template/do-template ~argv (test/testing ~testing-var (test/is ~expr)) ~@args))
    (throw (IllegalArgumentException.
             "The number of args doesn't match are2's argv or testing doc string may be missing."))))

(defn trunc
  "Returns the given string truncated to n characters."
  [s n]
  (when s
    (subs s 0 (min (count s) n))))

(defn sequence->fn
  [vals]
  "Creates a stateful function that returns individual values from the sequence. It returns the first
  value when called the first time, the second value on the second call and so on until the sequence
  is exhausted of values. Returns nil forever after that.

      user=> (def my-ints (sequence->fn [1 2 3]))
      user=> (my-ints)
      1
      user=> (my-ints)
      2
      user=> (my-ints)
      3
      user=> (my-ints)
      nil"
  (let [vals-atom (atom {:curr-val nil :next-vals (seq vals)})]
    (fn []
      (:curr-val (swap! vals-atom
                        (fn [{:keys [next-vals]}]
                          {:curr-val (first next-vals)
                           :next-vals (rest next-vals)}))))))

(defmacro future-with-logging
  "Creates a future that will log when a task starts and completes or if exceptions occur."
  [taskname & body ]
  `(future
    (info "Starting " ~taskname)
    (try
      (let [result# (do ~@body)]
        (info ~taskname " completed without exception")
        result#)
      (catch Throwable e#
        (error e# "Exception in " ~taskname)
        (throw e#))
      (finally
        (info ~taskname " complete.")))))

(defmacro time-execution
  "Times the execution of the body and returns a tuple of time it took and the results"
  [& body]
  `(let [start# (System/currentTimeMillis)
         result# (do ~@body)]
     [(- (System/currentTimeMillis) start#) result#]))

(defmacro defn-timed
  "Creates a function that logs how long it took to execute the body. It supports multiarity functions
  but only times how long the last listed arity version takes. This means it should be used with
  multiarity functions where it calls itself with the extra arguments."
  [fn-name & fn-tail]
  (let [fn-name-str (name fn-name)
        ns-str (str *ns*)
        ;; Extract the doc string from the function if present
        [doc-string fn-tail] (if (string? (first fn-tail))
                               [(first fn-tail) (next fn-tail)]
                               [nil fn-tail])
        ;; Wrap single arity functions in a list
        fn-tail (if (vector? (first fn-tail))
                  (list fn-tail)
                  fn-tail)
        ;; extract other arities defined in the function which will not be timed.
        other-arities (drop-last fn-tail)
        ;; extract the last arity definitions bindings and body
        [timed-arity-bindings & timed-arity-body] (last fn-tail)]
    `(defn ~fn-name
       ~@(when doc-string [doc-string])
       ~@other-arities
       (~timed-arity-bindings
         (let [start# (System/currentTimeMillis)]
           (try
{'fraction_non_alphanumeric': 0.11580821003448659, 'fraction_numerical': 0.006007342307264434, 'mean_word_length': 2.0819334933150495, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3203395', 'n_tokens_mistral': 2752, 'n_tokens_neox': 2630, 'n_words': 1002}