<commit_before>(ns yetibot.core.commands.collections
  (:require [clojure.pprint :refer [*print-right-margin* pprint]]
            [clojure.string :as s]
            [json-path :as jp]
            [taoensso.timbre :as timbre :refer [trace debug error info]]
            [yetibot.core.chat :refer [chat-data-structure]]
            [yetibot.core.hooks :refer [cmd-hook]]
            [yetibot.core.interpreter :refer [handle-cmd]]
            [yetibot.core.models.users :refer [min-user-keys]]
            [yetibot.core.util
             :refer
             [ensure-items-collection
              ensure-items-seqential
              psuedo-format
              split-kvs-with]]
            [yetibot.core.util.command :refer [error?]]
            [yetibot.core.util.command-info :refer [command-execution-info]]
            [yetibot.core.util.format :refer [format-exception-log]]))

(defn ensure-coll
  "Return nil if opts was set or return an error map otherwise"
  [{:keys [opts args]}]
  (or
    (ensure-items-collection opts)
    {:result/error
     (str "Expected a collection but you gave me `" args "`")}))

(defn coll-cmd
  "Helper to define commands that operate only on collections"
  [f]
  (with-meta
    (fn [cmd-args]
      (let [coll-or-error (ensure-coll cmd-args)]
        (if (error? coll-or-error)
          coll-or-error
          (let [result (f coll-or-error)]
            {:result/value result
             :result/data result}))))
    {:yb/cat #{:util :collection}}))

; random
(defn random
  "random <list> # returns a random item from <list>
   random # generate a random number"
  {:yb/cat #{:util :collection}}
  [{items :opts}]
  (if (not (empty? items))
    (rand-nth (ensure-items-collection items))
    (str (rand 100000))))

(cmd-hook #"random"
          _ random)

; shuffle
(def shuffle-cmd
  "shuffle <list>"
  (coll-cmd shuffle))

(cmd-hook #"shuffle"
          _ shuffle-cmd)

(def head-tail-regex #"(\d+).+")

; head / tail helpers
(defn head-or-tail
  [single-fn multi-fn n cmd-map]
  (let [coll-or-error (ensure-coll cmd-map)
        f (if (= 1 n) single-fn (partial multi-fn n))]
    (if (error? coll-or-error)
      coll-or-error
      (f coll-or-error))))

(def head (partial head-or-tail (comp str first) take))

(def tail (partial head-or-tail (comp str last) take-last))

; head
(defn head-1
  "head <list> # returns the first item from the <list>"
  {:yb/cat #{:util :collection}}
  [cmd-args]
  (head 1 cmd-args))

(defn head-n
  "head <n> <list> # return the first <n> items from the <list>"
  {:yb/cat #{:util :collection}}
  [{[_ n] :match :as cmd-args}]
  (head (read-string n) cmd-args))

(cmd-hook #"head"
          #"(\d+)" head-n
          _ head-1)

(cmd-hook #"take"
          #"(\d+)" head-n)

; tail
(defn tail-1
  "tail <list> # returns the last item from the <list>"
  {:yb/cat #{:util :collection}}
  [cmd-args] (tail 1 cmd-args))

(defn tail-n
  "tail <n> <list> # returns the last <n> items from the <list>"
  {:yb/cat #{:util :collection}}
  [{[_ n] :match :as cmd-args}]
  (tail (read-string n) cmd-args))

(cmd-hook #"tail"
          #"(\d+)" tail-n
          _ tail-1)

; droplast
(def drop-last-cmd
  "droplast <list> # drop the last item from <list>"
  (coll-cmd drop-last))

(cmd-hook #"droplast"
  _ drop-last-cmd)

; rest
(def rest-cmd
  "rest <list> # returns the last item from the <list>"
  (coll-cmd rest))

(cmd-hook #"rest"
  _ rest-cmd)

; xargs
; example usage: !users | xargs attack
(defn xargs
  "xargs <cmd> <list> # run <cmd> for every item in <list> in parallel; behavior is similar to xargs(1)'s xargs -n1"
  {:yb/cat #{:util :collection}}
  [{:keys [args opts user] :as cmd-params}]
  (if (s/blank? args)
    opts ; passthrough if no args
    (if-let [itms (ensure-items-collection opts)]
      (let [cat (-> (str args " " (first opts))
                    command-execution-info :matched-sub-cmd meta :yb/cat)
            cmd-runner (if (contains? cat :async) 'map 'pmap)]
        (debug "xargs using cmd-runner:" cmd-runner "for command" (pr-str args))
        ((resolve cmd-runner)
         (fn [item]
           (try
             (let [{:result/keys [value error] :as cmd-result}
                   (apply handle-cmd
                          ;; item could be a collection, such as when xargs is
                          ;; used on nested collections, e.g.:
                          ;; repeat 5 jargon | xargs words | xargs head
                          (if (coll? item)
                            [args (merge cmd-params {:raw item :opts item})]
                            [(psuedo-format args item)
                             (merge cmd-params {:raw item :opts nil})]))
                   _ (info "xargs cmd-result" (pr-str cmd-result))]
               (or error value cmd-result))
             (catch Exception ex
               (error "Exception in xargs cmd-runner:" cmd-runner
                      (format-exception-log ex))
               ex)))
         itms))<commit_msg>Allow non-map results in xargs (#79)<commit_after>(ns yetibot.core.commands.collections
  (:require [clojure.pprint :refer [*print-right-margin* pprint]]
            [clojure.string :as s]
            [json-path :as jp]
            [taoensso.timbre :as timbre :refer [trace debug error info]]
            [yetibot.core.chat :refer [chat-data-structure]]
            [yetibot.core.hooks :refer [cmd-hook]]
            [yetibot.core.interpreter :refer [handle-cmd]]
            [yetibot.core.models.users :refer [min-user-keys]]
            [yetibot.core.util
             :refer
             [ensure-items-collection
              ensure-items-seqential
              psuedo-format
              split-kvs-with]]
            [yetibot.core.util.command :refer [error?]]
            [yetibot.core.util.command-info :refer [command-execution-info]]
            [yetibot.core.util.format :refer [format-exception-log]]))

(defn ensure-coll
  "Return nil if opts was set or return an error map otherwise"
  [{:keys [opts args]}]
  (or
    (ensure-items-collection opts)
    {:result/error
     (str "Expected a collection but you gave me `" args "`")}))

(defn coll-cmd
  "Helper to define commands that operate only on collections"
  [f]
  (with-meta
    (fn [cmd-args]
      (let [coll-or-error (ensure-coll cmd-args)]
        (if (error? coll-or-error)
          coll-or-error
          (let [result (f coll-or-error)]
            {:result/value result
             :result/data result}))))
    {:yb/cat #{:util :collection}}))

; random
(defn random
  "random <list> # returns a random item from <list>
   random # generate a random number"
  {:yb/cat #{:util :collection}}
  [{items :opts}]
  (if (not (empty? items))
    (rand-nth (ensure-items-collection items))
    (str (rand 100000))))

(cmd-hook #"random"
          _ random)

; shuffle
(def shuffle-cmd
  "shuffle <list>"
  (coll-cmd shuffle))

(cmd-hook #"shuffle"
          _ shuffle-cmd)

(def head-tail-regex #"(\d+).+")

; head / tail helpers
(defn head-or-tail
  [single-fn multi-fn n cmd-map]
  (let [coll-or-error (ensure-coll cmd-map)
        f (if (= 1 n) single-fn (partial multi-fn n))]
    (if (error? coll-or-error)
      coll-or-error
      (f coll-or-error))))

(def head (partial head-or-tail (comp str first) take))

(def tail (partial head-or-tail (comp str last) take-last))

; head
(defn head-1
  "head <list> # returns the first item from the <list>"
  {:yb/cat #{:util :collection}}
  [cmd-args]
  (head 1 cmd-args))

(defn head-n
  "head <n> <list> # return the first <n> items from the <list>"
  {:yb/cat #{:util :collection}}
  [{[_ n] :match :as cmd-args}]
  (head (read-string n) cmd-args))

(cmd-hook #"head"
          #"(\d+)" head-n
          _ head-1)

(cmd-hook #"take"
          #"(\d+)" head-n)

; tail
(defn tail-1
  "tail <list> # returns the last item from the <list>"
  {:yb/cat #{:util :collection}}
  [cmd-args] (tail 1 cmd-args))

(defn tail-n
  "tail <n> <list> # returns the last <n> items from the <list>"
  {:yb/cat #{:util :collection}}
  [{[_ n] :match :as cmd-args}]
  (tail (read-string n) cmd-args))

(cmd-hook #"tail"
          #"(\d+)" tail-n
          _ tail-1)

; droplast
(def drop-last-cmd
  "droplast <list> # drop the last item from <list>"
  (coll-cmd drop-last))

(cmd-hook #"droplast"
  _ drop-last-cmd)

; rest
(def rest-cmd
  "rest <list> # returns the last item from the <list>"
  (coll-cmd rest))

(cmd-hook #"rest"
  _ rest-cmd)

; xargs
; example usage: !users | xargs attack
(defn xargs
  "xargs <cmd> <list> # run <cmd> for every item in <list> in parallel; behavior is similar to xargs(1)'s xargs -n1"
  {:yb/cat #{:util :collection}}
  [{:keys [args opts user] :as cmd-params}]
  (if (s/blank? args)
    opts ; passthrough if no args
    (if-let [itms (ensure-items-collection opts)]
      (let [cat (-> (str args " " (first opts))
                    command-execution-info :matched-sub-cmd meta :yb/cat)
            cmd-runner (if (contains? cat :async) 'map 'pmap)]
        (debug "xargs using cmd-runner:" cmd-runner "for command" (pr-str args))
        ((resolve cmd-runner)
         (fn [item]
           (try
             (let [cmd-result
                   (apply handle-cmd
                          ;; item could be a collection, such as when xargs is
                          ;; used on nested collections, e.g.:
                          ;; repeat 5 jargon | xargs words | xargs head
                          (if (coll? item)
                            [args (merge cmd-params {:raw item :opts item})]
                            [(psuedo-format args item)
                             (merge cmd-params {:raw item :opts nil})]))

                   [value error] (if (map? cmd-result)
                                   ((juxt :result/value :result/error)
                                    cmd-result)
                                   [cmd-result nil])
                   _ (info "xargs cmd-result" (pr-str cmd-result))]
               (or error value cmd-result))
             (catch Exception ex
               (error "Exception in xargs cmd-runner:" cmd-runner
                      (format-exception-log ex))
               ex)))
         itms))
{'fraction_non_alphanumeric': 0.1480935534591195, 'fraction_numerical': 0.0033411949685534592, 'mean_word_length': 2.0433612440191387, 'pattern_counts': {'":': 0, '<': 49, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3213283', 'n_tokens_mistral': 3451, 'n_tokens_neox': 3378, 'n_words': 981}