<commit_before>(ns onyx.generative.flow-conditions-gen-test
  (:require [clojure.test.check :as tc]
            [clojure.test.check.generators :as gen]
            [clojure.test.check.properties :as prop]
            [clojure.test :refer :all]
            [com.gfredericks.test.chuck :refer [times]]
            [com.gfredericks.test.chuck.clojure-test :refer [checking]]
            [onyx.peer.task-lifecycle :as t]
            [onyx.peer.task-compile :as c]
            [onyx.api]))

(def true-pred (constantly true))

(def false-pred (constantly false))

(def gen-tasks-and-subset
  "Returns a vector of two elements.
   The first represents a seq of downstream
   tasks, and the second represents a subset
   of that seq."
  (gen/bind
   (gen/vector gen/keyword)
   (fn [downstream]
     (gen/bind
      (gen/choose 0 (count downstream))
      (fn [size]
        (gen/bind
         (gen/shuffle (take size downstream))
         (fn [flow-to]
           (gen/return [downstream flow-to]))))))))

(deftest nil-flow-conditions
  "No flow condition routes to all downstream tasks"
  (let [downstream [:a :b]
        route (t/route-data nil nil nil nil downstream)]
    (is (= downstream (:flow route)))
    (is (nil? (:action route)))))

(deftest nil-flow-conditions-exception
  "No flow conditions with an exception rethrows the exception"
  (let [e (ex-info "hih" {})
        wrapped-e (ex-info "" {:exception e})]
    (is (thrown? clojure.lang.ExceptionInfo
                 (t/route-data nil nil wrapped-e nil [:a :b])))))

(deftest limit-downstream-results
  (checking
   "The matching flow condition limits the downstream selection"
   (times 50)
   [[downstream flow-to] gen-tasks-and-subset]
   (let [task :a
         fcs [{:flow/from task
               :flow/to flow-to
               :flow/predicate ::true-pred}]
         compiled (c/compile-fc-norms fcs task)
         results
         (t/route-data
          {:onyx.core/compiled-norm-fcs compiled} nil nil fcs downstream)]
     (is (= (into #{} flow-to) (:flow results)))
     (is (nil? (:action results))))))

(deftest conj-downstream-tasks-together
  (checking
   "It joins the :flow/to tasks together"
   (times 50)
   [downstream (gen/not-empty (gen/vector gen/keyword))
    size (gen/choose 0 (count downstream))
    flow-conditions (gen/not-empty
                     (gen/vector
                      (gen/hash-map
                       :flow/from (gen/return :a)
                       :flow/to (gen/fmap #(vec (take size %)) (gen/shuffle downstream))
                       :flow/predicate (gen/return ::true-pred))))]
   (let [compiled (c/compile-fc-norms flow-conditions :a)
         results
         (t/route-data
          {:onyx.core/compiled-norm-fcs compiled} nil nil flow-conditions downstream)]
     (is (= (into #{} (mapcat :flow/to flow-conditions)) (:flow results)))
     (is (nil? (:action results))))))

(def predicates
  {true ::true-red
   false ::false-pred})

(defn flow-condition-gen [from-task]
  (gen/hash-map
   :flow/from (gen/return from-task)
   :flow/to (gen/one-of [(gen/return :all) (gen/return :none) (gen/vector gen/keyword)])
   :flow/exclude-keys (gen/vector gen/keyword)))

(defn true-flow-condition-gen [base-gen]
  (gen/bind
   base-gen
   (fn [v]
     (let [g (gen/return ::true-pred)
           true-gen (gen/hash-map :flow/predicate g)]
       (gen/fmap #(merge v %) true-gen)))))

(defn false-flow-condition-gen [base-gen]
  (gen/bind
   base-gen
   (fn [v]
     (let [g (gen/return ::false-pred)
           true-gen (gen/hash-map :flow/predicate g)]
       (gen/fmap #(merge v %) true-gen)))))

(defn short-circuit-flow-condition-gen [base-gen]
  (gen/bind
   base-gen
   (fn [v]
     (let [g (gen/return true)
           true-gen (gen/hash-map :flow/short-circuit? g)]
       (gen/fmap #(merge v %) true-gen)))))

(defn exception-flow-condition-gen [base-gen]
  (gen/bind
   base-gen
   (fn [v]
     (let [g (gen/hash-map
              :flow/thrown-exception? (gen/return true)
              :flow/short-circuit? (gen/return true)
              :flow/post-transform (gen/one-of
                                    [(gen/return nil)
                                     (gen/return ::post-transform)]))]
       (gen/fmap #(merge v %) g)))))

(defn retry-flow-condition-gen [base-gen]
  (gen/bind
   base-gen
   (fn [v]
     (let [g (gen/hash-map :flow/action (gen/return :retry))]
       (gen/fmap #(merge v %) g)))))


(clojure.pprint/pprint
 (gen/sample
  (->> :A
       (flow-condition-gen)
       (true-flow-condition-gen)
       (exception-flow-condition-gen)
       (retry-flow-condition-gen))))





(deftest no-false-predicate-picks
  (checking
   "It doesn't pick any downstream tasks with false predicates"
   (times 50)
   [true-downstream (gen/fmap (fn [xs] (map (comp keyword str) xs)) (gen/not-empty (gen/vector gen/uuid)))
    false-downstream (gen/fmap (fn [xs] (map (comp keyword str) xs)) (gen/not-empty (gen/vector gen/uuid)))<commit_msg>Use new generators in some of the flow condition tests.<commit_after>(ns onyx.generative.flow-conditions-gen-test
  (:require [clojure.test.check :as tc]
            [clojure.test.check.generators :as gen]
            [clojure.test.check.properties :as prop]
            [clojure.test :refer :all]
            [com.gfredericks.test.chuck :refer [times]]
            [com.gfredericks.test.chuck.clojure-test :refer [checking]]
            [onyx.peer.task-lifecycle :as t]
            [onyx.peer.task-compile :as c]
            [onyx.api]))

(def true-pred (constantly true))

(def false-pred (constantly false))

(def predicates
  {true ::true-red
   false ::false-pred})

(defn flow-condition-gen [from-task]
  (gen/hash-map
   :flow/from (gen/return from-task)
   :flow/exclude-keys (gen/vector gen/keyword)))

(defn flow-to-all-gen [base-gen]
  (gen/bind
   base-gen
   (fn [v]
     (let [g (gen/hash-map :flow/to (gen/return :all))]
       (gen/fmap #(merge v %) g)))))

(defn flow-to-none-gen [base-gen]
  (gen/bind
   base-gen
   (fn [v]
     (let [g (gen/hash-map :flow/to (gen/return :none))]
       (gen/fmap #(merge v %) g)))))

(defn flow-to-tasks-gen [base-gen]
  (gen/bind
   base-gen
   (fn [v]
     (let [g (gen/hash-map :flow/to (gen/vector gen/keyword))]
       (gen/fmap #(merge v %) g)))))

(defn true-flow-condition-gen [base-gen]
  (gen/bind
   base-gen
   (fn [v]
     (let [g (gen/return ::true-pred)
           true-gen (gen/hash-map :flow/predicate g)]
       (gen/fmap #(merge v %) true-gen)))))

(defn false-flow-condition-gen [base-gen]
  (gen/bind
   base-gen
   (fn [v]
     (let [g (gen/return ::false-pred)
           false-gen (gen/hash-map :flow/predicate g)]
       (gen/fmap #(merge v %) false-gen)))))

(defn short-circuit-flow-condition-gen [base-gen]
  (gen/bind
   base-gen
   (fn [v]
     (let [g (gen/hash-map :flow/short-circuit? (gen/return true))]
       (gen/fmap #(merge v %) g)))))

(defn exception-flow-condition-gen [base-gen]
  (gen/bind
   base-gen
   (fn [v]
     (let [g (gen/hash-map
              :flow/thrown-exception? (gen/return true)
              :flow/short-circuit? (gen/return true)
              :flow/post-transform (gen/one-of
                                    [(gen/return nil)
                                     (gen/return ::post-transform)]))]
       (gen/fmap #(merge v %) g)))))

(defn retry-flow-condition-gen [base-gen]
  (gen/bind
   base-gen
   (fn [v]
     (let [g (gen/hash-map :flow/action (gen/return :retry))]
       (gen/fmap #(merge v %) g)))))

(deftest nil-flow-conditions
  "No flow condition routes to all downstream tasks"
  (let [downstream [:a :b]
        route (t/route-data nil nil nil nil downstream)]
    (is (= downstream (:flow route)))
    (is (nil? (:action route)))))

(deftest nil-flow-conditions-exception
  "No flow conditions with an exception rethrows the exception"
  (let [e (ex-info "hih" {})
        wrapped-e (ex-info "" {:exception e})]
    (is (thrown? clojure.lang.ExceptionInfo
                 (t/route-data nil nil wrapped-e nil [:a :b])))))

(deftest limit-downstream-results
  (checking
   "The matching flow condition limits the downstream selection"
   (times 50)
   [fcs (gen/vector (->> :a
                         (flow-condition-gen)
                         (flow-to-tasks-gen)
                         (true-flow-condition-gen))
                    1)
    other-downstream-tasks (gen/vector gen/keyword)]
   (let [task :a
         target-tasks (mapcat :flow/to fcs)
         downstream (into other-downstream-tasks target-tasks)
         compiled (c/compile-fc-norms fcs task)
         results
         (t/route-data
          {:onyx.core/compiled-norm-fcs compiled} nil nil fcs downstream)]
     (is (= (into #{} target-tasks) (:flow results)))
     (is (nil? (:action results))))))

(deftest conj-downstream-tasks-together
  (checking
   "It joins the :flow/to tasks together"
   (times 50)
   [downstream (gen/not-empty (gen/vector gen/keyword))
    size (gen/choose 0 (count downstream))
    flow-conditions (gen/not-empty
                     (gen/vector
                      (gen/hash-map
                       :flow/from (gen/return :a)
                       :flow/to (gen/fmap #(vec (take size %)) (gen/shuffle downstream))
                       :flow/predicate (gen/return ::true-pred))))]
   (let [compiled (c/compile-fc-norms flow-conditions :a)
         results
         (t/route-data
          {:onyx.core/compiled-norm-fcs compiled} nil nil flow-conditions downstream)]
     (is (= (into #{} (mapcat :flow/to flow-conditions)) (:flow results)))
     (is (nil? (:action results))))))





(deftest no-false-predicate-picks
  (checking
   "It doesn't pick any downstream tasks with false predicates"
   (times 50)
   [true-downstream (gen/fmap (fn [xs] (map (comp keyword str) xs)) (gen/not-empty (gen/vector gen/uuid)))
    false-downstream (gen/fmap (fn [xs] (map (comp keyword str) xs)) (gen/not-empty (gen/vector gen/uuid)))
{'fraction_non_alphanumeric': 0.1551377438166097, 'fraction_numerical': 0.0016086869093102755, 'mean_word_length': 2.2720394736842104, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5707166', 'n_tokens_mistral': 3495, 'n_tokens_neox': 3368, 'n_words': 839}