<commit_before>(ns vip.data-processor.pipeline
  (:require [clojure.tools.logging :as log]
            [clojure.stacktrace :as stacktrace]))

(defn try-validation [validation ctx]
  (try
    (validation ctx)
    (catch Throwable e
      (log/error e)
      (assoc ctx :stop "Exception caught"
                 :exception e))))

(defn run-pipeline [c]
  (loop [ctx c]
    (if-let [next-step (-> ctx :pipeline first)]
      (let [next-ctx (try-validation next-step ctx)]
        (if (:stop next-ctx)
          next-ctx
          (recur (assoc next-ctx :pipeline (rest (:pipeline ctx))))))
      ctx)))

(defn process [pipeline initial-input]
  (let [ctx {:input initial-input
             :pipeline pipeline}
        result (run-pipeline ctx)]
    (log/info result)
    (when-let [ex (:exception result)]
      (log/error (with-out-str (stacktrace/print-stack-trace ex)))
      (throw (ex-info "Exception during processing" {:exception ex
                                                     :initial-ctx ctx
                                                     :final-ctx result})))
    result))<commit_msg>[CS] Add warnings and errors keys to context<commit_after>(ns vip.data-processor.pipeline
  (:require [clojure.tools.logging :as log]
            [clojure.stacktrace :as stacktrace]))

(defn try-validation [validation ctx]
  (try
    (validation ctx)
    (catch Throwable e
      (log/error e)
      (assoc ctx :stop "Exception caught"
                 :exception e))))

(defn run-pipeline [c]
  (loop [ctx c]
    (if-let [next-step (-> ctx :pipeline first)]
      (let [next-ctx (try-validation next-step ctx)]
        (if (:stop next-ctx)
          next-ctx
          (recur (assoc next-ctx :pipeline (rest (:pipeline ctx))))))
      ctx)))

(defn process [pipeline initial-input]
  (let [ctx {:input initial-input
             :warnings []
             :errors []
             :pipeline pipeline}
        result (run-pipeline ctx)]
    (log/info result)
    (when-let [ex (:exception result)]
      (log/error (with-out-str (stacktrace/print-stack-trace ex)))
      (throw (ex-info "Exception during processing" {:exception ex
                                                     :initial-ctx ctx
                                                     :final-ctx result})))
    result))
{'fraction_non_alphanumeric': 0.1264718709114697, 'fraction_numerical': 0.0, 'mean_word_length': 1.8891687657430731, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5432267', 'n_tokens_mistral': 651, 'n_tokens_neox': 653, 'n_words': 190}