<commit_before>(ns desdemona.jobs.sample-submit-job
  (:require [desdemona.catalogs.sample-catalog :refer [build-catalog]]
            [desdemona.tasks.kafka :refer [add-kafka-input]]
            [desdemona.tasks.sql :refer [add-sql-insert-output]]
            [desdemona.lifecycles.sample-lifecycle :refer [build-lifecycles]]
            [desdemona.lifecycles.metrics :refer [add-metrics]]
            [desdemona.lifecycles.logging :refer [add-logging]]
            [desdemona.workflows.sample-workflow :refer [build-workflow]]
            [aero.core :refer [read-config]]
            [onyx.api]))

;;;;
;; Lets build a job
;; Since we always run in Docker Compose, kafka is added as an input, and onyx-sql is used as the output

(defn build-job []
  (let [batch-size 1
        batch-timeout 1000
        base-job {:catalog (build-catalog batch-size batch-timeout)
                  :lifecycles (build-lifecycles)
                  :workflow (build-workflow)
                  :task-scheduler :onyx.task-scheduler/balanced}]
    (-> base-job
        (add-kafka-input :read-lines {:onyx/batch-size batch-size
                                      :onyx/max-peers 1
                                      :kafka/topic "test1"<commit_msg>Remove unused import<commit_after>(ns desdemona.jobs.sample-submit-job
  (:require [desdemona.catalogs.sample-catalog :refer [build-catalog]]
            [desdemona.tasks.kafka :refer [add-kafka-input]]
            [desdemona.tasks.sql :refer [add-sql-insert-output]]
            [desdemona.lifecycles.sample-lifecycle :refer [build-lifecycles]]
            [desdemona.lifecycles.logging :refer [add-logging]]
            [desdemona.workflows.sample-workflow :refer [build-workflow]]
            [aero.core :refer [read-config]]
            [onyx.api]))

;;;;
;; Lets build a job
;; Since we always run in Docker Compose, kafka is added as an input, and onyx-sql is used as the output

(defn build-job []
  (let [batch-size 1
        batch-timeout 1000
        base-job {:catalog (build-catalog batch-size batch-timeout)
                  :lifecycles (build-lifecycles)
                  :workflow (build-workflow)
                  :task-scheduler :onyx.task-scheduler/balanced}]
    (-> base-job
        (add-kafka-input :read-lines {:onyx/batch-size batch-size
                                      :onyx/max-peers 1
                                      :kafka/topic "test1"
{'fraction_non_alphanumeric': 0.11935350186489847, 'fraction_numerical': 0.005801906340654786, 'mean_word_length': 2.0595690747782003, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6332234', 'n_tokens_mistral': 769, 'n_tokens_neox': 745, 'n_words': 156}