<commit_before>(ns pdok.featured.core
  (:require [pdok.featured
             [api :as api]
             [config :as config]
             [json-reader :refer [features-from-stream file-stream]]
             [processor :as processor :refer [consume shutdown]]
             [persistence :as pers]
             [timeline :as tl]
             [generator :refer [random-json-feature-stream]]
             [extracts :as extracts]
             [template :as template]]
            [clojure.tools.cli :refer [parse-opts]]
            [clojure.tools.logging :as log])
  (:gen-class))

(declare cli-options features-from-files)

(defn setup-processor [{:keys [dataset
                               no-projectors
                               no-timeline
                               no-state
                               projection] :as meta}]
  (log/info (str "Configuring:" (when dataset (str " - dataset: " dataset))
                 (when no-projectors " without projectors")
                 (when no-state " without state")
                 (when projection (str " as " projection))))
  (let [persistence (if no-state (pers/make-no-state) (config/persistence))
        projectors (cond-> [] (not no-projectors) (conj (config/projectors persistence :projection projection))
                           (not no-timeline) (conj (config/timeline persistence)))
        processor (processor/create meta dataset persistence projectors)]
    processor))

(defn process [{:keys [json-files dataset] :as options}]
  (log/info "Processing mode")
  (let [[meta features] (features-from-files json-files :dataset dataset)
        processor (setup-processor (merge meta options))]
    (dorun (consume processor features))
    (do (log/info "Shutting down.")
        (shutdown processor)))
  (log/info "Done processing")
  )

(defn int? [^java.lang.String s]
  (try
    (Integer/parseInt s)
    true
    (catch Exception e false)))

(defn replay [{:keys [replay dataset] :as options}]
  (log/info "Replay mode")
  (let [n (when (int? replay) (Integer/parseInt replay))
        root-col (when-not (int? replay) replay)
        processor (setup-processor options)]
    (processor/replay processor dataset n root-col)
    (log/info "Shutting down")
    (shutdown processor)
    (log/info "Done replaying")))

(defn exit [status msg]
  (println msg)
  (System/exit status))

(defn implementation-version []
  (-> ^java.lang.Class (eval 'pdok.featured.core) .getPackage .getImplementationVersion))

(defn -main [& args]
;  (println "ENV-test" (config/env :pdok-test))
  (let [{:keys [options arguments summary]} (parse-opts args cli-options)]
    (cond
      (:help options)
        (exit 0 summary)
      (:version options)
        (exit 0 (implementation-version))
      (:replay options)
        (if (not (:dataset options))
          (exit 0 "replaying requires dataset")
          (replay options))
      (:clear-timeline-changelog options)
        (if (not (:dataset options))
          (exit 0 "clearing changelog requires dataset")
          (tl/delete-changelog (config/timeline-for-dataset (:dataset options))))
      (not (or (seq arguments) (:std-in options)))
        (exit 0 "json-file or std-in required")
      :else
        (let [files (if (:std-in options)
                      [*in*]
                      arguments)]
          (process (assoc options :json-files files))))))

(def cli-options
  [[nil "--std-in" "Read from std-in"]
   ["-d" "--dataset DATASET" "dataset"]
   [nil "--no-projectors"]
   [nil "--no-timeline"]
   [nil "--no-state" "Use only with no nesting and action :new"]<commit_msg>feat: cmdline disable validation support<commit_after>(ns pdok.featured.core
  (:require [pdok.featured
             [api :as api]
             [config :as config]
             [json-reader :refer [features-from-stream file-stream]]
             [processor :as processor :refer [consume shutdown]]
             [persistence :as pers]
             [timeline :as tl]
             [generator :refer [random-json-feature-stream]]
             [extracts :as extracts]
             [template :as template]]
            [clojure.tools.cli :refer [parse-opts]]
            [clojure.tools.logging :as log])
  (:gen-class))

(declare cli-options features-from-files)

(defn setup-processor [{:keys [dataset
                               no-projectors
                               no-timeline
                               no-state
                               projection] :as meta}]
  (log/info (str "Configuring:" (when dataset (str " - dataset: " dataset))
                 (when no-projectors " without projectors")
                 (when no-state " without state")
                 (when projection (str " as " projection))))
  (let [persistence (if no-state (pers/make-no-state) (config/persistence))
        projectors (cond-> [] (not no-projectors) (conj (config/projectors persistence :projection projection))
                           (not no-timeline) (conj (config/timeline persistence)))
        processor (processor/create meta dataset persistence projectors)]
    processor))

(defn process [{:keys [json-files dataset] :as options}]
  (log/info "Processing mode")
  (let [[meta features] (features-from-files json-files :dataset dataset)
        processor (setup-processor (merge meta options))]
    (dorun (consume processor features))
    (do (log/info "Shutting down.")
        (shutdown processor)))
  (log/info "Done processing")
  )

(defn int? [^java.lang.String s]
  (try
    (Integer/parseInt s)
    true
    (catch Exception e false)))

(defn replay [{:keys [replay dataset] :as options}]
  (log/info "Replay mode")
  (let [n (when (int? replay) (Integer/parseInt replay))
        root-col (when-not (int? replay) replay)
        processor (setup-processor options)]
    (processor/replay processor dataset n root-col)
    (log/info "Shutting down")
    (shutdown processor)
    (log/info "Done replaying")))

(defn exit [status msg]
  (println msg)
  (System/exit status))

(defn implementation-version []
  (-> ^java.lang.Class (eval 'pdok.featured.core) .getPackage .getImplementationVersion))

(defn -main [& args]
;  (println "ENV-test" (config/env :pdok-test))
  (let [{:keys [options arguments summary]} (parse-opts args cli-options)]
    (cond
      (:help options)
        (exit 0 summary)
      (:version options)
        (exit 0 (implementation-version))
      (:replay options)
        (if (not (:dataset options))
          (exit 0 "replaying requires dataset")
          (replay options))
      (:clear-timeline-changelog options)
        (if (not (:dataset options))
          (exit 0 "clearing changelog requires dataset")
          (tl/delete-changelog (config/timeline-for-dataset (:dataset options))))
      (not (or (seq arguments) (:std-in options)))
        (exit 0 "json-file or std-in required")
      :else
        (let [files (if (:std-in options)
                      [*in*]
                      arguments)]
          (process (assoc options :json-files files))))))

(def cli-options
  [[nil "--std-in" "Read from std-in"]
   ["-d" "--dataset DATASET" "dataset"]
   [nil "--no-projectors"]
   [nil "--no-timeline"]
   [nil "--no-state" "Use only with no nesting and action :new"]
   [nil "--disable-validation"]
{'fraction_non_alphanumeric': 0.1346180316586373, 'fraction_numerical': 0.0013764624913971094, 'mean_word_length': 2.310250569476082, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '354464', 'n_tokens_mistral': 2146, 'n_tokens_neox': 2091, 'n_words': 621}