<commit_before>(ns clueweb-disqus.main
  "Main namespace from where we control the world"
  (:gen-class :main true)
  (:require [clojure.tools.cli :refer [parse-opts]]
            [clueweb-disqus.core :as core]
            [clueweb-disqus.posts :as posts]
            [clueweb-disqus.read-stats :as read-stats]
            [clueweb-disqus.recover :as recover]))

(def options [["-b" "--reboot" "Reboot the crawl by dumping restart points"]
              ["-r" "--recover TS" "Timestamp"]
              ["-s" "--stats" "Read stats"]
              ["-t" "--threads" "Dump a list of relevant thread-ids"]])

(defn -main
  [& args]
  (let [{options :options} (parse-opts args options)]
    (cond (:recover options)
          (core/threads-since (:recover options))
          
          (:stats options)
          (read-stats/-main)
          
          (:reboot options)
          (recover/dump-restart-points)

          (:threads options)
          (posts/dump-all-thread-ids core/disqus-jobs-dir))))<commit_msg>batching URLs option added<commit_after>(ns clueweb-disqus.main
  "Main namespace from where we control the world"
  (:gen-class :main true)
  (:require [clojure.tools.cli :refer [parse-opts]]
            [clueweb-disqus.core :as core]
            [clueweb-disqus.posts :as posts]
            [clueweb-disqus.read-stats :as read-stats]
            [clueweb-disqus.recover :as recover]))

(def options [["-b" "--reboot" "Reboot the crawl by dumping restart points"]
              ["-r" "--recover TS" "Timestamp"]
              ["-s" "--stats" "Read stats"]
              ["-t" "--threads" "Dump a list of relevant thread-ids"]
              ["-d" "--download-threads F" "Download a list of threads"]])

(defn -main
  [& args]
  (let [{options :options} (parse-opts args options)]
    (cond (:recover options)
          (core/threads-since (:recover options))
          
          (:stats options)
          (read-stats/-main)
          
          (:reboot options)
          (recover/dump-restart-points)

          (:threads options)
          (posts/break-up-threads-list)

          (:download-threads options)
          (posts/download-posts-for-threads (:download-threads options)))))
{'fraction_non_alphanumeric': 0.14870395634379263, 'fraction_numerical': 0.0, 'mean_word_length': 2.0985915492957745, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2105397', 'n_tokens_mistral': 705, 'n_tokens_neox': 675, 'n_words': 176}