<commit_before>(ns s7p.master.core
  (:require
   [clojure.core.async :refer [go go-loop timeout close! chan >! <! <!!]]
   [cheshire.core :as json]
   [zeromq.zmq :as zmq]
   [s7p.config :as config]))


(def qp100ms 100)

(defn timer [ms]
  (let [c (chan)]
    (go-loop [n 0]
      (>! c n)
      (<! (timeout ms))
      (recur (+ n 1)))
    c))

(defn enqueue-query-timer [sender reqs]
  (let [t (timer 100)]
   (go-loop [reqs reqs]
     (println (<! t))
     (doall
      (doseq [req (take qp100ms reqs)]
        (zmq/send-str sender (json/generate-string req))))
     (recur (drop qp100ms reqs)))))

(defn create-dsp [pub dsp]
  (zmq/send-str pub (json/generate-string {:command :create-dsp :data dsp})))

(defn remove-dsp [pub id]
  (zmq/send-str pub (json/generate-string {:command :remove-dsp :data id})))<commit_msg>use string rather than keyword to clarify that slave will decode it to string<commit_after>(ns s7p.master.core
  (:require
   [clojure.core.async :refer [go go-loop timeout close! chan >! <! <!!]]
   [cheshire.core :as json]
   [zeromq.zmq :as zmq]
   [s7p.config :as config]))


(def qp100ms 100)

(defn timer [ms]
  (let [c (chan)]
    (go-loop [n 0]
      (>! c n)
      (<! (timeout ms))
      (recur (+ n 1)))
    c))

(defn enqueue-query-timer [sender reqs]
  (let [t (timer 100)]
   (go-loop [reqs reqs]
     (println (<! t))
     (doall
      (doseq [req (take qp100ms reqs)]
        (zmq/send-str sender (json/generate-string req))))
     (recur (drop qp100ms reqs)))))

(defn create-dsp [pub dsp]
  (zmq/send-str pub (json/generate-string {:command "create-dsp" :data dsp})))

(defn remove-dsp [pub id]
  (zmq/send-str pub (json/generate-string {:command "remove-dsp" :data id})))
{'fraction_non_alphanumeric': 0.17736289381563594, 'fraction_numerical': 0.022170361726954493, 'mean_word_length': 2.988372093023256, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6476532', 'n_tokens_mistral': 733, 'n_tokens_neox': 687, 'n_words': 188}