<commit_before>(ns replme.web.repl
  (:require [clojure.core.async :refer [go <! >! <!! >!! chan close! go-loop]]
            [org.httpkit.server :refer [send! with-channel on-receive on-close open? websocket?]]
            [http.async.client :as http]
            [clojure.tools.logging :as log]
            [replme.web.resp :refer :all]
            [clojure.tools.nrepl :as repl]
            [docker.container :as container]))

(def nrepl-sentinel #"server started")

(defn- docker-attach
  [client id]
  (let [output (chan)
        url (str "http://" (:host client) "/containers/" id "/logs?stdout=1&follow=1")
        http-client (http/create-client)]
    (go
      (log/info "Reading logs from: " url)
      (doseq [msg (http/string (http/stream-seq http-client :get url))]
        (log/info "Container" id "logs:" msg)
        (>! output msg)))
    [output http-client]))

(defn- docker-cmd
  []
  {:Image "edpaget/lein"
   :Tty true
   :Cmd ["/opt/lein" "repl" ":headless" ":host" "0.0.0.0" ":port" "8081"]})

(defn- start-docker
  [client]
  (let [container (container/create client (docker-cmd))
        id (:Id container)
        logs (docker-attach client id)]
    (container/start client id)
    (log/info (str "Started container:" id))
    [id logs]))

(defn- docker-ip
  [client id]
  (-> (container/inspect client id)
      :NetworkSettings
      :IPAddress))

(defn- stop-docker
  [client id]
  (container/stop client id))

(defn- docker-repl
  [client in-chan out-chan]
  (let [[id [stdout http-client]] (start-docker client)
        ip (docker-ip client id)
        port 8081]
    (go-loop [msg (<! stdout)]
      (if (re-find nrepl-sentinel msg)
        (do
          (log/info (str "Connecting to docker nrepl at" ip ":" port))
          (>! out-chan "REPL OK")
          (go-loop [repl-conn (repl/connect :host ip :port port)
                    client (repl/client repl-conn 1000)
                    command (<! in-chan)]
            (->> (repl/message client {:op :eval :code command})
                 repl/response-values
                 prn-str
                 (>! out-chan)))
          (recur repl-conn (<! in-chan) client))
        (do (>! out-chan msg)
            (recur (<! stdout)))))
    [id http-client]))

(defn- handle-input
  [in-chan]
  (fn [data]
    (log/info (str "Received:" data))
    (>!! in-chan data)))

(defn- handle-close
  [client id http-client & chans]
  (fn [_]
    (log/info "Closing Repl Connnection")
    (stop-docker client id)
    (http/close http-client)
    (doseq [c chans] (close! c))))

(defn- handle-out
  [channel out-chan]
  (go-loop [msg (<! out-chan)]
    (send! channel msg false)
    (when (open? channel)
      (recur (<! out-chan)))))

(defn open-websocket
  [docker-client req]
  (let [in-chan (chan)
        out-chan (chan)]
    (with-channel req channel
      (if (websocket? channel)
        (let [[docker-id http-client] (docker-repl docker-client in-chan out-chan)
              send-chan (handle-out channel out-chan)]
          (log/info "Websocket Connected")
          (on-receive channel (handle-input in-chan))
          (on-close channel (handle-close docker-client docker-id http-client in-chan out-chan send-chan)))
        (send! channel (resp-bad-request))))))
<commit_msg>Send stdout to socket<commit_after>(ns replme.web.repl
  (:require [clojure.core.async :refer [go <! >! <!! >!! chan close! go-loop]]
            [org.httpkit.server :refer [send! with-channel on-receive on-close open? websocket?]]
            [http.async.client :as http]
            [clojure.tools.logging :as log]
            [replme.web.resp :refer :all]
            [clojure.tools.nrepl :as repl]
            [docker.container :as container]))

(def nrepl-sentinel #"server started")

(defn- docker-attach
  [client id]
  (let [output (chan)
        url (str "http://" (:host client) "/containers/" id "/logs?stdout=1&follow=1")
        http-client (http/create-client)]
    (go
      (log/info "Reading logs from: " url)
      (doseq [msg (http/string (http/stream-seq http-client :get url))]
        (log/info "Container" id "logs:" msg)
        (>! output msg)))
    [output http-client]))

(defn- docker-cmd
  []
  {:Image "edpaget/lein"
   :Tty true
   :Memory "256M"
   :Cmd ["/opt/lein" "repl" ":headless" ":host" "0.0.0.0" ":port" "8081"]})

(defn- start-docker
  [client]
  (let [container (container/create client (docker-cmd))
        id (:Id container)
        logs (docker-attach client id)]
    (container/start client id)
    (log/info (str "Started container:" id))
    [id logs]))

(defn- docker-ip
  [client id]
  (-> (container/inspect client id)
      :NetworkSettings
      :IPAddress))

(defn- stop-docker
  [client id]
  (container/stop client id))

(defn- vectorize
  [msg]
  (pr-str (vector msg)))

(defn- docker-repl
  [client in-chan out-chan]
  (let [[id [stdout http-client]] (start-docker client)
        ip (docker-ip client id)
        port 8081]
    (go-loop [msg (<! stdout)]
      (if (re-find nrepl-sentinel msg)
        (do
          (log/info (str "Connecting to docker nrepl at" ip ":" port))
          (>! out-chan (vectorize "REPL OK"))
          (go-loop [repl-conn (repl/connect :host ip :port port)
                    client (repl/client repl-conn 1000)
                    command (<! in-chan)]
            (->> (repl/message client {:op :eval :code command})
                 repl/response-values
                 prn-str
                 (>! out-chan))
            (recur repl-conn client (<! in-chan))))
        (do (>! out-chan (vectorize msg))
            (recur (<! stdout)))))
    [id http-client]))

(defn- handle-input
  [in-chan]
  (fn [data]
    (log/info (str "Received:" data))
    (>!! in-chan data)))

(defn- handle-close
  [client id http-client & chans]
  (fn [_]
    (log/info "Closing Repl Connnection")
    (stop-docker client id)
    (http/close http-client)
    (doseq [c chans] (close! c))))

(defn- handle-out
  [channel out-chan]
  (go-loop [msg (<! out-chan)]
    (send! channel msg false)
    (when (open? channel)
      (recur (<! out-chan)))))

(defn open-websocket
  [docker-client req]
  (let [in-chan (chan)
        out-chan (chan)]
    (with-channel req channel
      (if (websocket? channel)
        (let [[docker-id http-client] (docker-repl docker-client in-chan out-chan)
              send-chan (handle-out channel out-chan)]
          (log/info "Websocket Connected")
          (on-receive channel (handle-input in-chan))
          (on-close channel (handle-close docker-client docker-id http-client in-chan out-chan send-chan)))
        (send! channel (resp-bad-request))))))

{'fraction_non_alphanumeric': 0.16181571407027598, 'fraction_numerical': 0.00588146584225607, 'mean_word_length': 2.312687312687313, 'pattern_counts': {'":': 8, '<': 19, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1516249', 'n_tokens_mistral': 2373, 'n_tokens_neox': 2266, 'n_words': 599}