<commit_before>(ns nube.app
  (:require [bidi.bidi :refer [match-route]]
            [taoensso.carmine :as car :refer [wcar]]
            [clojure.data.json :as json]
            [org.httpkit.client :as http]
            [org.httpkit.server :refer [with-channel send!]]
            [pandect.core :refer [sha1]]
            [ring.middleware.basic-authentication :refer [basic-authentication-request authentication-failure]]))

(def env (atom {:controller "localhost"
                :port "8080"
                :redishost "127.0.0.1"
                :redisport "6379"
                :dockerport "4243"}))

(defn redis-conf []
  {:spec {:host (:redishost @env)
          :port (read-string (:redisport @env))
          :password (:redispassword @env)}})

(defmacro redis! [& body] `(car/wcar (redis-conf) ~@body))

(def port-range (set (range 8000 8999)))

(defn docker!
  ([method host uri] (docker! method host uri {}))
  ([method host uri options]
     (let [res @((if (= :get method) http/get http/post) (str "http://" host ":" (:dockerport @env) "/" uri) options)
           status (:status res)]
       (if (and status (< 199 status 300))
          (let [body (:body res)]
            (if (clojure.string/blank? body)
              {}
              (json/read-str body :key-fn keyword)))          
          (throw (Exception. (str "Docker remote api error. Status: " status)))))))

(defn ssplit [s] (when s (clojure.string/split s #":")))
(defn create-token [] (let [token (sha1 (pr-str (java.util.Date.)))] (redis! (car/set :token token)) token))
(defn get-token [] (if-let [token (redis! (car/get :token))] token (create-token)))

(defn notify-routers [] (redis! (car/publish "updates" (java.util.Date.))))

(defn load-apps [] (redis! (car/smembers :apps)))
(defn add-app [app] (redis! (car/sadd :apps app)
                            (redis! (car/rpush (str "frontend:" app) app))))
(defn remove-app [app] (redis! (car/srem :apps app)))

(defn add-app-env [app env val] (redis! (car/hset (str app ":envs") env val)))
(defn remove-app-env [app env] (redis! (car/hdel (str app ":envs") env)))
(defn load-app-envs [app] (apply hash-map (redis! (car/hgetall (str app ":envs")))))

(defn load-app-instances [app] (mapv #(subs % 7) (redis! (car/lrange (str "frontend:" app) 0 -1))))
(defn add-app-instance [app instance] (redis! (car/rpush (str "frontend:" app) (str "http://" instance))))
(defn remove-app-instance [app instance] (redis! (car/lrem (str "frontend:" app) 0 (str "http://" instance))))

(defn load-pending-app-instances [app] (redis! (car/smembers (str app ":pending-instances"))))
(defn add-pending-app-instance [app instance] (redis! (car/sadd (str app ":pending-instances") instance)))
(defn remove-pending-app-instance [app instance] (redis! (car/srem (str app ":pending-instances") instance)))

(defn load-hosts [] (redis! (car/smembers :hosts)))
(defn add-host [host] (redis! (car/sadd :hosts host)))
(defn remove-host [host] (redis! (car/srem :hosts host)))

(defn load-deployments [app] (redis! (car/lrange (str "deployments:" app) -100 -1)))
(defn save-deployments [app image count]
  (redis! (car/lpush (str "deployments:" app)
                     {:timestamp (java.util.Date.)
                      :app app
                      :instances (load-pending-app-instances app)
                      :image image
                      :count count})))

(defn get-containers [host] (docker! :get host "containers/json"))

(defn run-container [host port internal-port image envs]
  (let [internal-port (or internal-port "80/tcp")
        options {:Hostname "" :User "" :AttachStdin false :AttachStdout true :AttachStderr true
                 :Tty true :OpenStdin false :StdinOnce false :Cmd nil :Volumes {}
                 :Env (mapv (fn [[k v]] (str k "=" v)) envs)
                 :Image image :ExposedPorts {internal-port {}}}
        start-options {:PortBindings {internal-port [{:HostPort (str port)}]}}
        id (:Id (docker! :post host "containers/create"
                         {:headers {"Content-Type" "application/json"}
                          :body (json/write-str options)}))]
    (println "Container with id" id "created.")
    (docker! :post host (str "containers/" id "/start")
             {:headers {"Content-Type" "application/json"}
              :body (json/write-str start-options)})
    id))

(defn load-ports-in-use [host]
  (set (mapv #(:PublicPort (first (:Ports %))) (get-containers host))))

(defn find-available-port [host]
  (rand-nth (vec (clojure.set/difference port-range (load-ports-in-use host)))))

(defn load-container-by-host-and-port [host port]
  (first (filter #(= port (str (:PublicPort (first (:Ports %))))) (get-containers host))))

(defn stop-container [host id]
  (docker! :post host (str "containers/" id "/stop")))

(defn stop-container-by-port [host port]
  (when-let [id (:Id (load-container-by-host-and-port host port))]
    (stop-container host id)))

(defn health-check-host [host port]  
  (loop [n 15]
    (when (pos? n)
      (if (= 200 (:status @(http/get (str "http://" host ":" port "/") {:timeout 5000})))
        true
        (do
          (Thread/sleep (* (- 15 n) 500))
          (recur (dec n)))))))

(defn pull-docker-image [host image]
  (let [[image tag] (ssplit image)]
    (docker! :post host (str "images/create?fromImage=" image (if tag (str "&tag=" tag) "")))))

(defn kill-app-instance [app host port]
  (println "Killing instance at" (str host ":" port))
  (stop-container-by-port host port)
  (remove-app-instance app (str host ":" port)))

(defn deploy-app-instance [app host port internal-port image]
  (println "Pulling new tags for" image)
  (pull-docker-image host image)
  (println "Starting new container at" (str host ":" port))
  (let [id (run-container host port internal-port image (load-app-envs app))]
    (println "Checking host health")
    (if-not (health-check-host host port)
      (do
        (stop-container host id)
        (Exception. "Failed to deploy new instance."))
      (try
        (println "Adding" (str host ":" port) "pending")
        (add-pending-app-instance app (str host ":" port))
        :ok
        (catch Exception e
          (println "Deploy failed. Rolling back.")
          (try (kill-app-instance app host port)
               (Exception. "Deployment failed. Rolling back.")
               (catch Exception e (Exception. "Rollback failed. System may be in an invalid state."))))))))

(defn deploy-new-app-instances [app image count internal-port]
  (let [count (if (string? count) (read-string count) count)
        dist (into {} (map #(vector % (clojure.core/count (get-containers %))) (load-hosts)))
        total-containers (apply + (map second dist))
        hosts (vec (keys dist))<commit_msg>Working with hipache<commit_after>(ns nube.app
  (:require [bidi.bidi :refer [match-route]]
            [taoensso.carmine :as car :refer [wcar]]
            [clojure.data.json :as json]
            [org.httpkit.client :as http]
            [org.httpkit.server :refer [with-channel send!]]
            [pandect.core :refer [sha1]]
            [ring.middleware.basic-authentication :refer [basic-authentication-request authentication-failure]]))

(def env (atom {:controller "localhost"
                :port "8080"
                :redishost "127.0.0.1"
                :redisport "6379"
                :dockerport "4243"}))

(defn redis-conf []
  {:spec {:host (:redishost @env)
          :port (read-string (:redisport @env))
          :password (:redispassword @env)}})

(defmacro redis! [& body] `(car/wcar (redis-conf) ~@body))

(def port-range (set (range 8000 8999)))

(defn docker!
  ([method host uri] (docker! method host uri {}))
  ([method host uri options]
     (let [res @((if (= :get method) http/get http/post) (str "http://" host ":" (:dockerport @env) "/" uri) options)
           status (:status res)]
       (if (and status (< 199 status 300))
          (let [body (:body res)]
            (if (clojure.string/blank? body)
              {}
              (json/read-str body :key-fn keyword)))          
          (throw (Exception. (str "Docker remote api error. Status: " status)))))))

(defn ssplit [s] (when s (clojure.string/split s #":")))
(defn create-token [] (let [token (sha1 (pr-str (java.util.Date.)))] (redis! (car/set :token token)) token))
(defn get-token [] (if-let [token (redis! (car/get :token))] token (create-token)))

(defn notify-routers [] (redis! (car/publish "updates" (java.util.Date.))))

(defn load-apps [] (redis! (car/smembers :apps)))
(defn add-app [app] (redis! (car/sadd :apps app)
                            (redis! (car/rpush (str "frontend:" app) app))))
(defn remove-app [app] (redis! (car/srem :apps app)))

(defn add-app-env [app env val] (redis! (car/hset (str app ":envs") env val)))
(defn remove-app-env [app env] (redis! (car/hdel (str app ":envs") env)))
(defn load-app-envs [app] (apply hash-map (redis! (car/hgetall (str app ":envs")))))

(defn load-app-instances [app] (mapv #(subs % 7) (next (redis! (car/lrange (str "frontend:" app) 0 -1)))))
(defn add-app-instance [app instance] (redis! (car/rpush (str "frontend:" app) (str "http://" instance))))
(defn remove-app-instance [app instance] (redis! (car/lrem (str "frontend:" app) 0 (str "http://" instance))))

(defn load-pending-app-instances [app] (redis! (car/smembers (str app ":pending-instances"))))
(defn add-pending-app-instance [app instance] (redis! (car/sadd (str app ":pending-instances") instance)))
(defn remove-pending-app-instance [app instance] (redis! (car/srem (str app ":pending-instances") instance)))

(defn load-hosts [] (redis! (car/smembers :hosts)))
(defn add-host [host] (redis! (car/sadd :hosts host)))
(defn remove-host [host] (redis! (car/srem :hosts host)))

(defn load-deployments [app] (redis! (car/lrange (str "deployments:" app) -100 -1)))
(defn save-deployments [app image count]
  (redis! (car/lpush (str "deployments:" app)
                     {:timestamp (java.util.Date.)
                      :app app
                      :instances (load-pending-app-instances app)
                      :image image
                      :count count})))

(defn get-containers [host] (docker! :get host "containers/json"))

(defn run-container [host port internal-port image envs]
  (let [internal-port (or internal-port "80/tcp")
        options {:Hostname "" :User "" :AttachStdin false :AttachStdout true :AttachStderr true
                 :Tty true :OpenStdin false :StdinOnce false :Cmd nil :Volumes {}
                 :Env (mapv (fn [[k v]] (str k "=" v)) envs)
                 :Image image :ExposedPorts {internal-port {}}}
        start-options {:PortBindings {internal-port [{:HostPort (str port)}]}}
        id (:Id (docker! :post host "containers/create"
                         {:headers {"Content-Type" "application/json"}
                          :body (json/write-str options)}))]
    (println "Container with id" id "created.")
    (docker! :post host (str "containers/" id "/start")
             {:headers {"Content-Type" "application/json"}
              :body (json/write-str start-options)})
    id))

(defn load-ports-in-use [host]
  (set (mapv #(:PublicPort (first (:Ports %))) (get-containers host))))

(defn find-available-port [host]
  (rand-nth (vec (clojure.set/difference port-range (load-ports-in-use host)))))

(defn load-container-by-host-and-port [host port]
  (first (filter #(= port (str (:PublicPort (first (:Ports %))))) (get-containers host))))

(defn stop-container [host id]
  (docker! :post host (str "containers/" id "/stop")))

(defn stop-container-by-port [host port]
  (when-let [id (:Id (load-container-by-host-and-port host port))]
    (stop-container host id)))

(defn health-check-host [host port]  
  (loop [n 15]
    (when (pos? n)
      (if (= 200 (:status @(http/get (str "http://" host ":" port "/") {:timeout 5000})))
        true
        (do
          (Thread/sleep (* (- 15 n) 500))
          (recur (dec n)))))))

(defn pull-docker-image [host image]
  (let [[image tag] (ssplit image)]
    (docker! :post host (str "images/create?fromImage=" image (if tag (str "&tag=" tag) "")))))

(defn kill-app-instance [app host port]
  (try
    (println "Killing instance at" (str host ":" port))
    (stop-container-by-port host port)
    (remove-app-instance app (str host ":" port))
    (catch Exception e nil)))

(defn deploy-app-instance [app host port internal-port image]
  (println "Pulling new tags for" image)
  (pull-docker-image host image)
  (println "Starting new container at" (str host ":" port))
  (let [id (run-container host port internal-port image (load-app-envs app))]
    (println "Checking host health")
    (if-not (health-check-host host port)
      (do
        (stop-container host id)
        (Exception. "Failed to deploy new instance."))
      (try
        (println "Adding" (str host ":" port) "pending")
        (add-pending-app-instance app (str host ":" port))
        :ok
        (catch Exception e
          (println "Deploy failed. Rolling back.")
          (try (kill-app-instance app host port)
               (Exception. "Deployment failed. Rolling back.")
               (catch Exception e (Exception. "Rollback failed. System may be in an invalid state."))))))))

(defn deploy-new-app-instances [app image count internal-port]
  (let [count (if (string? count) (read-string count) count)
        dist (into {} (map #(vector % (clojure.core/count (get-containers %))) (load-hosts)))
        total-containers (apply + (map second dist))
        hosts (vec (keys dist))
{'fraction_non_alphanumeric': 0.17192568563845473, 'fraction_numerical': 0.008552049542907697, 'mean_word_length': 2.8504115810388875, 'pattern_counts': {'":': 28, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '573269', 'n_tokens_mistral': 4680, 'n_tokens_neox': 4478, 'n_words': 1341}