<commit_before>(ns ci3.controller
  (:require [ci3.k8s :as k8s]
            [ci3.github :as gh]
            [clojure.walk :as walk]))

(def cfg {:apiVersion "ci3.io/v1" :ns "default"})

(defonce stop (atom nil))

(defn process-build [builds]
  (when-let [items (:items builds)]
    (doseq [i items]
      (when-not (:pod i)
        (let [id (get-in i [:metadata :name])]
          (println "Process " id)
          (let [pod (k8s/create
                     {:prefix "api" :ns "default" :apiVersion "v1"}
                     :pods {:apiVersion "v1"
                            :kind "Pod"
                            :metadata {:name id
                                       :annotations {:system "ci3"}
                                       :lables {:system "ci3"}}
                            :spec {:restartPolicy "Never"
                                   :volumes
                                   [{:name "docker-sock"
                                     :hostPath {:path "/var/run/docker.sock"}}]
                                   :containers
                                   [{:name "agent"
                                     :image "eu.gcr.io/aidbox-next/ci3:latest"
                                     :imagePullPolicy "Always"
                                     :args ["agent"]
                                     :volumeMounts
                                     [{:name "docker-sock"
                                       :mountPath "/var/run/docker.sock"}]
                                     :env [{:name "BUILD_ID" :value id}
                                           {:name "REPOSITORY" :value (get-in i [:payload :repository :full_name])}
                                           {:name "DOCKER_KEY" :valueFrom {:secretKeyRef {:name "docker-registry" :key "key"}}}
                                           {:name "SERVICE_ACCOUNT" :valueFrom {:secretKeyRef {:name "docker-registry" :key "key"}}}]}
                                    ]}})]
            (println "Create pod" pod)
            (when-not (= "Failure" (get pod "status"))
              (println "Update build:"
                       (k8s/patch k8s/cfg :builds
                                  id {:pod pod
                                      :scheduledAt (str (java.util.Date.))
                                      :status "sche"})))))))))

(defn watch-resource [cfg rt process]
  (if @stop
    (println "Stop watching " (name rt))
    (future
      (process (walk/keywordize-keys (k8s/list cfg rt)))
      (Thread/sleep 5000)
      (watch-resource cfg rt process))))
<commit_msg>mount gsutil config into agent<commit_after>(ns ci3.controller
  (:require [ci3.k8s :as k8s]
            [ci3.github :as gh]
            [clojure.walk :as walk]))

(def cfg {:apiVersion "ci3.io/v1" :ns "default"})

(defonce stop (atom nil))

(defn process-build [builds]
  (when-let [items (:items builds)]
    (doseq [i items]
      (when-not (:pod i)
        (let [id (get-in i [:metadata :name])]
          (println "Process " id)
          (let [pod (k8s/create
                     {:prefix "api" :ns "default" :apiVersion "v1"}
                     :pods {:apiVersion "v1"
                            :kind "Pod"
                            :metadata {:name id
                                       :annotations {:system "ci3"}
                                       :lables {:system "ci3"}}
                            :spec {:restartPolicy "Never"
                                   :volumes
                                   [{:name "docker-sock"
                                     :hostPath {:path "/var/run/docker.sock"}}
                                    {:name "gsutil"
                                     :secret {:secretName "storage"
                                              :items {:key "boto" :path ".boto"}}}]
                                   :containers
                                   [{:name "agent"
                                     :image "eu.gcr.io/aidbox-next/ci3:latest"
                                     :imagePullPolicy "Always"
                                     :args ["agent"]
                                     :volumeMounts
                                     [{:name "docker-sock"
                                       :mountPath "/var/run/docker.sock"}
                                      {:name "gsutil"
                                       :mountPath "/gsutil"
                                       :readOnly true}]
                                     :env [{:name "BUILD_ID" :value id}
                                           {:name "BOTO_CONFIG" :value "/gsutil/.boto"}
                                           {:name "REPOSITORY" :value (get-in i [:payload :repository :full_name])}
                                           {:name "DOCKER_KEY" :valueFrom {:secretKeyRef {:name "docker-registry" :key "key"}}}
                                           {:name "SERVICE_ACCOUNT" :valueFrom {:secretKeyRef {:name "docker-registry" :key "key"}}}]}]}})]
            (println "Create pod" pod)
            (when-not (= "Failure" (get pod "status"))
              (println "Update build:"
                       (k8s/patch k8s/cfg :builds
                                  id {:pod pod
                                      :scheduledAt (str (java.util.Date.))
                                      :status "sche"})))))))))

(defn watch-resource [cfg rt process]
  (if @stop
    (println "Stop watching " (name rt))
    (future
      (process (walk/keywordize-keys (k8s/list cfg rt)))
      (Thread/sleep 5000)
      (watch-resource cfg rt process))))

{'fraction_non_alphanumeric': 0.1193019943019943, 'fraction_numerical': 0.007122507122507123, 'mean_word_length': 0.974340949033392, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 58, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3063206', 'n_tokens_mistral': 1496, 'n_tokens_neox': 1379, 'n_words': 356}