<commit_before>; Copyright (C) 2013, 2014, 2015 Dr. <NAME>  (<EMAIL>, <EMAIL>)
; Licensed under the terms of the GNU Affero General Public License v3.
; See the "LICENSE.txt" file provided with this software.

(ns cider-ci.dispatcher.dispatch
  (:require
    [cider-ci.dispatcher.executor :as executor-utils]
    [cider-ci.dispatcher.task :as task]
    [cider-ci.dispatcher.trial :as trial-utils]
    [cider-ci.utils.config :refer [get-config]]
    [cider-ci.utils.daemon :as daemon]
    [drtom.logbug.debug :as debug]
    [cider-ci.utils.http :as http]
    [cider-ci.utils.rdbms :as rdbms]
    [drtom.logbug.catcher :as catcher]
    [clj-http.client :as http-client]
    [clj-logging-config.log4j :as logging-config]
    [clojure.data.json :as json]
    [clojure.java.jdbc :as jdbc]
    [clojure.tools.logging :as logging]
    [honeysql.core :as hc]
    [honeysql.helpers :as hh]
    [robert.hooke :as hooke]
    ))


;### build urls ###############################################################

(defn- git-path [repository-id]
  (http/build-service-path :repository (str "/" repository-id "/git")))

(defn- git-url [repository-id]
  ( ->
    (jdbc/query
      (rdbms/get-ds)
      ["SELECT git_url FROM repositories WHERE id = ?" repository-id])
    first
    :git_url))

(defn- trial-attachments-path [trial-id]
  (http/build-service-path :storage  (str "/trial-attachments/" trial-id "/")))

(defn- tree-attachments-path [tree-id]
  (http/build-service-path :storage  (str "/tree-attachments/" tree-id "/")))

(defn- patch-path [executor trial-id]
  (http/build-service-path :dispatcher (str "/trials/" trial-id )))

;### dispatch data ############################################################
(defn get-branch-and-commit [job-id]
  (first (jdbc/query (rdbms/get-ds)
           ["SELECT branches.name, branches.repository_id,
              commits.tree_id as tree_id,
              commits.id as git_commit_id FROM branches
            INNER JOIN branches_commits ON branches.id = branches_commits.branch_id
            INNER JOIN commits ON branches_commits.commit_id = commits.id
            INNER JOIN jobs ON commits.tree_id = jobs.tree_id
            WHERE jobs.id = ?
            ORDER BY branches.updated_at DESC" job-id])))

(defn add-git-url [data repository-id]
  (conj data
        {:git_path (git-path repository-id)
         :git_url (git-url repository-id) }))

(defn build-dispatch-data [trial executor]
  (let [task (first (jdbc/query (rdbms/get-ds)
                                ["SELECT * FROM tasks WHERE tasks.id = ?" (:task_id trial)]))
        task-spec (task/get-task-spec (:id task))
        job-id (:job_id task)
        branch-and-commit (get-branch-and-commit job-id)
        tree-id (:tree_id branch-and-commit)
        repository-id (:repository_id branch-and-commit)
        trial-id (:id trial)
        environment-variables (conj (or (:environment-variables task-spec) {})
                                    {:CIDER_CI_JOB_ID job-id
                                     :CIDER_CI_TASK_ID (:task_id trial)
                                     :CIDER_CI_TRIAL_ID trial-id
                                     :CIDER_CI_TREE_ID (:tree_id branch-and-commit)})
        data {
              :environment-variables environment-variables
              :job_id job-id
              :git_branch_name (:name branch-and-commit)
              :git_commit_id (:git_commit_id branch-and-commit)
              :git-options (or (:git-options task-spec) {})
              :git_tree_id (:tree_id branch-and-commit)
              :patch_path (patch-path executor trial-id)
              :ports (:ports task-spec)
              :repository_id repository-id
              :scripts (:scripts trial)
              :task_id (:task_id trial)
              :tree-attachments (:tree-attachments task-spec)
              :tree-attachments-path (tree-attachments-path tree-id)
              :trial-attachments (:trial-attachments task-spec)
              :trial-attachments-path (trial-attachments-path trial-id)
              :trial_id trial-id
              }]
    (-> data
        (add-git-url repository-id))))


;### dispatch #################################################################

(defn choose-executor-to-dispatch-to [trial]
  (->> (-> (hh/select :executors_with_load.*)
           (hh/from :trials)
           (hh/where [:= :trials.id (:id trial)])
           (hh/merge-join :tasks [:= :tasks.id :trials.task_id])
           (hh/merge-join :executors_with_load (hc/raw "(tasks.traits <@ executors_with_load.traits)"))
           (hh/merge-where (hc/raw "(last_ping_at > (now() - interval '1 Minutes'))"))
           (hh/merge-where [:= :enabled true])
           (hh/merge-where [:< :relative_load 1])
           (hh/merge-join :jobs [:= :tasks.job_id :jobs.id])
           (hh/merge-join :commits [:= :jobs.tree_id :commits.tree_id])
           (hh/merge-join :branches_commits [:= :commits.id :branches_commits.commit_id])
           (hh/merge-join :branches [:= :branches_commits.branch_id :branches.id])
           (hh/merge-join :repositories [:= :branches.repository_id :repositories.id])
           (hh/merge-join [:or
                           (hc/raw "(executors_with_load.accepted_repositories = '{}')")
                           (hc/raw " repositories.git_url = ANY(executors_with_load.accepted_repositories) ")])
           hc/format)
       (jdbc/query (rdbms/get-ds))
       (map (fn [e] (repeat (- (:max_load e) (:current_load e)) e)))
       flatten rand-nth))

(def ^:private available-executor-exists-query
  (->
    (hh/select 1)
    (hh/from :executors_with_load)
    (hh/merge-where [:< :relative_load 1])
    (hh/merge-where [:= :enabled true])
    (hh/merge-where (hc/raw "(tasks.traits <@ executors_with_load.traits)"))
    (hh/merge-where (hc/raw "(last_ping_at > (now() - interval '1 Minutes'))"))))

(def ^:private join-trial-to-repo<commit_msg>Fix dispatching to executors which do not accept the repo<commit_after>; Copyright (C) 2013, 2014, 2015 Dr. <NAME>  (<EMAIL>, <EMAIL>)
; Licensed under the terms of the GNU Affero General Public License v3.
; See the "LICENSE.txt" file provided with this software.

(ns cider-ci.dispatcher.dispatch
  (:require
    [cider-ci.dispatcher.executor :as executor-utils]
    [cider-ci.dispatcher.task :as task]
    [cider-ci.dispatcher.trial :as trial-utils]
    [cider-ci.utils.config :refer [get-config]]
    [cider-ci.utils.daemon :as daemon]
    [drtom.logbug.debug :as debug]
    [cider-ci.utils.http :as http]
    [cider-ci.utils.rdbms :as rdbms]
    [drtom.logbug.catcher :as catcher]
    [clj-http.client :as http-client]
    [clj-logging-config.log4j :as logging-config]
    [clojure.data.json :as json]
    [clojure.java.jdbc :as jdbc]
    [clojure.tools.logging :as logging]
    [honeysql.core :as hc]
    [honeysql.helpers :as hh]
    [robert.hooke :as hooke]
    ))


;### build urls ###############################################################

(defn- git-path [repository-id]
  (http/build-service-path :repository (str "/" repository-id "/git")))

(defn- git-url [repository-id]
  ( ->
    (jdbc/query
      (rdbms/get-ds)
      ["SELECT git_url FROM repositories WHERE id = ?" repository-id])
    first
    :git_url))

(defn- trial-attachments-path [trial-id]
  (http/build-service-path :storage  (str "/trial-attachments/" trial-id "/")))

(defn- tree-attachments-path [tree-id]
  (http/build-service-path :storage  (str "/tree-attachments/" tree-id "/")))

(defn- patch-path [executor trial-id]
  (http/build-service-path :dispatcher (str "/trials/" trial-id )))

;### dispatch data ############################################################
(defn get-branch-and-commit [job-id]
  (first (jdbc/query (rdbms/get-ds)
           ["SELECT branches.name, branches.repository_id,
              commits.tree_id as tree_id,
              commits.id as git_commit_id FROM branches
            INNER JOIN branches_commits ON branches.id = branches_commits.branch_id
            INNER JOIN commits ON branches_commits.commit_id = commits.id
            INNER JOIN jobs ON commits.tree_id = jobs.tree_id
            WHERE jobs.id = ?
            ORDER BY branches.updated_at DESC" job-id])))

(defn add-git-url [data repository-id]
  (conj data
        {:git_path (git-path repository-id)
         :git_url (git-url repository-id) }))

(defn build-dispatch-data [trial executor]
  (let [task (first (jdbc/query (rdbms/get-ds)
                                ["SELECT * FROM tasks WHERE tasks.id = ?" (:task_id trial)]))
        task-spec (task/get-task-spec (:id task))
        job-id (:job_id task)
        branch-and-commit (get-branch-and-commit job-id)
        tree-id (:tree_id branch-and-commit)
        repository-id (:repository_id branch-and-commit)
        trial-id (:id trial)
        environment-variables (conj (or (:environment-variables task-spec) {})
                                    {:CIDER_CI_JOB_ID job-id
                                     :CIDER_CI_TASK_ID (:task_id trial)
                                     :CIDER_CI_TRIAL_ID trial-id
                                     :CIDER_CI_TREE_ID (:tree_id branch-and-commit)})
        data {
              :environment-variables environment-variables
              :job_id job-id
              :git_branch_name (:name branch-and-commit)
              :git_commit_id (:git_commit_id branch-and-commit)
              :git-options (or (:git-options task-spec) {})
              :git_tree_id (:tree_id branch-and-commit)
              :patch_path (patch-path executor trial-id)
              :ports (:ports task-spec)
              :repository_id repository-id
              :scripts (:scripts trial)
              :task_id (:task_id trial)
              :tree-attachments (:tree-attachments task-spec)
              :tree-attachments-path (tree-attachments-path tree-id)
              :trial-attachments (:trial-attachments task-spec)
              :trial-attachments-path (trial-attachments-path trial-id)
              :trial_id trial-id
              }]
    (-> data
        (add-git-url repository-id))))


;### dispatch #################################################################

(defn choose-executor-to-dispatch-to [trial]
  (->> (-> (hh/select :executors_with_load.*)
           (hh/from :trials)
           (hh/where [:= :trials.id (:id trial)])
           (hh/merge-join :tasks [:= :tasks.id :trials.task_id])
           (hh/merge-join :executors_with_load (hc/raw "(tasks.traits <@ executors_with_load.traits)"))
           (hh/merge-where (hc/raw "(last_ping_at > (now() - interval '1 Minutes'))"))
           (hh/merge-where [:= :enabled true])
           (hh/merge-where [:< :relative_load 1])
           (hh/merge-join :jobs [:= :tasks.job_id :jobs.id])
           (hh/merge-join :commits [:= :jobs.tree_id :commits.tree_id])
           (hh/merge-join :branches_commits [:= :commits.id :branches_commits.commit_id])
           (hh/merge-join :branches [:= :branches_commits.branch_id :branches.id])
           (hh/merge-join :repositories [:= :branches.repository_id :repositories.id])
           (hh/merge-where [:or
                           (hc/raw "(executors_with_load.accepted_repositories = '{}')")
                           (hc/raw " repositories.git_url = ANY(executors_with_load.accepted_repositories) ")])
           hc/format)
       (jdbc/query (rdbms/get-ds))
       (map (fn [e] (repeat (- (:max_load e) (:current_load e)) e)))
       flatten rand-nth))

(def ^:private available-executor-exists-query
  (->
    (hh/select 1)
    (hh/from :executors_with_load)
    (hh/merge-where [:< :relative_load 1])
    (hh/merge-where [:= :enabled true])
    (hh/merge-where (hc/raw "(tasks.traits <@ executors_with_load.traits)"))
    (hh/merge-where (hc/raw "(last_ping_at > (now() - interval '1 Minutes'))"))))

(def ^:private join-trial-to-repo
{'fraction_non_alphanumeric': 0.16946294578871934, 'fraction_numerical': 0.003203777084562853, 'mean_word_length': 2.4283236994219655, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5378848', 'n_tokens_mistral': 4028, 'n_tokens_neox': 3831, 'n_words': 790}