<commit_before>(ns lambdacd-git.core
  (:require [clojure.core.async :as async]
            [clojure.tools.logging :as log]
            [lambdacd.steps.support :as support]
            [lambdacd.presentation.pipeline-state :as pipeline-state]
            [lambdacd-git.git :as git]
            [clojure.data :refer [diff]]
            [clojure.java.io :as io]
            [lambdacd.event-bus :as event-bus]
            [clojure.walk :as walk]
            [ring.middleware.params :as ring-params]
            [ring.util.response :as ring-response]
            [compojure.core :as compojure]
            [lambdacd-git.ssh-agent-support :as ssh-agent-support]
            [me.raynes.fs :as fs])
  (:import (java.util.regex Pattern)
           (java.util Date)
           (java.text SimpleDateFormat)
           (org.eclipse.jgit.transport SshSessionFactory)
           (java.io SequenceInputStream FileInputStream File)
           (com.jcraft.jsch JSch)
           (clojure.lang SeqEnumeration)))

(defn- find-changed-revision [old-revisions new-revisions]
  (let [[_ new-entries _] (diff old-revisions new-revisions)
        changed-ref        (->> new-entries
                                (first)
                                (key))
        last-seen-revision (get old-revisions changed-ref)
        new-revision       (get new-revisions changed-ref)]
    {:changed-ref  changed-ref
     :revision     new-revision
     :old-revision last-seen-revision}))

(defn- report-git-exception [ref remote e]
  (log/warn e (str "could not get current revision for ref " ref " on " remote))
  (println "could not get current revision for ref" ref "on" remote ":" (.getMessage e)))

(defn- current-revision-or-nil [remote ref]
  (try
    (git/current-revisions remote ref)
    (catch Exception e
      (report-git-exception ref remote e)
      nil)))

(defn- found-new-commit [remote last-seen-revisions current-revisions]
  (let [changes (find-changed-revision last-seen-revisions current-revisions)]
    (println "Found new commit: " (:revision changes) "on" (:changed-ref changes))
    {:status         :success
     :changed-ref    (:changed-ref changes)
     :changed-remote remote
     :revision       (:revision changes)
     :old-revision   (:old-revision changes)
     :all-revisions  current-revisions}))

(defn- report-waiting-status [ctx]
  (async/>!! (:result-channel ctx) [:status :waiting]))

(defn- wait-for-next-poll [poll-notifications ms-between-polls kill-channel]
  (async/alt!!
    kill-channel nil
    poll-notifications ([_] (println "Received notification. Polling out of schedule"))
    (async/timeout ms-between-polls) :poll))

(defn- kill-switch->ch [ctx]
  (let [ch       (async/chan)
        notifier (fn [key reference old new]
                   (if (and (not= old new)
                            (= true new))
                     (async/>!! ch :killed)))]
    (add-watch (:is-killed ctx) ::to-channel-watcher notifier)
    ch))

(defn- clean-up-kill-switch->ch [a]
  (remove-watch a ::to-channel-watcher))

(defn- wait-for-revision-changed [last-seen-revisions remote ref ctx ms-between-polls poll-notifications]
  (println "Last seen revisions:" (or last-seen-revisions "None") ". Waiting for new commit...")
  (let [kill-channel (kill-switch->ch ctx)
        result       (loop [last-seen-revisions last-seen-revisions]
                       (support/if-not-killed ctx
                         (let [current-revisions (current-revision-or-nil remote ref)]
                           (if (and
                                 (not (nil? current-revisions))
                                 (not= current-revisions last-seen-revisions))
                             (found-new-commit remote last-seen-revisions current-revisions)
                             (do
                               (report-waiting-status ctx)
                               (wait-for-next-poll poll-notifications ms-between-polls kill-channel)
                               (recur current-revisions))))))]
    (clean-up-kill-switch->ch (:is-killed ctx))
    result))

(defn- last-seen-revisions-from-history [ctx]
  (let [last-step-result (pipeline-state/most-recent-step-result-with :_git-last-seen-revisions ctx)]
    (:_git-last-seen-revisions last-step-result)))

(defn- initial-revisions [ctx remote ref]
  (or
    (last-seen-revisions-from-history ctx)
    (current-revision-or-nil remote ref)))

(defn- persist-last-seen-revisions [wait-for-result last-seen-revisions ctx]
  (let [current-revisions    (:all-revisions wait-for-result)
        revisions-to-persist (or current-revisions last-seen-revisions)]
    (async/>!! (:result-channel ctx) [:_git-last-seen-revisions revisions-to-persist]) ; by sending it through the result-channel, we can be pretty sure users don't overwrite it<commit_msg>Fixed bug where, if fetching of a new revision failed, the last seen revision would be replaced by nil, so the pipeline would always trigger after a failed revision fetch<commit_after>(ns lambdacd-git.core
  (:require [clojure.core.async :as async]
            [clojure.tools.logging :as log]
            [lambdacd.steps.support :as support]
            [lambdacd.presentation.pipeline-state :as pipeline-state]
            [lambdacd-git.git :as git]
            [clojure.data :refer [diff]]
            [clojure.java.io :as io]
            [lambdacd.event-bus :as event-bus]
            [clojure.walk :as walk]
            [ring.middleware.params :as ring-params]
            [ring.util.response :as ring-response]
            [compojure.core :as compojure]
            [lambdacd-git.ssh-agent-support :as ssh-agent-support]
            [me.raynes.fs :as fs])
  (:import (java.util.regex Pattern)
           (java.util Date)
           (java.text SimpleDateFormat)
           (org.eclipse.jgit.transport SshSessionFactory)
           (java.io SequenceInputStream FileInputStream File)
           (com.jcraft.jsch JSch)
           (clojure.lang SeqEnumeration)))

(defn- find-changed-revision [old-revisions new-revisions]
  (let [[_ new-entries _] (diff old-revisions new-revisions)
        changed-ref        (->> new-entries
                                (first)
                                (key))
        last-seen-revision (get old-revisions changed-ref)
        new-revision       (get new-revisions changed-ref)]
    {:changed-ref  changed-ref
     :revision     new-revision
     :old-revision last-seen-revision}))

(defn- report-git-exception [ref remote e]
  (log/warn e (str "could not get current revision for ref " ref " on " remote))
  (println "could not get current revision for ref" ref "on" remote ":" (.getMessage e)))

(defn- current-revision-or-nil [remote ref]
  (try
    (git/current-revisions remote ref)
    (catch Exception e
      (report-git-exception ref remote e)
      nil)))

(defn- found-new-commit [remote last-seen-revisions current-revisions]
  (let [changes (find-changed-revision last-seen-revisions current-revisions)]
    (println "Found new commit: " (:revision changes) "on" (:changed-ref changes))
    {:status         :success
     :changed-ref    (:changed-ref changes)
     :changed-remote remote
     :revision       (:revision changes)
     :old-revision   (:old-revision changes)
     :all-revisions  current-revisions}))

(defn- report-waiting-status [ctx]
  (async/>!! (:result-channel ctx) [:status :waiting]))

(defn- wait-for-next-poll [poll-notifications ms-between-polls kill-channel]
  (async/alt!!
    kill-channel nil
    poll-notifications ([_] (println "Received notification. Polling out of schedule"))
    (async/timeout ms-between-polls) :poll))

(defn- kill-switch->ch [ctx]
  (let [ch       (async/chan)
        notifier (fn [key reference old new]
                   (if (and (not= old new)
                            (= true new))
                     (async/>!! ch :killed)))]
    (add-watch (:is-killed ctx) ::to-channel-watcher notifier)
    ch))

(defn- clean-up-kill-switch->ch [a]
  (remove-watch a ::to-channel-watcher))

(defn- wait-for-revision-changed [last-seen-revisions remote ref ctx ms-between-polls poll-notifications]
  (println "Last seen revisions:" (or last-seen-revisions "None") ". Waiting for new commit...")
  (let [kill-channel (kill-switch->ch ctx)
        result       (loop [last-seen-revisions last-seen-revisions]
                       (support/if-not-killed ctx
                         (let [current-revisions (current-revision-or-nil remote ref)]
                           (if (and
                                 (not (nil? current-revisions))
                                 (not= current-revisions last-seen-revisions))
                             (found-new-commit remote last-seen-revisions current-revisions)
                             (do
                               (report-waiting-status ctx)
                               (wait-for-next-poll poll-notifications ms-between-polls kill-channel)
                               (recur last-seen-revisions))))))]
    (clean-up-kill-switch->ch (:is-killed ctx))
    result))

(defn- last-seen-revisions-from-history [ctx]
  (let [last-step-result (pipeline-state/most-recent-step-result-with :_git-last-seen-revisions ctx)]
    (:_git-last-seen-revisions last-step-result)))

(defn- initial-revisions [ctx remote ref]
  (or
    (last-seen-revisions-from-history ctx)
    (current-revision-or-nil remote ref)))

(defn- persist-last-seen-revisions [wait-for-result last-seen-revisions ctx]
  (let [current-revisions    (:all-revisions wait-for-result)
        revisions-to-persist (or current-revisions last-seen-revisions)]
    (async/>!! (:result-channel ctx) [:_git-last-seen-revisions revisions-to-persist]) ; by sending it through the result-channel, we can be pretty sure users don't overwrite it
{'fraction_non_alphanumeric': 0.12608562378665578, 'fraction_numerical': 0.0, 'mean_word_length': 2.218678066425518, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2455966', 'n_tokens_mistral': 3112, 'n_tokens_neox': 2979, 'n_words': 716}