<commit_before>/*
derived from:
  https://gist.github.com/akomakom/481507c0dd79ec52a395

which is itself a fork of:
  https://gist.github.com/rb2k/8372402
*/

/**
Jenkins System Groovy script to clean up workspaces on all slaves.

Check if a slave has < X GB of free space, perform cleanup if it's less.
If slave is idle, wipe out everything in the workspace directory as well any extra configured directories.
If slave is busy, wipe out individual job workspace directories for jobs that aren't running.
Either way, remove custom workspaces also if they aren't in use.
**/

import hudson.model.*
import hudson.util.*
import jenkins.model.*
import hudson.FilePath.FileCallable
import hudson.slaves.OfflineCause
import hudson.node_monitors.*
import groovy.transform.InheritConstructors

@InheritConstructors
class CleanupException extends Exception {}

@InheritConstructors
class Node extends CleanupException {
  hudson.model.Slave node

  Node(hudson.model.Slave node) {
    super()
    this.node = node
  }

  Node(hudson.model.Slave node, String m) {
    super(m)
    this.node = node
  }

  Node(hudson.model.Slave node, String m, Throwable t) {
    super(m, t)
    this.node = node
  }
}

@InheritConstructors
class Failed extends Node {}
@InheritConstructors
class Offline extends Node {}
@InheritConstructors
class Skipped extends Node {}

// threshold is in GB and comes from a job parameter
def threshold = 100
def skippedLabels = [
  'lsst-dev',
  'snowflake',
]
// additional paths under slave's root path that should be removed if found
def extraDirectoriesToDelete = []


def deleteRemote(def path, boolean deleteContentsOnly) {
  boolean result = true
  def pathAsString = path.getRemote()
  if (path.exists()) {
    try {
      if (deleteContentsOnly) {
        path.deleteContents()
        println ".... deleted ALL contents of ${pathAsString}"
      } else {
        path.deleteRecursive()
        println ".... deleted directory ${pathAsString}"
      }
    } catch (Throwable t) {
      println "Failed to delete ${pathAsString}: ${t}"
      result = false
    }
  }
  return result
}

/*
 * find all jobs which are not folders or pipelines
 *
*/
ArrayList allJobs() {
  Jenkins.instance.getAllItems(TopLevelItem).findAll{ item ->
    item instanceof Job && !("${item.class}".contains('WorkflowJob'))
  }
}

def nodeStatus = [:].withDefault {[]}
def prevOffline = false

for (node in Jenkins.instance.nodes) {
  try {
    println "found ${node.displayName}"

    def computer = node.toComputer()
    // a null channel indicates that the node is offline
    if (computer.getChannel() == null) {
      throw new Offline(node)
    }

    if (node.assignedLabels.find{ it.expression in skippedLabels }) {
      throw new Skipped(node, "based on label(s)")
    }

    def size = DiskSpaceMonitor.DESCRIPTOR.get(computer).size
    def roundedSize = size / (1024 * 1024 * 1024) as int

    println("node: " + node.getDisplayName()
            + ", free space: " + roundedSize
            + "GB. Idle: ${computer.isIdle()}")

    prevOffline = computer.isOffline()
    if (prevOffline && computer.getOfflineCauseReason().startsWith('disk cleanup from job')) {
      // previous run screwed up, ignore it and clear it at the end
      prevOffline = false
    }

    // skip nodes with sufficent disk space
    if (roundedSize >= threshold) {
      throw new Skipped(node, "disk threshhold")
    }

    // mark node as offline
    if (!prevOffline) {
      //don't override any previosly set temporarily offline causes (set by humans possibly)
      computer.setTemporarilyOffline(true,<commit_msg>check that DiskSpaceMonitorDescriptor is not null<commit_after>/*
derived from:
  https://gist.github.com/akomakom/481507c0dd79ec52a395

which is itself a fork of:
  https://gist.github.com/rb2k/8372402
*/

/**
Jenkins System Groovy script to clean up workspaces on all slaves.

Check if a slave has < X GB of free space, perform cleanup if it's less.
If slave is idle, wipe out everything in the workspace directory as well any extra configured directories.
If slave is busy, wipe out individual job workspace directories for jobs that aren't running.
Either way, remove custom workspaces also if they aren't in use.
**/

import hudson.model.*
import hudson.util.*
import jenkins.model.*
import hudson.FilePath.FileCallable
import hudson.slaves.OfflineCause
import hudson.node_monitors.*
import groovy.transform.InheritConstructors

@InheritConstructors
class CleanupException extends Exception {}

@InheritConstructors
class Node extends CleanupException {
  hudson.model.Slave node

  Node(hudson.model.Slave node) {
    super()
    this.node = node
  }

  Node(hudson.model.Slave node, String m) {
    super(m)
    this.node = node
  }

  Node(hudson.model.Slave node, String m, Throwable t) {
    super(m, t)
    this.node = node
  }
}

@InheritConstructors
class Failed extends Node {}
@InheritConstructors
class Offline extends Node {}
@InheritConstructors
class Skipped extends Node {}

// threshold is in GB and comes from a job parameter
def threshold = 100
def skippedLabels = [
  'lsst-dev',
  'snowflake',
]
// additional paths under slave's root path that should be removed if found
def extraDirectoriesToDelete = []


def deleteRemote(def path, boolean deleteContentsOnly) {
  boolean result = true
  def pathAsString = path.getRemote()
  if (path.exists()) {
    try {
      if (deleteContentsOnly) {
        path.deleteContents()
        println ".... deleted ALL contents of ${pathAsString}"
      } else {
        path.deleteRecursive()
        println ".... deleted directory ${pathAsString}"
      }
    } catch (Throwable t) {
      println "Failed to delete ${pathAsString}: ${t}"
      result = false
    }
  }
  return result
}

/*
 * find all jobs which are not folders or pipelines
 *
*/
ArrayList allJobs() {
  Jenkins.instance.getAllItems(TopLevelItem).findAll{ item ->
    item instanceof Job && !("${item.class}".contains('WorkflowJob'))
  }
}

def nodeStatus = [:].withDefault {[]}
def prevOffline = false

for (node in Jenkins.instance.nodes) {
  try {
    println "found ${node.displayName}"

    def computer = node.toComputer()
    // a null channel indicates that the node is offline
    if (computer.getChannel() == null) {
      throw new Offline(node)
    }

    if (node.assignedLabels.find{ it.expression in skippedLabels }) {
      throw new Skipped(node, "based on label(s)")
    }

    def dsm = DiskSpaceMonitor.DESCRIPTOR.get(computer)
    if (dsm) {
      def size = dsm.size
    } else {
      throw new Failed(node, "unable to determine disk usage (this is bad)")
    }
    def roundedSize = size / (1024 * 1024 * 1024) as int

    println("node: " + node.getDisplayName()
            + ", free space: " + roundedSize
            + "GB. Idle: ${computer.isIdle()}")

    prevOffline = computer.isOffline()
    if (prevOffline && computer.getOfflineCauseReason().startsWith('disk cleanup from job')) {
      // previous run screwed up, ignore it and clear it at the end
      prevOffline = false
    }

    // skip nodes with sufficent disk space
    if (roundedSize >= threshold) {
      throw new Skipped(node, "disk threshhold")
    }

    // mark node as offline
    if (!prevOffline) {
      //don't override any previosly set temporarily offline causes (set by humans possibly)
      computer.setTemporarilyOffline(true,
{'fraction_non_alphanumeric': 0.10212650684003792, 'fraction_numerical': 0.010023025870242449, 'mean_word_length': 3.248561565017261, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '412819', 'n_tokens_mistral': 2355, 'n_tokens_neox': 2190, 'n_words': 721}