<commit_before># -*- python -*-
# ex: set syntax=python:

from buildbot.plugins import *
from twisted.python import log
from password import *
from buildslaves import *
from buildbot.buildslave import BuildSlave

bb_slave_port = 9989
bb_try_port = 8033
bb_web_port = 8010
bb_master = "build.zfsonlinux.org:9989"
bb_url = "https://raw.githubusercontent.com/zfsonlinux/zfs-buildbot/master/scripts/"
zol_url = "http://zfsonlinux.org"
web_url = "http://build.zfsonlinux.org/"
spl_repo = "https://github.com/zfsonlinux/spl.git"
zfs_repo = "https://github.com/zfsonlinux/zfs.git"
linux_repo = "https://github.com/torvalds/linux.git"
zfs_path = "/usr/libexec/zfs:/usr/share/zfs:/usr/lib/rpm/zfs:/usr/lib/zfs"
bin_path = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"

all_repositories = {
    "https://github.com/torvalds/linux" : 'linux',
    "https://github.com/zfsonlinux/spl" : 'spl',
    "https://github.com/zfsonlinux/zfs" : 'zfs',
    "https://github.com/torvalds/linux.git" : 'linux',
    "https://github.com/zfsonlinux/spl.git" : 'spl',
    "https://github.com/zfsonlinux/zfs.git" : 'zfs',
}

c = BuildmasterConfig = {}

c['caches'] = {
    'Changes' : 10,
    'Builds' : 15,
    'chdicts' : 10,
    'BuildRequests' : 10,
    'SourceStamps' : 10,
    'ssdicts' : 20,
    'objectids' : 10,
    'usdicts' : 10,
}


####### BUILD FACTORIES

# A builder also has a BuildFactory, which is responsible for creating new
# Build instances: because the Build instance is what actually performs each
# build, choosing the BuildFactory is the way to specify what happens each
# time a build is done (Builds).

from buildbot.steps.source.git import Git
from buildbot.steps.shell import ShellCommand
from buildbot.steps.master import MasterShellCommand
from buildbot.status.results import SUCCESS
from buildbot.status.results import FAILURE
from buildbot.status.results import WARNINGS
from buildbot.status.results import SKIPPED

def do_step_build(step, name):
    props = step.build.getProperties()
    if props.hasProperty(name) and props[name] == "yes":
        return True
    else:
        return False

def do_step_build_linux(step):
    return do_step_build(step, 'buildlinux')

def do_step_build_spl(step):
    return do_step_build(step, 'buildspl')

def do_step_build_zfs(step):
    return do_step_build(step, 'buildzfs')

def do_step_check_lint(step):
    return do_step_build(step, 'checklint')

#
# Build factory - Build source in-tree
#
# Perform an in-tree build using the default options.  This is used to verify
# the build accross a range of distributions and architectures.
#
style_factory = util.BuildFactory()

style_factory.addStep(ShellCommand(
    env={'PATH' : bin_path},
    command=["runurl", bb_url + "bb-dependencies.sh"],
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    haltOnFailure=True, logEnviron=False,
    lazylogfiles=True,
    description=["installing dependencies"],
    descriptionDone=["installed dependencies"]))
style_factory.addStep(Git(repourl=zfs_repo, workdir="build/zfs",
    mode="full", method="clobber", codebase="zfs",
    logEnviron=False, getDescription=True,
    lazylogfiles=True,
    description=["cloning"], descriptionDone=["cloned"]))
style_factory.addStep(ShellCommand(
    workdir="build/zfs", env={'PATH' : bin_path,
        'CONFIG_OPTIONS' : '--with-config=srpm'},
    command=["runurl", bb_url + "bb-build-zfs.sh"],
    haltOnFailure=True, logEnviron=False,
    lazylogfiles=True,
    logfiles={"configure"  : { "filename" : "configure.log", "follow" : True },
              "config.log" : { "filename" : "config.log",    "follow" : True },
              "make"       : { "filename" : "make.log",      "follow" : True }},
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    description=["building zfs"], descriptionDone=["built zfs"]))
style_factory.addStep(ShellCommand(command=["make", "checkstyle"],
    env={'PATH' : bin_path},
    workdir="build/zfs", logEnviron=False,
    haltOnFailure=False, flunkOnWarnings=True,
    lazylogfiles=True,
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    description=["checking style"], descriptionDone=["checked style"]))
style_factory.addStep(ShellCommand(command=["make", "lint"],
    env={'PATH' : bin_path},
    workdir="build/zfs", logEnviron=False,
    haltOnFailure=False, flunkOnWarnings=True,
    lazylogfiles=True,
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    description=["checking lint"], descriptionDone=["checked lint"],
    doStepIf = do_step_check_lint,
    hideStepIf=lambda results, s: results==SKIPPED))

build_factory = util.BuildFactory()

build_factory.addStep(ShellCommand(
    env={'PATH' : bin_path},
    command=["runurl", bb_url + "bb-dependencies.sh"],
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    haltOnFailure=True, logEnviron=False,
    lazylogfiles=True,
    description=["installing dependencies"],
    descriptionDone=["installed dependencies"]))

build_factory.addStep(Git(repourl=linux_repo, workdir="build/linux",
    mode="full", method="clobber", shallow=True, codebase="linux",
    logEnviron=False, getDescription=True,
    lazylogfiles=True,
    description=["cloning"], descriptionDone=["cloned"],
    doStepIf = do_step_build_linux,
    hideStepIf=lambda results, s: results==SKIPPED))
build_factory.addStep(Git(repourl=spl_repo, workdir="build/spl",
    mode="full", method="clobber", codebase="spl",
    logEnviron=False, getDescription=True,
    lazylogfiles=True,
    description=["cloning"], descriptionDone=["cloned"]))
build_factory.addStep(Git(repourl=zfs_repo, workdir="build/zfs",
    mode="full", method="clobber", codebase="zfs",
    logEnviron=False, getDescription=True,
    lazylogfiles=True,
    description=["cloning"], descriptionDone=["cloned"]))

build_factory.addStep(ShellCommand(
    workdir="build/linux", env={'PATH' : bin_path,
        'LINUX_BUILTIN' : util.Interpolate('%(prop:builtin:-no)s') },
    command=["runurl", bb_url + "bb-build-linux.sh"],
    haltOnFailure=True, logEnviron=False,
    lazylogfiles=True,
    logfiles={"configure"  : { "filename" : "configure.log", "follow" : True },
              ".config"    : { "filename" : ".config",       "follow" : False},
              "make"       : { "filename" : "make.log",      "follow" : True }},
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    description=["building linux"], descriptionDone=["built linux"],
    doStepIf = do_step_build_linux,
    hideStepIf=lambda results, s: results==SKIPPED))
build_factory.addStep(ShellCommand(
    workdir="build/spl", env={'PATH' : bin_path,
        'CONFIG_OPTIONS' : util.Interpolate('%(prop:configspl:-"")s'),
        'LINUX_CUSTOM' : util.Interpolate('%(prop:buildlinux:-no)s') },
    command=["runurl", bb_url + "bb-build-zfs.sh"],
    haltOnFailure=True, logEnviron=False,
    lazylogfiles=True,
    logfiles={"configure"  : { "filename" : "configure.log", "follow" : True },
              "config.log" : { "filename" : "config.log",    "follow" : True },
              "make"       : { "filename" : "make.log",      "follow" : True }},
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    description=["building spl"], descriptionDone=["built spl"],
    doStepIf = do_step_build_spl,
    hideStepIf=lambda results, s: results==SKIPPED))
build_factory.addStep(ShellCommand(
    workdir="build/zfs", env={'PATH' : bin_path,
        'CONFIG_OPTIONS' : util.Interpolate('%(prop:configzfs:-"")s'),
        'LINUX_CUSTOM' : util.Interpolate('%(prop:buildlinux:-no)s') },
    command=["runurl", bb_url + "bb-build-zfs.sh"],
    haltOnFailure=True, logEnviron=False,
    lazylogfiles=True,
    logfiles={"configure"  : { "filename" : "configure.log", "follow" : True },
              "config.log" : { "filename" : "config.log",    "follow" : True },
              "make"       : { "filename" : "make.log",      "follow" : True }},
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    description=["building zfs"], descriptionDone=["built zfs"],
    doStepIf = do_step_build_zfs,
    hideStepIf=lambda results, s: results==SKIPPED))
build_factory.addStep(ShellCommand(command=["make", "lint"],
    workdir="build/zfs", logEnviron=False, env={'PATH' : bin_path},
    haltOnFailure=False, flunkOnWarnings=True,
    lazylogfiles=True,
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    description=["checking lint"], descriptionDone=["checked lint"],
    doStepIf = do_step_check_lint,
    hideStepIf=lambda results, s: results==SKIPPED))

#
# Test factory - Build packages and run the test suites.
#
# Perform a package build with debugging enabled, install the packages,
# and run of the test suite.  The tests themselves have been seperated from
# the buildbot configuration to minimize the need for restarting the master.
#
test_factory = util.BuildFactory()

test_factory.addStep(ShellCommand(
    env={'PATH' : bin_path},
    command=["runurl", bb_url + "bb-dependencies.sh"],
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    haltOnFailure=True, logEnviron=False,
    lazylogfiles=True,
    description=["installing dependencies"],
    descriptionDone=["installed dependencies"]))
test_factory.addStep(Git(repourl=spl_repo, workdir="build/spl",
    mode="full", method="clobber", codebase="spl",
    logEnviron=False, getDescription=True,
    lazylogfiles=True,
    description=["cloning"], descriptionDone=["cloned"]))
test_factory.addStep(ShellCommand(
    env={'PATH' : bin_path,
        'CONFIG_OPTIONS' : util.Interpolate('%(prop:configspl:-"")s'),
        'LINUX_CUSTOM' : util.Interpolate('%(prop:buildlinux:-no)s') },
    workdir="build/spl",
    command=["runurl", bb_url + "bb-build-packages.sh"],
    haltOnFailure=True, logEnviron=False,
    lazylogfiles=True,
    logfiles={"configure"  : { "filename" : "configure.log", "follow" : True },
              "config.log" : { "filename" : "config.log",    "follow" : True },
              "make"       : { "filename" : "make.log",      "follow" : True },
              "install"    : { "filename" : "install.log",   "follow" : True }},
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    description=["building spl"], descriptionDone=["built spl"]))
test_factory.addStep(Git(repourl=zfs_repo, workdir="build/zfs",
    mode="full", method="clobber", codebase="zfs",
    logEnviron=False, getDescription=True,
    lazylogfiles=True,
    description=["cloning"], descriptionDone=["cloned"]))
test_factory.addStep(ShellCommand(
    env={'PATH' : bin_path,
        'CONFIG_OPTIONS' : util.Interpolate('%(prop:configzfs:-"")s'),
        'LINUX_CUSTOM' : util.Interpolate('%(prop:buildlinux:-no)s') },
    workdir="build/zfs",
    command=["runurl", bb_url + "bb-build-packages.sh"],
    haltOnFailure=True, logEnviron=False,
    lazylogfiles=True,
    logfiles={"configure"  : { "filename" : "configure.log", "follow" : True },
              "config.log" : { "filename" : "config.log",    "follow" : True },
              "make"       : { "filename" : "make.log",      "follow" : True },
              "install"    : { "filename" : "install.log",   "follow" : True }},
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    description=["building zfs"], descriptionDone=["built zfs"]))
test_factory.addStep(ShellCommand(
    workdir="build/tests",
    env={'PATH' : bin_path + ":" + zfs_path},
    command=["runurl", bb_url + "bb-test-prepare.sh"],
    haltOnFailure=False, maxTime=120, sigtermTime=30, logEnviron=False,
    lazylogfiles=True,
    logfiles={"test"     : { "filename" : "TEST",        "follow" : False }},
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    description=["preparing env"], descriptionDone=["prepared env"]))
test_factory.addStep(ShellCommand(
    workdir="build/tests/splat",
    env={'PATH' : bin_path + ":" + zfs_path},
    command=["runurl", bb_url + "bb-test-splat.sh"],
    haltOnFailure=False, maxTime=600, sigtermTime=30, logEnviron=False,
    lazylogfiles=True,
    logfiles={"console" : { "filename" : "console.log", "follow" : False }},
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    description=["splat"], descriptionDone=["splat"],
    hideStepIf=lambda results, s: results==SKIPPED))
test_factory.addStep(ShellCommand(
    workdir="build/tests/ztest",
    env={'PATH' : bin_path + ":" + zfs_path},
    command=["runurl", bb_url + "bb-test-ztest.sh"],
    haltOnFailure=False, maxTime=14400, sigtermTime=30, logEnviron=False,
    lazylogfiles=True,
    logfiles={"ztest.out" : { "filename" : "ztest.out", "follow" : False },
              "console" : { "filename" : "console.log", "follow" : False }},
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    description=["ztest"], descriptionDone=["ztest"],
    hideStepIf=lambda results, s: results==SKIPPED))
test_factory.addStep(ShellCommand(
    workdir="build/tests/zimport",
    env={'PATH' : bin_path + ":" + zfs_path},
    command=["runurl", bb_url + "bb-test-zimport.sh"],
    haltOnFailure=False, maxTime=1200, sigtermTime=30, logEnviron=False,
    lazylogfiles=True,
    logfiles={"console" : { "filename" : "console.log", "follow" : False }},
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    description=["zimport"], descriptionDone=["zimport"],
    hideStepIf=lambda results, s: results==SKIPPED))
test_factory.addStep(ShellCommand(
    workdir="build/tests/xfstests",
    env={'PATH' : bin_path + ":" + zfs_path},
    command=["runurl", bb_url + "bb-test-xfstests.sh"],
    haltOnFailure=False, flunkOnWarnings=True,
    maxTime=1200, sigtermTime=30, logEnviron=False,
    lazylogfiles=True,
    logfiles={"configure" : { "filename" : "configure.log", "follow" : True },
              "make"      : { "filename" : "make.log",      "follow" : True },
              "console"   : { "filename" : "console.log",   "follow" : False },
              "kmemleak"  : { "filename" : "kmemleak.log",  "follow" : False }},
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    description=["xfstest"], descriptionDone=["xfstests"],
    hideStepIf=lambda results, s: results==SKIPPED))
test_factory.addStep(ShellCommand(
    workdir="build/tests/zfstests",
    env={'PATH' : bin_path + ":" + zfs_path},
    command=["runurl", bb_url + "bb-test-zfstests.sh"],
    haltOnFailure=False, flunkOnWarnings=True,
    maxTime=18000, sigtermTime=30, logEnviron=False,
    lazylogfiles=True, timeout=4200,
    logfiles={"console"  : { "filename" : "console.log",  "follow" : False },
              "log"      : { "filename" : "log",          "follow" : True },
              "kmemleak" : { "filename" : "kmemleak.log", "follow" : False }},
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    description=["zfstests"], descriptionDone=["zfstests"],
    hideStepIf=lambda results, s: results==SKIPPED))
test_factory.addStep(ShellCommand(
    workdir="build/tests/zfsstress",
    env={'PATH' : bin_path + ":" + zfs_path},
    command=["runurl", bb_url + "bb-test-zfsstress.sh"],
    haltOnFailure=False, flunkOnWarnings=True,
    maxTime=14400, sigtermTime=30, logEnviron=False,
    lazylogfiles=True,
    logfiles={"console"  : { "filename" : "console.log",  "follow" : False },
              "kmemleak" : { "filename" : "kmemleak.log", "follow" : False }},
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    description=["zfsstress"], descriptionDone=["zfsstress"],
    hideStepIf=lambda results, s: results==SKIPPED))

test_factory.addStep(ShellCommand(command=["runurl", bb_url + "bb-cleanup.sh"],
    env={'PATH' : bin_path},
    haltOnFailure=False, logEnviron=False,
    lazylogfiles=True,
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    description=["removing zfs"], descriptionDone=["removed zfs"]))

#
# Performance factory - Build packages and run the performance suite.
#
# Perform a package build with debugging disabled, install the packages,
# and run a battery of performance tests.  The tests themselves have been
# seperated from the buildbot configuration to minimize the need for
# restarting the master.
#
perf_factory = util.BuildFactory()

perf_factory.addStep(ShellCommand(
    env={'PATH' : bin_path},
    command=["runurl", bb_url + "bb-dependencies.sh"],
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    haltOnFailure=True, logEnviron=False,
    lazylogfiles=True,
    description=["installing dependencies"],
    descriptionDone=["installed dependencies"]))
perf_factory.addStep(Git(repourl=spl_repo, workdir="build/spl",
    mode="full", method="clobber", codebase="spl",
    logEnviron=False, getDescription=True,
    lazylogfiles=True,
    description=["cloning"], descriptionDone=["cloned"]))
perf_factory.addStep(ShellCommand(
    env={'PATH' : bin_path,
        'CONFIG_OPTIONS' : util.Interpolate('%(prop:configspl:-"")s') },
    workdir="build/spl",
    command=["runurl", bb_url + "bb-build-packages.sh"],
    haltOnFailure=True, logEnviron=False,
    lazylogfiles=True,
    logfiles={"configure"  : { "filename" : "configure.log", "follow" : True },
              "config.log" : { "filename" : "config.log",    "follow" : True },
              "make"       : { "filename" : "make.log",      "follow" : True },
              "install"    : { "filename" : "install.log",   "follow" : True }},
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    description=["building spl"], descriptionDone=["built spl"]))
perf_factory.addStep(Git(repourl=zfs_repo, workdir="build/zfs",
    mode="full", method="clobber", codebase="zfs",
    logEnviron=False, getDescription=True,
    lazylogfiles=True,
    description=["cloning"], descriptionDone=["cloned"]))
perf_factory.addStep(ShellCommand(
    env={'PATH' : bin_path,
        'CONFIG_OPTIONS' : util.Interpolate('%(prop:configzfs:-"")s') },
    workdir="build/zfs",
    command=["runurl", bb_url + "bb-build-packages.sh"],
    haltOnFailure=True, logEnviron=False,
    lazylogfiles=True,
    logfiles={"configure"  : { "filename" : "configure.log", "follow" : True },
              "config.log" : { "filename" : "config.log",    "follow" : True },
              "make"       : { "filename" : "make.log",      "follow" : True },
              "install"    : { "filename" : "install.log",   "follow" : True }},
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    description=["building zfs"], descriptionDone=["built zfs"]))
perf_factory.addStep(ShellCommand(
    workdir="build/tests",
    env={'PATH' : bin_path + ":" + zfs_path,
         'AVAILABLE_DISKS' : 'xvdb xvdc xvdd'},
    command=["runurl", bb_url + "bb-test-prepare.sh"],
    haltOnFailure=False, maxTime=3600, sigtermTime=30, logEnviron=False,
    lazylogfiles=True, timeout=3600,
    logfiles={"test"     : { "filename" : "TEST",        "follow" : False }},
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    description=["preparing env"], descriptionDone=["prepared env"]))
perf_factory.addStep(ShellCommand(
    workdir="build/tests/zfstests",
    env={'PATH' : bin_path + ":" + zfs_path,
         'DEFAULT_ZFSTESTS_RUNFILE' : 'perf-regression.run',
         'DEFAULT_ZFSTESTS_DISKS' : 'xvdb1 xvdc1 xvdd1'},
    command=["runurl", bb_url + "bb-test-zfstests.sh"],
    haltOnFailure=False, maxTime=21600, sigtermTime=30, logEnviron=False,
    lazylogfiles=True,
    logfiles={"console" : { "filename" : "console.log", "follow" : False },
              "log"     : { "filename" : "log",         "follow" : True }},
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    timeout=5400, description=["zfstests"], descriptionDone=["zfstests"],
    hideStepIf=lambda results, s: results==SKIPPED))
perf_factory.addStep(ShellCommand(command=["runurl", bb_url + "bb-cleanup.sh"],
    env={'PATH' : bin_path},
    haltOnFailure=False, logEnviron=False,
    lazylogfiles=True,
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    description=["removing zfs"], descriptionDone=["removed zfs"]))


####### WORKERS / SLAVES

# Each builder is associated with one of more workers / slaves.  They are
# responsible for carrying out the steps described by a build factory.
# Note that any particular build will only take place on one slave.
#
# AMI Reference:
#
# ami-10260870 - Amazon 07-12-2017
# ami-ca2608aa - CentOS 6 07-12-2017
# ami-4f27092f - CentOS 7 07-12-2017
# ami-12240a72 - CentOS 7 07-12-2017 Mainline
# ami-3d240a5d - Debian 8 07-12-2017
# ami-b4d2fbd4 - Fedora 26 07-24-2017
# ami-46270926 - Ubuntu 14.04 07-12-2017
# ami-1efadd7e - Ubuntu 14.04 05-01-2017 PV
# ami-1b29077b - Ubuntu 16.04 07-12-2017
# ami-473d1727 - Ubuntu 16.04 08-15-2017 Kmemleak
# ami-f9250b99 - Ubuntu 17.04 07-12-2017

# Style slaves
numstyleslaves = 2

style_slaves = [
    ZFSEC2StyleSlave(
        name="Ubuntu-16.04-x86_64-styleslave%s" % (str(i+1)),
        ami="ami-1b29077b"
    ) for i in range(0, numstyleslaves)
]

# Distribution slaves
numslaves = 1

kernel_default_x86_64_slave = [
    ZFSEC2BuildSlave(
        name="Amazon-KernelDefault-x86_64-buildslave%s" % (str(i+1)),
        ami="ami-10260870"
    ) for i in range(0, numslaves)
]

kernel_builtin_x86_64_slave = [
    ZFSEC2BuildSlave(
        name="Amazon-KernelBuiltin-x86_64-buildslave%s" % (str(i+1)),
        ami="ami-10260870"
    ) for i in range(0, numslaves)
]

debian8_x86_64_slave = [
    ZFSEC2BuildSlave(
        name="Debian-8-x86_64-buildslave%s" % (str(i+1)),
        ami="ami-3d240a5d"
    ) for i in range(0, numslaves)
]

fedora26_x86_64_slave = [
    ZFSEC2BuildSlave(
        name="Fedora-26-x86_64-buildslave%s" % (str(i+1)),
        ami="ami-b4d2fbd4"
    ) for i in range(0, numslaves)
]

centos6_x86_64_slave = [
    ZFSEC2BuildSlave(
        name="CentOS-6.7-x86_64-buildslave%s" % (str(i+1)),
        ami="ami-ca2608aa"
    ) for i in range(0, numslaves)
]

centos7_x86_64_slave = [
    ZFSEC2BuildSlave(
        name="CentOS-7.1-x86_64-buildslave%s" % (str(i+1)),
        ami="ami-4f27092f"
    ) for i in range(0, numslaves)
]

ubuntu14_x86_64_slave = [
    ZFSEC2BuildSlave(
        name="Ubuntu-14.04-x86_64-buildslave%s" % (str(i+1)),
        ami="ami-46270926"
    ) for i in range(0, numslaves)
]

ubuntu16_x86_64_slave = [
    ZFSEC2BuildSlave(
        name="Ubuntu-16.04-x86_64-buildslave%s" % (str(i+1)),
        ami="ami-1b29077b"
    ) for i in range(0, numslaves)
]

distro_slaves = \
    kernel_default_x86_64_slave + \
    kernel_builtin_x86_64_slave + \
    debian8_x86_64_slave + \
    centos6_x86_64_slave + \
    centos7_x86_64_slave + \
    fedora26_x86_64_slave + \
    ubuntu14_x86_64_slave + \
    ubuntu16_x86_64_slave

# Architecture slaves
amazon_x86_64_slave = [
    ZFSEC2BuildSlave(
        name="Amazon-2015.09-x86_64-buildslave%s" % (str(i+1)),
        ami="ami-10260870"
    ) for i in range(0, numslaves)
]

ubuntu14_i686_slave = [
    ZFSEC2PVSlave(
        name="Ubuntu-14.04-i686-buildslave%s" % (str(i+1)),
        ami="ami-1efadd7e"
    ) for i in range(0, numslaves)
]

debian8_arm_slave = [
    BuildSlave("Debian-8-arm-buildslave1",
        slave_userpass["Debian-8-arm-buildslave1"]),
    BuildSlave("Debian-8-arm-buildslave2",
        slave_userpass["Debian-8-arm-buildslave2"]),
]

ubuntu16_aarch64_slave = [
    BuildSlave("Ubuntu-16.04-aarch64-buildslave1",
        slave_userpass["Ubuntu-16.04-aarch64-buildslave1"]),
    BuildSlave("Ubuntu-16.04-aarch64-buildslave2",
        slave_userpass["Ubuntu-16.04-aarch64-buildslave2"]),
]

debian8_ppc_slave = [
    BuildSlave("Debian-8-ppc-buildslave",
        slave_userpass["Debian-8-ppc-buildslave"]),
]

debian8_ppc64_slave = [
    BuildSlave("Debian-8-ppc64-buildslave1",
        slave_userpass["Debian-8-ppc64-buildslave1"]),
    BuildSlave("Debian-8-ppc64-buildslave2",
        slave_userpass["Debian-8-ppc64-buildslave2"]),
]

arch_slaves = \
    amazon_x86_64_slave + \
    ubuntu14_i686_slave + \
    debian8_arm_slave + \
    ubuntu16_aarch64_slave + \
    debian8_ppc_slave + \
    debian8_ppc64_slave

# Test slaves
numtestslaves = 16

amazon_x86_64_testslave = [
    ZFSEC2TestSlave(
        name="Amazon-2015.09-x86_64-testslave%s" % (str(i+1)),
        ami="ami-10260870"
    ) for i in range(0, numtestslaves)
]

centos6_x86_64_testslave = [
    ZFSEC2TestSlave(
        name="CentOS-6-x86_64-testslave%s" % (str(i+1)),
        ami="ami-ca2608aa"
    ) for i in range(0, numtestslaves)
]

centos7_x86_64_testslave = [
    ZFSEC2TestSlave(
        name="CentOS-7-x86_64-testslave%s" % (str(i+1)),
        ami="ami-4f27092f"
    ) for i in range(0, numtestslaves)
]

centos7_x86_64_ml_testslave = [
    ZFSEC2TestSlave(
        name="CentOS-7-x86_64-ml-testslave%s" % (str(i+1)),
        ami="ami-12240a72"
    ) for i in range(0, numtestslaves)
]

fedora26_x86_64_testslave = [
    ZFSEC2TestSlave(
        name="Fedora-26-x86_64-testslave%s" % (str(i+1)),
        ami="ami-b4d2fbd4"
    ) for i in range(0, numtestslaves)
]

ubuntu16_x86_64_testslave = [
    ZFSEC2TestSlave(
        name="Ubuntu-16.04-x86_64-testslave%s" % (str(i+1)),
        ami="ami-1b29077b"
    ) for i in range(0, numtestslaves)
]

ubuntu17_x86_64_testslave = [
    ZFSEC2TestSlave(
        name="Ubuntu-17.04-x86_64-testslave%s" % (str(i+1)),
        ami="ami-f9250b99"
    ) for i in range(0, numtestslaves)
]

ubuntu16_x86_64_kmemleak_testslave = [
    ZFSEC2TestSlave(
        name="Ubuntu-16.04-x86_64-kmemleak-testslave%s" % (str(i+1)),
        ami="ami-473d1727"
    ) for i in range(0, numtestslaves)
]

ubuntu14_i686_testslave = [
    ZFSEC2PVTestSlave(
        name="Ubuntu-14.04-i686-testslave%s" % (str(i+1)),
        ami="ami-1efadd7e"
    ) for i in range(0, numtestslaves)
]

test_slaves = \
    amazon_x86_64_testslave + \
    centos6_x86_64_testslave + \
    centos7_x86_64_testslave + \
    centos7_x86_64_ml_testslave + \
    fedora26_x86_64_testslave + \
    ubuntu16_x86_64_testslave + \
    ubuntu17_x86_64_testslave + \
    ubuntu16_x86_64_kmemleak_testslave + \
    ubuntu14_i686_testslave

# Performance slaves
numperfslaves = 2

amazon_x86_64_simd_perfslave = [
    ZFSEC2VectorTestSlave(
        name="Amazon-2015.09-x86_64-simd-perfslave%s" % (str(i+1)),
        ami="ami-10260870"
    ) for i in range(0, numtestslaves)
]

amazon_x86_64_perfslave = [
    ZFSEC2PerfTestSlave(
        name="Amazon-2015.09-x86_64-perfslave%s" % (str(i+1)),
        ami="ami-10260870"
    ) for i in range(0, numperfslaves)
]

perf_slaves = \
    amazon_x86_64_perfslave + \
    amazon_x86_64_simd_perfslave

# All slaves
all_slaves = \
    style_slaves + \
    distro_slaves + \
    arch_slaves + \
    test_slaves + \
    perf_slaves

style_tags = [ "Style" ]
build_distro_tags = [ "Distributions" ]
build_arch_tags = [ "Architectures" ]
test_tags = [ "Tests" ]
perf_tags = [ "Performance" ]


####### BUILDERS

# The Buildmaster runs a collection of Builders, each of which handles a single
# type of build (e.g. full versus quick), on one or more workers. Builders
# serve as a kind of queue for a particular type of build. Each Builder gets a
# separate column in the waterfall display. In general, each Builder runs
# independently (although various kinds of interlocks can cause one Builder
# to have an effect on another).

# Properties are used to control how a build is performed.
builder_default_properties = {
    "buildlinux":    "no",
    "buildspl":      "yes",
    "buildzfs":      "yes",
    "builtin":       "no",
    "configspl":     "--enable-debug",
    "configzfs":     "--enable-debug",
    "repoowner":     "zfsonlinux",
    "reponame":      "zfs",
    "checklint":     "no",
}

builder_redhat_properties = {
    "buildlinux":    "no",
    "buildspl":      "yes",
    "buildzfs":      "yes",
    "builtin":       "no",
    "configspl":     "--enable-debug --with-spec=redhat",
    "configzfs":     "--enable-debug --with-spec=redhat",
    "repoowner":     "zfsonlinux",
    "reponame":      "zfs",
    "checklint":     "no",
}

builder_release_properties = {
    "buildlinux":    "no",
    "buildspl":      "yes",
    "buildzfs":      "yes",
    "builtin":       "no",
    "configspl":     "",
    "configzfs":     "",
    "repoowner":     "zfsonlinux",
    "reponame":      "zfs",
    "checklint":     "no",
}

builder_linux_properties = {
    "buildlinux":    "yes",
    "buildspl":      "yes",
    "buildzfs":      "yes",
    "builtin":       "no",
    "configspl":     "--enable-debug",
    "configzfs":     "--enable-debug",
    "repoowner":     "zfsonlinux",
    "reponame":      "zfs",
    "checklint":     "no",
}

builder_builtin_properties = {
    "buildlinux":    "yes",
    "buildspl":      "no",
    "buildzfs":      "no",
    "builtin":       "yes",
    "configspl":     "--enable-debug",
    "configzfs":     "--enable-debug",
    "repoowner":     "zfsonlinux",
    "reponame":      "zfs",
    "checklint":     "no",
}

builder_style_properties = {
    "buildlinux":    "no",
    "buildspl":      "yes",
    "buildzfs":      "yes",
    "builtin":       "no",
    "configspl":     "--enable-debug",
    "configzfs":     "--enable-debug",
    "repoowner":     "zfsonlinux",
    "reponame":      "zfs",
    "checklint":     "yes",
}

builder_arch_properties = {
    "buildlinux":    "no",
    "buildspl":      "yes",
    "buildzfs":      "yes",
    "builtin":       "no",
    "configspl":     "--enable-debug",
    "configzfs":     "--enable-debug",
    "repoowner":     "zfsonlinux",
    "reponame":      "zfs",
    "checklint":     "yes",
}

builder_perf_properties = {
    "buildlinux":    "no",
    "buildspl":      "yes",
    "buildzfs":      "yes",
    "builtin":       "no",
    "configspl":     "",
    "configzfs":     "",
    "repoowner":     "zfsonlinux",
    "reponame":      "zfs",
    "checklint":     "no",
}

def prioritizeBuilders(buildmaster, builders):
    """
    Called by the buildmaster to prioritize the builders.  Returns a sorted
    array of builders designed to improve ec2 utilization. Builders with
    substantiated, idle slaves are given priority. Followed by builders with no
    substantiated slaves. The lowest priority is a builder that is busy.
    This helps keep all buildslaves busy while new latent buildslaves are
    bootstrapped, a process which can take several minutes.
    """

    idle_builders = []
    busy_builders = []
    avail_builders = []

    for b in builders:
        idle = False
        busy = False
        for s in b.slaves:
            if s.isIdle():
               idle = True
               break

            if s.isBusy():
               busy = True

        if idle is True:
            if re.search('BUILD', b.name):
                idle_builders.insert(0, b)
            else:
                idle_builders.append(b)
        elif busy is True:
            if re.search('BUILD', b.name):
                busy_builders.insert(0, b)
            else:
                busy_builders.append(b)
        else:
            if re.search('BUILD', b.name):
                avail_builders.insert(0, b)
            else:
                avail_builders.append(b)

    sorted_builders = idle_builders + avail_builders + busy_builders

    log.msg("prioritized %i builder(s): %s" % (len(sorted_builders),
        [b.name for b in sorted_builders]))

    return sorted_builders

c['prioritizeBuilders'] = prioritizeBuilders

# Style builders
style_builders = [
    ZFSBuilderConfig(
        name="Ubuntu 16.04 x86_64 (STYLE)",
        factory=style_factory,
        slavenames=[slave.slavename for slave in style_slaves],
        tags=style_tags,
        properties=builder_style_properties,
    ),
]

# Distro builders
distro_builders = [
    ZFSBuilderConfig(
        name="Kernel.org Default x86_64 (BUILD)",
        factory=build_factory,
        slavenames=[slave.name for slave in kernel_default_x86_64_slave],
        tags=build_distro_tags,
        properties=builder_linux_properties,
    ),
    ZFSBuilderConfig(
        name="Kernel.org Built-in x86_64 (BUILD)",
        factory=build_factory,
        slavenames=[slave.name for slave in kernel_builtin_x86_64_slave],
        tags=build_distro_tags,
        properties=builder_builtin_properties,
    ),
    ZFSBuilderConfig(
        name="Debian 8 x86_64 (BUILD)",
        factory=build_factory,
        slavenames=[slave.name for slave in debian8_x86_64_slave],
        tags=build_distro_tags,
        properties=builder_default_properties,
    ),
    ZFSBuilderConfig(
        name="Fedora 26 x86_64 (BUILD)",
        factory=build_factory,
        slavenames=[slave.name for slave in fedora26_x86_64_slave],
        tags=build_distro_tags,
        properties=builder_default_properties,
    ),
    ZFSBuilderConfig(
        name="CentOS 6.7 x86_64 (BUILD)",
        factory=build_factory,
        slavenames=[slave.name for slave in centos6_x86_64_slave],
        tags=build_distro_tags,
        properties=builder_redhat_properties,
    ),
    ZFSBuilderConfig(
        name="CentOS 7.1 x86_64 (BUILD)",
        factory=build_factory,
        slavenames=[slave.name for slave in centos7_x86_64_slave],
        tags=build_distro_tags,
        properties=builder_redhat_properties,
    ),
    ZFSBuilderConfig(
        name="Ubuntu 14.04 x86_64 (BUILD)",
        factory=build_factory,
        slavenames=[slave.name for slave in ubuntu14_x86_64_slave],
        tags=build_distro_tags,
        properties=builder_default_properties,
    ),
    ZFSBuilderConfig(
        name="Ubuntu 16.04 x86_64 (BUILD)",
        factory=build_factory,
        slavenames=[slave.name for slave in ubuntu16_x86_64_slave],
        tags=build_distro_tags,
        properties=builder_default_properties,
    ),
]

# Architecture builders
arch_builders = [
    ZFSBuilderConfig(
        name="Amazon 2015.09 x86_64 (BUILD)",
        factory=build_factory,
        slavenames=[slave.name for slave in amazon_x86_64_slave],
        tags=build_arch_tags,
        properties=builder_arch_properties,
    ),
    ZFSBuilderConfig(
        name="Ubuntu 14.04 i686 (BUILD)",
        factory=build_factory,
        slavenames=[slave.name for slave in ubuntu14_i686_slave],
        tags=build_arch_tags,
        properties=builder_arch_properties,
    ),
    ZFSBuilderConfig(
        name="Debian 8 arm (BUILD)",
        factory=build_factory,
        slavenames=[slave.slavename for slave in debian8_arm_slave],
        tags=build_arch_tags,
        properties=builder_arch_properties,
    ),
    ZFSBuilderConfig(
        name="Ubuntu 16.04 aarch64 (BUILD)",
        factory=build_factory,
        slavenames=[slave.slavename for slave in ubuntu16_aarch64_slave],
        tags=build_arch_tags,
        properties=builder_arch_properties,
    ),
    ZFSBuilderConfig(
        name="Debian 8 ppc (BUILD)",
        factory=build_factory,
        slavenames=[slave.slavename for slave in debian8_ppc_slave],
        tags=build_arch_tags,
        properties=builder_arch_properties,
    ),
    ZFSBuilderConfig(
        name="Debian 8 ppc64 (BUILD)",
        factory=build_factory,
        slavenames=[slave.slavename for slave in debian8_ppc64_slave],
        tags=build_arch_tags,
        properties=builder_arch_properties,
    ),
]

# Test builders
test_builders = [
    ZFSBuilderConfig(
        name="Amazon 2015.09 x86_64 Release (TEST)",
        factory=test_factory,
        slavenames=[slave.name for slave in amazon_x86_64_testslave],
        tags=test_tags,
        properties=builder_release_properties,
    ),
    ZFSBuilderConfig(
        name="CentOS 6.7 x86_64 (TEST)",
        factory=test_factory,
        slavenames=[slave.name for slave in centos6_x86_64_testslave],
        tags=test_tags,
        properties=builder_redhat_properties,
    ),
    ZFSBuilderConfig(
        name="CentOS 7.1 x86_64 (TEST)",
        factory=test_factory,
        slavenames=[slave.name for slave in centos7_x86_64_testslave],
        tags=test_tags,
        properties=builder_redhat_properties,
    ),
    ZFSBuilderConfig(
        name="CentOS 7 x86_64 Mainline (TEST)",
        factory=test_factory,
        slavenames=[slave.name for slave in centos7_x86_64_ml_testslave],
        tags=test_tags,
        properties=builder_default_properties,
    ),
    ZFSBuilderConfig(
        name="Fedora 26 x86_64 (TEST)",
        factory=test_factory,
        slavenames=[slave.name for slave in fedora26_x86_64_testslave],
        tags=test_tags,
        properties=builder_default_properties,
    ),
    ZFSBuilderConfig(
        name="Ubuntu 16.04 x86_64 (TEST)",
        factory=test_factory,
        slavenames=[slave.name for slave in ubuntu16_x86_64_testslave],
        tags=test_tags,
        properties=builder_default_properties,
    ),
    ZFSBuilderConfig(
        name="Ubuntu 17.04 x86_64 (TEST)",<commit_msg>Remove CentOS minor version from builder names (#83)<commit_after># -*- python -*-
# ex: set syntax=python:

from buildbot.plugins import *
from twisted.python import log
from password import *
from buildslaves import *
from buildbot.buildslave import BuildSlave

bb_slave_port = 9989
bb_try_port = 8033
bb_web_port = 8010
bb_master = "build.zfsonlinux.org:9989"
bb_url = "https://raw.githubusercontent.com/zfsonlinux/zfs-buildbot/master/scripts/"
zol_url = "http://zfsonlinux.org"
web_url = "http://build.zfsonlinux.org/"
spl_repo = "https://github.com/zfsonlinux/spl.git"
zfs_repo = "https://github.com/zfsonlinux/zfs.git"
linux_repo = "https://github.com/torvalds/linux.git"
zfs_path = "/usr/libexec/zfs:/usr/share/zfs:/usr/lib/rpm/zfs:/usr/lib/zfs"
bin_path = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"

all_repositories = {
    "https://github.com/torvalds/linux" : 'linux',
    "https://github.com/zfsonlinux/spl" : 'spl',
    "https://github.com/zfsonlinux/zfs" : 'zfs',
    "https://github.com/torvalds/linux.git" : 'linux',
    "https://github.com/zfsonlinux/spl.git" : 'spl',
    "https://github.com/zfsonlinux/zfs.git" : 'zfs',
}

c = BuildmasterConfig = {}

c['caches'] = {
    'Changes' : 10,
    'Builds' : 15,
    'chdicts' : 10,
    'BuildRequests' : 10,
    'SourceStamps' : 10,
    'ssdicts' : 20,
    'objectids' : 10,
    'usdicts' : 10,
}


####### BUILD FACTORIES

# A builder also has a BuildFactory, which is responsible for creating new
# Build instances: because the Build instance is what actually performs each
# build, choosing the BuildFactory is the way to specify what happens each
# time a build is done (Builds).

from buildbot.steps.source.git import Git
from buildbot.steps.shell import ShellCommand
from buildbot.steps.master import MasterShellCommand
from buildbot.status.results import SUCCESS
from buildbot.status.results import FAILURE
from buildbot.status.results import WARNINGS
from buildbot.status.results import SKIPPED

def do_step_build(step, name):
    props = step.build.getProperties()
    if props.hasProperty(name) and props[name] == "yes":
        return True
    else:
        return False

def do_step_build_linux(step):
    return do_step_build(step, 'buildlinux')

def do_step_build_spl(step):
    return do_step_build(step, 'buildspl')

def do_step_build_zfs(step):
    return do_step_build(step, 'buildzfs')

def do_step_check_lint(step):
    return do_step_build(step, 'checklint')

#
# Build factory - Build source in-tree
#
# Perform an in-tree build using the default options.  This is used to verify
# the build accross a range of distributions and architectures.
#
style_factory = util.BuildFactory()

style_factory.addStep(ShellCommand(
    env={'PATH' : bin_path},
    command=["runurl", bb_url + "bb-dependencies.sh"],
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    haltOnFailure=True, logEnviron=False,
    lazylogfiles=True,
    description=["installing dependencies"],
    descriptionDone=["installed dependencies"]))
style_factory.addStep(Git(repourl=zfs_repo, workdir="build/zfs",
    mode="full", method="clobber", codebase="zfs",
    logEnviron=False, getDescription=True,
    lazylogfiles=True,
    description=["cloning"], descriptionDone=["cloned"]))
style_factory.addStep(ShellCommand(
    workdir="build/zfs", env={'PATH' : bin_path,
        'CONFIG_OPTIONS' : '--with-config=srpm'},
    command=["runurl", bb_url + "bb-build-zfs.sh"],
    haltOnFailure=True, logEnviron=False,
    lazylogfiles=True,
    logfiles={"configure"  : { "filename" : "configure.log", "follow" : True },
              "config.log" : { "filename" : "config.log",    "follow" : True },
              "make"       : { "filename" : "make.log",      "follow" : True }},
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    description=["building zfs"], descriptionDone=["built zfs"]))
style_factory.addStep(ShellCommand(command=["make", "checkstyle"],
    env={'PATH' : bin_path},
    workdir="build/zfs", logEnviron=False,
    haltOnFailure=False, flunkOnWarnings=True,
    lazylogfiles=True,
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    description=["checking style"], descriptionDone=["checked style"]))
style_factory.addStep(ShellCommand(command=["make", "lint"],
    env={'PATH' : bin_path},
    workdir="build/zfs", logEnviron=False,
    haltOnFailure=False, flunkOnWarnings=True,
    lazylogfiles=True,
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    description=["checking lint"], descriptionDone=["checked lint"],
    doStepIf = do_step_check_lint,
    hideStepIf=lambda results, s: results==SKIPPED))

build_factory = util.BuildFactory()

build_factory.addStep(ShellCommand(
    env={'PATH' : bin_path},
    command=["runurl", bb_url + "bb-dependencies.sh"],
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    haltOnFailure=True, logEnviron=False,
    lazylogfiles=True,
    description=["installing dependencies"],
    descriptionDone=["installed dependencies"]))

build_factory.addStep(Git(repourl=linux_repo, workdir="build/linux",
    mode="full", method="clobber", shallow=True, codebase="linux",
    logEnviron=False, getDescription=True,
    lazylogfiles=True,
    description=["cloning"], descriptionDone=["cloned"],
    doStepIf = do_step_build_linux,
    hideStepIf=lambda results, s: results==SKIPPED))
build_factory.addStep(Git(repourl=spl_repo, workdir="build/spl",
    mode="full", method="clobber", codebase="spl",
    logEnviron=False, getDescription=True,
    lazylogfiles=True,
    description=["cloning"], descriptionDone=["cloned"]))
build_factory.addStep(Git(repourl=zfs_repo, workdir="build/zfs",
    mode="full", method="clobber", codebase="zfs",
    logEnviron=False, getDescription=True,
    lazylogfiles=True,
    description=["cloning"], descriptionDone=["cloned"]))

build_factory.addStep(ShellCommand(
    workdir="build/linux", env={'PATH' : bin_path,
        'LINUX_BUILTIN' : util.Interpolate('%(prop:builtin:-no)s') },
    command=["runurl", bb_url + "bb-build-linux.sh"],
    haltOnFailure=True, logEnviron=False,
    lazylogfiles=True,
    logfiles={"configure"  : { "filename" : "configure.log", "follow" : True },
              ".config"    : { "filename" : ".config",       "follow" : False},
              "make"       : { "filename" : "make.log",      "follow" : True }},
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    description=["building linux"], descriptionDone=["built linux"],
    doStepIf = do_step_build_linux,
    hideStepIf=lambda results, s: results==SKIPPED))
build_factory.addStep(ShellCommand(
    workdir="build/spl", env={'PATH' : bin_path,
        'CONFIG_OPTIONS' : util.Interpolate('%(prop:configspl:-"")s'),
        'LINUX_CUSTOM' : util.Interpolate('%(prop:buildlinux:-no)s') },
    command=["runurl", bb_url + "bb-build-zfs.sh"],
    haltOnFailure=True, logEnviron=False,
    lazylogfiles=True,
    logfiles={"configure"  : { "filename" : "configure.log", "follow" : True },
              "config.log" : { "filename" : "config.log",    "follow" : True },
              "make"       : { "filename" : "make.log",      "follow" : True }},
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    description=["building spl"], descriptionDone=["built spl"],
    doStepIf = do_step_build_spl,
    hideStepIf=lambda results, s: results==SKIPPED))
build_factory.addStep(ShellCommand(
    workdir="build/zfs", env={'PATH' : bin_path,
        'CONFIG_OPTIONS' : util.Interpolate('%(prop:configzfs:-"")s'),
        'LINUX_CUSTOM' : util.Interpolate('%(prop:buildlinux:-no)s') },
    command=["runurl", bb_url + "bb-build-zfs.sh"],
    haltOnFailure=True, logEnviron=False,
    lazylogfiles=True,
    logfiles={"configure"  : { "filename" : "configure.log", "follow" : True },
              "config.log" : { "filename" : "config.log",    "follow" : True },
              "make"       : { "filename" : "make.log",      "follow" : True }},
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    description=["building zfs"], descriptionDone=["built zfs"],
    doStepIf = do_step_build_zfs,
    hideStepIf=lambda results, s: results==SKIPPED))
build_factory.addStep(ShellCommand(command=["make", "lint"],
    workdir="build/zfs", logEnviron=False, env={'PATH' : bin_path},
    haltOnFailure=False, flunkOnWarnings=True,
    lazylogfiles=True,
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    description=["checking lint"], descriptionDone=["checked lint"],
    doStepIf = do_step_check_lint,
    hideStepIf=lambda results, s: results==SKIPPED))

#
# Test factory - Build packages and run the test suites.
#
# Perform a package build with debugging enabled, install the packages,
# and run of the test suite.  The tests themselves have been seperated from
# the buildbot configuration to minimize the need for restarting the master.
#
test_factory = util.BuildFactory()

test_factory.addStep(ShellCommand(
    env={'PATH' : bin_path},
    command=["runurl", bb_url + "bb-dependencies.sh"],
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    haltOnFailure=True, logEnviron=False,
    lazylogfiles=True,
    description=["installing dependencies"],
    descriptionDone=["installed dependencies"]))
test_factory.addStep(Git(repourl=spl_repo, workdir="build/spl",
    mode="full", method="clobber", codebase="spl",
    logEnviron=False, getDescription=True,
    lazylogfiles=True,
    description=["cloning"], descriptionDone=["cloned"]))
test_factory.addStep(ShellCommand(
    env={'PATH' : bin_path,
        'CONFIG_OPTIONS' : util.Interpolate('%(prop:configspl:-"")s'),
        'LINUX_CUSTOM' : util.Interpolate('%(prop:buildlinux:-no)s') },
    workdir="build/spl",
    command=["runurl", bb_url + "bb-build-packages.sh"],
    haltOnFailure=True, logEnviron=False,
    lazylogfiles=True,
    logfiles={"configure"  : { "filename" : "configure.log", "follow" : True },
              "config.log" : { "filename" : "config.log",    "follow" : True },
              "make"       : { "filename" : "make.log",      "follow" : True },
              "install"    : { "filename" : "install.log",   "follow" : True }},
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    description=["building spl"], descriptionDone=["built spl"]))
test_factory.addStep(Git(repourl=zfs_repo, workdir="build/zfs",
    mode="full", method="clobber", codebase="zfs",
    logEnviron=False, getDescription=True,
    lazylogfiles=True,
    description=["cloning"], descriptionDone=["cloned"]))
test_factory.addStep(ShellCommand(
    env={'PATH' : bin_path,
        'CONFIG_OPTIONS' : util.Interpolate('%(prop:configzfs:-"")s'),
        'LINUX_CUSTOM' : util.Interpolate('%(prop:buildlinux:-no)s') },
    workdir="build/zfs",
    command=["runurl", bb_url + "bb-build-packages.sh"],
    haltOnFailure=True, logEnviron=False,
    lazylogfiles=True,
    logfiles={"configure"  : { "filename" : "configure.log", "follow" : True },
              "config.log" : { "filename" : "config.log",    "follow" : True },
              "make"       : { "filename" : "make.log",      "follow" : True },
              "install"    : { "filename" : "install.log",   "follow" : True }},
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    description=["building zfs"], descriptionDone=["built zfs"]))
test_factory.addStep(ShellCommand(
    workdir="build/tests",
    env={'PATH' : bin_path + ":" + zfs_path},
    command=["runurl", bb_url + "bb-test-prepare.sh"],
    haltOnFailure=False, maxTime=120, sigtermTime=30, logEnviron=False,
    lazylogfiles=True,
    logfiles={"test"     : { "filename" : "TEST",        "follow" : False }},
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    description=["preparing env"], descriptionDone=["prepared env"]))
test_factory.addStep(ShellCommand(
    workdir="build/tests/splat",
    env={'PATH' : bin_path + ":" + zfs_path},
    command=["runurl", bb_url + "bb-test-splat.sh"],
    haltOnFailure=False, maxTime=600, sigtermTime=30, logEnviron=False,
    lazylogfiles=True,
    logfiles={"console" : { "filename" : "console.log", "follow" : False }},
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    description=["splat"], descriptionDone=["splat"],
    hideStepIf=lambda results, s: results==SKIPPED))
test_factory.addStep(ShellCommand(
    workdir="build/tests/ztest",
    env={'PATH' : bin_path + ":" + zfs_path},
    command=["runurl", bb_url + "bb-test-ztest.sh"],
    haltOnFailure=False, maxTime=14400, sigtermTime=30, logEnviron=False,
    lazylogfiles=True,
    logfiles={"ztest.out" : { "filename" : "ztest.out", "follow" : False },
              "console" : { "filename" : "console.log", "follow" : False }},
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    description=["ztest"], descriptionDone=["ztest"],
    hideStepIf=lambda results, s: results==SKIPPED))
test_factory.addStep(ShellCommand(
    workdir="build/tests/zimport",
    env={'PATH' : bin_path + ":" + zfs_path},
    command=["runurl", bb_url + "bb-test-zimport.sh"],
    haltOnFailure=False, maxTime=1200, sigtermTime=30, logEnviron=False,
    lazylogfiles=True,
    logfiles={"console" : { "filename" : "console.log", "follow" : False }},
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    description=["zimport"], descriptionDone=["zimport"],
    hideStepIf=lambda results, s: results==SKIPPED))
test_factory.addStep(ShellCommand(
    workdir="build/tests/xfstests",
    env={'PATH' : bin_path + ":" + zfs_path},
    command=["runurl", bb_url + "bb-test-xfstests.sh"],
    haltOnFailure=False, flunkOnWarnings=True,
    maxTime=1200, sigtermTime=30, logEnviron=False,
    lazylogfiles=True,
    logfiles={"configure" : { "filename" : "configure.log", "follow" : True },
              "make"      : { "filename" : "make.log",      "follow" : True },
              "console"   : { "filename" : "console.log",   "follow" : False },
              "kmemleak"  : { "filename" : "kmemleak.log",  "follow" : False }},
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    description=["xfstest"], descriptionDone=["xfstests"],
    hideStepIf=lambda results, s: results==SKIPPED))
test_factory.addStep(ShellCommand(
    workdir="build/tests/zfstests",
    env={'PATH' : bin_path + ":" + zfs_path},
    command=["runurl", bb_url + "bb-test-zfstests.sh"],
    haltOnFailure=False, flunkOnWarnings=True,
    maxTime=18000, sigtermTime=30, logEnviron=False,
    lazylogfiles=True, timeout=4200,
    logfiles={"console"  : { "filename" : "console.log",  "follow" : False },
              "log"      : { "filename" : "log",          "follow" : True },
              "kmemleak" : { "filename" : "kmemleak.log", "follow" : False }},
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    description=["zfstests"], descriptionDone=["zfstests"],
    hideStepIf=lambda results, s: results==SKIPPED))
test_factory.addStep(ShellCommand(
    workdir="build/tests/zfsstress",
    env={'PATH' : bin_path + ":" + zfs_path},
    command=["runurl", bb_url + "bb-test-zfsstress.sh"],
    haltOnFailure=False, flunkOnWarnings=True,
    maxTime=14400, sigtermTime=30, logEnviron=False,
    lazylogfiles=True,
    logfiles={"console"  : { "filename" : "console.log",  "follow" : False },
              "kmemleak" : { "filename" : "kmemleak.log", "follow" : False }},
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    description=["zfsstress"], descriptionDone=["zfsstress"],
    hideStepIf=lambda results, s: results==SKIPPED))

test_factory.addStep(ShellCommand(command=["runurl", bb_url + "bb-cleanup.sh"],
    env={'PATH' : bin_path},
    haltOnFailure=False, logEnviron=False,
    lazylogfiles=True,
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    description=["removing zfs"], descriptionDone=["removed zfs"]))

#
# Performance factory - Build packages and run the performance suite.
#
# Perform a package build with debugging disabled, install the packages,
# and run a battery of performance tests.  The tests themselves have been
# seperated from the buildbot configuration to minimize the need for
# restarting the master.
#
perf_factory = util.BuildFactory()

perf_factory.addStep(ShellCommand(
    env={'PATH' : bin_path},
    command=["runurl", bb_url + "bb-dependencies.sh"],
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    haltOnFailure=True, logEnviron=False,
    lazylogfiles=True,
    description=["installing dependencies"],
    descriptionDone=["installed dependencies"]))
perf_factory.addStep(Git(repourl=spl_repo, workdir="build/spl",
    mode="full", method="clobber", codebase="spl",
    logEnviron=False, getDescription=True,
    lazylogfiles=True,
    description=["cloning"], descriptionDone=["cloned"]))
perf_factory.addStep(ShellCommand(
    env={'PATH' : bin_path,
        'CONFIG_OPTIONS' : util.Interpolate('%(prop:configspl:-"")s') },
    workdir="build/spl",
    command=["runurl", bb_url + "bb-build-packages.sh"],
    haltOnFailure=True, logEnviron=False,
    lazylogfiles=True,
    logfiles={"configure"  : { "filename" : "configure.log", "follow" : True },
              "config.log" : { "filename" : "config.log",    "follow" : True },
              "make"       : { "filename" : "make.log",      "follow" : True },
              "install"    : { "filename" : "install.log",   "follow" : True }},
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    description=["building spl"], descriptionDone=["built spl"]))
perf_factory.addStep(Git(repourl=zfs_repo, workdir="build/zfs",
    mode="full", method="clobber", codebase="zfs",
    logEnviron=False, getDescription=True,
    lazylogfiles=True,
    description=["cloning"], descriptionDone=["cloned"]))
perf_factory.addStep(ShellCommand(
    env={'PATH' : bin_path,
        'CONFIG_OPTIONS' : util.Interpolate('%(prop:configzfs:-"")s') },
    workdir="build/zfs",
    command=["runurl", bb_url + "bb-build-packages.sh"],
    haltOnFailure=True, logEnviron=False,
    lazylogfiles=True,
    logfiles={"configure"  : { "filename" : "configure.log", "follow" : True },
              "config.log" : { "filename" : "config.log",    "follow" : True },
              "make"       : { "filename" : "make.log",      "follow" : True },
              "install"    : { "filename" : "install.log",   "follow" : True }},
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    description=["building zfs"], descriptionDone=["built zfs"]))
perf_factory.addStep(ShellCommand(
    workdir="build/tests",
    env={'PATH' : bin_path + ":" + zfs_path,
         'AVAILABLE_DISKS' : 'xvdb xvdc xvdd'},
    command=["runurl", bb_url + "bb-test-prepare.sh"],
    haltOnFailure=False, maxTime=3600, sigtermTime=30, logEnviron=False,
    lazylogfiles=True, timeout=3600,
    logfiles={"test"     : { "filename" : "TEST",        "follow" : False }},
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    description=["preparing env"], descriptionDone=["prepared env"]))
perf_factory.addStep(ShellCommand(
    workdir="build/tests/zfstests",
    env={'PATH' : bin_path + ":" + zfs_path,
         'DEFAULT_ZFSTESTS_RUNFILE' : 'perf-regression.run',
         'DEFAULT_ZFSTESTS_DISKS' : 'xvdb1 xvdc1 xvdd1'},
    command=["runurl", bb_url + "bb-test-zfstests.sh"],
    haltOnFailure=False, maxTime=21600, sigtermTime=30, logEnviron=False,
    lazylogfiles=True,
    logfiles={"console" : { "filename" : "console.log", "follow" : False },
              "log"     : { "filename" : "log",         "follow" : True }},
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    timeout=5400, description=["zfstests"], descriptionDone=["zfstests"],
    hideStepIf=lambda results, s: results==SKIPPED))
perf_factory.addStep(ShellCommand(command=["runurl", bb_url + "bb-cleanup.sh"],
    env={'PATH' : bin_path},
    haltOnFailure=False, logEnviron=False,
    lazylogfiles=True,
    decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
    description=["removing zfs"], descriptionDone=["removed zfs"]))


####### WORKERS / SLAVES

# Each builder is associated with one of more workers / slaves.  They are
# responsible for carrying out the steps described by a build factory.
# Note that any particular build will only take place on one slave.
#
# AMI Reference:
#
# ami-10260870 - Amazon 07-12-2017
# ami-ca2608aa - CentOS 6 07-12-2017
# ami-4f27092f - CentOS 7 07-12-2017
# ami-12240a72 - CentOS 7 07-12-2017 Mainline
# ami-3d240a5d - Debian 8 07-12-2017
# ami-b4d2fbd4 - Fedora 26 07-24-2017
# ami-46270926 - Ubuntu 14.04 07-12-2017
# ami-1efadd7e - Ubuntu 14.04 05-01-2017 PV
# ami-1b29077b - Ubuntu 16.04 07-12-2017
# ami-473d1727 - Ubuntu 16.04 08-15-2017 Kmemleak
# ami-f9250b99 - Ubuntu 17.04 07-12-2017

# Style slaves
numstyleslaves = 2

style_slaves = [
    ZFSEC2StyleSlave(
        name="Ubuntu-16.04-x86_64-styleslave%s" % (str(i+1)),
        ami="ami-1b29077b"
    ) for i in range(0, numstyleslaves)
]

# Distribution slaves
numslaves = 1

kernel_default_x86_64_slave = [
    ZFSEC2BuildSlave(
        name="Amazon-KernelDefault-x86_64-buildslave%s" % (str(i+1)),
        ami="ami-10260870"
    ) for i in range(0, numslaves)
]

kernel_builtin_x86_64_slave = [
    ZFSEC2BuildSlave(
        name="Amazon-KernelBuiltin-x86_64-buildslave%s" % (str(i+1)),
        ami="ami-10260870"
    ) for i in range(0, numslaves)
]

debian8_x86_64_slave = [
    ZFSEC2BuildSlave(
        name="Debian-8-x86_64-buildslave%s" % (str(i+1)),
        ami="ami-3d240a5d"
    ) for i in range(0, numslaves)
]

fedora26_x86_64_slave = [
    ZFSEC2BuildSlave(
        name="Fedora-26-x86_64-buildslave%s" % (str(i+1)),
        ami="ami-b4d2fbd4"
    ) for i in range(0, numslaves)
]

centos6_x86_64_slave = [
    ZFSEC2BuildSlave(
        name="CentOS-6-x86_64-buildslave%s" % (str(i+1)),
        ami="ami-ca2608aa"
    ) for i in range(0, numslaves)
]

centos7_x86_64_slave = [
    ZFSEC2BuildSlave(
        name="CentOS-7-x86_64-buildslave%s" % (str(i+1)),
        ami="ami-4f27092f"
    ) for i in range(0, numslaves)
]

ubuntu14_x86_64_slave = [
    ZFSEC2BuildSlave(
        name="Ubuntu-14.04-x86_64-buildslave%s" % (str(i+1)),
        ami="ami-46270926"
    ) for i in range(0, numslaves)
]

ubuntu16_x86_64_slave = [
    ZFSEC2BuildSlave(
        name="Ubuntu-16.04-x86_64-buildslave%s" % (str(i+1)),
        ami="ami-1b29077b"
    ) for i in range(0, numslaves)
]

distro_slaves = \
    kernel_default_x86_64_slave + \
    kernel_builtin_x86_64_slave + \
    debian8_x86_64_slave + \
    centos6_x86_64_slave + \
    centos7_x86_64_slave + \
    fedora26_x86_64_slave + \
    ubuntu14_x86_64_slave + \
    ubuntu16_x86_64_slave

# Architecture slaves
amazon_x86_64_slave = [
    ZFSEC2BuildSlave(
        name="Amazon-2015.09-x86_64-buildslave%s" % (str(i+1)),
        ami="ami-10260870"
    ) for i in range(0, numslaves)
]

ubuntu14_i686_slave = [
    ZFSEC2PVSlave(
        name="Ubuntu-14.04-i686-buildslave%s" % (str(i+1)),
        ami="ami-1efadd7e"
    ) for i in range(0, numslaves)
]

debian8_arm_slave = [
    BuildSlave("Debian-8-arm-buildslave1",
        slave_userpass["Debian-8-arm-buildslave1"]),
    BuildSlave("Debian-8-arm-buildslave2",
        slave_userpass["Debian-8-arm-buildslave2"]),
]

ubuntu16_aarch64_slave = [
    BuildSlave("Ubuntu-16.04-aarch64-buildslave1",
        slave_userpass["Ubuntu-16.04-aarch64-buildslave1"]),
    BuildSlave("Ubuntu-16.04-aarch64-buildslave2",
        slave_userpass["Ubuntu-16.04-aarch64-buildslave2"]),
]

debian8_ppc_slave = [
    BuildSlave("Debian-8-ppc-buildslave",
        slave_userpass["Debian-8-ppc-buildslave"]),
]

debian8_ppc64_slave = [
    BuildSlave("Debian-8-ppc64-buildslave1",
        slave_userpass["Debian-8-ppc64-buildslave1"]),
    BuildSlave("Debian-8-ppc64-buildslave2",
        slave_userpass["Debian-8-ppc64-buildslave2"]),
]

arch_slaves = \
    amazon_x86_64_slave + \
    ubuntu14_i686_slave + \
    debian8_arm_slave + \
    ubuntu16_aarch64_slave + \
    debian8_ppc_slave + \
    debian8_ppc64_slave

# Test slaves
numtestslaves = 16

amazon_x86_64_testslave = [
    ZFSEC2TestSlave(
        name="Amazon-2015.09-x86_64-testslave%s" % (str(i+1)),
        ami="ami-10260870"
    ) for i in range(0, numtestslaves)
]

centos6_x86_64_testslave = [
    ZFSEC2TestSlave(
        name="CentOS-6-x86_64-testslave%s" % (str(i+1)),
        ami="ami-ca2608aa"
    ) for i in range(0, numtestslaves)
]

centos7_x86_64_testslave = [
    ZFSEC2TestSlave(
        name="CentOS-7-x86_64-testslave%s" % (str(i+1)),
        ami="ami-4f27092f"
    ) for i in range(0, numtestslaves)
]

centos7_x86_64_ml_testslave = [
    ZFSEC2TestSlave(
        name="CentOS-7-x86_64-ml-testslave%s" % (str(i+1)),
        ami="ami-12240a72"
    ) for i in range(0, numtestslaves)
]

fedora26_x86_64_testslave = [
    ZFSEC2TestSlave(
        name="Fedora-26-x86_64-testslave%s" % (str(i+1)),
        ami="ami-b4d2fbd4"
    ) for i in range(0, numtestslaves)
]

ubuntu16_x86_64_testslave = [
    ZFSEC2TestSlave(
        name="Ubuntu-16.04-x86_64-testslave%s" % (str(i+1)),
        ami="ami-1b29077b"
    ) for i in range(0, numtestslaves)
]

ubuntu17_x86_64_testslave = [
    ZFSEC2TestSlave(
        name="Ubuntu-17.04-x86_64-testslave%s" % (str(i+1)),
        ami="ami-f9250b99"
    ) for i in range(0, numtestslaves)
]

ubuntu16_x86_64_kmemleak_testslave = [
    ZFSEC2TestSlave(
        name="Ubuntu-16.04-x86_64-kmemleak-testslave%s" % (str(i+1)),
        ami="ami-473d1727"
    ) for i in range(0, numtestslaves)
]

ubuntu14_i686_testslave = [
    ZFSEC2PVTestSlave(
        name="Ubuntu-14.04-i686-testslave%s" % (str(i+1)),
        ami="ami-1efadd7e"
    ) for i in range(0, numtestslaves)
]

test_slaves = \
    amazon_x86_64_testslave + \
    centos6_x86_64_testslave + \
    centos7_x86_64_testslave + \
    centos7_x86_64_ml_testslave + \
    fedora26_x86_64_testslave + \
    ubuntu16_x86_64_testslave + \
    ubuntu17_x86_64_testslave + \
    ubuntu16_x86_64_kmemleak_testslave + \
    ubuntu14_i686_testslave

# Performance slaves
numperfslaves = 2

amazon_x86_64_simd_perfslave = [
    ZFSEC2VectorTestSlave(
        name="Amazon-2015.09-x86_64-simd-perfslave%s" % (str(i+1)),
        ami="ami-10260870"
    ) for i in range(0, numtestslaves)
]

amazon_x86_64_perfslave = [
    ZFSEC2PerfTestSlave(
        name="Amazon-2015.09-x86_64-perfslave%s" % (str(i+1)),
        ami="ami-10260870"
    ) for i in range(0, numperfslaves)
]

perf_slaves = \
    amazon_x86_64_perfslave + \
    amazon_x86_64_simd_perfslave

# All slaves
all_slaves = \
    style_slaves + \
    distro_slaves + \
    arch_slaves + \
    test_slaves + \
    perf_slaves

style_tags = [ "Style" ]
build_distro_tags = [ "Distributions" ]
build_arch_tags = [ "Architectures" ]
test_tags = [ "Tests" ]
perf_tags = [ "Performance" ]


####### BUILDERS

# The Buildmaster runs a collection of Builders, each of which handles a single
# type of build (e.g. full versus quick), on one or more workers. Builders
# serve as a kind of queue for a particular type of build. Each Builder gets a
# separate column in the waterfall display. In general, each Builder runs
# independently (although various kinds of interlocks can cause one Builder
# to have an effect on another).

# Properties are used to control how a build is performed.
builder_default_properties = {
    "buildlinux":    "no",
    "buildspl":      "yes",
    "buildzfs":      "yes",
    "builtin":       "no",
    "configspl":     "--enable-debug",
    "configzfs":     "--enable-debug",
    "repoowner":     "zfsonlinux",
    "reponame":      "zfs",
    "checklint":     "no",
}

builder_redhat_properties = {
    "buildlinux":    "no",
    "buildspl":      "yes",
    "buildzfs":      "yes",
    "builtin":       "no",
    "configspl":     "--enable-debug --with-spec=redhat",
    "configzfs":     "--enable-debug --with-spec=redhat",
    "repoowner":     "zfsonlinux",
    "reponame":      "zfs",
    "checklint":     "no",
}

builder_release_properties = {
    "buildlinux":    "no",
    "buildspl":      "yes",
    "buildzfs":      "yes",
    "builtin":       "no",
    "configspl":     "",
    "configzfs":     "",
    "repoowner":     "zfsonlinux",
    "reponame":      "zfs",
    "checklint":     "no",
}

builder_linux_properties = {
    "buildlinux":    "yes",
    "buildspl":      "yes",
    "buildzfs":      "yes",
    "builtin":       "no",
    "configspl":     "--enable-debug",
    "configzfs":     "--enable-debug",
    "repoowner":     "zfsonlinux",
    "reponame":      "zfs",
    "checklint":     "no",
}

builder_builtin_properties = {
    "buildlinux":    "yes",
    "buildspl":      "no",
    "buildzfs":      "no",
    "builtin":       "yes",
    "configspl":     "--enable-debug",
    "configzfs":     "--enable-debug",
    "repoowner":     "zfsonlinux",
    "reponame":      "zfs",
    "checklint":     "no",
}

builder_style_properties = {
    "buildlinux":    "no",
    "buildspl":      "yes",
    "buildzfs":      "yes",
    "builtin":       "no",
    "configspl":     "--enable-debug",
    "configzfs":     "--enable-debug",
    "repoowner":     "zfsonlinux",
    "reponame":      "zfs",
    "checklint":     "yes",
}

builder_arch_properties = {
    "buildlinux":    "no",
    "buildspl":      "yes",
    "buildzfs":      "yes",
    "builtin":       "no",
    "configspl":     "--enable-debug",
    "configzfs":     "--enable-debug",
    "repoowner":     "zfsonlinux",
    "reponame":      "zfs",
    "checklint":     "yes",
}

builder_perf_properties = {
    "buildlinux":    "no",
    "buildspl":      "yes",
    "buildzfs":      "yes",
    "builtin":       "no",
    "configspl":     "",
    "configzfs":     "",
    "repoowner":     "zfsonlinux",
    "reponame":      "zfs",
    "checklint":     "no",
}

def prioritizeBuilders(buildmaster, builders):
    """
    Called by the buildmaster to prioritize the builders.  Returns a sorted
    array of builders designed to improve ec2 utilization. Builders with
    substantiated, idle slaves are given priority. Followed by builders with no
    substantiated slaves. The lowest priority is a builder that is busy.
    This helps keep all buildslaves busy while new latent buildslaves are
    bootstrapped, a process which can take several minutes.
    """

    idle_builders = []
    busy_builders = []
    avail_builders = []

    for b in builders:
        idle = False
        busy = False
        for s in b.slaves:
            if s.isIdle():
               idle = True
               break

            if s.isBusy():
               busy = True

        if idle is True:
            if re.search('BUILD', b.name):
                idle_builders.insert(0, b)
            else:
                idle_builders.append(b)
        elif busy is True:
            if re.search('BUILD', b.name):
                busy_builders.insert(0, b)
            else:
                busy_builders.append(b)
        else:
            if re.search('BUILD', b.name):
                avail_builders.insert(0, b)
            else:
                avail_builders.append(b)

    sorted_builders = idle_builders + avail_builders + busy_builders

    log.msg("prioritized %i builder(s): %s" % (len(sorted_builders),
        [b.name for b in sorted_builders]))

    return sorted_builders

c['prioritizeBuilders'] = prioritizeBuilders

# Style builders
style_builders = [
    ZFSBuilderConfig(
        name="Ubuntu 16.04 x86_64 (STYLE)",
        factory=style_factory,
        slavenames=[slave.slavename for slave in style_slaves],
        tags=style_tags,
        properties=builder_style_properties,
    ),
]

# Distro builders
distro_builders = [
    ZFSBuilderConfig(
        name="Kernel.org Default x86_64 (BUILD)",
        factory=build_factory,
        slavenames=[slave.name for slave in kernel_default_x86_64_slave],
        tags=build_distro_tags,
        properties=builder_linux_properties,
    ),
    ZFSBuilderConfig(
        name="Kernel.org Built-in x86_64 (BUILD)",
        factory=build_factory,
        slavenames=[slave.name for slave in kernel_builtin_x86_64_slave],
        tags=build_distro_tags,
        properties=builder_builtin_properties,
    ),
    ZFSBuilderConfig(
        name="Debian 8 x86_64 (BUILD)",
        factory=build_factory,
        slavenames=[slave.name for slave in debian8_x86_64_slave],
        tags=build_distro_tags,
        properties=builder_default_properties,
    ),
    ZFSBuilderConfig(
        name="Fedora 26 x86_64 (BUILD)",
        factory=build_factory,
        slavenames=[slave.name for slave in fedora26_x86_64_slave],
        tags=build_distro_tags,
        properties=builder_default_properties,
    ),
    ZFSBuilderConfig(
        name="CentOS 6 x86_64 (BUILD)",
        factory=build_factory,
        slavenames=[slave.name for slave in centos6_x86_64_slave],
        tags=build_distro_tags,
        properties=builder_redhat_properties,
    ),
    ZFSBuilderConfig(
        name="CentOS 7 x86_64 (BUILD)",
        factory=build_factory,
        slavenames=[slave.name for slave in centos7_x86_64_slave],
        tags=build_distro_tags,
        properties=builder_redhat_properties,
    ),
    ZFSBuilderConfig(
        name="Ubuntu 14.04 x86_64 (BUILD)",
        factory=build_factory,
        slavenames=[slave.name for slave in ubuntu14_x86_64_slave],
        tags=build_distro_tags,
        properties=builder_default_properties,
    ),
    ZFSBuilderConfig(
        name="Ubuntu 16.04 x86_64 (BUILD)",
        factory=build_factory,
        slavenames=[slave.name for slave in ubuntu16_x86_64_slave],
        tags=build_distro_tags,
        properties=builder_default_properties,
    ),
]

# Architecture builders
arch_builders = [
    ZFSBuilderConfig(
        name="Amazon 2015.09 x86_64 (BUILD)",
        factory=build_factory,
        slavenames=[slave.name for slave in amazon_x86_64_slave],
        tags=build_arch_tags,
        properties=builder_arch_properties,
    ),
    ZFSBuilderConfig(
        name="Ubuntu 14.04 i686 (BUILD)",
        factory=build_factory,
        slavenames=[slave.name for slave in ubuntu14_i686_slave],
        tags=build_arch_tags,
        properties=builder_arch_properties,
    ),
    ZFSBuilderConfig(
        name="Debian 8 arm (BUILD)",
        factory=build_factory,
        slavenames=[slave.slavename for slave in debian8_arm_slave],
        tags=build_arch_tags,
        properties=builder_arch_properties,
    ),
    ZFSBuilderConfig(
        name="Ubuntu 16.04 aarch64 (BUILD)",
        factory=build_factory,
        slavenames=[slave.slavename for slave in ubuntu16_aarch64_slave],
        tags=build_arch_tags,
        properties=builder_arch_properties,
    ),
    ZFSBuilderConfig(
        name="Debian 8 ppc (BUILD)",
        factory=build_factory,
        slavenames=[slave.slavename for slave in debian8_ppc_slave],
        tags=build_arch_tags,
        properties=builder_arch_properties,
    ),
    ZFSBuilderConfig(
        name="Debian 8 ppc64 (BUILD)",
        factory=build_factory,
        slavenames=[slave.slavename for slave in debian8_ppc64_slave],
        tags=build_arch_tags,
        properties=builder_arch_properties,
    ),
]

# Test builders
test_builders = [
    ZFSBuilderConfig(
        name="Amazon 2015.09 x86_64 Release (TEST)",
        factory=test_factory,
        slavenames=[slave.name for slave in amazon_x86_64_testslave],
        tags=test_tags,
        properties=builder_release_properties,
    ),
    ZFSBuilderConfig(
        name="CentOS 6 x86_64 (TEST)",
        factory=test_factory,
        slavenames=[slave.name for slave in centos6_x86_64_testslave],
        tags=test_tags,
        properties=builder_redhat_properties,
    ),
    ZFSBuilderConfig(
        name="CentOS 7 x86_64 (TEST)",
        factory=test_factory,
        slavenames=[slave.name for slave in centos7_x86_64_testslave],
        tags=test_tags,
        properties=builder_redhat_properties,
    ),
    ZFSBuilderConfig(
        name="CentOS 7 x86_64 Mainline (TEST)",
        factory=test_factory,
        slavenames=[slave.name for slave in centos7_x86_64_ml_testslave],
        tags=test_tags,
        properties=builder_default_properties,
    ),
    ZFSBuilderConfig(
        name="Fedora 26 x86_64 (TEST)",
        factory=test_factory,
        slavenames=[slave.name for slave in fedora26_x86_64_testslave],
        tags=test_tags,
        properties=builder_default_properties,
    ),
    ZFSBuilderConfig(
        name="Ubuntu 16.04 x86_64 (TEST)",
        factory=test_factory,
        slavenames=[slave.name for slave in ubuntu16_x86_64_testslave],
        tags=test_tags,
        properties=builder_default_properties,
    ),
    ZFSBuilderConfig(
        name="Ubuntu 17.04 x86_64 (TEST)",
{'fraction_non_alphanumeric': 0.14481321625241853, 'fraction_numerical': 0.03369007833957975, 'mean_word_length': 3.3532807162209917, 'pattern_counts': {'":': 164, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 20, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5349874', 'n_tokens_mistral': 27663, 'n_tokens_neox': 25827, 'n_words': 4680}