<commit_before># -*- python -*-
# ex: set syntax=python:
# vim: ft=python
# NIpy buildBOT Master Instance config
#
# [ TODO ]      (in no particular order)
# - combine multiple steps into one (if possible)
# - rename steps to be more concise and informative
# - generate numpy version based templates
# - what's up with the buildout not using ~/.buildout/default.cfg and not caching
#       * we may need to set some properties for this to work
# - install more versions of python on the slaves
# - have slaves report back their configuration (so we have them doing the right builders)
# - if possible, forcebuild every builder on a particular slave (to check that it can)
# - hookup with git post-commit hooks (instead of relying on poller)

""" A brief tour of buildbot

To remind myself

A ChangeSource looks for Changes.

In our case we are using GitPoller ChangeSource to look for changes in our git
repositories.

When ChangeSource finds changes, it broadcasts them to all Schedulers.  Each
Scheduler must decide if it wants to react to a Change.  It generally does this
by filtering changes to select changes it will act on.

We use SingleBranchSchedulers for this.  These obviously respond only to
changes in one branch of a repository.

A BuildSet is the association of (information about what changes need to be
tested) and (Builders on which to test)

If a Scheduler selects changes for action, it sends BuildSets to the system
(the BuildMaster).  Therefore, a Scheduler needs to know its Builders. It gets
these by name (string).

For each BuildSet, the (changes to be tested) are sent to each Builder.  Each
of these requests is BuildRequest.

A Builder accepts a BuildRequest and runs it on a BuildSlave.  It does this by
using its own BuildFactory instance to make Builds.

A Builder can have more than one associated BuildSlave.  In this case each
associated BuildSlave is assumed to produce identical results, and the Builder
can therefore choose at whim which BuildSlave to use.

A Build has one or more Steps that should be run with a given (changes to be
tested).

One type of build Step is a Trigger, which activates a triggerable Scheduler,
which can react any way it likes, including sending further BuildSets.
"""

from os.path import join as pjoin

# This is convenient for splitting sh commands into lists
import shlex

# Buildbot objects
from buildbot.buildslave import BuildSlave
from buildbot.changes.gitpoller import GitPoller
from buildbot.schedulers.filter import ChangeFilter
from buildbot.scheduler import Scheduler
from buildbot.schedulers import timed
from buildbot.process.factory import BuildFactory
from buildbot.steps.source import Git
from buildbot.steps.shell import ShellCommand, SetProperty
from buildbot.steps.transfer import FileDownload, DirectoryUpload
from buildbot.process.properties import WithProperties
from buildbot.config import BuilderConfig

# This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing.
c = BuildmasterConfig = {}

####### BUILDSLAVES

# Get passwords from file not in repository.  `slave_pass.py` defines a dict,
# `slave_passes` with key, value pairs defining (slave_name, slave_password)
import slave_pass as s_p
reload(s_p)
# Check that slave names are unique
if len(set(s_p.slave_passes)) != len(s_p.slave_passes):
    raise ValueError('Slave names must be unique')

# The 'slaves' list defines the set of recognized buildslaves. Each element is
# a BuildSlave object, specifying a unique slave name and password.  The same
# slave name and password must be configured on the slave.
slaves = [BuildSlave(slave_name, slave_pass)
    for slave_name, slave_pass in s_p.slave_passes.items()]

experimental = ['pi-ApxuMed',]

linux_64_slaves = ['tom-standard']
linux_32_slaves = ['bongoslave']
debian_ppc_slaves = ['debian-ppc-32']
sparc_slaves_sid = 'nd-bb-slave-sparc-sid'
sparc_slaves_wheezy = 'nd-bb-slave-sparc-wheezy'
sparc_slaves = ['nd-bb-slave-sparc']
osx_leopard_ppc_slaves = ['osx-10.5-ppc']
osx_tiger_ppc_slaves = ['osx-mini']
osx_leopard_intel_slaves = ['osx-10.5']
osx_snowleopard_slaves = ['osx-10.6']
osx_mountainlion_slaves = ['osx-10.8']
xp_32_slaves = ['xp-mini']
win7_64_slaves = ['mike-win7-64']
arm_slaves = ['arm-raspberry-pi']

# slave names must be unique
c['slaves'] = slaves

# 'slavePortnum' defines the TCP port to listen on for connections from slaves.
# This must match the value configured into the buildslaves (with their
# --master option)
c['slavePortnum'] = 9989

# Helper functions and constants
class GithubBot(object):

    need_nose = True

    def __init__(self,
                 organization,
                 repo,
                 test_cmd,
                 doc_build_cmd=None,
                 built_doc_src=None,
                 easy_depends=(),
                 pip_depends=(),
                 doc_easy_depends=('sphinx',),
                 src_subdir=None,
                 download_dir=None,
                 build_timeout=None):
        self.organization = organization
        self.repo = repo
        self.git_url = "git://github.com/%s/%s.git" % (organization, repo)
        self.test_cmd = test_cmd
        self.doc_build_cmd = doc_build_cmd
        self.built_doc_src = built_doc_src
        # Dependencies for install / test
        self.easy_depends = easy_depends # easy_install installs
        self.pip_depends = pip_depends # pip
        self.doc_easy_depends = doc_easy_depends # easy_installs for docs
        self.src_subdir = src_subdir # Subdirectory containing source
        if download_dir is None:
            download_dir = '%s-dist' % repo
        self.download_dir = download_dir
        self.build_timeout = build_timeout

    @property
    def project(self):
        return "%s" % (self.repo,)

    def poller(self, branch="master", interval=300):
        return GitPoller(
            self.git_url,
            project=self.project,
            category=self.project,
            workdir = "%s-%s-%s" % (self.organization, self.repo, branch),
            branch=branch,
            pollinterval=interval)

    def scheduler(self, buildernames):
        if isinstance(buildernames, basestring):
            buildernames = [buildernames]
        name = self.project
        changefilter = ChangeFilter(repository=self.git_url,
                                    project=self.project,
                                    category=self.project)
        return Scheduler(name=name,
                        change_filter=changefilter,
                        treeStableTimer=None,
                        builderNames=buildernames)

    def nightly_scheduler(self,
                          buildernames,
                          name_suff,
                          hour = 2, minute = 23,
                          ifchanged=True
                         ):
        """ Make a timed scheduler for this repo """
        if isinstance(buildernames, basestring):
            buildernames = [buildernames]
        name = "%s-%s-nightly-%s" % (self.organization, self.repo, name_suff)
        changefilter = ChangeFilter(repository=self.git_url,
                                    project=self.project,
                                    category=self.project)
        return timed.Nightly(name=name,
                branch="master",
                builderNames=buildernames,
                change_filter=changefilter,
                hour=hour,
                minute=minute,
                onlyIfChanged=ifchanged)

    def _code_build_factory(self,
                            python=None,
                            easy_depends=None,
                            pip_depends=None,
                            pre_cmds=()):
        if easy_depends is None:
            easy_depends = self.easy_depends
        if pip_depends is None:
            pip_depends = self.pip_depends
        factory = BuildFactory()
        factory.addStep(Git(repourl=self.git_url, mode='copy', retry=(15, 5)))
        # Any pre-building commands
        for cmd in pre_cmds:
            factory.addStep(ShellCommand(command=shlex.split(cmd, posix=False)))
        # Obviously requires virtualenv on the slave
        venv_params = ['--distribute', '--system-site-packages', 'venv']
        if not python is None:
            venv_params = ['--python=' + python] + venv_params
        factory.addStep(ShellCommand(command=['virtualenv'] + venv_params))
        # Now we need the venv binary directory
        factory.addStep(FileDownload('slave_scripts/bs_tools.py',
                                     'bs_tools.py'))
        factory.addStep(SetProperty(command=shlex.split(
            'python bs_tools.py path-prepend-str venv'),
            property='venv_script_dir'))
        factory.addStep(SetProperty(command=shlex.split(
            'python bs_tools.py abspath venv'),
            property='venv_dir'))
        # Set env, with virtualenv in front of the path
        my_env={'PATH': WithProperties("%(venv_script_dir)s${PATH}")}
        workdir = WithProperties("%(venv_dir)s")
        # Add install and runtime dependencies
        for pkg_spec in easy_depends:
            factory.addStep(ShellCommand(command=[
                        'easy_install', pkg_spec], env=my_env))
        for pkg_spec in pip_depends:
            factory.addStep(ShellCommand(command=[
                        'pip', 'install', pkg_spec], env=my_env))
        # Work out working directory for install
        if not self.src_subdir is None:
            factory.addStep(SetProperty(command=[
                'python', 'bs_tools.py', 'abspath', self.src_subdir],
                property='src_path'))
            build_workdir = WithProperties("%(src_path)s")
        else:
            build_workdir = None
        return factory, my_env, workdir, build_workdir

    def factory(self,
                test_cmd=None,
                python=None,
                easy_depends=None,
                pip_depends=None,
                pre_cmds=()):
        if test_cmd is None:
            test_cmd = self.test_cmd
        factory, env, workdir, build_workdir = self._code_build_factory(
            python, easy_depends, pip_depends, pre_cmds)
        # Additional options for ShellCommands, if provided.  To not
        # rely on knowing internal logic (either handles None as the
        # default value), we explicitly construct a dictionary only
        # with the specified options
        kwargs = {}
        if self.build_timeout:
            kwargs['timeout'] = self.build_timeout
        # Install into virtualenv
        factory.addStep(ShellCommand(
            command=['python', 'setup.py', 'install'],
            env=env,
            workdir=build_workdir,
            **kwargs))
        # Test
        factory.addStep(ShellCommand(command=test_cmd,
            env=env, workdir=workdir, **kwargs))
        return factory

    def bdist_factory(self,
                      bdist_type,
                      test_cmd=None,
                      python=None,
                      easy_depends=None,
                      pip_depends=None,
                      pre_cmds=()):
        if test_cmd is None:
            test_cmd = self.test_cmd
        factory, env, workdir, build_workdir = self._code_build_factory(
            python, easy_depends, pip_depends, pre_cmds)
        # Build binary format(s)
        if python is None: # Use virtualenv python
            my_python = 'python'
            build_env = env
        else: # Use specified python
            my_python = python
            # Insert virtualenv into PYTHONPATH for building using virtualenv
            # dependencies
            factory.addStep(SetProperty(command=shlex.split(
                'python bs_tools.py get-python-lib'),
                property='venv_python_lib', env=env))
            build_env = env.copy()
            build_env['PYTHONPATH'] = WithProperties("%(venv_python_lib)s")
        if bdist_type == 'egg':
            factory.addStep(ShellCommand(
                command=[my_python, 'setup.py', 'bdist_egg'],
                env=build_env,
                workdir=build_workdir))
            factory.addStep(ShellCommand(
                command=[my_python, 'setup.py', 'bdist_mpkg'],
                env=build_env,
                workdir=build_workdir))
            factory.addStep(SetProperty(command=shlex.split(
                'python bs_tools.py glob dist "*.egg"'),
                property='bdist_file'))
        elif bdist_type == 'exe':
            factory.addStep(ShellCommand(
                command=[my_python, 'setup.py', 'bdist', '--formats=wininst'],
                env=build_env,
                workdir=build_workdir))
            factory.addStep(SetProperty(command=shlex.split(
                'python bs_tools.py glob dist "*.exe"'),
                property='bdist_file'))
        else:
            raise ValueError('What is this?: ' + bdist_type)
        # Install into virtualenv
        factory.addStep(ShellCommand(
            command=[ 'easy_install', WithProperties('%(bdist_file)s')],
            env=env))
        # Test
        factory.addStep(ShellCommand(command=test_cmd,
            env=env, workdir=workdir))
        # Upload
        factory.addStep(DirectoryUpload(slavesrc="dist",
                masterdest="public_html/" + self.download_dir))
        return factory

    def doc_factory(self,
                    doc_build_cmd=None,
                    built_doc_src=None,
                    doc_doctest_cmd=None):
        if doc_build_cmd is None:
            doc_build_cmd = self.doc_build_cmd
        if built_doc_src is None:
            built_doc_src = self.built_doc_src
        if None in (doc_build_cmd, built_doc_src):
            raise ValueError("Need defined build command and doc source")
        factory, env, workdir, build_workdir = self._code_build_factory()
        # Install into virtualenv
        factory.addStep(ShellCommand(
            command=['python', 'setup.py', 'install'],
            env=env,
            workdir=build_workdir))
        for pkg_spec in self.doc_easy_depends:
            factory.addStep(ShellCommand(command=[
                        'easy_install', pkg_spec], env=env))
        if not doc_doctest_cmd is None:
            factory.addStep(ShellCommand(command=doc_doctest_cmd, env=env))
        factory.addStep(ShellCommand(command=doc_build_cmd, env=env))
        factory.addStep(DirectoryUpload(slavesrc=built_doc_src,
                    masterdest="public_html/nightly/" + self.repo + '/doc'))
        return factory

    def build_builders(self, name_slavenames,
                       test_cmd=None,
                       python=None,
                       easy_depends=None,
                       pip_depends=None):
        factory = self.factory(test_cmd, python, easy_depends, pip_depends)
        return self._factory_builders(factory, name_slavenames)

    def _factory_builders(self, factory, name_slavenames):
        builders = []
        for name, slavelist in name_slavenames:
            if isinstance(slavelist, basestring):
                slavelist = [slavelist]
            builders.append(BuilderConfig(
                name=name,
                slavenames=slavelist,
                factory=factory))
        return builders

    def build_doc_builders(self,
            name_slavenames,
            doc_build_cmd=None,
            built_doc_src=None,
            doc_doctest_cmd=None):
        factory = self.doc_factory(doc_build_cmd, built_doc_src, doc_doctest_cmd)
        return self._factory_builders(factory, name_slavenames)


def nptest_command(pkg_name,
                   verbose=10,
                   doctests=False,
                   python='python',
                   cd=False,
                   stdout=False,
                   exe=False):
    """ Run tests from new directory """
    pystr = ''
    if cd:
        pystr += "import os; os.chdir('..');"
    pystr += ("import sys; import %s; sys.exit(not %s.test(doctests=%s"
              % (pkg_name, pkg_name, doctests))
    if verbose != None:
        pystr += ', verbose=%d' % verbose
    if exe:
        pystr += ', extra_argv=["--exe"]'
    pystr += ").wasSuccessful())"
    return [python, '-c', "%s" % pystr]


nibabel_bot = GithubBot('nipy', 'nibabel',
    test_cmd = shlex.split('nosetests --verbosity=3 --exe --with-doctest nibabel'),
    doc_build_cmd = "cd doc && make html",
    built_doc_src = "build/html",
    easy_depends = ('nose',)
)
nitime_bot = GithubBot('nipy', 'nitime',
    test_cmd = nptest_command('nitime', verbose=None),
    easy_depends = ('nose',)
)
nitime_arokem_bot = GithubBot('arokem','nitime',
    test_cmd = nptest_command('nitime', verbose=None),
    easy_depends = ('nose',)
)
nipy_bot = GithubBot('nipy', 'nipy',
    test_cmd = nptest_command('nipy', doctests=True),
    doc_build_cmd = "cd doc && make dist",
    built_doc_src = "doc/dist",
    easy_depends = ('nose',),
    pip_depends = ('sympy', 'nibabel')
)
dipy_bot = GithubBot('nipy', 'dipy',
    test_cmd = shlex.split('nosetests --verbosity=3 --exe --with-doctest dipy'),
    easy_depends = ('nose',),
    pip_depends = ('cython>=0.17', 'nibabel')
)
# dipy bot for linux machines with optional easy_depends that need an X window
# for testing
dipy_bot_nox = GithubBot('nipy', 'dipy',
    test_cmd = shlex.split('xvfb-run --auto-servernum --server-args="-screen 0 1024x768x24" nosetests --verbosity=3 --with-doctest dipy'),
    easy_depends = ('nose', 'nibabel')
)
nipype_bot = GithubBot('nipy', 'nipype',
    test_cmd = nptest_command('nipype', doctests=True),
    easy_depends = ('nose', 'nibabel', 'networkx', 'Traits')
)
regreg_bot = GithubBot('regreg', 'regreg',
    test_cmd = shlex.split('nosetests --verbosity=3 --exe --with-doctest regreg'),
    easy_depends = ('nose',),
    src_subdir = 'code'
)
# pandas bot, primarily for sparc testing
pandas_bot = GithubBot('pydata', 'pandas',
    test_cmd = shlex.split('xvfb-run --auto-servernum --server-args="-screen 0 1024x768x24" nosetests --verbosity=3 --exe pandas'),
    doc_build_cmd = "make doc",
    built_doc_src = "doc/dist",
    easy_depends = ('nose',),
    pip_depends = ('apiclient',),
    build_timeout = 3600, # allow for its longer builds/testing especially for sparcs
)
# statsmodels bot, primarily for sparc testing
statsmodels_bot = GithubBot('statsmodels', 'statsmodels',
    test_cmd = shlex.split('xvfb-run --auto-servernum --server-args="-screen 0 1024x768x24" nosetests --verbosity=3 --exe statsmodels'),
    doc_build_cmd = "make -C docs html",
    built_doc_src = "docs/build",
    easy_depends = ('nose',),
    pip_depends = ('patsy>=0.1.0',),
)
####### CHANGESOURCES

# the 'change_source' setting tells the buildmaster how it should find out
# about source code changes.  Here we point to the buildbot clone of nibabel.

#TODO: We should probably change the GitPoller to a PBChangeSource to react to
# commits http://buildbot.net/buildbot/docs/latest/manual/cfg-changesources.html#chsrc-PBChangeSource
c['change_source'] = [bot.poller() for bot in (nibabel_bot,
                                               nitime_bot,
                                               nipy_bot,
                                               dipy_bot,
                                               nipype_bot,
                                               regreg_bot,
                                               pandas_bot,
                                               statsmodels_bot,
                                              )]


####### SCHEDULERS
# Configure the Schedulers, which decide how to react to incoming changes.

c['schedulers'] = [
    nibabel_bot.scheduler(['nibabel-py2.6',
                           'nibabel-py2.6-32',
                           'nibabel-py2.6-xp',
                           'nibabel-py2.7-win7',
                           'nibabel-py2.6-sparc',
                           'nibabel-py2.7-ppc',
                           'nibabel-py2.6-arm',
                           'nibabel-py2.7-osx-10.4-ppc',
                           'nibabel-py2.7-osx-10.6',
                           'nibabel-py2.7-osx-10.8',
                           'nibabel-py2.6-osx-10.5-ppc',
                           'nibabel-py2.6-osx-10.5-intel',
                           'nibabel-py3.2',
                           'nibabel-py3.3',
                           'nibabel-py2.7-pyd0.9.7',
                           'nibabel-py2.7-pyd0.9.8',
                          ]),
    nitime_bot.scheduler(['nitime-py2.6',
                          'nitime-py2.6-32',
                          'nitime-py2.6-xp',
                          'nitime-py2.7-osx-10.4-ppc',
                          'nitime-py2.7-osx-10.6',
                          'nitime-py2.7-osx-10.8',
                          'nitime-py2.6-osx-10.5-ppc',
                          'nitime-py2.6-osx-10.5-intel',
                         ]),
    nipy_bot.scheduler(['nipy-py2.6',
                        'nipy-py2.6-32',
                        'nipy-py2.6-xp',
                        'nipy-py2.6-sparc',
                        'nipy-py2.7-ppc',
                        'nipy-py2.6-arm',
                        'nipy-py2.7-osx-10.4-ppc',
                        'nipy-py2.7-osx-10.6',
                        'nipy-py2.7-osx-10.8',
                        'nipy-py2.6-osx-10.5-ppc',
                        'nipy-py2.6-osx-10.5-intel',
                        'nipy-py3.2',
                        'nipy-py3.3',
                        'nipy-py2.7-sym0.6.6-nib1.2.0',
                       ]),
    dipy_bot.scheduler(['dipy-py2.6',
                        'dipy-py2.6-32',
                        'dipy-py2.6-xp',
                        'dipy-py2.7-osx-10.6',
                        'dipy-py2.7-osx-10.8',
                        'dipy-py2.6-osx-10.5-ppc',
                        'dipy-py2.6-osx-10.5-intel',
                        'dipy-py3.2',
                        'dipy-py3.3',
                       ]),
    nipype_bot.scheduler(['nipype-py2.6',
                          'nipype-py2.6-32',
                          'nipype-py2.6-xp',
                          'nipype-py2.7-osx-10.6',
                          'nipype-py2.7-osx-10.8',
                          'nipype-py2.7-osx-10.4-ppc',
                          'nipype-py2.6-osx-10.5-ppc',
                          'nipype-py2.6-osx-10.5-intel',
                       ]),
    regreg_bot.scheduler(['regreg-py2.6',
                          'regreg-py2.6-32',
                          'regreg-py2.6-xp',
                          'regreg-py2.7-osx-10.6',
                          'regreg-py2.7-osx-10.8',
                          'regreg-py2.7-osx-10.4-ppc',
                          'regreg-py2.6-osx-10.5-ppc',
                          'regreg-py2.6-osx-10.5-intel',
                       ]),
    pandas_bot.scheduler([
                        'pandas-py2.x-sid-sparc',
                        'pandas-py3.x-sid-sparc',
                        'pandas-py2.x-wheezy-sparc',
                        'pandas-py3.x-wheezy-sparc',
                        ]),
    statsmodels_bot.scheduler([
                        'statsmodels-py2.x-sid-sparc',
                        # not yet
                        #'statsmodels-py3.x-sid-sparc',
                        'statsmodels-py2.x-wheezy-sparc',
                        #'statsmodels-py3.x-wheezy-sparc',
                        ]),
    nipy_bot.nightly_scheduler(['nipy-doc-builder'], 'doc', 2, 01, True),
    nibabel_bot.nightly_scheduler(['nibabel-doc-builder'], 'doc', 2, 20, True),
    pandas_bot.nightly_scheduler(['pandas-doc-builder'], 'doc', 2, 20, True),
    statsmodels_bot.nightly_scheduler(['statsmodels-doc-builder'], 'doc', 3, 20, True),
    # Windows 32 nightly builds
    nipy_bot.nightly_scheduler(['nipy-bdist32-26'], 'exe-26', 2, 10, True),
    nipy_bot.nightly_scheduler(['nipy-bdist32-27'], 'exe-27', 2, 20, True),
    nipy_bot.nightly_scheduler(['nipy-bdist32-32'], 'exe-32', 2, 30, True),
    nibabel_bot.nightly_scheduler(['nibabel-bdist32-26'],
                                  'exe-26', 2, 40, True),
    nibabel_bot.nightly_scheduler(['nibabel-bdist32-32'],
                                  'exe-32', 2, 50, True),
    # Windows 64 nightly builds
    nibabel_bot.nightly_scheduler(['nibabel-bdist64-26'],
                                  'exe64-26', 2, 00, True),
    # OSX nightlies
    nipy_bot.nightly_scheduler(['nipy-bdist-mpkg-2.6'], 'egg-26', 2, 10, True),
    nipy_bot.nightly_scheduler(['nipy-bdist-mpkg-2.7'], 'egg-27', 2, 20, True),
    nipy_bot.nightly_scheduler(['nipy-bdist-mpkg-3.3'], 'egg-33', 2, 30, True),
    # Dipy
    # Windows 32 nightly builds
    dipy_bot.nightly_scheduler(['dipy-bdist32-26'], 'dipy-exe-26', 3, 10, True),
    dipy_bot.nightly_scheduler(['dipy-bdist32-27'], 'dipy-exe-27', 3, 20, True),
    dipy_bot.nightly_scheduler(['dipy-bdist32-32'], 'dipy-exe-32', 3, 30, True),
    # OSX nightlies
    dipy_bot.nightly_scheduler(['dipy-bdist-mpkg-2.6'], 'dipy-egg-26', 3, 10, True),
    dipy_bot.nightly_scheduler(['dipy-bdist-mpkg-2.7'], 'dipy-egg-27', 3, 20, True),
    dipy_bot.nightly_scheduler(['dipy-bdist-mpkg-3.3'], 'dipy-egg-33', 3, 30, True),
]

# Some factories

def factory_buildout_nibabel(python='python2.6',numpy='1.6.1'):
    f = BuildFactory()
    # check out the source
    # XXX: the mastersrc should make appropriate changes based on numpy version
    f.addStep(Git(repourl='git://github.com/nipy/nibabel.git', mode='copy'))
    f.addStep(FileDownload(mastersrc="bootstrap.py",
                                          slavedest="bootstrap.py"))
    f.addStep(FileDownload(mastersrc="nibabel_buildout.cfg",
                                          slavedest="buildout.cfg"))
    f.addStep(ShellCommand(command=[python, "bootstrap.py"]))
    f.addStep(ShellCommand(command=["bin/buildout"]))
    f.addStep(ShellCommand(nptest_command('nibabel',python='bin/buildoutpython')))
    # run the tests (note that this will require that 'nosetests' is installed)
    #f.addStep(nosetests_command('nibabel', doctests=True))
    return f


def shell_cmd_factory(git_url, cmds, posix=True):
    # Run a series of commands over a git repo
    factory = BuildFactory()
    factory.addStep(Git(
        repourl=git_url,
        mode='copy', retry=(15, 5)))
    for cmd in cmds:
        factory.addStep(ShellCommand(command=shlex.split(cmd, posix=posix)))
    return factory


def example_factory(git_url, py_path, out_dir, python_cmd='python'):
    factory = BuildFactory()
    factory.addStep(Git(
        repourl=git_url,
        mode='copy', retry=(15, 5)))
    # Touch c files to make sure build doesn't fail because of c file checkout
    # times
    factory.addStep(ShellCommand(command=shlex.split(
        'python tools/touch_cython_cs.py')))
    # Make virtualenv into which to install
    factory.addStep(
        ShellCommand(command=shlex.split(
            py_path + r'/bin/virtualenv --distribute --system-site-packages venv')
            ))
    # Install into virtualenv
    factory.addStep(
        ShellCommand(command=[python_cmd, 'setup.py', 'install'],
            env={'PATH': WithProperties("%(workdir)s/build/venv/bin:${PATH}")}
            ))
    # Make directory for examples
    factory.addStep(
        ShellCommand(command=['mkdir', 'eg_logs']))
    # Run examples
    factory.addStep(
        ShellCommand(command=[
            python_cmd,
            '../tools/run_log_examples.py',
            '../examples',
            '--log-path=../eg_logs'],
            workdir=r'build/venv',
            env={'PATH': WithProperties("%(workdir)s/build/venv/bin:${PATH}")}
            ))
    # Upload logs
    factory.addStep(DirectoryUpload(slavesrc="eg_logs",
            masterdest="public_html/" + out_dir))
    return factory

####### BUILDERS

# The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
# what steps, and which slaves can execute them.  Note that any particular build will
# only take place on one slave.

c['builders'] = []
# nibabel exe builders
nibabel_test_cmd = shlex.split('nosetests --verbosity=3 --with-doctest nibabel')
c['builders'].append(BuilderConfig(
    name='nibabel-bdist32-26',
    slavenames=xp_32_slaves,
    factory=nibabel_bot.bdist_factory(
        'exe',
        test_cmd = nibabel_test_cmd,
        python = r'c:\Python26\python.exe')
))
# The Python32 numpy 1.6.2 installation needs patching according to
# git diff ad9c2f4~1..ad9c2f4
# to remove use of __stdout__ and __stderr__ in numpy distutils
c['builders'].append(BuilderConfig(
    name='nibabel-bdist32-32',
    slavenames=xp_32_slaves,
    factory=nibabel_bot.bdist_factory(
        'exe',
        test_cmd = nibabel_test_cmd,
        python = r'c:\Python32\python.exe')
))
c['builders'].append(BuilderConfig(
    name='nibabel-bdist64-26',
    slavenames=win7_64_slaves,
    factory=nibabel_bot.bdist_factory(
        'exe',
        test_cmd = nibabel_test_cmd,
        python = r'c:\Python26\python.exe')
))
# nipy exe builders
nipy_test_cmd = nptest_command('nipy', doctests=True)
c['builders'].append(BuilderConfig(
    name='nipy-bdist32-26',
    slavenames=xp_32_slaves,
    factory=nipy_bot.bdist_factory(
        'exe',
        test_cmd = nipy_test_cmd,
        python = r'c:\Python26\python.exe',
        pre_cmds = [r'python tools\touch_cython_cs.py'])
))
c['builders'].append(BuilderConfig(
    name='nipy-bdist32-27',
    slavenames=xp_32_slaves,
    factory=nipy_bot.bdist_factory(
        'exe',
        test_cmd = nipy_test_cmd,
        python = r'c:\Python27\python.exe',
        pre_cmds = [r'python tools\touch_cython_cs.py'])
))
c['builders'].append(BuilderConfig(
    name='nipy-bdist32-32',
    slavenames=xp_32_slaves,
    factory=nipy_bot.bdist_factory(
        'exe',
        test_cmd = nipy_test_cmd,
        python = r'c:\Python32\python.exe',
        pre_cmds = [r'python tools\touch_cython_cs.py'])
))
# Add bdist builders for OSX
# Need exe because easy_install makes tests executable on 3.3
nipy_test_cmd = nptest_command('nipy', doctests=True, exe=True)
c['builders'].append(BuilderConfig(
    name='nipy-bdist-mpkg-2.6',
    slavenames=['osx-10.6'],
    factory=nipy_bot.bdist_factory(
        'egg',
        test_cmd=nipy_test_cmd,
        python='/Library/Frameworks/Python.framework/Versions/2.6/bin/python')
))
c['builders'].append(BuilderConfig(
    name='nipy-bdist-mpkg-2.7',
    slavenames=['osx-10.6'],
    factory=nipy_bot.bdist_factory(
        'egg',
        test_cmd=nipy_test_cmd,
        python='/Library/Frameworks/Python.framework/Versions/2.7/bin/python')
))
c['builders'].append(BuilderConfig(
    name='nipy-bdist-mpkg-3.3',
    slavenames=['osx-10.6'],
    factory=nipy_bot.bdist_factory(
        'egg',
        test_cmd=nipy_test_cmd,
        python='/Library/Frameworks/Python.framework/Versions/3.3/bin/python3')
))
##############################
# Dipy builders
##############################
win_dipy_test_cmd = ['python',  r'..\tools\dipnost', 'dipy']
c['builders'].append(BuilderConfig(
    name='dipy-bdist32-26',
    slavenames=xp_32_slaves,
    factory=dipy_bot.bdist_factory(
        'exe',
        test_cmd=win_dipy_test_cmd,
        python=r'c:\Python26\python.exe')
))
c['builders'].append(BuilderConfig(
    name='dipy-bdist32-27',
    slavenames=xp_32_slaves,
    factory=dipy_bot.bdist_factory(
        'exe',
        test_cmd=win_dipy_test_cmd,
        python=r'c:\Python27\python.exe')
))
c['builders'].append(BuilderConfig(
    name='dipy-bdist32-32',
    slavenames=xp_32_slaves,
    factory=dipy_bot.bdist_factory(
        'exe',
        test_cmd=win_dipy_test_cmd,
        python=r'c:\Python32\python.exe')
))
# Add bdist builders for OSX
# Each of the tested _system_ pythons will need:
# * numpy, scipy, virtualenv, bdist_mpkg
osx_dipy_test_cmd = ['python',  '../tools/dipnost', '--exe', '--verbosity=3', 'dipy']
c['builders'].append(BuilderConfig(
    name='dipy-bdist-mpkg-2.6',
    slavenames=['osx-10.6'],
    factory=dipy_bot.bdist_factory(
        'egg',
        test_cmd=osx_dipy_test_cmd,
        python='/Library/Frameworks/Python.framework/Versions/2.6/bin/python')
))
c['builders'].append(BuilderConfig(
    name='dipy-bdist-mpkg-2.7',
    slavenames=['osx-10.6'],
    factory=dipy_bot.bdist_factory(
        'egg',
        test_cmd=osx_dipy_test_cmd,
        python='/Library/Frameworks/Python.framework/Versions/2.7/bin/python')
))
c['builders'].append(BuilderConfig(
    name='dipy-bdist-mpkg-3.3',
    slavenames=['osx-10.6'],
    factory=dipy_bot.bdist_factory(
        'egg',
        test_cmd=osx_dipy_test_cmd,
        python='/Library/Frameworks/Python.framework/Versions/3.3/bin/python3')
))
# Add nightly docs builders
c['builders'] += nipy_bot.build_doc_builders(
    (('nipy-doc-builder', linux_64_slaves),),
    doc_doctest_cmd = "cd doc && make doctest"
)
c['builders'] += nibabel_bot.build_doc_builders(
    (('nibabel-doc-builder', linux_64_slaves),))
c['builders'] += pandas_bot.build_doc_builders(
    (('pandas-doc-builder', sparc_slaves_sid),))
c['builders'] += statsmodels_bot.build_doc_builders(
    (('statsmodels-doc-builder', sparc_slaves_sid),))
# pre-release builders
c['builders'].append(BuilderConfig(
    name='nipy-release-checks',
    slavenames=['osx-10.6'],
    factory=shell_cmd_factory(
        'git://github.com/nipy/nipy.git',
        ('make distclean',
         'python -m compileall .',
         'make sdist-tests',
         'make bdist-egg-tests',
         'make check-version-info',
         'make check-files'))))
c['builders'].append(BuilderConfig(
    name='dipy-release-checks',
    slavenames=['osx-10.6'],
    factory=shell_cmd_factory(
        'git://github.com/nipy/dipy.git',
        ('make distclean',
         'python -m compileall .',
         'make sdist-tests',
         'make bdist-egg-tests',
         'make check-version-info',
         'make check-files'))))
# Example builders
c['builders'].append(BuilderConfig(
    name='nipy-examples-2.6',
    slavenames=['osx-10.6'],
    factory=example_factory(
        'git://github.com/nipy/nipy.git',
        '/Library/Frameworks/Python.framework/Versions/2.6',
        "nipy-examples-2.6")
))
c['builders'].append(BuilderConfig(
    name='nipy-examples-3.3',
    slavenames=['osx-10.6'],
    factory=example_factory(
        'git://github.com/nipy/nipy.git',
        '/Library/Frameworks/Python.framework/Versions/3.3',
        "nipy-examples-3.3",
        "python3")
))

# Code builders
c['builders'] += nibabel_bot.build_builders(
    (('nibabel-py2.6', linux_64_slaves),
     ('nibabel-py2.6-32', linux_32_slaves),
     ('nibabel-py2.7-osx-10.4-ppc', osx_tiger_ppc_slaves),
     ('nibabel-py2.7-osx-10.6', osx_snowleopard_slaves),
     ('nibabel-py2.7-osx-10.8', osx_mountainlion_slaves),
     ('nibabel-py2.6-osx-10.5-ppc', osx_leopard_ppc_slaves),
     ('nibabel-py2.6-osx-10.5-intel', osx_leopard_intel_slaves),
     ('nibabel-py2.7-ppc', debian_ppc_slaves),
     ('nibabel-py2.6-arm', arm_slaves),
     ('nibabel-py2.6-xp', xp_32_slaves),
     ('nibabel-py2.7-win7', win7_64_slaves),
     ('nibabel-py2.6-sparc', sparc_slaves)))
# Python 3.2 into virtualenv
c['builders'] += nibabel_bot.build_builders(
    (('nibabel-py3.2', linux_64_slaves),),
    python='python3.2')
c['builders'] += nibabel_bot.build_builders(
    (('nibabel-py3.3', osx_snowleopard_slaves),),
    python='python3.3')
# Different versions of Pydicom
c['builders'] += nibabel_bot.build_builders(
    (('nibabel-py2.7-pyd0.9.7', osx_mountainlion_slaves),),
    python='python2.7',
    easy_depends = ('nose', 'pydicom==0.9.7'))
c['builders'] += nibabel_bot.build_builders(
    (('nibabel-py2.7-pyd0.9.8', osx_mountainlion_slaves),),
    python='python2.7',
    easy_depends = ('nose', 'pydicom==0.9.8'))
# Nitime
c['builders'] += nitime_bot.build_builders(
    (('nitime-py2.6', linux_64_slaves),
     ('nitime-py2.6-32', linux_32_slaves),
     ('nitime-py2.7-osx-10.4-ppc', osx_tiger_ppc_slaves),
     ('nitime-py2.7-osx-10.6', osx_snowleopard_slaves),
     ('nitime-py2.7-osx-10.8', osx_mountainlion_slaves),
     ('nitime-py2.6-osx-10.5-ppc', osx_leopard_ppc_slaves),
     ('nitime-py2.6-osx-10.5-intel', osx_leopard_intel_slaves),
     ('nitime-py2.6-arm', arm_slaves),
     ('nitime-py2.6-xp', xp_32_slaves)))
c['builders'] += nitime_arokem_bot.build_builders(
    (('nitime-py2.6-arokem', linux_64_slaves),
     ('nitime-py2.6-arokem-32', linux_32_slaves),
     ('nitime-py2.7-arokem-osx-10.4-ppc', osx_tiger_ppc_slaves),
     ('nitime-py2.7-arokem-osx-10.6', osx_snowleopard_slaves),
     ('nitime-py2.6-arokem-osx-10.5-ppc', osx_leopard_ppc_slaves),
     ('nitime-py2.6-arokem-osx-10.5-intel', osx_leopard_intel_slaves),
     ('nitime-py2.6-arokem-xp', xp_32_slaves)))
c['builders'] += nipy_bot.build_builders(
    (('nipy-py2.6', linux_64_slaves),
     ('nipy-py2.6-32', linux_32_slaves),
     ('nipy-py2.7-osx-10.4-ppc', osx_tiger_ppc_slaves),
     ('nipy-py2.7-osx-10.6', osx_snowleopard_slaves),
     ('nipy-py2.7-osx-10.8', osx_mountainlion_slaves),
     ('nipy-py2.6-osx-10.5-ppc', osx_leopard_ppc_slaves),
     ('nipy-py2.6-osx-10.5-intel', osx_leopard_intel_slaves),
     ('nipy-py2.7-ppc', debian_ppc_slaves),
     ('nipy-py2.6-arm', arm_slaves),
     ('nipy-py2.6-xp', xp_32_slaves),
     ('nipy-py2.6-sparc', sparc_slaves)))
# Python 3.2 into virtualenv
c['builders'] += nipy_bot.build_builders(
    (('nipy-py3.2', linux_64_slaves),),
    python='python3.2')
c['builders'] += nipy_bot.build_builders(
    (('nipy-py3.3', osx_snowleopard_slaves),),
    python='python3.3')
# Different versions of Sympy, nibabel
c['builders'] += nipy_bot.build_builders(
    (('nipy-py2.7-sym0.6.6-nib1.2.0', osx_mountainlion_slaves),),
    pip_depends = ('sympy==0.6.6', 'nibabel==1.2.0'))
# The dipy builder seems to fail early sometimes, on OSX, maybe due to:
# http://bugs.python.org/issue8458
c['builders'] += dipy_bot_nox.build_builders(
    (('dipy-py2.6', linux_64_slaves),
     ('dipy-py2.6-32', linux_32_slaves),
))
c['builders'] += dipy_bot.build_builders((
     ('dipy-py2.7-osx-10.6', osx_snowleopard_slaves),
     ('dipy-py2.7-osx-10.8', osx_mountainlion_slaves),
     ('dipy-py2.6-osx-10.5-ppc', osx_leopard_ppc_slaves),
     ('dipy-py2.6-osx-10.5-intel', osx_leopard_intel_slaves),
     ('dipy-py2.6-arm', arm_slaves),
     ('dipy-py2.6-xp', xp_32_slaves),
     ('dipy-py2.6-sparc', sparc_slaves)))
# Python 3.2 into virtualenv
c['builders'] += dipy_bot.build_builders(
    (('dipy-py3.2', linux_64_slaves),),
    python='python3.2')
c['builders'] += dipy_bot.build_builders(
    (('dipy-py3.3', osx_snowleopard_slaves),),
    python='python3.3')
# Nipype
c['builders'] += nipype_bot.build_builders(
    (('nipype-py2.6', linux_64_slaves),
     ('nipype-py2.6-32', linux_32_slaves),
     ('nipype-py2.7-osx-10.6', osx_snowleopard_slaves),
     ('nipype-py2.7-osx-10.8', osx_mountainlion_slaves),
     ('nipype-py2.7-osx-10.4-ppc', osx_tiger_ppc_slaves),
     ('nipype-py2.6-osx-10.5-ppc', osx_leopard_ppc_slaves),
     ('nipype-py2.6-osx-10.5-intel', osx_leopard_intel_slaves),
     ('nipype-py2.6-arm', arm_slaves),
     ('nipype-py2.6-xp', xp_32_slaves),
     ('nipype-py2.6-sparc', sparc_slaves)))
c['builders'] += regreg_bot.build_builders(
    (('regreg-py2.6', linux_64_slaves),
     ('regreg-py2.6-32', linux_32_slaves),
     ('regreg-py2.7-osx-10.6', osx_snowleopard_slaves),
     ('regreg-py2.7-osx-10.8', osx_mountainlion_slaves),
     ('regreg-py2.7-osx-10.4-ppc', osx_tiger_ppc_slaves),
     ('regreg-py2.6-osx-10.5-ppc', osx_leopard_ppc_slaves),
     ('regreg-py2.6-osx-10.5-intel', osx_leopard_intel_slaves),
     ('regreg-py2.6-arm', arm_slaves),
     ('regreg-py2.6-xp', xp_32_slaves),
     ('regreg-py2.6-sparc', sparc_slaves)))
# Pandas - can be tested only on recent debian releases
c['builders'] += pandas_bot.build_builders(
    (('pandas-py2.x-sid-sparc', sparc_slaves_sid),
     ('pandas-py2.x-wheezy-sparc', sparc_slaves_wheezy)))
# and python3 builders
c['builders'] += pandas_bot.build_builders(
    (('pandas-py3.x-sid-sparc', sparc_slaves_sid),
     ('pandas-py3.x-wheezy-sparc', sparc_slaves_wheezy)),
    python='python3')
# Statsmodels - can be tested only on recent debian releases
c['builders'] += statsmodels_bot.build_builders(
    (('statsmodels-py2.x-sid-sparc', sparc_slaves_sid),
     ('statsmodels-py2.x-wheezy-sparc', sparc_slaves_wheezy)))
## and python3 builders
#c['builders'] += statsmodels_bot.build_builders(
#    (('statsmodels-py3.x-sid-sparc', sparc_slaves_sid),
#     ('statsmodels-py3.x-wheezy-sparc', sparc_slaves_wheezy)),
#    python='python3')
""" Disabling these guys; they haven't worked for a while
c['builders'].append(
    BuilderConfig(name="buildout-test2.6",
      slavenames=experimental,
      factory=factory_buildout_nibabel()))
c['builders'].append(
    BuilderConfig(name="buildout-test2.5",
      slavenames=experimental,
      factory=factory_buildout_nibabel('python2.5')))
c['builders'].append(
    BuilderConfig(name="buildout-test2.4",
      slavenames=experimental,
      factory=factory_buildout_nibabel('python2.4')))
"""

# Assign all builders categories -- project names, to improve status visualizations
for b in c['builders']:
    b.category = b.name.split('-')[0]

####### STATUS TARGETS

# 'status' is a list of Status Targets. The results of each build will be
# pushed to these targets. buildbot/status/*.py has a variety to choose from,
# including web pages, email senders, and IRC bots.

c['status'] = []

from buildbot.status import html
from buildbot.status.web import auth, authz

http_auth = auth.HTPasswdAuth('<PASSWORD>')

authz_cfg=authz.Authz(
    auth = http_auth,
    # change any of these to True to enable; 'auth' to require authorization'
    # see the manual for more options
    gracefulShutdown = False,
    forceBuild = 'auth',
    forceAllBuilds = 'auth',
    pingBuilder = False,
    stopBuild = 'auth',
    stopAllBuilds = False,
    cancelPendingBuild = False,
)
c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg,
                                  #change_hook_dialects={ 'base': True, 'github' : True }
                  ))

####### PROJECT IDENTITY
# the 'title' string will appear at the top of this buildbot
# installation's html.WebStatus home page (linked to the
# 'titleURL') and is embedded in the title of the waterfall HTML page.

c['title'] = "NiPy"
c['titleURL'] = "http://nipy.org"

# the 'buildbotURL' string should point to the location where the buildbot's
# internal web server (usually the html.WebStatus page) is visible. This
# typically uses the port number set in the Waterfall 'status' entry, but
# with an externally-visible host name which the buildbot cannot figure out
# without some help.

c['buildbotURL'] = "http://localhost:8010/"

####### DB URL

# This specifies what database buildbot uses to store change and scheduler
# state.  You can leave this at its default for all but the largest
# installations.
c['db_url'] = "sqlite:///state.sqlite"

<commit_msg>BF: adding use of build_timeout when building documentation<commit_after># -*- python -*-
# ex: set syntax=python:
# vim: ft=python
# NIpy buildBOT Master Instance config
#
# [ TODO ]      (in no particular order)
# - combine multiple steps into one (if possible)
# - rename steps to be more concise and informative
# - generate numpy version based templates
# - what's up with the buildout not using ~/.buildout/default.cfg and not caching
#       * we may need to set some properties for this to work
# - install more versions of python on the slaves
# - have slaves report back their configuration (so we have them doing the right builders)
# - if possible, forcebuild every builder on a particular slave (to check that it can)
# - hookup with git post-commit hooks (instead of relying on poller)

""" A brief tour of buildbot

To remind myself

A ChangeSource looks for Changes.

In our case we are using GitPoller ChangeSource to look for changes in our git
repositories.

When ChangeSource finds changes, it broadcasts them to all Schedulers.  Each
Scheduler must decide if it wants to react to a Change.  It generally does this
by filtering changes to select changes it will act on.

We use SingleBranchSchedulers for this.  These obviously respond only to
changes in one branch of a repository.

A BuildSet is the association of (information about what changes need to be
tested) and (Builders on which to test)

If a Scheduler selects changes for action, it sends BuildSets to the system
(the BuildMaster).  Therefore, a Scheduler needs to know its Builders. It gets
these by name (string).

For each BuildSet, the (changes to be tested) are sent to each Builder.  Each
of these requests is BuildRequest.

A Builder accepts a BuildRequest and runs it on a BuildSlave.  It does this by
using its own BuildFactory instance to make Builds.

A Builder can have more than one associated BuildSlave.  In this case each
associated BuildSlave is assumed to produce identical results, and the Builder
can therefore choose at whim which BuildSlave to use.

A Build has one or more Steps that should be run with a given (changes to be
tested).

One type of build Step is a Trigger, which activates a triggerable Scheduler,
which can react any way it likes, including sending further BuildSets.
"""

from os.path import join as pjoin

# This is convenient for splitting sh commands into lists
import shlex

# Buildbot objects
from buildbot.buildslave import BuildSlave
from buildbot.changes.gitpoller import GitPoller
from buildbot.schedulers.filter import ChangeFilter
from buildbot.scheduler import Scheduler
from buildbot.schedulers import timed
from buildbot.process.factory import BuildFactory
from buildbot.steps.source import Git
from buildbot.steps.shell import ShellCommand, SetProperty
from buildbot.steps.transfer import FileDownload, DirectoryUpload
from buildbot.process.properties import WithProperties
from buildbot.config import BuilderConfig

# This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing.
c = BuildmasterConfig = {}

####### BUILDSLAVES

# Get passwords from file not in repository.  `slave_pass.py` defines a dict,
# `slave_passes` with key, value pairs defining (slave_name, slave_password)
import slave_pass as s_p
reload(s_p)
# Check that slave names are unique
if len(set(s_p.slave_passes)) != len(s_p.slave_passes):
    raise ValueError('Slave names must be unique')

# The 'slaves' list defines the set of recognized buildslaves. Each element is
# a BuildSlave object, specifying a unique slave name and password.  The same
# slave name and password must be configured on the slave.
slaves = [BuildSlave(slave_name, slave_pass)
    for slave_name, slave_pass in s_p.slave_passes.items()]

experimental = ['pi-ApxuMed',]

linux_64_slaves = ['tom-standard']
linux_32_slaves = ['bongoslave']
debian_ppc_slaves = ['debian-ppc-32']
sparc_slaves_sid = 'nd-bb-slave-sparc-sid'
sparc_slaves_wheezy = 'nd-bb-slave-sparc-wheezy'
sparc_slaves = ['nd-bb-slave-sparc']
osx_leopard_ppc_slaves = ['osx-10.5-ppc']
osx_tiger_ppc_slaves = ['osx-mini']
osx_leopard_intel_slaves = ['osx-10.5']
osx_snowleopard_slaves = ['osx-10.6']
osx_mountainlion_slaves = ['osx-10.8']
xp_32_slaves = ['xp-mini']
win7_64_slaves = ['mike-win7-64']
arm_slaves = ['arm-raspberry-pi']

# slave names must be unique
c['slaves'] = slaves

# 'slavePortnum' defines the TCP port to listen on for connections from slaves.
# This must match the value configured into the buildslaves (with their
# --master option)
c['slavePortnum'] = 9989

# Helper functions and constants
class GithubBot(object):

    need_nose = True

    def __init__(self,
                 organization,
                 repo,
                 test_cmd,
                 doc_build_cmd=None,
                 built_doc_src=None,
                 easy_depends=(),
                 pip_depends=(),
                 doc_easy_depends=('sphinx',),
                 src_subdir=None,
                 download_dir=None,
                 build_timeout=None):
        self.organization = organization
        self.repo = repo
        self.git_url = "git://github.com/%s/%s.git" % (organization, repo)
        self.test_cmd = test_cmd
        self.doc_build_cmd = doc_build_cmd
        self.built_doc_src = built_doc_src
        # Dependencies for install / test
        self.easy_depends = easy_depends # easy_install installs
        self.pip_depends = pip_depends # pip
        self.doc_easy_depends = doc_easy_depends # easy_installs for docs
        self.src_subdir = src_subdir # Subdirectory containing source
        if download_dir is None:
            download_dir = '%s-dist' % repo
        self.download_dir = download_dir
        self.build_timeout = build_timeout

    @property
    def project(self):
        return "%s" % (self.repo,)

    def poller(self, branch="master", interval=300):
        return GitPoller(
            self.git_url,
            project=self.project,
            category=self.project,
            workdir = "%s-%s-%s" % (self.organization, self.repo, branch),
            branch=branch,
            pollinterval=interval)

    def scheduler(self, buildernames):
        if isinstance(buildernames, basestring):
            buildernames = [buildernames]
        name = self.project
        changefilter = ChangeFilter(repository=self.git_url,
                                    project=self.project,
                                    category=self.project)
        return Scheduler(name=name,
                        change_filter=changefilter,
                        treeStableTimer=None,
                        builderNames=buildernames)

    def nightly_scheduler(self,
                          buildernames,
                          name_suff,
                          hour = 2, minute = 23,
                          ifchanged=True
                         ):
        """ Make a timed scheduler for this repo """
        if isinstance(buildernames, basestring):
            buildernames = [buildernames]
        name = "%s-%s-nightly-%s" % (self.organization, self.repo, name_suff)
        changefilter = ChangeFilter(repository=self.git_url,
                                    project=self.project,
                                    category=self.project)
        return timed.Nightly(name=name,
                branch="master",
                builderNames=buildernames,
                change_filter=changefilter,
                hour=hour,
                minute=minute,
                onlyIfChanged=ifchanged)

    def _code_build_factory(self,
                            python=None,
                            easy_depends=None,
                            pip_depends=None,
                            pre_cmds=()):
        if easy_depends is None:
            easy_depends = self.easy_depends
        if pip_depends is None:
            pip_depends = self.pip_depends
        factory = BuildFactory()
        factory.addStep(Git(repourl=self.git_url, mode='copy', retry=(15, 5)))
        # Any pre-building commands
        for cmd in pre_cmds:
            factory.addStep(ShellCommand(command=shlex.split(cmd, posix=False)))
        # Obviously requires virtualenv on the slave
        venv_params = ['--distribute', '--system-site-packages', 'venv']
        if not python is None:
            venv_params = ['--python=' + python] + venv_params
        factory.addStep(ShellCommand(command=['virtualenv'] + venv_params))
        # Now we need the venv binary directory
        factory.addStep(FileDownload('slave_scripts/bs_tools.py',
                                     'bs_tools.py'))
        factory.addStep(SetProperty(command=shlex.split(
            'python bs_tools.py path-prepend-str venv'),
            property='venv_script_dir'))
        factory.addStep(SetProperty(command=shlex.split(
            'python bs_tools.py abspath venv'),
            property='venv_dir'))
        # Set env, with virtualenv in front of the path
        my_env={'PATH': WithProperties("%(venv_script_dir)s${PATH}")}
        workdir = WithProperties("%(venv_dir)s")
        # Add install and runtime dependencies
        for pkg_spec in easy_depends:
            factory.addStep(ShellCommand(command=[
                        'easy_install', pkg_spec], env=my_env))
        for pkg_spec in pip_depends:
            factory.addStep(ShellCommand(command=[
                        'pip', 'install', pkg_spec], env=my_env))
        # Work out working directory for install
        if not self.src_subdir is None:
            factory.addStep(SetProperty(command=[
                'python', 'bs_tools.py', 'abspath', self.src_subdir],
                property='src_path'))
            build_workdir = WithProperties("%(src_path)s")
        else:
            build_workdir = None
        return factory, my_env, workdir, build_workdir

    def factory(self,
                test_cmd=None,
                python=None,
                easy_depends=None,
                pip_depends=None,
                pre_cmds=()):
        if test_cmd is None:
            test_cmd = self.test_cmd
        factory, env, workdir, build_workdir = self._code_build_factory(
            python, easy_depends, pip_depends, pre_cmds)
        # Additional options for ShellCommands, if provided.  To not
        # rely on knowing internal logic (either handles None as the
        # default value), we explicitly construct a dictionary only
        # with the specified options
        kwargs = {}
        if self.build_timeout:
            kwargs['timeout'] = self.build_timeout
        # Install into virtualenv
        factory.addStep(ShellCommand(
            command=['python', 'setup.py', 'install'],
            env=env,
            workdir=build_workdir,
            **kwargs))
        # Test
        factory.addStep(ShellCommand(command=test_cmd,
            env=env, workdir=workdir, **kwargs))
        return factory

    def bdist_factory(self,
                      bdist_type,
                      test_cmd=None,
                      python=None,
                      easy_depends=None,
                      pip_depends=None,
                      pre_cmds=()):
        if test_cmd is None:
            test_cmd = self.test_cmd
        factory, env, workdir, build_workdir = self._code_build_factory(
            python, easy_depends, pip_depends, pre_cmds)
        # Build binary format(s)
        if python is None: # Use virtualenv python
            my_python = 'python'
            build_env = env
        else: # Use specified python
            my_python = python
            # Insert virtualenv into PYTHONPATH for building using virtualenv
            # dependencies
            factory.addStep(SetProperty(command=shlex.split(
                'python bs_tools.py get-python-lib'),
                property='venv_python_lib', env=env))
            build_env = env.copy()
            build_env['PYTHONPATH'] = WithProperties("%(venv_python_lib)s")
        if bdist_type == 'egg':
            factory.addStep(ShellCommand(
                command=[my_python, 'setup.py', 'bdist_egg'],
                env=build_env,
                workdir=build_workdir))
            factory.addStep(ShellCommand(
                command=[my_python, 'setup.py', 'bdist_mpkg'],
                env=build_env,
                workdir=build_workdir))
            factory.addStep(SetProperty(command=shlex.split(
                'python bs_tools.py glob dist "*.egg"'),
                property='bdist_file'))
        elif bdist_type == 'exe':
            factory.addStep(ShellCommand(
                command=[my_python, 'setup.py', 'bdist', '--formats=wininst'],
                env=build_env,
                workdir=build_workdir))
            factory.addStep(SetProperty(command=shlex.split(
                'python bs_tools.py glob dist "*.exe"'),
                property='bdist_file'))
        else:
            raise ValueError('What is this?: ' + bdist_type)
        # Install into virtualenv
        factory.addStep(ShellCommand(
            command=[ 'easy_install', WithProperties('%(bdist_file)s')],
            env=env))
        # Test
        factory.addStep(ShellCommand(command=test_cmd,
            env=env, workdir=workdir))
        # Upload
        factory.addStep(DirectoryUpload(slavesrc="dist",
                masterdest="public_html/" + self.download_dir))
        return factory

    def doc_factory(self,
                    doc_build_cmd=None,
                    built_doc_src=None,
                    doc_doctest_cmd=None):
        if doc_build_cmd is None:
            doc_build_cmd = self.doc_build_cmd
        if built_doc_src is None:
            built_doc_src = self.built_doc_src
        if None in (doc_build_cmd, built_doc_src):
            raise ValueError("Need defined build command and doc source")
        factory, env, workdir, build_workdir = self._code_build_factory()
        # Install into virtualenv
        kwargs = {}
        if self.build_timeout:
            kwargs['timeout'] = self.build_timeout
        factory.addStep(ShellCommand(
            command=['python', 'setup.py', 'install'],
            env=env,
            workdir=build_workdir,
            **kwargs))
        for pkg_spec in self.doc_easy_depends:
            factory.addStep(ShellCommand(command=[
                        'easy_install', pkg_spec], env=env))
        if not doc_doctest_cmd is None:
            factory.addStep(ShellCommand(command=doc_doctest_cmd, env=env))
        factory.addStep(ShellCommand(command=doc_build_cmd, env=env))
        factory.addStep(DirectoryUpload(slavesrc=built_doc_src,
                    masterdest="public_html/nightly/" + self.repo + '/doc'))
        return factory

    def build_builders(self, name_slavenames,
                       test_cmd=None,
                       python=None,
                       easy_depends=None,
                       pip_depends=None):
        factory = self.factory(test_cmd, python, easy_depends, pip_depends)
        return self._factory_builders(factory, name_slavenames)

    def _factory_builders(self, factory, name_slavenames):
        builders = []
        for name, slavelist in name_slavenames:
            if isinstance(slavelist, basestring):
                slavelist = [slavelist]
            builders.append(BuilderConfig(
                name=name,
                slavenames=slavelist,
                factory=factory))
        return builders

    def build_doc_builders(self,
            name_slavenames,
            doc_build_cmd=None,
            built_doc_src=None,
            doc_doctest_cmd=None):
        factory = self.doc_factory(doc_build_cmd, built_doc_src, doc_doctest_cmd)
        return self._factory_builders(factory, name_slavenames)


def nptest_command(pkg_name,
                   verbose=10,
                   doctests=False,
                   python='python',
                   cd=False,
                   stdout=False,
                   exe=False):
    """ Run tests from new directory """
    pystr = ''
    if cd:
        pystr += "import os; os.chdir('..');"
    pystr += ("import sys; import %s; sys.exit(not %s.test(doctests=%s"
              % (pkg_name, pkg_name, doctests))
    if verbose != None:
        pystr += ', verbose=%d' % verbose
    if exe:
        pystr += ', extra_argv=["--exe"]'
    pystr += ").wasSuccessful())"
    return [python, '-c', "%s" % pystr]


nibabel_bot = GithubBot('nipy', 'nibabel',
    test_cmd = shlex.split('nosetests --verbosity=3 --exe --with-doctest nibabel'),
    doc_build_cmd = "cd doc && make html",
    built_doc_src = "build/html",
    easy_depends = ('nose',)
)
nitime_bot = GithubBot('nipy', 'nitime',
    test_cmd = nptest_command('nitime', verbose=None),
    easy_depends = ('nose',)
)
nitime_arokem_bot = GithubBot('arokem','nitime',
    test_cmd = nptest_command('nitime', verbose=None),
    easy_depends = ('nose',)
)
nipy_bot = GithubBot('nipy', 'nipy',
    test_cmd = nptest_command('nipy', doctests=True),
    doc_build_cmd = "cd doc && make dist",
    built_doc_src = "doc/dist",
    easy_depends = ('nose',),
    pip_depends = ('sympy', 'nibabel')
)
dipy_bot = GithubBot('nipy', 'dipy',
    test_cmd = shlex.split('nosetests --verbosity=3 --exe --with-doctest dipy'),
    easy_depends = ('nose',),
    pip_depends = ('cython>=0.17', 'nibabel')
)
# dipy bot for linux machines with optional easy_depends that need an X window
# for testing
dipy_bot_nox = GithubBot('nipy', 'dipy',
    test_cmd = shlex.split('xvfb-run --auto-servernum --server-args="-screen 0 1024x768x24" nosetests --verbosity=3 --with-doctest dipy'),
    easy_depends = ('nose', 'nibabel')
)
nipype_bot = GithubBot('nipy', 'nipype',
    test_cmd = nptest_command('nipype', doctests=True),
    easy_depends = ('nose', 'nibabel', 'networkx', 'Traits')
)
regreg_bot = GithubBot('regreg', 'regreg',
    test_cmd = shlex.split('nosetests --verbosity=3 --exe --with-doctest regreg'),
    easy_depends = ('nose',),
    src_subdir = 'code'
)
# pandas bot, primarily for sparc testing
pandas_bot = GithubBot('pydata', 'pandas',
    test_cmd = shlex.split('xvfb-run --auto-servernum --server-args="-screen 0 1024x768x24" nosetests --verbosity=3 --exe pandas'),
    doc_build_cmd = "make doc",
    built_doc_src = "doc/dist",
    easy_depends = ('nose',),
    pip_depends = ('apiclient',),
    build_timeout = 3600, # allow for its longer builds/testing especially for sparcs
)
# statsmodels bot, primarily for sparc testing
statsmodels_bot = GithubBot('statsmodels', 'statsmodels',
    test_cmd = shlex.split('xvfb-run --auto-servernum --server-args="-screen 0 1024x768x24" nosetests --verbosity=3 --exe statsmodels'),
    doc_build_cmd = "make -C docs html",
    built_doc_src = "docs/build",
    easy_depends = ('nose',),
    pip_depends = ('patsy>=0.1.0',),
)
####### CHANGESOURCES

# the 'change_source' setting tells the buildmaster how it should find out
# about source code changes.  Here we point to the buildbot clone of nibabel.

#TODO: We should probably change the GitPoller to a PBChangeSource to react to
# commits http://buildbot.net/buildbot/docs/latest/manual/cfg-changesources.html#chsrc-PBChangeSource
c['change_source'] = [bot.poller() for bot in (nibabel_bot,
                                               nitime_bot,
                                               nipy_bot,
                                               dipy_bot,
                                               nipype_bot,
                                               regreg_bot,
                                               pandas_bot,
                                               statsmodels_bot,
                                              )]


####### SCHEDULERS
# Configure the Schedulers, which decide how to react to incoming changes.

c['schedulers'] = [
    nibabel_bot.scheduler(['nibabel-py2.6',
                           'nibabel-py2.6-32',
                           'nibabel-py2.6-xp',
                           'nibabel-py2.7-win7',
                           'nibabel-py2.6-sparc',
                           'nibabel-py2.7-ppc',
                           'nibabel-py2.6-arm',
                           'nibabel-py2.7-osx-10.4-ppc',
                           'nibabel-py2.7-osx-10.6',
                           'nibabel-py2.7-osx-10.8',
                           'nibabel-py2.6-osx-10.5-ppc',
                           'nibabel-py2.6-osx-10.5-intel',
                           'nibabel-py3.2',
                           'nibabel-py3.3',
                           'nibabel-py2.7-pyd0.9.7',
                           'nibabel-py2.7-pyd0.9.8',
                          ]),
    nitime_bot.scheduler(['nitime-py2.6',
                          'nitime-py2.6-32',
                          'nitime-py2.6-xp',
                          'nitime-py2.7-osx-10.4-ppc',
                          'nitime-py2.7-osx-10.6',
                          'nitime-py2.7-osx-10.8',
                          'nitime-py2.6-osx-10.5-ppc',
                          'nitime-py2.6-osx-10.5-intel',
                         ]),
    nipy_bot.scheduler(['nipy-py2.6',
                        'nipy-py2.6-32',
                        'nipy-py2.6-xp',
                        'nipy-py2.6-sparc',
                        'nipy-py2.7-ppc',
                        'nipy-py2.6-arm',
                        'nipy-py2.7-osx-10.4-ppc',
                        'nipy-py2.7-osx-10.6',
                        'nipy-py2.7-osx-10.8',
                        'nipy-py2.6-osx-10.5-ppc',
                        'nipy-py2.6-osx-10.5-intel',
                        'nipy-py3.2',
                        'nipy-py3.3',
                        'nipy-py2.7-sym0.6.6-nib1.2.0',
                       ]),
    dipy_bot.scheduler(['dipy-py2.6',
                        'dipy-py2.6-32',
                        'dipy-py2.6-xp',
                        'dipy-py2.7-osx-10.6',
                        'dipy-py2.7-osx-10.8',
                        'dipy-py2.6-osx-10.5-ppc',
                        'dipy-py2.6-osx-10.5-intel',
                        'dipy-py3.2',
                        'dipy-py3.3',
                       ]),
    nipype_bot.scheduler(['nipype-py2.6',
                          'nipype-py2.6-32',
                          'nipype-py2.6-xp',
                          'nipype-py2.7-osx-10.6',
                          'nipype-py2.7-osx-10.8',
                          'nipype-py2.7-osx-10.4-ppc',
                          'nipype-py2.6-osx-10.5-ppc',
                          'nipype-py2.6-osx-10.5-intel',
                       ]),
    regreg_bot.scheduler(['regreg-py2.6',
                          'regreg-py2.6-32',
                          'regreg-py2.6-xp',
                          'regreg-py2.7-osx-10.6',
                          'regreg-py2.7-osx-10.8',
                          'regreg-py2.7-osx-10.4-ppc',
                          'regreg-py2.6-osx-10.5-ppc',
                          'regreg-py2.6-osx-10.5-intel',
                       ]),
    pandas_bot.scheduler([
                        'pandas-py2.x-sid-sparc',
                        'pandas-py3.x-sid-sparc',
                        'pandas-py2.x-wheezy-sparc',
                        'pandas-py3.x-wheezy-sparc',
                        ]),
    statsmodels_bot.scheduler([
                        'statsmodels-py2.x-sid-sparc',
                        # not yet
                        #'statsmodels-py3.x-sid-sparc',
                        'statsmodels-py2.x-wheezy-sparc',
                        #'statsmodels-py3.x-wheezy-sparc',
                        ]),
    nipy_bot.nightly_scheduler(['nipy-doc-builder'], 'doc', 2, 01, True),
    nibabel_bot.nightly_scheduler(['nibabel-doc-builder'], 'doc', 2, 20, True),
    pandas_bot.nightly_scheduler(['pandas-doc-builder'], 'doc', 2, 20, True),
    statsmodels_bot.nightly_scheduler(['statsmodels-doc-builder'], 'doc', 3, 20, True),
    # Windows 32 nightly builds
    nipy_bot.nightly_scheduler(['nipy-bdist32-26'], 'exe-26', 2, 10, True),
    nipy_bot.nightly_scheduler(['nipy-bdist32-27'], 'exe-27', 2, 20, True),
    nipy_bot.nightly_scheduler(['nipy-bdist32-32'], 'exe-32', 2, 30, True),
    nibabel_bot.nightly_scheduler(['nibabel-bdist32-26'],
                                  'exe-26', 2, 40, True),
    nibabel_bot.nightly_scheduler(['nibabel-bdist32-32'],
                                  'exe-32', 2, 50, True),
    # Windows 64 nightly builds
    nibabel_bot.nightly_scheduler(['nibabel-bdist64-26'],
                                  'exe64-26', 2, 00, True),
    # OSX nightlies
    nipy_bot.nightly_scheduler(['nipy-bdist-mpkg-2.6'], 'egg-26', 2, 10, True),
    nipy_bot.nightly_scheduler(['nipy-bdist-mpkg-2.7'], 'egg-27', 2, 20, True),
    nipy_bot.nightly_scheduler(['nipy-bdist-mpkg-3.3'], 'egg-33', 2, 30, True),
    # Dipy
    # Windows 32 nightly builds
    dipy_bot.nightly_scheduler(['dipy-bdist32-26'], 'dipy-exe-26', 3, 10, True),
    dipy_bot.nightly_scheduler(['dipy-bdist32-27'], 'dipy-exe-27', 3, 20, True),
    dipy_bot.nightly_scheduler(['dipy-bdist32-32'], 'dipy-exe-32', 3, 30, True),
    # OSX nightlies
    dipy_bot.nightly_scheduler(['dipy-bdist-mpkg-2.6'], 'dipy-egg-26', 3, 10, True),
    dipy_bot.nightly_scheduler(['dipy-bdist-mpkg-2.7'], 'dipy-egg-27', 3, 20, True),
    dipy_bot.nightly_scheduler(['dipy-bdist-mpkg-3.3'], 'dipy-egg-33', 3, 30, True),
]

# Some factories

def factory_buildout_nibabel(python='python2.6',numpy='1.6.1'):
    f = BuildFactory()
    # check out the source
    # XXX: the mastersrc should make appropriate changes based on numpy version
    f.addStep(Git(repourl='git://github.com/nipy/nibabel.git', mode='copy'))
    f.addStep(FileDownload(mastersrc="bootstrap.py",
                                          slavedest="bootstrap.py"))
    f.addStep(FileDownload(mastersrc="nibabel_buildout.cfg",
                                          slavedest="buildout.cfg"))
    f.addStep(ShellCommand(command=[python, "bootstrap.py"]))
    f.addStep(ShellCommand(command=["bin/buildout"]))
    f.addStep(ShellCommand(nptest_command('nibabel',python='bin/buildoutpython')))
    # run the tests (note that this will require that 'nosetests' is installed)
    #f.addStep(nosetests_command('nibabel', doctests=True))
    return f


def shell_cmd_factory(git_url, cmds, posix=True):
    # Run a series of commands over a git repo
    factory = BuildFactory()
    factory.addStep(Git(
        repourl=git_url,
        mode='copy', retry=(15, 5)))
    for cmd in cmds:
        factory.addStep(ShellCommand(command=shlex.split(cmd, posix=posix)))
    return factory


def example_factory(git_url, py_path, out_dir, python_cmd='python'):
    factory = BuildFactory()
    factory.addStep(Git(
        repourl=git_url,
        mode='copy', retry=(15, 5)))
    # Touch c files to make sure build doesn't fail because of c file checkout
    # times
    factory.addStep(ShellCommand(command=shlex.split(
        'python tools/touch_cython_cs.py')))
    # Make virtualenv into which to install
    factory.addStep(
        ShellCommand(command=shlex.split(
            py_path + r'/bin/virtualenv --distribute --system-site-packages venv')
            ))
    # Install into virtualenv
    factory.addStep(
        ShellCommand(command=[python_cmd, 'setup.py', 'install'],
            env={'PATH': WithProperties("%(workdir)s/build/venv/bin:${PATH}")}
            ))
    # Make directory for examples
    factory.addStep(
        ShellCommand(command=['mkdir', 'eg_logs']))
    # Run examples
    factory.addStep(
        ShellCommand(command=[
            python_cmd,
            '../tools/run_log_examples.py',
            '../examples',
            '--log-path=../eg_logs'],
            workdir=r'build/venv',
            env={'PATH': WithProperties("%(workdir)s/build/venv/bin:${PATH}")}
            ))
    # Upload logs
    factory.addStep(DirectoryUpload(slavesrc="eg_logs",
            masterdest="public_html/" + out_dir))
    return factory

####### BUILDERS

# The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
# what steps, and which slaves can execute them.  Note that any particular build will
# only take place on one slave.

c['builders'] = []
# nibabel exe builders
nibabel_test_cmd = shlex.split('nosetests --verbosity=3 --with-doctest nibabel')
c['builders'].append(BuilderConfig(
    name='nibabel-bdist32-26',
    slavenames=xp_32_slaves,
    factory=nibabel_bot.bdist_factory(
        'exe',
        test_cmd = nibabel_test_cmd,
        python = r'c:\Python26\python.exe')
))
# The Python32 numpy 1.6.2 installation needs patching according to
# git diff ad9c2f4~1..ad9c2f4
# to remove use of __stdout__ and __stderr__ in numpy distutils
c['builders'].append(BuilderConfig(
    name='nibabel-bdist32-32',
    slavenames=xp_32_slaves,
    factory=nibabel_bot.bdist_factory(
        'exe',
        test_cmd = nibabel_test_cmd,
        python = r'c:\Python32\python.exe')
))
c['builders'].append(BuilderConfig(
    name='nibabel-bdist64-26',
    slavenames=win7_64_slaves,
    factory=nibabel_bot.bdist_factory(
        'exe',
        test_cmd = nibabel_test_cmd,
        python = r'c:\Python26\python.exe')
))
# nipy exe builders
nipy_test_cmd = nptest_command('nipy', doctests=True)
c['builders'].append(BuilderConfig(
    name='nipy-bdist32-26',
    slavenames=xp_32_slaves,
    factory=nipy_bot.bdist_factory(
        'exe',
        test_cmd = nipy_test_cmd,
        python = r'c:\Python26\python.exe',
        pre_cmds = [r'python tools\touch_cython_cs.py'])
))
c['builders'].append(BuilderConfig(
    name='nipy-bdist32-27',
    slavenames=xp_32_slaves,
    factory=nipy_bot.bdist_factory(
        'exe',
        test_cmd = nipy_test_cmd,
        python = r'c:\Python27\python.exe',
        pre_cmds = [r'python tools\touch_cython_cs.py'])
))
c['builders'].append(BuilderConfig(
    name='nipy-bdist32-32',
    slavenames=xp_32_slaves,
    factory=nipy_bot.bdist_factory(
        'exe',
        test_cmd = nipy_test_cmd,
        python = r'c:\Python32\python.exe',
        pre_cmds = [r'python tools\touch_cython_cs.py'])
))
# Add bdist builders for OSX
# Need exe because easy_install makes tests executable on 3.3
nipy_test_cmd = nptest_command('nipy', doctests=True, exe=True)
c['builders'].append(BuilderConfig(
    name='nipy-bdist-mpkg-2.6',
    slavenames=['osx-10.6'],
    factory=nipy_bot.bdist_factory(
        'egg',
        test_cmd=nipy_test_cmd,
        python='/Library/Frameworks/Python.framework/Versions/2.6/bin/python')
))
c['builders'].append(BuilderConfig(
    name='nipy-bdist-mpkg-2.7',
    slavenames=['osx-10.6'],
    factory=nipy_bot.bdist_factory(
        'egg',
        test_cmd=nipy_test_cmd,
        python='/Library/Frameworks/Python.framework/Versions/2.7/bin/python')
))
c['builders'].append(BuilderConfig(
    name='nipy-bdist-mpkg-3.3',
    slavenames=['osx-10.6'],
    factory=nipy_bot.bdist_factory(
        'egg',
        test_cmd=nipy_test_cmd,
        python='/Library/Frameworks/Python.framework/Versions/3.3/bin/python3')
))
##############################
# Dipy builders
##############################
win_dipy_test_cmd = ['python',  r'..\tools\dipnost', 'dipy']
c['builders'].append(BuilderConfig(
    name='dipy-bdist32-26',
    slavenames=xp_32_slaves,
    factory=dipy_bot.bdist_factory(
        'exe',
        test_cmd=win_dipy_test_cmd,
        python=r'c:\Python26\python.exe')
))
c['builders'].append(BuilderConfig(
    name='dipy-bdist32-27',
    slavenames=xp_32_slaves,
    factory=dipy_bot.bdist_factory(
        'exe',
        test_cmd=win_dipy_test_cmd,
        python=r'c:\Python27\python.exe')
))
c['builders'].append(BuilderConfig(
    name='dipy-bdist32-32',
    slavenames=xp_32_slaves,
    factory=dipy_bot.bdist_factory(
        'exe',
        test_cmd=win_dipy_test_cmd,
        python=r'c:\Python32\python.exe')
))
# Add bdist builders for OSX
# Each of the tested _system_ pythons will need:
# * numpy, scipy, virtualenv, bdist_mpkg
osx_dipy_test_cmd = ['python',  '../tools/dipnost', '--exe', '--verbosity=3', 'dipy']
c['builders'].append(BuilderConfig(
    name='dipy-bdist-mpkg-2.6',
    slavenames=['osx-10.6'],
    factory=dipy_bot.bdist_factory(
        'egg',
        test_cmd=osx_dipy_test_cmd,
        python='/Library/Frameworks/Python.framework/Versions/2.6/bin/python')
))
c['builders'].append(BuilderConfig(
    name='dipy-bdist-mpkg-2.7',
    slavenames=['osx-10.6'],
    factory=dipy_bot.bdist_factory(
        'egg',
        test_cmd=osx_dipy_test_cmd,
        python='/Library/Frameworks/Python.framework/Versions/2.7/bin/python')
))
c['builders'].append(BuilderConfig(
    name='dipy-bdist-mpkg-3.3',
    slavenames=['osx-10.6'],
    factory=dipy_bot.bdist_factory(
        'egg',
        test_cmd=osx_dipy_test_cmd,
        python='/Library/Frameworks/Python.framework/Versions/3.3/bin/python3')
))
# Add nightly docs builders
c['builders'] += nipy_bot.build_doc_builders(
    (('nipy-doc-builder', linux_64_slaves),),
    doc_doctest_cmd = "cd doc && make doctest"
)
c['builders'] += nibabel_bot.build_doc_builders(
    (('nibabel-doc-builder', linux_64_slaves),))
c['builders'] += pandas_bot.build_doc_builders(
    (('pandas-doc-builder', sparc_slaves_sid),))
c['builders'] += statsmodels_bot.build_doc_builders(
    (('statsmodels-doc-builder', sparc_slaves_sid),))
# pre-release builders
c['builders'].append(BuilderConfig(
    name='nipy-release-checks',
    slavenames=['osx-10.6'],
    factory=shell_cmd_factory(
        'git://github.com/nipy/nipy.git',
        ('make distclean',
         'python -m compileall .',
         'make sdist-tests',
         'make bdist-egg-tests',
         'make check-version-info',
         'make check-files'))))
c['builders'].append(BuilderConfig(
    name='dipy-release-checks',
    slavenames=['osx-10.6'],
    factory=shell_cmd_factory(
        'git://github.com/nipy/dipy.git',
        ('make distclean',
         'python -m compileall .',
         'make sdist-tests',
         'make bdist-egg-tests',
         'make check-version-info',
         'make check-files'))))
# Example builders
c['builders'].append(BuilderConfig(
    name='nipy-examples-2.6',
    slavenames=['osx-10.6'],
    factory=example_factory(
        'git://github.com/nipy/nipy.git',
        '/Library/Frameworks/Python.framework/Versions/2.6',
        "nipy-examples-2.6")
))
c['builders'].append(BuilderConfig(
    name='nipy-examples-3.3',
    slavenames=['osx-10.6'],
    factory=example_factory(
        'git://github.com/nipy/nipy.git',
        '/Library/Frameworks/Python.framework/Versions/3.3',
        "nipy-examples-3.3",
        "python3")
))

# Code builders
c['builders'] += nibabel_bot.build_builders(
    (('nibabel-py2.6', linux_64_slaves),
     ('nibabel-py2.6-32', linux_32_slaves),
     ('nibabel-py2.7-osx-10.4-ppc', osx_tiger_ppc_slaves),
     ('nibabel-py2.7-osx-10.6', osx_snowleopard_slaves),
     ('nibabel-py2.7-osx-10.8', osx_mountainlion_slaves),
     ('nibabel-py2.6-osx-10.5-ppc', osx_leopard_ppc_slaves),
     ('nibabel-py2.6-osx-10.5-intel', osx_leopard_intel_slaves),
     ('nibabel-py2.7-ppc', debian_ppc_slaves),
     ('nibabel-py2.6-arm', arm_slaves),
     ('nibabel-py2.6-xp', xp_32_slaves),
     ('nibabel-py2.7-win7', win7_64_slaves),
     ('nibabel-py2.6-sparc', sparc_slaves)))
# Python 3.2 into virtualenv
c['builders'] += nibabel_bot.build_builders(
    (('nibabel-py3.2', linux_64_slaves),),
    python='python3.2')
c['builders'] += nibabel_bot.build_builders(
    (('nibabel-py3.3', osx_snowleopard_slaves),),
    python='python3.3')
# Different versions of Pydicom
c['builders'] += nibabel_bot.build_builders(
    (('nibabel-py2.7-pyd0.9.7', osx_mountainlion_slaves),),
    python='python2.7',
    easy_depends = ('nose', 'pydicom==0.9.7'))
c['builders'] += nibabel_bot.build_builders(
    (('nibabel-py2.7-pyd0.9.8', osx_mountainlion_slaves),),
    python='python2.7',
    easy_depends = ('nose', 'pydicom==0.9.8'))
# Nitime
c['builders'] += nitime_bot.build_builders(
    (('nitime-py2.6', linux_64_slaves),
     ('nitime-py2.6-32', linux_32_slaves),
     ('nitime-py2.7-osx-10.4-ppc', osx_tiger_ppc_slaves),
     ('nitime-py2.7-osx-10.6', osx_snowleopard_slaves),
     ('nitime-py2.7-osx-10.8', osx_mountainlion_slaves),
     ('nitime-py2.6-osx-10.5-ppc', osx_leopard_ppc_slaves),
     ('nitime-py2.6-osx-10.5-intel', osx_leopard_intel_slaves),
     ('nitime-py2.6-arm', arm_slaves),
     ('nitime-py2.6-xp', xp_32_slaves)))
c['builders'] += nitime_arokem_bot.build_builders(
    (('nitime-py2.6-arokem', linux_64_slaves),
     ('nitime-py2.6-arokem-32', linux_32_slaves),
     ('nitime-py2.7-arokem-osx-10.4-ppc', osx_tiger_ppc_slaves),
     ('nitime-py2.7-arokem-osx-10.6', osx_snowleopard_slaves),
     ('nitime-py2.6-arokem-osx-10.5-ppc', osx_leopard_ppc_slaves),
     ('nitime-py2.6-arokem-osx-10.5-intel', osx_leopard_intel_slaves),
     ('nitime-py2.6-arokem-xp', xp_32_slaves)))
c['builders'] += nipy_bot.build_builders(
    (('nipy-py2.6', linux_64_slaves),
     ('nipy-py2.6-32', linux_32_slaves),
     ('nipy-py2.7-osx-10.4-ppc', osx_tiger_ppc_slaves),
     ('nipy-py2.7-osx-10.6', osx_snowleopard_slaves),
     ('nipy-py2.7-osx-10.8', osx_mountainlion_slaves),
     ('nipy-py2.6-osx-10.5-ppc', osx_leopard_ppc_slaves),
     ('nipy-py2.6-osx-10.5-intel', osx_leopard_intel_slaves),
     ('nipy-py2.7-ppc', debian_ppc_slaves),
     ('nipy-py2.6-arm', arm_slaves),
     ('nipy-py2.6-xp', xp_32_slaves),
     ('nipy-py2.6-sparc', sparc_slaves)))
# Python 3.2 into virtualenv
c['builders'] += nipy_bot.build_builders(
    (('nipy-py3.2', linux_64_slaves),),
    python='python3.2')
c['builders'] += nipy_bot.build_builders(
    (('nipy-py3.3', osx_snowleopard_slaves),),
    python='python3.3')
# Different versions of Sympy, nibabel
c['builders'] += nipy_bot.build_builders(
    (('nipy-py2.7-sym0.6.6-nib1.2.0', osx_mountainlion_slaves),),
    pip_depends = ('sympy==0.6.6', 'nibabel==1.2.0'))
# The dipy builder seems to fail early sometimes, on OSX, maybe due to:
# http://bugs.python.org/issue8458
c['builders'] += dipy_bot_nox.build_builders(
    (('dipy-py2.6', linux_64_slaves),
     ('dipy-py2.6-32', linux_32_slaves),
))
c['builders'] += dipy_bot.build_builders((
     ('dipy-py2.7-osx-10.6', osx_snowleopard_slaves),
     ('dipy-py2.7-osx-10.8', osx_mountainlion_slaves),
     ('dipy-py2.6-osx-10.5-ppc', osx_leopard_ppc_slaves),
     ('dipy-py2.6-osx-10.5-intel', osx_leopard_intel_slaves),
     ('dipy-py2.6-arm', arm_slaves),
     ('dipy-py2.6-xp', xp_32_slaves),
     ('dipy-py2.6-sparc', sparc_slaves)))
# Python 3.2 into virtualenv
c['builders'] += dipy_bot.build_builders(
    (('dipy-py3.2', linux_64_slaves),),
    python='python3.2')
c['builders'] += dipy_bot.build_builders(
    (('dipy-py3.3', osx_snowleopard_slaves),),
    python='python3.3')
# Nipype
c['builders'] += nipype_bot.build_builders(
    (('nipype-py2.6', linux_64_slaves),
     ('nipype-py2.6-32', linux_32_slaves),
     ('nipype-py2.7-osx-10.6', osx_snowleopard_slaves),
     ('nipype-py2.7-osx-10.8', osx_mountainlion_slaves),
     ('nipype-py2.7-osx-10.4-ppc', osx_tiger_ppc_slaves),
     ('nipype-py2.6-osx-10.5-ppc', osx_leopard_ppc_slaves),
     ('nipype-py2.6-osx-10.5-intel', osx_leopard_intel_slaves),
     ('nipype-py2.6-arm', arm_slaves),
     ('nipype-py2.6-xp', xp_32_slaves),
     ('nipype-py2.6-sparc', sparc_slaves)))
c['builders'] += regreg_bot.build_builders(
    (('regreg-py2.6', linux_64_slaves),
     ('regreg-py2.6-32', linux_32_slaves),
     ('regreg-py2.7-osx-10.6', osx_snowleopard_slaves),
     ('regreg-py2.7-osx-10.8', osx_mountainlion_slaves),
     ('regreg-py2.7-osx-10.4-ppc', osx_tiger_ppc_slaves),
     ('regreg-py2.6-osx-10.5-ppc', osx_leopard_ppc_slaves),
     ('regreg-py2.6-osx-10.5-intel', osx_leopard_intel_slaves),
     ('regreg-py2.6-arm', arm_slaves),
     ('regreg-py2.6-xp', xp_32_slaves),
     ('regreg-py2.6-sparc', sparc_slaves)))
# Pandas - can be tested only on recent debian releases
c['builders'] += pandas_bot.build_builders(
    (('pandas-py2.x-sid-sparc', sparc_slaves_sid),
     ('pandas-py2.x-wheezy-sparc', sparc_slaves_wheezy)))
# and python3 builders
c['builders'] += pandas_bot.build_builders(
    (('pandas-py3.x-sid-sparc', sparc_slaves_sid),
     ('pandas-py3.x-wheezy-sparc', sparc_slaves_wheezy)),
    python='python3')
# Statsmodels - can be tested only on recent debian releases
c['builders'] += statsmodels_bot.build_builders(
    (('statsmodels-py2.x-sid-sparc', sparc_slaves_sid),
     ('statsmodels-py2.x-wheezy-sparc', sparc_slaves_wheezy)))
## and python3 builders
#c['builders'] += statsmodels_bot.build_builders(
#    (('statsmodels-py3.x-sid-sparc', sparc_slaves_sid),
#     ('statsmodels-py3.x-wheezy-sparc', sparc_slaves_wheezy)),
#    python='python3')
""" Disabling these guys; they haven't worked for a while
c['builders'].append(
    BuilderConfig(name="buildout-test2.6",
      slavenames=experimental,
      factory=factory_buildout_nibabel()))
c['builders'].append(
    BuilderConfig(name="buildout-test2.5",
      slavenames=experimental,
      factory=factory_buildout_nibabel('python2.5')))
c['builders'].append(
    BuilderConfig(name="buildout-test2.4",
      slavenames=experimental,
      factory=factory_buildout_nibabel('python2.4')))
"""

# Assign all builders categories -- project names, to improve status visualizations
for b in c['builders']:
    b.category = b.name.split('-')[0]

####### STATUS TARGETS

# 'status' is a list of Status Targets. The results of each build will be
# pushed to these targets. buildbot/status/*.py has a variety to choose from,
# including web pages, email senders, and IRC bots.

c['status'] = []

from buildbot.status import html
from buildbot.status.web import auth, authz

http_auth = auth.HTPasswdAuth('<PASSWORD>')

authz_cfg=authz.Authz(
    auth = http_auth,
    # change any of these to True to enable; 'auth' to require authorization'
    # see the manual for more options
    gracefulShutdown = False,
    forceBuild = 'auth',
    forceAllBuilds = 'auth',
    pingBuilder = False,
    stopBuild = 'auth',
    stopAllBuilds = False,
    cancelPendingBuild = False,
)
c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg,
                                  #change_hook_dialects={ 'base': True, 'github' : True }
                  ))

####### PROJECT IDENTITY
# the 'title' string will appear at the top of this buildbot
# installation's html.WebStatus home page (linked to the
# 'titleURL') and is embedded in the title of the waterfall HTML page.

c['title'] = "NiPy"
c['titleURL'] = "http://nipy.org"

# the 'buildbotURL' string should point to the location where the buildbot's
# internal web server (usually the html.WebStatus page) is visible. This
# typically uses the port number set in the Waterfall 'status' entry, but
# with an externally-visible host name which the buildbot cannot figure out
# without some help.

c['buildbotURL'] = "http://localhost:8010/"

####### DB URL

# This specifies what database buildbot uses to store change and scheduler
# state.  You can leave this at its default for all but the largest
# installations.
c['db_url'] = "sqlite:///state.sqlite"


{'fraction_non_alphanumeric': 0.1322016222479722, 'fraction_numerical': 0.024310544611819234, 'mean_word_length': 2.4310420228203395, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 78, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '700627', 'n_tokens_mistral': 30742, 'n_tokens_neox': 28884, 'n_words': 5379}