<commit_before># -*- python -*-
# ex: set syntax=python:

# This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing.
c = BuildmasterConfig = {}
c['protocols'] = {'pb': {'port': 9989}}




####### BUILDSLAVES
from buildbot.buildslave import BuildSlave

ubuntu_names = []
for version in ["14.04", "13.10"]:
    for arch in ["x64", "x86"]:
        ubuntu_names += ["ubuntu%s-%s"%(version, arch)]

osx_names = ["osx10.7", "osx10.8", "osx10.9"]

c['slaves'] = []
for name in ubuntu_names + osx_names:
    arch = 'amd64'
    if name[-3:] == 'x86':
        arch = 'i386'
    c['slaves'] += [BuildSlave(name, 'julialang42', properties={'arch':arch, 'release':name})]




####### CHANGESOURCES
from buildbot.changes.pb import PBChangeSource
c['change_source'] = PBChangeSource(user='github_source', passwd='<PASSWORD>')




####### SCHEDULERS
from buildbot.schedulers.basic import AnyBranchScheduler, Dependent
from buildbot.schedulers.triggerable import Triggerable
from buildbot.schedulers.forcesched import *
from buildbot.changes import filter
c['schedulers'] = []

# Add a scheduler for julia continuous deployment
julia_builders = ["package_" + z for z in ubuntu_names + [osx_names[-1]]]
c['schedulers'].append(AnyBranchScheduler( name="julia build", treeStableTimer=None, builderNames=julia_builders))

# Add a scheduler for building bottles
bottle_builders = ["bottle_" + z for z in osx_names]
bottle_scheduler = ForceScheduler(
    name="bottle build",
    builderNames=bottle_builders,
    reason=FixedParameter(name="reason", default=""),
    branch=FixedParameter(name="branch", default=""),
    revision=FixedParameter(name="revision", default=""),
    repository=FixedParameter(name="repository", default=""),
    project=FixedParameter(name="project", default="Bottling"),
    properties=[
        StringParameter(name="formula", label="Formula", default="")
    ]
)
c['schedulers'].append(bottle_scheduler)

c['schedulers'].append(Triggerable( name="bottle upload", builderNames=["bottle_host"] ))


####### BUILD FACTORIES
from buildbot.process.factory import BuildFactory
from buildbot.steps.source.git import Git
from buildbot.steps.shell import ShellCommand, SetPropertyFromCommand
from buildbot.steps.trigger import Trigger
from buildbot.steps.master import MasterShellCommand
from buildbot.process.properties import Property, Interpolate
from buildbot.steps.package.deb.pbuilder import DebPbuilder
from buildbot.steps.transfer import FileUpload, FileDownload


# Steps to build a Mac .dmg Julia package
osx_julia_package_factory = BuildFactory()
osx_julia_package_factory.useProgress = True
BUILD_OPTS=["LLVM_CONFIG=llvm-config-3.3", "LLVM_LLC=llc-3.3", "VERBOSE=1", "USE_BLAS64=0", "SUITESPARSE_INC='-I /usr/local/include'",
            "LIBBLAS=-lopenblas", "LIBBLASNAME=libopenblas", "LIBLAPACK=-lopenblas", "LIBLAPACKNAME=libopenblas"]
for lib in ["LLVM","ZLIB","SUITESPARSE","ARPACK","BLAS","FFTW","LAPACK","GMP","MPFR","PCRE","LIBUNWIND"]:
    BUILD_OPTS += ["USE_SYSTEM_%s=1"%(lib)]
osx_julia_package_factory.addSteps([
    # Clone julia
    Git(name="Julia checkout", repourl=Property('repository', default='git://github.com/JuliaLang/julia.git'), mode='full', method='fresh', submodules=True, clobberOnFailure=True, progress=True),

    ShellCommand(command=["brew", "install", "--only-dependencies", "julia"]),

    # Repackage suite-sparse
    ShellCommand(command=["make", "-C", "contrib", "-f", "repackage_system_suitesparse4.make"] + BUILD_OPTS, haltOnFailure=True),

    # Make!
    ShellCommand(command=["make"] + BUILD_OPTS, haltOnFailure=True),

    # Do the tests
    ShellCommand(command=["make", "testall"] + BUILD_OPTS, haltOnFailure=True),

    # Actually package the darn thing
    ShellCommand(command=["make", "-C", "contrib/mac/app"] + BUILD_OPTS),

    # Upload the package to the host!
    SetPropertyFromCommand(name="Get dmg filename", command=["/bin/bash", "-c", "cd contrib/mac/app && echo *.dmg"], property="filename"),
    MasterShellCommand(name="Make julia_package", command=["mkdir", "-p", "/tmp/julia_package"]),
    FileUpload(slavesrc=Interpolate("contrib/mac/app/%(prop:filename)s.dmg"), masterdest=Interpolate("/tmp/julia_package/%(prop:filename)s.dmg")),

    # Now upload to AWS!
    MasterShellCommand(name="Upload to AWS", command=["aws", "put", "x-amz-acl: public-read", "julianightlies", Interpolate("/tmp/julia_package/%(prop:filename)s.dmg")])
])


# Steps to build a Homebrew Bottle
osx_bottle_factory = BuildFactory()
osx_bottle_factory.useProgress = True
osx_bottle_factory.addSteps([
    # Clean everything out that's in the directory!
    ShellCommand(name="precleanup", command=["/bin/bash", "-c", "rm -f *.{sh,gz}"]),
    # Copy our build_bottle.sh script over to the slave:
    FileDownload(mastersrc="../commands/build_bottle.sh", slavedest="build_bottle.sh"),
    
    # Next, invoke build_bottle.sh!
    ShellCommand(command=["/bin/bash", "build_bottle.sh", Property('formula')], haltOnFailure=True),

    # Grab the output and transfer it back!
    SetPropertyFromCommand(name="Get bottle filename", command=["/bin/bash", "-c", "echo *.tar.gz"], property="filename"),
    MasterShellCommand(name="Make bottle_cache", command=["mkdir", "-p", "/tmp/bottle_cache"]),
    FileUpload(slavesrc=Interpolate("%(prop:filename)s"), masterdest=Interpolate("/tmp/bottle_cache/%(prop:filename)s")),

    # Trigger our host bottle upload step!
    Trigger(schedulerNames=['bottle upload'], set_properties={'formula': Property('formula')})
])


# Steps to upload a Homebrew bottle from the master
host_bottle_upload = BuildFactory()
host_bottle_upload.useProgress = True
host_bottle_upload.addSteps([
    # Run upload_bottle_archive.sh
    MasterShellCommand(name="Upload Bottle", command=["/bin/bash", "../commands/upload_bottle.sh", Interpolate("%(prop:formula)s")])
])



# Steps to build an Ubuntu .deb Julia Package
ubuntu_package_julia_factory = BuildFactory()
ubuntu_package_julia_factory.useProgress = True
ubuntu_package_julia_factory.addSteps([
    # Clone julia
    Git(name="Julia checkout", repourl=Property('repository', default='git://github.com/JuliaLang/julia.git'), mode='full', method='fresh', submodules=True, clobberOnFailure=True, progress=True),

    # Perform pre-tarball steps
    ShellCommand(name="version_git.jl", command=["make", "-C", "base", "version_git.jl.phony"]),
    ShellCommand(name="Get dependencies", command=["make", "-C", "deps", "get-dsfmt", "get-utf8proc"]),

    # Save the combined version string and release
    SetPropertyFromCommand(name="Compute debversion", command=["/bin/bash", "-c", "echo $(cat ./VERSION | cut -f1 -d'-')~pre+$(git rev-list HEAD ^$(git describe --tags --abbrev=0) | wc -l | sed -e 's/[^[:digit:]]//g')"], property="debversion"),

    # Make the source tarball
    ShellCommand(name="Source Tarball", command=["/bin/bash", "-c", Interpolate("tar --exclude .git -czf ../julia_%(prop:debversion)s.orig.tar.gz .")]),

    # Get the debian/ directory
    ShellCommand(name="Get debian/", command=["/bin/bash", "-c", "rm -rf debian; wget https://github.com/staticfloat/julia-debian/archive/master.tar.gz -O- | tar -zx --exclude README.md --strip-components=1"]),
    
    # Bump the version
    ShellCommand(name="Bump debian version", command=["/bin/bash", "-c", Interpolate("EMAIL='<NAME> <<EMAIL>>' dch -v %(prop:debversion)s 'nightly git build'")]),
    # Build the .deb!
    ShellCommand(name="debuild", command=["debuild"]),
    
    # Upload the result!
    FileUpload(slavesrc=Interpolate("../julia_%(prop:debversion)s_%(prop:arch)s.deb"), masterdest=Interpolate("../julia_%(prop:debversion)s~%(prop:release)s_%(prop:arch)s.deb")),

    # Since this stuff is getting thrown into the parent directory, we need to clean up after ourselves!
    ShellCommand(name="cleanup", command=["/bin/bash", "-c", "rm -f ../*.{gz,build,dsc,deb,changes}"])
])



####### BUILDERS
from buildbot.config import BuilderConfig
c['builders'] = []

# Add all the ubuntu julia packagers
for name in ubuntu_names:
    c['builders'].append(BuilderConfig(
        name="package_%s"%(name),
        slavenames=[name],
        category="Packaging",
        factory=ubuntu_package_julia_factory
    ))

# Add my osx julia packager (we only need one right now!)
c['builders'].append(BuilderConfig(
    name="package_%s"%(osx_names[-1]),
    slavenames=["osx10.9"],
    category="Packaging",
    factory=osx_julia_package_factory<commit_msg>Upload to the proper location, (/tmp/julia_package) not the parent directory on the host!<commit_after># -*- python -*-
# ex: set syntax=python:

# This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing.
c = BuildmasterConfig = {}
c['protocols'] = {'pb': {'port': 9989}}




####### BUILDSLAVES
from buildbot.buildslave import BuildSlave

ubuntu_names = []
for version in ["14.04", "13.10"]:
    for arch in ["x64", "x86"]:
        ubuntu_names += ["ubuntu%s-%s"%(version, arch)]

osx_names = ["osx10.7", "osx10.8", "osx10.9"]

c['slaves'] = []
for name in ubuntu_names + osx_names:
    arch = 'amd64'
    if name[-3:] == 'x86':
        arch = 'i386'
    c['slaves'] += [BuildSlave(name, 'julialang42', properties={'arch':arch, 'release':name})]




####### CHANGESOURCES
from buildbot.changes.pb import PBChangeSource
c['change_source'] = PBChangeSource(user='github_source', passwd='<PASSWORD>')




####### SCHEDULERS
from buildbot.schedulers.basic import AnyBranchScheduler, Dependent
from buildbot.schedulers.triggerable import Triggerable
from buildbot.schedulers.forcesched import *
from buildbot.changes import filter
c['schedulers'] = []

# Add a scheduler for julia continuous deployment
julia_builders = ["package_" + z for z in ubuntu_names + [osx_names[-1]]]
c['schedulers'].append(AnyBranchScheduler( name="julia build", treeStableTimer=None, builderNames=julia_builders))

# Add a scheduler for building bottles
bottle_builders = ["bottle_" + z for z in osx_names]
bottle_scheduler = ForceScheduler(
    name="bottle build",
    builderNames=bottle_builders,
    reason=FixedParameter(name="reason", default=""),
    branch=FixedParameter(name="branch", default=""),
    revision=FixedParameter(name="revision", default=""),
    repository=FixedParameter(name="repository", default=""),
    project=FixedParameter(name="project", default="Bottling"),
    properties=[
        StringParameter(name="formula", label="Formula", default="")
    ]
)
c['schedulers'].append(bottle_scheduler)

c['schedulers'].append(Triggerable( name="bottle upload", builderNames=["bottle_host"] ))


####### BUILD FACTORIES
from buildbot.process.factory import BuildFactory
from buildbot.steps.source.git import Git
from buildbot.steps.shell import ShellCommand, SetPropertyFromCommand
from buildbot.steps.trigger import Trigger
from buildbot.steps.master import MasterShellCommand
from buildbot.process.properties import Property, Interpolate
from buildbot.steps.package.deb.pbuilder import DebPbuilder
from buildbot.steps.transfer import FileUpload, FileDownload


# Steps to build a Mac .dmg Julia package
osx_julia_package_factory = BuildFactory()
osx_julia_package_factory.useProgress = True
BUILD_OPTS=["LLVM_CONFIG=llvm-config-3.3", "LLVM_LLC=llc-3.3", "VERBOSE=1", "USE_BLAS64=0", "SUITESPARSE_INC='-I /usr/local/include'",
            "LIBBLAS=-lopenblas", "LIBBLASNAME=libopenblas", "LIBLAPACK=-lopenblas", "LIBLAPACKNAME=libopenblas"]
for lib in ["LLVM","ZLIB","SUITESPARSE","ARPACK","BLAS","FFTW","LAPACK","GMP","MPFR","PCRE","LIBUNWIND"]:
    BUILD_OPTS += ["USE_SYSTEM_%s=1"%(lib)]
osx_julia_package_factory.addSteps([
    # Clone julia
    Git(name="Julia checkout", repourl=Property('repository', default='git://github.com/JuliaLang/julia.git'), mode='full', method='fresh', submodules=True, clobberOnFailure=True, progress=True),

    ShellCommand(command=["brew", "install", "--only-dependencies", "julia"]),

    # Repackage suite-sparse
    ShellCommand(command=["make", "-C", "contrib", "-f", "repackage_system_suitesparse4.make"] + BUILD_OPTS, haltOnFailure=True),

    # Make!
    ShellCommand(command=["make"] + BUILD_OPTS, haltOnFailure=True),

    # Do the tests
    ShellCommand(command=["make", "testall"] + BUILD_OPTS, haltOnFailure=True),

    # Actually package the darn thing
    ShellCommand(command=["make", "-C", "contrib/mac/app"] + BUILD_OPTS),

    # Upload the package to the host!
    SetPropertyFromCommand(name="Get dmg filename", command=["/bin/bash", "-c", "cd contrib/mac/app && echo *.dmg"], property="filename"),
    MasterShellCommand(name="Make julia_package", command=["mkdir", "-p", "/tmp/julia_package"]),
    FileUpload(slavesrc=Interpolate("contrib/mac/app/%(prop:filename)s"), masterdest=Interpolate("/tmp/julia_package/%(prop:filename)s")),

    # Now upload to AWS!
    MasterShellCommand(name="Upload to AWS", command=["aws", "put", "x-amz-acl: public-read", "julianightlies", Interpolate("/tmp/julia_package/%(prop:filename)s.dmg")])
])


# Steps to build a Homebrew Bottle
osx_bottle_factory = BuildFactory()
osx_bottle_factory.useProgress = True
osx_bottle_factory.addSteps([
    # Clean everything out that's in the directory!
    ShellCommand(name="precleanup", command=["/bin/bash", "-c", "rm -f *.{sh,gz}"]),
    # Copy our build_bottle.sh script over to the slave:
    FileDownload(mastersrc="../commands/build_bottle.sh", slavedest="build_bottle.sh"),
    
    # Next, invoke build_bottle.sh!
    ShellCommand(command=["/bin/bash", "build_bottle.sh", Property('formula')], haltOnFailure=True),

    # Grab the output and transfer it back!
    SetPropertyFromCommand(name="Get bottle filename", command=["/bin/bash", "-c", "echo *.tar.gz"], property="filename"),
    MasterShellCommand(name="Make bottle_cache", command=["mkdir", "-p", "/tmp/bottle_cache"]),
    FileUpload(slavesrc=Interpolate("%(prop:filename)s"), masterdest=Interpolate("/tmp/bottle_cache/%(prop:filename)s")),

    # Trigger our host bottle upload step!
    Trigger(schedulerNames=['bottle upload'], set_properties={'formula': Property('formula')})
])


# Steps to upload a Homebrew bottle from the master
host_bottle_upload = BuildFactory()
host_bottle_upload.useProgress = True
host_bottle_upload.addSteps([
    # Run upload_bottle_archive.sh
    MasterShellCommand(name="Upload Bottle", command=["/bin/bash", "../commands/upload_bottle.sh", Interpolate("%(prop:formula)s")])
])



# Steps to build an Ubuntu .deb Julia Package
ubuntu_package_julia_factory = BuildFactory()
ubuntu_package_julia_factory.useProgress = True
ubuntu_package_julia_factory.addSteps([
    # Clone julia
    Git(name="Julia checkout", repourl=Property('repository', default='git://github.com/JuliaLang/julia.git'), mode='full', method='fresh', submodules=True, clobberOnFailure=True, progress=True),

    # Perform pre-tarball steps
    ShellCommand(name="version_git.jl", command=["make", "-C", "base", "version_git.jl.phony"]),
    ShellCommand(name="Get dependencies", command=["make", "-C", "deps", "get-dsfmt", "get-utf8proc"]),

    # Save the combined version string and release
    SetPropertyFromCommand(name="Compute debversion", command=["/bin/bash", "-c", "echo $(cat ./VERSION | cut -f1 -d'-')~pre+$(git rev-list HEAD ^$(git describe --tags --abbrev=0) | wc -l | sed -e 's/[^[:digit:]]//g')"], property="debversion"),

    # Make the source tarball
    ShellCommand(name="Source Tarball", command=["/bin/bash", "-c", Interpolate("tar --exclude .git -czf ../julia_%(prop:debversion)s.orig.tar.gz .")]),

    # Get the debian/ directory
    ShellCommand(name="Get debian/", command=["/bin/bash", "-c", "rm -rf debian; wget https://github.com/staticfloat/julia-debian/archive/master.tar.gz -O- | tar -zx --exclude README.md --strip-components=1"]),
    
    # Bump the version
    ShellCommand(name="Bump debian version", command=["/bin/bash", "-c", Interpolate("EMAIL='<NAME> <<EMAIL>>' dch -v %(prop:debversion)s 'nightly git build'")]),
    # Build the .deb!
    ShellCommand(name="debuild", command=["debuild"]),
    
    # Upload the result!
    MasterShellCommand(name="Make julia_package", command=["mkdir", "-p", "/tmp/julia_package"]),
    FileUpload(slavesrc=Interpolate("../julia_%(prop:debversion)s_%(prop:arch)s.deb"), masterdest=Interpolate("/tmp/julia_package/julia_%(prop:debversion)s~%(prop:release)s_%(prop:arch)s.deb")),

    # Since this stuff is getting thrown into the parent directory, we need to clean up after ourselves!
    ShellCommand(name="cleanup", command=["/bin/bash", "-c", "rm -f ../*.{gz,build,dsc,deb,changes}"])
])



####### BUILDERS
from buildbot.config import BuilderConfig
c['builders'] = []

# Add all the ubuntu julia packagers
for name in ubuntu_names:
    c['builders'].append(BuilderConfig(
        name="package_%s"%(name),
        slavenames=[name],
        category="Packaging",
        factory=ubuntu_package_julia_factory
    ))

# Add my osx julia packager (we only need one right now!)
c['builders'].append(BuilderConfig(
    name="package_%s"%(osx_names[-1]),
    slavenames=["osx10.9"],
    category="Packaging",
    factory=osx_julia_package_factory
{'fraction_non_alphanumeric': 0.16672490100163057, 'fraction_numerical': 0.006289308176100629, 'mean_word_length': 5.424616535727647, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3346340', 'n_tokens_mistral': 5980, 'n_tokens_neox': 5665, 'n_words': 1176}