<commit_before># -*- python -*-
# ex: set filetype=python:

import random

from buildbot.process import remotecommand
from buildbot.plugins import *
from buildbot.plugins import util, steps
from buildbot.process import buildstep, logobserver
from twisted.internet import defer


# This is a sample buildmaster config file. It must be installed as
# 'master.cfg' in your buildmaster's base directory.

# This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing.
c = BuildmasterConfig = {}

####### WORKERS

# The 'workers' list defines the set of recognized workers. Each element is
# a Worker object, specifying a unique worker name and password.  The same
# worker name and password must be configured on the worker.
# c['workers'] = [worker.Worker("example-worker", "pass", max_builds=30),
#                 worker.Worker("example-worker2", "pass", max_builds=30)]

workers = ["bot1","bot2"]

c['workers'] = [worker.LocalWorker(bot) for bot in workers]
# 'protocols' contains information about protocols which master will use for
# communicating with workers. You must define at least 'port' option that workers
# could connect to your master with this protocol.
# 'port' must match the value configured into the workers (with their
# --master option)
c['protocols'] = {'pb': {'port': 9989}}

####### CHANGESOURCES

# the 'change_source' setting tells the buildmaster how it should find out
# about source code changes.  Here we point to the buildbot clone of pyflakes.

c['change_source'] = []
c['change_source'].append(changes.GitPoller(
        'git://github.com/Foxboron/PKGBUILDS.git',
        workdir='gitpoller-workdir', branch='master',
        pollinterval=10))

####### SCHEDULERS

# Configure the Schedulers, which decide how to react to incoming changes.  In this
# case, just kick off a 'runtests' build

c['schedulers'] = []
#c['schedulers'].append(schedulers.SingleBranchScheduler(
#                            name="all",
#                            change_filter=util.ChangeFilter(branch='master'),
#                            treeStableTimer=None,
#                            builderNames=["unpack-packages"]))



sch = schedulers.ForceScheduler(
        name="force",
        buttonName="Build package",
        label="Build package",
        builderNames=["unpack-packages"],

        codebases=[
            util.CodebaseParameter(
                "",
                name="Main repository",
                branch=util.ChoiceStringParameter(
                    name="branch",
                    choices=["master"],
                    default="master"),

                revision=util.FixedParameter(name="revision", default=""),
                repository=util.FixedParameter(name="repository", default=""),
                project=util.FixedParameter(name="project", default=""),
                ),
            ],

        properties=[
            util.StringParameter(name="build_package",
                label="Package:", default="",
                required=False, size=80),
            util.StringParameter(name="repository",
                label="Repository:", default="foxboron",
                required=False, size=80),
            ]
    )


c['schedulers'].append(sch)     

c['schedulers'].append(schedulers.Triggerable(name="build", builderNames=["build-package"]))

####### BUILDERS



class TriggerWithPackageNames(steps.Trigger, buildstep.ShellMixin):

    def assign_build_props(self, name, dependencies, worker=False):
        props = self.set_properties.copy()
        props["virtual_builder_name"] = name 
        props["package"] = name 
        props["dependencies"] = dependencies 
        if worker:
            props["workername"] = worker
        return ["build", props]

    def add_build(self, package, needed, worker=False):
        """I was scared for multipe triggers for the same package :c"""
        if package not in self._build_requests:
            self.sp.append(self.assign_build_props(package, needed))

    def resolve(self, name, packages, graph=[]):
        """Poor mans dependency resolver """
        available = list(packages.keys())
        for i in packages[name]:
            if i in available and i not in graph:
                graph.append(i)
                self.resolve(i, packages, graph=graph)
        return graph

    def extract_deps(self, srcinfo):
        """ Poor mans .SRCINFO parser """
        packages={}
        pkgname=""

        for i in srcinfo.split("\n"):
            if not i:
                continue
            if i[0] == "#":
                continue
            option = i.strip() 
            key,value = option.split(" = ")
            if key == "pkgbase":
                pkgname=value
                packages[pkgname] = []
            if key == "makedepends":
                packages[pkgname].append(value)
            if key == "depends":
                packages[pkgname].append(value)
        return packages

    @defer.inlineCallbacks
    def getSchedulersAndProperties(self):
        """Bread and butter.
        This triggers all the needed PKGBUILDS and resolves the dependencies"""
        self.sp = []
        self._build_requests = []

        # If we have dependant builds, they need to go on the same worker
        worker = False


        # I simply dont know how to make the yield stuff recursive
        # so we find all .SRCINFOs available and make a list
        self.observer = logobserver.BufferLogObserver()
        self.addLogObserver('stdio', self.observer)
        cmd = yield self.makeRemoteShellCommand(command="cat */.SRCINFO")
        yield self.runCommand(cmd)
        dependencies = self.extract_deps(self.observer.getStdout())


        # Build all packages
        if self.getProperty("build_package"):
            package = self.getProperty("build_package")
            needed = self.resolve(package, dependencies)
            # Run over all needed packages and find their deps
            for i in needed:
                _needed = self.resolve(i, dependencies)
                self.add_build(i, _needed)
            self.add_build(package, needed)
            return self.sp
       
       # Build all
        if self.getProperty("build_all"):
            for package in dependencies.keys():
                needed = self.resolve(package, dependencies)
                # Run over all needed packages and find their deps
                for i in needed:
                    worker = random.choice(workers)
                    _needed = self.resolve(i, dependencie, worker=worker)
                    self.add_build(i, _needed, worker=worker)
                self.add_build(package, needed, worker=worker)
            return self.sp

        # Only build changes PKGBUILDS
        # And no, i don't know how to remove the output from
        # self.observer :c
        self.observer = logobserver.BufferLogObserver()
        self.addLogObserver('stdio', self.observer)
        cmd = yield self.makeRemoteShellCommand(command=["git", "diff","--name-only","HEAD~1"])
        yield self.runCommand(cmd)

        packages = []<commit_msg>Fixed single package build<commit_after># -*- python -*-
# ex: set filetype=python:

import random

from buildbot.process import remotecommand
from buildbot.plugins import *
from buildbot.plugins import util, steps
from buildbot.process import buildstep, logobserver
from twisted.internet import defer


# This is a sample buildmaster config file. It must be installed as
# 'master.cfg' in your buildmaster's base directory.

# This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing.
c = BuildmasterConfig = {}

####### WORKERS

# The 'workers' list defines the set of recognized workers. Each element is
# a Worker object, specifying a unique worker name and password.  The same
# worker name and password must be configured on the worker.
# c['workers'] = [worker.Worker("example-worker", "pass", max_builds=30),
#                 worker.Worker("example-worker2", "pass", max_builds=30)]

workers = ["bot1","bot2"]

c['workers'] = [worker.LocalWorker(bot) for bot in workers]
# 'protocols' contains information about protocols which master will use for
# communicating with workers. You must define at least 'port' option that workers
# could connect to your master with this protocol.
# 'port' must match the value configured into the workers (with their
# --master option)
c['protocols'] = {'pb': {'port': 9989}}

####### CHANGESOURCES

# the 'change_source' setting tells the buildmaster how it should find out
# about source code changes.  Here we point to the buildbot clone of pyflakes.

c['change_source'] = []
c['change_source'].append(changes.GitPoller(
        'git://github.com/Foxboron/PKGBUILDS.git',
        workdir='gitpoller-workdir', branch='master',
        pollinterval=10))

####### SCHEDULERS

# Configure the Schedulers, which decide how to react to incoming changes.  In this
# case, just kick off a 'runtests' build

c['schedulers'] = []
#c['schedulers'].append(schedulers.SingleBranchScheduler(
#                            name="all",
#                            change_filter=util.ChangeFilter(branch='master'),
#                            treeStableTimer=None,
#                            builderNames=["unpack-packages"]))



sch = schedulers.ForceScheduler(
        name="force",
        buttonName="Build package",
        label="Build package",
        builderNames=["unpack-packages"],

        codebases=[
            util.CodebaseParameter(
                "",
                name="Main repository",
                branch=util.ChoiceStringParameter(
                    name="branch",
                    choices=["master"],
                    default="master"),

                revision=util.FixedParameter(name="revision", default=""),
                repository=util.FixedParameter(name="repository", default=""),
                project=util.FixedParameter(name="project", default=""),
                ),
            ],

        properties=[
            util.StringParameter(name="build_package",
                label="Package:", default="",
                required=False, size=80),
            util.StringParameter(name="repository",
                label="Repository:", default="foxboron",
                required=False, size=80),
            ]
    )


c['schedulers'].append(sch)     

c['schedulers'].append(schedulers.Triggerable(name="build", builderNames=["build-package"]))

####### BUILDERS



class TriggerWithPackageNames(steps.Trigger, buildstep.ShellMixin):

    def assign_build_props(self, name, dependencies, worker=False):
        props = self.set_properties.copy()
        props["virtual_builder_name"] = name 
        props["package"] = name 
        props["dependencies"] = dependencies 
        if worker:
            props["workername"] = worker
        return ["build", props]

    def add_build(self, package, needed, worker=False):
        """I was scared for multipe triggers for the same package :c"""
        if package not in self._build_requests:
            self.sp.append(self.assign_build_props(package, needed))

    def resolve(self, name, packages, graph=[]):
        """Poor mans dependency resolver """
        available = list(packages.keys())
        for i in packages[name]:
            if i in available and i not in graph:
                graph.append(i)
                self.resolve(i, packages, graph=graph)
        return graph

    def extract_deps(self, srcinfo):
        """ Poor mans .SRCINFO parser """
        packages={}
        pkgname=""

        for i in srcinfo.split("\n"):
            if not i:
                continue
            if i[0] == "#":
                continue
            option = i.strip() 
            key,value = option.split(" = ")
            if key == "pkgbase":
                pkgname=value
                packages[pkgname] = []
            if key == "makedepends":
                packages[pkgname].append(value)
            if key == "depends":
                packages[pkgname].append(value)
        return packages

    @defer.inlineCallbacks
    def getSchedulersAndProperties(self):
        """Bread and butter.
        This triggers all the needed PKGBUILDS and resolves the dependencies"""
        self.sp = []
        self._build_requests = []

        # If we have dependant builds, they need to go on the same worker
        worker = False


        # I simply dont know how to make the yield stuff recursive
        # so we find all .SRCINFOs available and make a list
        self.observer = logobserver.BufferLogObserver()
        self.addLogObserver('stdio', self.observer)
        cmd = yield self.makeRemoteShellCommand(command="cat */.SRCINFO")
        yield self.runCommand(cmd)
        dependencies = self.extract_deps(self.observer.getStdout())


        # Build all packages
        if self.getProperty("build_package"):
            package = self.getProperty("build_package")
            needed = self.resolve(package, dependencies)
            # Run over all needed packages and find their deps
            for i in needed:
                worker = random.choice(workers)
                _needed = self.resolve(i, dependencies, worker=worker)
                self.add_build(i, _needed, worker=worker)
            self.add_build(package, needed, worker=worker)
            return self.sp
       
       # Build all
        if self.getProperty("build_all"):
            for package in dependencies.keys():
                needed = self.resolve(package, dependencies)
                # Run over all needed packages and find their deps
                for i in needed:
                    worker = random.choice(workers)
                    _needed = self.resolve(i, dependencie, worker=worker)
                    self.add_build(i, _needed, worker=worker)
                self.add_build(package, needed, worker=worker)
            return self.sp

        # Only build changes PKGBUILDS
        # And no, i don't know how to remove the output from
        # self.observer :c
        self.observer = logobserver.BufferLogObserver()
        self.addLogObserver('stdio', self.observer)
        cmd = yield self.makeRemoteShellCommand(command=["git", "diff","--name-only","HEAD~1"])
        yield self.runCommand(cmd)

        packages = []
{'fraction_non_alphanumeric': 0.10795770728992765, 'fraction_numerical': 0.0026432943795214247, 'mean_word_length': 2.3042978625603308, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7177737', 'n_tokens_mistral': 3841, 'n_tokens_neox': 3659, 'n_words': 1084}