<commit_before>#
# This file is part of Hawaii.
#
# Copyright (C) 2015 <NAME> <<EMAIL>>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

import os, re

from buildbot.plugins import steps
from buildbot.process.buildstep import ShellMixin
from buildbot.process.buildstep import BuildStep
from buildbot.status.results import *

from twisted.internet import defer

class BinaryPackageBuild(ShellMixin, BuildStep):
    """
    Build a package in a clean chroot.
    See https://wiki.archlinux.org/index.php/DeveloperWiki:Building_in_a_Clean_Chroot
    """
    description = "Build a package in a clean chroot"
    artifacts = []

    def __init__(self, name, arch, depends, provides, **kwargs):
        BuildStep.__init__(self, haltOnFailure=True, **kwargs)
        self.name = "BinaryPackage {} {}".format(name, arch)
        self.pkgname = name
        self.arch = arch
        self.depends = depends
        self.provides = provides

    @defer.inlineCallbacks
    def run(self):
        log = yield self.addLog("logs")

        yield log.addStdout(u"Depends: {}\n".format(self.depends))
        yield log.addStdout(u"Provides: {}\n".format(self.provides))

        # Package directory
        workdir = os.path.join(self.workdir, self.pkgname)

        # Check whether we already built the latest version
        cmd = yield self._makeCommand("../../helpers/pkgversion -l PKGBUILD", workdir=workdir)
        yield self.runCommand(cmd)
        if cmd.didFail():
            defer.returnValue(FAILURE)
        self.latest_version = cmd.stdout.strip()
        if self.latest_version == "?":
            yield log.addStderr(u"Unable to determine the {} version".format(self.pkgname))
            defer.returnValue(FAILURE)
        yield log.addStdout(u"Latest Version: {}\n".format(self.latest_version))

        # Determine the package file name
        self.pkgfilename = "{}-{}-{}.pkg.tar.xz".format(self.pkgname, self.latest_version, self.arch)
        yield log.addStdout(u"Expected package file name: {}\n".format(self.pkgfilename))

        # Already built packages
        r = re.compile(r'{}.*\-.*\-{}\.pkg\.tar\.xz'.format(self.pkgname, self.arch))
        already_built_packages = filter(r.match, self.getProperty("existing_packages"))
        yield log.addStdout(u"Existing packages: {}\n".format(already_built_packages))

        # Did we have this package already built?
        r = re.compile(r'{}.*\-{}\-{}\.pkg\.tar\.xz'.format(self.pkgname, self.latest_version, self.arch))
        already_built_same_version = filter(r.match, self.getProperty("existing_packages"))

        if len(already_built_same_version) > 0:
            self.current = True
            yield log.addStdout(u"Package already built, skipping!\n")
        else:
            # Retrieve the chroot path
            chrootdir = self.getProperty("chroot_basedir")
            yield log.addStdout(u"Building from chroot: {}\n".format(chrootdir))

            # Actually build the package
            repodir = os.path.join(self.workdir, "..", "repository")
            cmd = yield self._makeCommand("sudo makechrootpkg -cu -D {}:/var/tmp/repository -r {}".format(repodir, chrootdir), workdir=workdir)
            yield self.runCommand(cmd)
            if cmd.didFail():
                defer.returnValue(FAILURE)

            # Find the artifacts
            cmd = yield self._makeCommand(["/usr/bin/find", ".", "-type", "f", "-name", "*.pkg.tar.xz", "-printf", "%f "], workdir=workdir)
            yield self.runCommand(cmd)
            if cmd.didFail():
                defer.returnValue(FAILURE)

            # Add artifact to the list<commit_msg>archlinux/pkgactions: Use ccm to build packages<commit_after>#
# This file is part of Hawaii.
#
# Copyright (C) 2015 <NAME> <<EMAIL>>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

import os, re

from buildbot.plugins import steps
from buildbot.process.buildstep import ShellMixin
from buildbot.process.buildstep import BuildStep
from buildbot.status.results import *

from twisted.internet import defer

from chrootactions import CcmAction

class BinaryPackageBuild(CcmAction):
    """
    Build a package in a clean chroot.
    See https://wiki.archlinux.org/index.php/DeveloperWiki:Building_in_a_Clean_Chroot
    """
    description = "Build a package in a clean chroot"
    artifacts = []
    ccm = True

    def __init__(self, name, arch, depends, provides, **kwargs):
        CcmAction.__init__(self, arch=arch, action="s", workdir=os.path.join(self.workdir, name), **kwargs)
        self.name = "BinaryPackage {} {}".format(name, arch)
        self.pkgname = name
        self.arch = arch
        self.depends = depends
        self.provides = provides

    @defer.inlineCallbacks
    def run(self):
        log = yield self.addLog("logs")

        yield log.addStdout(u"Depends: {}\n".format(self.depends))
        yield log.addStdout(u"Provides: {}\n".format(self.provides))

        # Check whether we already built the latest version
        cmd = yield self._makeCommand("../../helpers/pkgversion -l PKGBUILD")
        yield self.runCommand(cmd)
        if cmd.didFail():
            defer.returnValue(FAILURE)
        self.latest_version = cmd.stdout.strip()
        if self.latest_version == "?":
            yield log.addStderr(u"Unable to determine the {} version".format(self.pkgname))
            defer.returnValue(FAILURE)
        yield log.addStdout(u"Latest Version: {}\n".format(self.latest_version))

        # Determine the package file name
        self.pkgfilename = "{}-{}-{}.pkg.tar.xz".format(self.pkgname, self.latest_version, self.arch)
        yield log.addStdout(u"Expected package file name: {}\n".format(self.pkgfilename))

        # Already built packages
        r = re.compile(r'{}.*\-.*\-{}\.pkg\.tar\.xz'.format(self.pkgname, self.arch))
        already_built_packages = filter(r.match, self.getProperty("existing_packages"))
        yield log.addStdout(u"Existing packages: {}\n".format(already_built_packages))

        # Did we have this package already built?
        r = re.compile(r'{}.*\-{}\-{}\.pkg\.tar\.xz'.format(self.pkgname, self.latest_version, self.arch))
        already_built_same_version = filter(r.match, self.getProperty("existing_packages"))

        if len(already_built_same_version) > 0:
            self.current = True
            yield log.addStdout(u"Package already built, skipping!\n")
        else:
            if self.ccm is True:
                # Build the package with ccm
                cmd = yield self._makeCcmCommand("s")
                yield self.runCommand(cmd)
                if cmd.didFail():
                    defer.returnValue(FAILURE)
            else:
                # Retrieve the chroot path
                chrootdir = self.getProperty("chroot_basedir")
                yield log.addStdout(u"Building from chroot: {}\n".format(chrootdir))

                # Actually build the package
                repodir = os.path.join(self.workdir, "..", "repository")
                cmd = yield self._makeCommand("sudo makechrootpkg -cu -D {}:/var/tmp/repository -r {}".format(repodir, chrootdir))
                yield self.runCommand(cmd)
                if cmd.didFail():
                    defer.returnValue(FAILURE)

            # Find the artifacts
            cmd = yield self._makeCommand(["/usr/bin/find", ".", "-type", "f", "-name", "*.pkg.tar.xz", "-printf", "%f "])
            yield self.runCommand(cmd)
            if cmd.didFail():
                defer.returnValue(FAILURE)

            # Add artifact to the list
{'fraction_non_alphanumeric': 0.11929864771945908, 'fraction_numerical': 0.0013752005500802201, 'mean_word_length': 2.6916243654822334, 'pattern_counts': {'":': 2, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '921636', 'n_tokens_mistral': 2574, 'n_tokens_neox': 2467, 'n_words': 696}