<commit_before>#
# Copyright (c) SAS Institute, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""
Module for finding artifactory packages and updating them
"""

from collections import deque
import logging
import time

from conary import conarycfg
from rmake.build import buildcfg
from rmake.cmdline import helper

from . import cmdline
from . import pkgsource
from .bot import Bot as BotSuperClass
from .build import Builder
from .errors import JobFailedError
from .lib import util
from .update import Updater as UpdaterSuperClass


log = logging.getLogger('updatebot.artifactory')


class Bot(BotSuperClass):

    _updateMode = 'artifactory'

    def __init__(self, cfg):
        self._validateMode(cfg)

        self._cfg = cfg

        self._clientcfg = cmdline.UpdateBotClientConfig()
        self._ui = cmdline.UserInterface(self._clientcfg)

        self._pkgSource = pkgsource.PackageSource(self._cfg, self._ui)
        self._updater = Updater(self._cfg, self._ui, self._pkgSource)

    def create(self, rebuild=False, recreate=None):
        """
        Do initial imports.

        :param bool rebuild: build all packages, even if source is the same
        :param bool recreate: recreate all source packages
        """
        start = time.time()
        log.info('starting import')

        # Populate rpm source object from yum metadata.
        self._pkgSource.load()

        # Import sources into repository.
        trvMap, fail = self._updater.create(buildAll=rebuild, recreate=recreate)

        if fail:
            log.error('failed to create %s packages:' % len(fail))
            for pkg, e in fail:
                log.error('failed to import %s: %s' % (pkg, e))
            return {}, fail

        log.info('elapsed time %s' % (time.time() - start, ))
        return trvMap, fail


class Updater(UpdaterSuperClass):
    """Class for finding and updating packages sourced from artifactory
    """

    def _buildLeaf(self, leaf, cache, buildAll=False, recreate=False):
        failedImports = set()
        srcVersion = cache.get((
            '%s:source' % leaf.name,
            leaf.getConaryVersion(),
            None,
            ))
        binVersion = cache.get((
            leaf.name,
            leaf.getConaryVersion(),
            None,
            ))

        # determine if leaf needs to be imported, and update srcVersion
        srcVersion = self._importPackage(leaf, srcVersion, recreate)

        # if buildAll is true, or there is no existing binary or the
        # binary was built from a different source, then build leaf
        if (buildAll or not binVersion
                or binVersion.getSourceVersion() != srcVersion):
            return srcVersion, True
        else:
            log.info('not building %s' % leaf)
            return srcVersion, False

    def _build(self, buildSet, buildReqs, cache):
        """Helper function to do some repetivite pre-build processing

        :param buildSet: list of name, version, flavor tuples and packages to
            build
        :type buildSet: [((name, version, flavor), package), ...]
        :param dict cache: conary version cache
        """
        # unpack buildSet into nvf tuples
        nvfs = []
        buildPackages = []
        resolveTroves = set()

        for package, version in buildSet:
            nvfs.append((package.name, version, None))
            buildPackages.append(package)

        # get our base rmakeCfg
        rmakeCfg = Builder(self._cfg, self._ui)._getRmakeConfig()

        # create resolve troves for deps not in the current chunk
        resolveTroves.update(set([
            (dep.name, rmakeCfg.buildLabel, dep.getConaryVersion())
            for dep in buildReqs
            if dep not in buildPackages
            ]))

        rmakeCfg.configKey(
            'resolveTroves',
            ' '.join('%s=%s/%s' % r for r in resolveTroves),
            )

        # make a new buidler with rmakeCfg to do the actual build
        builder = Builder(self._cfg, self._ui, rmakeCfg=rmakeCfg)

        # Build all newly imported packages.<commit_msg>Fix empty resolveTroves breaking rmake builds<commit_after>#
# Copyright (c) SAS Institute, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""
Module for finding artifactory packages and updating them
"""

from collections import deque
import logging
import time

from conary import conarycfg
from rmake.build import buildcfg
from rmake.cmdline import helper

from . import cmdline
from . import pkgsource
from .bot import Bot as BotSuperClass
from .build import Builder
from .errors import JobFailedError
from .lib import util
from .update import Updater as UpdaterSuperClass


log = logging.getLogger('updatebot.artifactory')


class Bot(BotSuperClass):

    _updateMode = 'artifactory'

    def __init__(self, cfg):
        self._validateMode(cfg)

        self._cfg = cfg

        self._clientcfg = cmdline.UpdateBotClientConfig()
        self._ui = cmdline.UserInterface(self._clientcfg)

        self._pkgSource = pkgsource.PackageSource(self._cfg, self._ui)
        self._updater = Updater(self._cfg, self._ui, self._pkgSource)

    def create(self, rebuild=False, recreate=None):
        """
        Do initial imports.

        :param bool rebuild: build all packages, even if source is the same
        :param bool recreate: recreate all source packages
        """
        start = time.time()
        log.info('starting import')

        # Populate rpm source object from yum metadata.
        self._pkgSource.load()

        # Import sources into repository.
        trvMap, fail = self._updater.create(buildAll=rebuild, recreate=recreate)

        if fail:
            log.error('failed to create %s packages:' % len(fail))
            for pkg, e in fail:
                log.error('failed to import %s: %s' % (pkg, e))
            return {}, fail

        log.info('elapsed time %s' % (time.time() - start, ))
        return trvMap, fail


class Updater(UpdaterSuperClass):
    """Class for finding and updating packages sourced from artifactory
    """

    def _buildLeaf(self, leaf, cache, buildAll=False, recreate=False):
        failedImports = set()
        srcVersion = cache.get((
            '%s:source' % leaf.name,
            leaf.getConaryVersion(),
            None,
            ))
        binVersion = cache.get((
            leaf.name,
            leaf.getConaryVersion(),
            None,
            ))

        # determine if leaf needs to be imported, and update srcVersion
        srcVersion = self._importPackage(leaf, srcVersion, recreate)

        # if buildAll is true, or there is no existing binary or the
        # binary was built from a different source, then build leaf
        if (buildAll or not binVersion
                or binVersion.getSourceVersion() != srcVersion):
            return srcVersion, True
        else:
            log.info('not building %s', leaf)
            return srcVersion, False

    def _build(self, buildSet, buildReqs, cache):
        """Helper function to do some repetivite pre-build processing

        :param buildSet: list of name, version, flavor tuples and packages to
            build
        :type buildSet: [((name, version, flavor), package), ...]
        :param dict cache: conary version cache
        """
        # unpack buildSet into nvf tuples
        nvfs = []
        buildPackages = []
        resolveTroves = set()

        for package, version in buildSet:
            nvfs.append((package.name, version, None))
            buildPackages.append(package)

        # get our base rmakeCfg
        rmakeCfg = Builder(self._cfg, self._ui)._getRmakeConfig()

        # create resolve troves for deps not in the current chunk
        resolveTroves.update(set([
            (dep.name, rmakeCfg.buildLabel, dep.getConaryVersion())
            for dep in buildReqs
            if dep not in buildPackages
            ]))

        if resolveTroves:
            rmakeCfg.configKey(
                'resolveTroves',
                ' '.join('%s=%s/%s' % r for r in resolveTroves),
                )

        # make a new buidler with rmakeCfg to do the actual build
        builder = Builder(self._cfg, self._ui, rmakeCfg=rmakeCfg)

        # Build all newly imported packages.
{'fraction_non_alphanumeric': 0.08553554093409557, 'fraction_numerical': 0.0008629058354007119, 'mean_word_length': 2.4000733406674, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3186884', 'n_tokens_mistral': 2527, 'n_tokens_neox': 2453, 'n_words': 884}