<commit_before>#   Copyright 2015 Red Hat, Inc.
#
#   Licensed under the Apache License, Version 2.0 (the "License"); you may
#   not use this file except in compliance with the License. You may obtain
#   a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#   License for the specific language governing permissions and limitations
#   under the License.
#
from __future__ import print_function

import argparse
import glob
import logging
import os
import os.path
import re
import shutil
import six
import tempfile
import time
import yaml

from heatclient.common import template_utils
from heatclient import exc as hc_exc
from keystoneclient import exceptions as kscexc
from os_cloud_config import keystone
from os_cloud_config import keystone_pki
from os_cloud_config.utils import clients as occ_clients
from osc_lib.command import command
from osc_lib import exceptions as oscexc
from osc_lib.i18n import _
from osc_lib import utils as osc_utils
from swiftclient.exceptions import ClientException
from tripleo_common import update

from tripleoclient import constants
from tripleoclient import exceptions
from tripleoclient import utils
from tripleoclient.workflows import deployment
from tripleoclient.workflows import parameters as workflow_params
from tripleoclient.workflows import plan_management


class DeployOvercloud(command.Command):
    """Deploy Overcloud"""

    log = logging.getLogger(__name__ + ".DeployOvercloud")
    predeploy_errors = 0
    predeploy_warnings = 0

    def _update_parameters(self, args, network_client, stack):
        parameters = {}

        stack_is_new = stack is None

        timestamp = int(time.time())
        parameters['DeployIdentifier'] = timestamp
        parameters['UpdateIdentifier'] = ''
        parameters['StackAction'] = 'CREATE' if stack_is_new else 'UPDATE'

        # Update parameters from answers file:
        if args.answers_file is not None:
            with open(args.answers_file, 'r') as answers_file:
                answers = yaml.load(answers_file)

            if args.templates is None:
                args.templates = answers['templates']
            if 'environments' in answers:
                if args.environment_files is not None:
                    answers['environments'].extend(args.environment_files)
                args.environment_files = answers['environments']

        param_args = (
            ('NtpServer', 'ntp_server'),
            ('ControllerCount', 'control_scale'),
            ('ComputeCount', 'compute_scale'),
            ('ObjectStorageCount', 'swift_storage_scale'),
            ('BlockStorageCount', 'block_storage_scale'),
            ('CephStorageCount', 'ceph_storage_scale'),
            ('OvercloudControlFlavor', 'control_flavor'),
            ('OvercloudComputeFlavor', 'compute_flavor'),
            ('OvercloudBlockStorageFlavor', 'block_storage_flavor'),
            ('OvercloudSwiftStorageFlavor', 'swift_storage_flavor'),
            ('OvercloudCephStorageFlavor', 'ceph_storage_flavor'),
        )

        if stack_is_new:
            new_stack_args = (
                ('NovaComputeLibvirtType', 'libvirt_type'),
            )
            param_args = param_args + new_stack_args

        # Update parameters from commandline
        for param, arg in param_args:
            if getattr(args, arg, None) is not None:
                parameters[param] = getattr(args, arg)

        # Scaling needs extra parameters
        number_controllers = int(parameters.get('ControllerCount', 0))
        dhcp_agents_per_network = (min(number_controllers, 3) if
                                   number_controllers else 1)

        parameters.update({
            'NeutronDhcpAgentsPerNetwork': dhcp_agents_per_network,
        })

        return parameters

    def _create_registration_env(self, args):
        tht_root = args.templates

        env_file = os.path.join(
            tht_root,
            constants.RHEL_REGISTRATION_EXTRACONFIG_NAME,
            'environment-rhel-registration.yaml')
        registry = os.path.join(
            tht_root,
            constants.RHEL_REGISTRATION_EXTRACONFIG_NAME,
            'rhel-registration-resource-registry.yaml')
        user_env = {'rhel_reg_method': args.reg_method,
                    'rhel_reg_org': args.reg_org,
                    'rhel_reg_force': args.reg_force,
                    'rhel_reg_sat_url': args.reg_sat_url,
                    'rhel_reg_activation_key': args.reg_activation_key}
        return [registry, env_file], {"parameter_defaults": user_env}

    def _create_parameters_env(self, parameters):
        parameter_defaults = {"parameter_defaults": parameters}
        return parameter_defaults

    def _process_multiple_environments(self, created_env_files, added_files,
                                       tht_root, user_tht_root, cleanup=True):
        env_files = {}
        localenv = {}
        for env_path in created_env_files:
            self.log.debug("Processing environment files %s" % env_path)
            abs_env_path = os.path.abspath(env_path)
            if abs_env_path.startswith(user_tht_root):
                new_env_path = abs_env_path.replace(user_tht_root, tht_root)
                self.log.debug("Redirecting env file %s to %s"
                               % (abs_env_path, new_env_path))
                env_path = new_env_path
            try:
                files, env = template_utils.process_environment_and_files(
                    env_path=env_path)
            except hc_exc.CommandError as ex:
                # This provides fallback logic so that we can reference files
                # inside the resource_registry values that may be rendered via
                # j2.yaml templates, where the above will fail because the
                # file doesn't exist in user_tht_root, but it is in tht_root
                # See bug https://bugs.launchpad.net/tripleo/+bug/1625783
                # for details on why this is needed (backwards-compatibility)
                self.log.debug("Error %s processing environment file %s"
                               % (six.text_type(ex), env_path))
                with open(abs_env_path, 'r') as f:
                    env_map = yaml.safe_load(f)
                env_registry = env_map.get('resource_registry', {})
                env_dirname = os.path.dirname(os.path.abspath(env_path))
                for rsrc, rsrc_path in six.iteritems(env_registry):
                    # We need to calculate the absolute path relative to
                    # env_path not cwd (which is what abspath uses).
                    abs_rsrc_path = os.path.normpath(
                        os.path.join(env_dirname, rsrc_path))
                    # If the absolute path matches user_tht_root, rewrite
                    # a temporary environment pointing at tht_root instead
                    if abs_rsrc_path.startswith(user_tht_root):
                        new_rsrc_path = abs_rsrc_path.replace(user_tht_root,
                                                              tht_root)
                        self.log.debug("Rewriting %s %s path to %s"
                                       % (env_path, rsrc, new_rsrc_path))
                        env_registry[rsrc] = new_rsrc_path
                    else:
                        env_registry[rsrc] = rsrc_path
                env_map['resource_registry'] = env_registry
                f_name = os.path.basename(os.path.splitext(abs_env_path)[0])<commit_msg>Use the absolute path for temp templates<commit_after>#   Copyright 2015 Red Hat, Inc.
#
#   Licensed under the Apache License, Version 2.0 (the "License"); you may
#   not use this file except in compliance with the License. You may obtain
#   a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#   License for the specific language governing permissions and limitations
#   under the License.
#
from __future__ import print_function

import argparse
import glob
import logging
import os
import os.path
import re
import shutil
import six
import tempfile
import time
import yaml

from heatclient.common import template_utils
from heatclient import exc as hc_exc
from keystoneclient import exceptions as kscexc
from os_cloud_config import keystone
from os_cloud_config import keystone_pki
from os_cloud_config.utils import clients as occ_clients
from osc_lib.command import command
from osc_lib import exceptions as oscexc
from osc_lib.i18n import _
from osc_lib import utils as osc_utils
from swiftclient.exceptions import ClientException
from tripleo_common import update

from tripleoclient import constants
from tripleoclient import exceptions
from tripleoclient import utils
from tripleoclient.workflows import deployment
from tripleoclient.workflows import parameters as workflow_params
from tripleoclient.workflows import plan_management


class DeployOvercloud(command.Command):
    """Deploy Overcloud"""

    log = logging.getLogger(__name__ + ".DeployOvercloud")
    predeploy_errors = 0
    predeploy_warnings = 0

    def _update_parameters(self, args, network_client, stack):
        parameters = {}

        stack_is_new = stack is None

        timestamp = int(time.time())
        parameters['DeployIdentifier'] = timestamp
        parameters['UpdateIdentifier'] = ''
        parameters['StackAction'] = 'CREATE' if stack_is_new else 'UPDATE'

        # Update parameters from answers file:
        if args.answers_file is not None:
            with open(args.answers_file, 'r') as answers_file:
                answers = yaml.load(answers_file)

            if args.templates is None:
                args.templates = answers['templates']
            if 'environments' in answers:
                if args.environment_files is not None:
                    answers['environments'].extend(args.environment_files)
                args.environment_files = answers['environments']

        param_args = (
            ('NtpServer', 'ntp_server'),
            ('ControllerCount', 'control_scale'),
            ('ComputeCount', 'compute_scale'),
            ('ObjectStorageCount', 'swift_storage_scale'),
            ('BlockStorageCount', 'block_storage_scale'),
            ('CephStorageCount', 'ceph_storage_scale'),
            ('OvercloudControlFlavor', 'control_flavor'),
            ('OvercloudComputeFlavor', 'compute_flavor'),
            ('OvercloudBlockStorageFlavor', 'block_storage_flavor'),
            ('OvercloudSwiftStorageFlavor', 'swift_storage_flavor'),
            ('OvercloudCephStorageFlavor', 'ceph_storage_flavor'),
        )

        if stack_is_new:
            new_stack_args = (
                ('NovaComputeLibvirtType', 'libvirt_type'),
            )
            param_args = param_args + new_stack_args

        # Update parameters from commandline
        for param, arg in param_args:
            if getattr(args, arg, None) is not None:
                parameters[param] = getattr(args, arg)

        # Scaling needs extra parameters
        number_controllers = int(parameters.get('ControllerCount', 0))
        dhcp_agents_per_network = (min(number_controllers, 3) if
                                   number_controllers else 1)

        parameters.update({
            'NeutronDhcpAgentsPerNetwork': dhcp_agents_per_network,
        })

        return parameters

    def _create_registration_env(self, args):
        tht_root = args.templates

        env_file = os.path.join(
            tht_root,
            constants.RHEL_REGISTRATION_EXTRACONFIG_NAME,
            'environment-rhel-registration.yaml')
        registry = os.path.join(
            tht_root,
            constants.RHEL_REGISTRATION_EXTRACONFIG_NAME,
            'rhel-registration-resource-registry.yaml')
        user_env = {'rhel_reg_method': args.reg_method,
                    'rhel_reg_org': args.reg_org,
                    'rhel_reg_force': args.reg_force,
                    'rhel_reg_sat_url': args.reg_sat_url,
                    'rhel_reg_activation_key': args.reg_activation_key}
        return [registry, env_file], {"parameter_defaults": user_env}

    def _create_parameters_env(self, parameters):
        parameter_defaults = {"parameter_defaults": parameters}
        return parameter_defaults

    def _process_multiple_environments(self, created_env_files, added_files,
                                       tht_root, user_tht_root, cleanup=True):
        env_files = {}
        localenv = {}
        for env_path in created_env_files:
            self.log.debug("Processing environment files %s" % env_path)
            abs_env_path = os.path.abspath(env_path)
            if abs_env_path.startswith(user_tht_root):
                new_env_path = abs_env_path.replace(user_tht_root, tht_root)
                self.log.debug("Redirecting env file %s to %s"
                               % (abs_env_path, new_env_path))
                env_path = new_env_path
            try:
                files, env = template_utils.process_environment_and_files(
                    env_path=env_path)
            except hc_exc.CommandError as ex:
                # This provides fallback logic so that we can reference files
                # inside the resource_registry values that may be rendered via
                # j2.yaml templates, where the above will fail because the
                # file doesn't exist in user_tht_root, but it is in tht_root
                # See bug https://bugs.launchpad.net/tripleo/+bug/1625783
                # for details on why this is needed (backwards-compatibility)
                self.log.debug("Error %s processing environment file %s"
                               % (six.text_type(ex), env_path))
                with open(abs_env_path, 'r') as f:
                    env_map = yaml.safe_load(f)
                env_registry = env_map.get('resource_registry', {})
                env_dirname = os.path.dirname(os.path.abspath(env_path))
                for rsrc, rsrc_path in six.iteritems(env_registry):
                    # We need to calculate the absolute path relative to
                    # env_path not cwd (which is what abspath uses).
                    abs_rsrc_path = os.path.normpath(
                        os.path.join(env_dirname, rsrc_path))
                    # If the absolute path matches user_tht_root, rewrite
                    # a temporary environment pointing at tht_root instead
                    if abs_rsrc_path.startswith(user_tht_root):
                        new_rsrc_path = abs_rsrc_path.replace(user_tht_root,
                                                              tht_root)
                        self.log.debug("Rewriting %s %s path to %s"
                                       % (env_path, rsrc, new_rsrc_path))
                        env_registry[rsrc] = new_rsrc_path
                    else:
                        env_registry[rsrc] = abs_rsrc_path
                env_map['resource_registry'] = env_registry
                f_name = os.path.basename(os.path.splitext(abs_env_path)[0])
{'fraction_non_alphanumeric': 0.07311578811868442, 'fraction_numerical': 0.0030893994979725816, 'mean_word_length': 2.23775786622213, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2348395', 'n_tokens_mistral': 4323, 'n_tokens_neox': 4130, 'n_words': 1091}