<commit_before>import importlib
import os
import json

import sh
import yaml
from path import path

from helpers import sh_bake
from helpers import update_config

git = sh_bake(sh.git)
pip = sh_bake(sh.pip)
nosetests = sh_bake(sh.nosetests)


CLOUDIFY_SYSTEM_TESTS = 'cloudify-system-tests'


class HandlerPackage(object):

    def __init__(self, handler, external, directory=None):
        if not external:
            self.init = importlib.import_module(
                'helpers.handlers.{0}'.format(handler))
            self.requirements_path = os.path.join(
                os.path.dirname(__file__), 'helpers', 'handlers',
                handler, 'requirements.txt')
            self.update_config = importlib.import_module(
                'helpers.handlers.{0}.update_config'.format(handler))
        else:
            main_package, handler_name = handler.split('.')
            self.init = importlib.import_module(
                '{0}.system_tests.handlers.{1}'.format(main_package,
                                                       handler_name))
            self.requirements_path = os.path.join(
                directory, main_package, 'system_tests', 'handlers',
                handler_name, 'requirements.txt')
            self.update_config = importlib.import_module(
                '{0}.system_tests.handlers.{1}.update_config'.format(
                    main_package, handler_name))


class SuiteRunner(object):

    def __init__(self):
        self.base_dir = os.environ['BASE_HOST_DIR']
        self.work_dir = os.environ['WORK_DIR']
        self.branch_name_core = os.environ['BRANCH_NAME_CORE']
        self.branch_name_plugins = os.environ['BRANCH_NAME_PLUGINS']
        self.branch_name_system_tests = os.environ['BRANCH_NAME_SYSTEM_TESTS']
        self.opencm_git_pwd = os.environ['OPENCM_GIT_PWD']

        self.test_suite_name = os.environ['TEST_SUITE_NAME']
        self.test_suite = json.loads(os.environ['TEST_SUITE'])
        with open(os.path.join(self.base_dir, 'suites', 'suites.yaml')) as f:
            self.suites_yaml = yaml.load(f.read())

        self.handler_configuration = self.suites_yaml[
            'handler_configurations'][self.test_suite['handler_configuration']]
        self.bootstrap_using_providers = self.handler_configuration.get(
            'bootstrap_using_providers', False)
        self.bootstrap_using_docker = self.handler_configuration.get(
            'bootstrap_using_docker', True)

        inputs = self.handler_configuration['inputs']
        self.original_inputs_path = os.path.join(self.base_dir,
                                                 'configurations', inputs)
        self.generated_inputs_path = os.path.join(
            self.work_dir, 'generated-config.{0}'.format(
                'yaml' if self.bootstrap_using_providers else 'json'))
        self.generated_suites_yaml_path = os.path.join(
            self.work_dir, 'generated-suites.yaml')
        self.manager_blueprints_dir = os.path.join(
            self.work_dir, 'cloudify-manager-blueprints')

        self.handler = self.handler_configuration['handler']
        self.handler_package = None

    def set_env_variables(self):
        os.environ['WORKFLOW_TASK_RETRIES'] = os.environ.get(
            'WORKFLOW_TASK_RETRIES', '20')
        os.environ['HANDLER_CONFIGURATION'] = self.test_suite[
            'handler_configuration']
        os.environ['SUITES_YAML_PATH'] = self.generated_suites_yaml_path

    def clone_and_install_packages(self):
        with path(self.work_dir):
            self._clone_and_checkout_repo(repo=CLOUDIFY_SYSTEM_TESTS,
                                          branch=self.branch_name_system_tests)
            self._clone_and_checkout_repo(repo='cloudify-cli',
                                          branch=self.branch_name_core)
            self._clone_and_checkout_repo(repo='cloudify-manager-blueprints',
                                          branch=self.branch_name_core)
            pip.install('./cloudify-cli',
                        '-r', './cloudify-cli/requirements.txt').wait()
            pip.install('-e', './{0}'.format(CLOUDIFY_SYSTEM_TESTS)).wait()

            if 'external' in self.handler_configuration:
                external = self.handler_configuration['external']
                repo = external['repo']
                branch = external.get('branch', self.branch_name_plugins)
                organization = external.get('organization', 'cloudify-cosmo')
                self._clone_and_checkout_repo(repo=repo,
                                              branch=branch,
                                              organization=organization)
                pip.install('-e', './{0}'.format(repo)).wait()

                self.handler_package = HandlerPackage(
                    self.handler,
                    external=True,
                    directory=os.path.join(self.work_dir, repo))
            else:
                self.handler_package = HandlerPackage(self.handler,
                                                      external=False)

            pip.install('-r', self.handler_package.requirements_path).wait()

            handler_init = self.handler_package.init
            if self.bootstrap_using_providers:
                provider_repo = handler_init.provider_repo
                self._clone_and_checkout_repo(repo=provider_repo,
                                              branch=self.branch_name_plugins)
                pip.install('./{0}'.format(provider_repo)).wait()

            # TODO: this logic only exists for the vpshere handler
            # move handler into plugin code and install it like any
            # other external handler
            if hasattr(handler_init, 'plugin_repo'):
                plugin_repo = handler_init.plugin_repo
                private_repo = getattr(handler_init, 'private', False)
                self._clone_and_checkout_repo(repo=plugin_repo,
                                              branch=self.branch_name_plugins,
                                              private_repo=private_repo)
                pip.install('./{0}'.format(plugin_repo)).wait()
                if getattr(handler_init, 'has_manager_blueprint', False):
                    self.manager_blueprints_dir = os.path.join(
                        self.work_dir, plugin_repo)

    def _clone_and_checkout_repo(self,
                                 repo,
                                 branch,
                                 organization='cloudify-cosmo',
                                 private_repo=False):
        with path(self.work_dir):
            if private_repo:
                git.clone('https://opencm:{0}@github.com/{1}/{2}'
                          .format(self.opencm_git_pwd, organization, repo),
                          depth=1).wait()
            else:
                git.clone('https://github.com/{0}/{1}'
                          .format(organization, repo), depth=1).wait()

            with path(repo):
                git.checkout(branch).wait()

    def generate_config(self):
        with open(self.original_inputs_path) as rf:
            with open(self.generated_inputs_path, 'w') as wf:
                wf.write(rf.read())
        update_config.update_config(
            config_path=self.generated_inputs_path,
            bootstrap_using_providers=self.bootstrap_using_providers,
            bootstrap_using_docker=self.bootstrap_using_docker,<commit_msg>CFY-1850 move import inside function so that it only gets evaluated after code has bee properly installed<commit_after>import importlib
import os
import json

import sh
import yaml
from path import path

from helpers import sh_bake
# don't put imports that may include system tests code here
# put them inside functions that use them only after cloudify-system-tests
# have been installed

git = sh_bake(sh.git)
pip = sh_bake(sh.pip)
nosetests = sh_bake(sh.nosetests)


CLOUDIFY_SYSTEM_TESTS = 'cloudify-system-tests'


class HandlerPackage(object):

    def __init__(self, handler, external, directory=None):
        if not external:
            self.init = importlib.import_module(
                'helpers.handlers.{0}'.format(handler))
            self.requirements_path = os.path.join(
                os.path.dirname(__file__), 'helpers', 'handlers',
                handler, 'requirements.txt')
            self.update_config = importlib.import_module(
                'helpers.handlers.{0}.update_config'.format(handler))
        else:
            main_package, handler_name = handler.split('.')
            self.init = importlib.import_module(
                '{0}.system_tests.handlers.{1}'.format(main_package,
                                                       handler_name))
            self.requirements_path = os.path.join(
                directory, main_package, 'system_tests', 'handlers',
                handler_name, 'requirements.txt')
            self.update_config = importlib.import_module(
                '{0}.system_tests.handlers.{1}.update_config'.format(
                    main_package, handler_name))


class SuiteRunner(object):

    def __init__(self):
        self.base_dir = os.environ['BASE_HOST_DIR']
        self.work_dir = os.environ['WORK_DIR']
        self.branch_name_core = os.environ['BRANCH_NAME_CORE']
        self.branch_name_plugins = os.environ['BRANCH_NAME_PLUGINS']
        self.branch_name_system_tests = os.environ['BRANCH_NAME_SYSTEM_TESTS']
        self.opencm_git_pwd = os.environ['OPENCM_GIT_PWD']

        self.test_suite_name = os.environ['TEST_SUITE_NAME']
        self.test_suite = json.loads(os.environ['TEST_SUITE'])
        with open(os.path.join(self.base_dir, 'suites', 'suites.yaml')) as f:
            self.suites_yaml = yaml.load(f.read())

        self.handler_configuration = self.suites_yaml[
            'handler_configurations'][self.test_suite['handler_configuration']]
        self.bootstrap_using_providers = self.handler_configuration.get(
            'bootstrap_using_providers', False)
        self.bootstrap_using_docker = self.handler_configuration.get(
            'bootstrap_using_docker', True)

        inputs = self.handler_configuration['inputs']
        self.original_inputs_path = os.path.join(self.base_dir,
                                                 'configurations', inputs)
        self.generated_inputs_path = os.path.join(
            self.work_dir, 'generated-config.{0}'.format(
                'yaml' if self.bootstrap_using_providers else 'json'))
        self.generated_suites_yaml_path = os.path.join(
            self.work_dir, 'generated-suites.yaml')
        self.manager_blueprints_dir = os.path.join(
            self.work_dir, 'cloudify-manager-blueprints')

        self.handler = self.handler_configuration['handler']
        self.handler_package = None

    def set_env_variables(self):
        os.environ['WORKFLOW_TASK_RETRIES'] = os.environ.get(
            'WORKFLOW_TASK_RETRIES', '20')
        os.environ['HANDLER_CONFIGURATION'] = self.test_suite[
            'handler_configuration']
        os.environ['SUITES_YAML_PATH'] = self.generated_suites_yaml_path

    def clone_and_install_packages(self):
        with path(self.work_dir):
            self._clone_and_checkout_repo(repo=CLOUDIFY_SYSTEM_TESTS,
                                          branch=self.branch_name_system_tests)
            self._clone_and_checkout_repo(repo='cloudify-cli',
                                          branch=self.branch_name_core)
            self._clone_and_checkout_repo(repo='cloudify-manager-blueprints',
                                          branch=self.branch_name_core)
            pip.install('./cloudify-cli',
                        '-r', './cloudify-cli/requirements.txt').wait()
            pip.install('-e', './{0}'.format(CLOUDIFY_SYSTEM_TESTS)).wait()

            if 'external' in self.handler_configuration:
                external = self.handler_configuration['external']
                repo = external['repo']
                branch = external.get('branch', self.branch_name_plugins)
                organization = external.get('organization', 'cloudify-cosmo')
                self._clone_and_checkout_repo(repo=repo,
                                              branch=branch,
                                              organization=organization)
                pip.install('-e', './{0}'.format(repo)).wait()

                self.handler_package = HandlerPackage(
                    self.handler,
                    external=True,
                    directory=os.path.join(self.work_dir, repo))
            else:
                self.handler_package = HandlerPackage(self.handler,
                                                      external=False)

            pip.install('-r', self.handler_package.requirements_path).wait()

            handler_init = self.handler_package.init
            if self.bootstrap_using_providers:
                provider_repo = handler_init.provider_repo
                self._clone_and_checkout_repo(repo=provider_repo,
                                              branch=self.branch_name_plugins)
                pip.install('./{0}'.format(provider_repo)).wait()

            # TODO: this logic only exists for the vpshere handler
            # move handler into plugin code and install it like any
            # other external handler
            if hasattr(handler_init, 'plugin_repo'):
                plugin_repo = handler_init.plugin_repo
                private_repo = getattr(handler_init, 'private', False)
                self._clone_and_checkout_repo(repo=plugin_repo,
                                              branch=self.branch_name_plugins,
                                              private_repo=private_repo)
                pip.install('./{0}'.format(plugin_repo)).wait()
                if getattr(handler_init, 'has_manager_blueprint', False):
                    self.manager_blueprints_dir = os.path.join(
                        self.work_dir, plugin_repo)

    def _clone_and_checkout_repo(self,
                                 repo,
                                 branch,
                                 organization='cloudify-cosmo',
                                 private_repo=False):
        with path(self.work_dir):
            if private_repo:
                git.clone('https://opencm:{0}@github.com/{1}/{2}'
                          .format(self.opencm_git_pwd, organization, repo),
                          depth=1).wait()
            else:
                git.clone('https://github.com/{0}/{1}'
                          .format(organization, repo), depth=1).wait()

            with path(repo):
                git.checkout(branch).wait()

    def generate_config(self):
        from helpers import update_config
        with open(self.original_inputs_path) as rf:
            with open(self.generated_inputs_path, 'w') as wf:
                wf.write(rf.read())
        update_config.update_config(
            config_path=self.generated_inputs_path,
            bootstrap_using_providers=self.bootstrap_using_providers,
            bootstrap_using_docker=self.bootstrap_using_docker,
{'fraction_non_alphanumeric': 0.09837467921300257, 'fraction_numerical': 0.0028953082845298416, 'mean_word_length': 1.8926532165968786, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 52, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4899295', 'n_tokens_mistral': 4137, 'n_tokens_neox': 4018, 'n_words': 614}