<commit_before>from __future__ import print_function, unicode_literals

import logging
import ssl

import botocore.exceptions
from cliff.lister import Lister

from madcore import const
from madcore.base import JenkinsBase
from madcore import configs
from madcore.configs import config
from madcore.configure import MadcoreConfigure
from madcore.const import DOMAIN_REGISTRATION
from madcore.libs import timeouts
from madcore.libs.cloudformation import StackCreate


class Configure(JenkinsBase, Lister):
    _description = "Configure madcore"
    logger = logging.getLogger(__name__)

    @classmethod
    def add_repos_params(cls, parser):
        for repo in const.REPO_CLONE:
            for option in ['branch', 'commit']:
                cmd = "--%s_%s" % (option, repo)
                hlp = "Set %s %s" % (repo, option)
                parser.add_argument(cmd, required=False,  default=None, help=hlp)

    def get_parser(self, prog_name):
        parser = super(Configure, self).get_parser(prog_name)

        envs = [const.ENVIRONMENT_PROD, const.ENVIRONMENT_DEV]
        parser.add_argument('env', nargs='?', default=const.ENVIRONMENT_PROD, choices=envs,
                            help="Select the env used to build the project.")
        parser.add_argument('--force', default=False, action='store_true', dest='force',
                            help="Skip input prompt, use defaults.")
        parser.add_argument('--reset', default=False, action='store_true', dest='reset',
                            help="Reset config.")

        self.add_repos_params(parser=parser)

        return parser

    @classmethod
    def enable_ssl(cls):
        ssl._create_default_https_context = ssl.create_default_context

    @classmethod
    def disable_ssl(cls):
        ssl._create_default_https_context = ssl._create_unverified_context

    def is_backup_found(self):
        # TODO@geo maybe we should check if specific files exists in backup?

        s3_client = self.get_aws_client('s3')
        s3_bucket_name = self.get_s3_bucket_name()

        try:
            s3_objects = s3_client.list_objects(
                Bucket=s3_bucket_name,
                Delimiter='/'
            )
            for item in s3_objects.get('CommonPrefixes', []):
                if item['Prefix'] == 'backup/':
                    return True

        except botocore.exceptions.ClientError as s3_error:
            self.logger.error(s3_error)

        return False

    def read_domain_from_backup(self):
        s3_client = self.get_aws_client('s3')
        s3_bucket_name = self.get_s3_bucket_name()

        list_response = s3_client.list_objects(
            Bucket=s3_bucket_name,
            Prefix='backup/certs/openssl.cnf',
            MaxKeys=1
        )

        if list_response.get('Contents', None):
            list_response = list_response['Contents'][0]
            s3_object = s3_client.get_object(
                Bucket=s3_bucket_name,
                IfMatch=list_response['ETag'],
                Key=list_response['Key']
            )

            if s3_object.get('Body', None):
                cnf = configs.OpensslConfig()
                cnf.from_string(s3_object['Body'].read())

                return cnf.get_main_domain_name()

        return None

    def wait_until_madcore_is_up(self):
        self.log_figlet("Wait until madcore is up")
        if self.wait_until_jenkins_is_up(log_msg='Wait until madcore is up...'):
            self.logger.info("Madcore is up, continue.")
        else:
            self.logger.error("Error while waiting for madcore.")
            self.exit()

    def wait_until_endpoint_is_up(self, endpoint):
        msg = "Wait until '%s' is up" % endpoint
        endpoint_url = self.get_endpoint_url(endpoint)

        self.log_figlet(msg)
        endpoint_up = self.wait_until_url_is_up(endpoint_url, log_msg=msg, timeout=timeouts.ENDPOINT_UP_TIMEOUT,
                                                verify=True)
        if endpoint_up:
            self.logger.info("[%s] Endpoint is up: %s.", endpoint, endpoint_url)
            self.log_figlet("%s UP!", endpoint)
        else:
            self.logger.error("[%s] Error while waiting for endpoint.", endpoint)

        return endpoint_up

    def wait_until_all_endpoints_are_up(self):
        endpoints = ['kubeapi', 'kubedash', 'grafana', 'jenkins']

        results = []
        for endpoint in endpoints:
            endpoint_up = self.wait_until_endpoint_is_up(endpoint)
            results.append((endpoint, 'OK' if endpoint_up else 'NOT OK'))

        return results

    def wait_until_all_plugin_endpoints_are_up(self):
        # TODO@geo fix this after we install proper plugins
        endpoints = ['spark', 'zeppelin', 'influxdb']

        results = []
        for endpoint in endpoints:
            results.append((endpoint, 'NOT OK'))

        return results

    def take_action(self, parsed_args):
        config.set_env(parsed_args.env)

        configure = MadcoreConfigure(self.app, self.app_args)
        configure.take_action(parsed_args)

        self.log_figlet("Cloudformation")
        stack_create = StackCreate(self.app, self.app_args)
        stack_create.take_action(parsed_args)<commit_msg>local configure only (repos)<commit_after>from __future__ import print_function, unicode_literals

import logging
import ssl

import botocore.exceptions
from cliff.lister import Lister

from madcore import const
from madcore.base import JenkinsBase
from madcore import configs
from madcore.configs import config
from madcore.configure import MadcoreConfigure
from madcore.const import DOMAIN_REGISTRATION
from madcore.libs import timeouts
from madcore.libs.cloudformation import StackCreate


class Configure(JenkinsBase, Lister):
    _description = "Configure madcore"
    logger = logging.getLogger(__name__)

    @classmethod
    def add_repos_params(cls, parser):
        for repo in const.REPO_CLONE:
            for option in ['branch', 'commit']:
                cmd = "--%s_%s" % (option, repo)
                hlp = "Set %s %s" % (repo, option)
                parser.add_argument(cmd, required=False,  default=None, help=hlp)

    def get_parser(self, prog_name):
        parser = super(Configure, self).get_parser(prog_name)

        envs = [const.ENVIRONMENT_PROD, const.ENVIRONMENT_DEV]
        parser.add_argument('env', nargs='?', default=const.ENVIRONMENT_PROD, choices=envs,
                            help="Select the env used to build the project.")
        parser.add_argument('--force', default=False, action='store_true', dest='force',
                            help="Skip input prompt, use defaults.")
        parser.add_argument('--reset', default=False, action='store_true', dest='reset',
                            help="Reset config.")
        parser.add_argument('--local', default=False, action='store_true', dest='local',
                            help="Local config only.")

        self.add_repos_params(parser=parser)

        return parser

    @classmethod
    def enable_ssl(cls):
        ssl._create_default_https_context = ssl.create_default_context

    @classmethod
    def disable_ssl(cls):
        ssl._create_default_https_context = ssl._create_unverified_context

    def is_backup_found(self):
        # TODO@geo maybe we should check if specific files exists in backup?

        s3_client = self.get_aws_client('s3')
        s3_bucket_name = self.get_s3_bucket_name()

        try:
            s3_objects = s3_client.list_objects(
                Bucket=s3_bucket_name,
                Delimiter='/'
            )
            for item in s3_objects.get('CommonPrefixes', []):
                if item['Prefix'] == 'backup/':
                    return True

        except botocore.exceptions.ClientError as s3_error:
            self.logger.error(s3_error)

        return False

    def read_domain_from_backup(self):
        s3_client = self.get_aws_client('s3')
        s3_bucket_name = self.get_s3_bucket_name()

        list_response = s3_client.list_objects(
            Bucket=s3_bucket_name,
            Prefix='backup/certs/openssl.cnf',
            MaxKeys=1
        )

        if list_response.get('Contents', None):
            list_response = list_response['Contents'][0]
            s3_object = s3_client.get_object(
                Bucket=s3_bucket_name,
                IfMatch=list_response['ETag'],
                Key=list_response['Key']
            )

            if s3_object.get('Body', None):
                cnf = configs.OpensslConfig()
                cnf.from_string(s3_object['Body'].read())

                return cnf.get_main_domain_name()

        return None

    def wait_until_madcore_is_up(self):
        self.log_figlet("Wait until madcore is up")
        if self.wait_until_jenkins_is_up(log_msg='Wait until madcore is up...'):
            self.logger.info("Madcore is up, continue.")
        else:
            self.logger.error("Error while waiting for madcore.")
            self.exit()

    def wait_until_endpoint_is_up(self, endpoint):
        msg = "Wait until '%s' is up" % endpoint
        endpoint_url = self.get_endpoint_url(endpoint)

        self.log_figlet(msg)
        endpoint_up = self.wait_until_url_is_up(endpoint_url, log_msg=msg, timeout=timeouts.ENDPOINT_UP_TIMEOUT,
                                                verify=True)
        if endpoint_up:
            self.logger.info("[%s] Endpoint is up: %s.", endpoint, endpoint_url)
            self.log_figlet("%s UP!", endpoint)
        else:
            self.logger.error("[%s] Error while waiting for endpoint.", endpoint)

        return endpoint_up

    def wait_until_all_endpoints_are_up(self):
        endpoints = ['kubeapi', 'kubedash', 'grafana', 'jenkins']

        results = []
        for endpoint in endpoints:
            endpoint_up = self.wait_until_endpoint_is_up(endpoint)
            results.append((endpoint, 'OK' if endpoint_up else 'NOT OK'))

        return results

    def wait_until_all_plugin_endpoints_are_up(self):
        # TODO@geo fix this after we install proper plugins
        endpoints = ['spark', 'zeppelin', 'influxdb']

        results = []
        for endpoint in endpoints:
            results.append((endpoint, 'NOT OK'))

        return results

    def take_action(self, parsed_args):
        config.set_env(parsed_args.env)

        configure = MadcoreConfigure(self.app, self.app_args)
        configure.take_action(parsed_args)

        if parsed_args.local:
            self.logger.info("Local Registration Completed")
            return ('Name','Value'), [('configure --local','Local Repos Updated')]

        self.log_figlet("Cloudformation")
        stack_create = StackCreate(self.app, self.app_args)
        stack_create.take_action(parsed_args)
{'fraction_non_alphanumeric': 0.09703429101019463, 'fraction_numerical': 0.004263206672845227, 'mean_word_length': 2.423540609137056, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4860328', 'n_tokens_mistral': 3217, 'n_tokens_neox': 3097, 'n_words': 668}