<commit_before>#!/usr/bin/env python
# Copyright 2015-2016 Yelp Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import datetime
import logging

import humanize
import isodate
from requests.exceptions import ReadTimeout

from paasta_tools import marathon_tools
from paasta_tools.autoscaling.autoscaling_service_lib import get_autoscaling_info
from paasta_tools.autoscaling.autoscaling_service_lib import ServiceAutoscalingInfo
from paasta_tools.mesos_tools import get_all_slaves_for_blacklist_whitelist
from paasta_tools.mesos_tools import get_cached_list_of_running_tasks_from_frameworks
from paasta_tools.mesos_tools import get_mesos_slaves_grouped_by_attribute
from paasta_tools.mesos_tools import select_tasks_by_id
from paasta_tools.mesos_tools import status_mesos_tasks_verbose
from paasta_tools.smartstack_tools import backend_is_up
from paasta_tools.smartstack_tools import get_backends
from paasta_tools.smartstack_tools import match_backends_and_tasks
from paasta_tools.utils import _log
from paasta_tools.utils import calculate_tail_lines
from paasta_tools.utils import compose_job_id
from paasta_tools.utils import datetime_from_utc_to_local
from paasta_tools.utils import format_table
from paasta_tools.utils import is_under_replicated
from paasta_tools.utils import load_system_paasta_config
from paasta_tools.utils import NoDockerImageError
from paasta_tools.utils import paasta_print
from paasta_tools.utils import PaastaColors
from paasta_tools.utils import remove_ansi_escape_sequences

log = logging.getLogger(__name__)
log.addHandler(logging.NullHandler())


def restart_marathon_job(service, instance, app_id, client, cluster):
    name = PaastaColors.cyan(compose_job_id(service, instance))
    _log(
        service=service,
        line="EmergencyRestart: Scaling %s down to 0 instances, then letting them scale back up" % (name),
        component='deploy',
        level='event',
        cluster=cluster,
        instance=instance,
    )
    client.scale_app(app_id, instances=0, force=True)


def bouncing_status_human(app_count, bounce_method):
    if app_count == 0:
        return PaastaColors.red("Disabled")
    elif app_count == 1:
        return PaastaColors.green("Configured")
    elif app_count > 1:
        return PaastaColors.yellow("Bouncing (%s)" % bounce_method)
    else:
        return PaastaColors.red("Unknown (count: %s)" % app_count)


def get_bouncing_status(service, instance, client, job_config):
    apps = marathon_tools.get_matching_appids(service, instance, client)
    app_count = len(apps)
    bounce_method = job_config.get_bounce_method()
    return bouncing_status_human(app_count, bounce_method)


def desired_state_human(desired_state, instances):
    if desired_state == 'start' and instances != 0:
        return PaastaColors.bold('Started')
    elif desired_state == 'start' and instances == 0:
        return PaastaColors.bold('Stopped')
    elif desired_state == 'stop':
        return PaastaColors.red('Stopped')
    else:
        return PaastaColors.red('Unknown (desired_state: %s)' % desired_state)


def status_desired_state(service, instance, client, job_config):
    status = get_bouncing_status(service, instance, client, job_config)
    desired_state = desired_state_human(job_config.get_desired_state(), job_config.get_instances())
    return "Desired State:      %s and %s" % (status, desired_state)


def status_marathon_job_human(
    service, instance, deploy_status, app_id,
    running_instances, normal_instance_count, unused_offers_summary=None,
):
    name = PaastaColors.cyan(compose_job_id(service, instance))
    if unused_offers_summary is not None and len(unused_offers_summary) > 0:
        stalled_str = "\n    ".join(["%s: %s times" % (k, n) for k, n in unused_offers_summary.items()])
        stall_reason = "\n  Possibly stalled for:\n    %s" % stalled_str
    else:
        stall_reason = ""
    if deploy_status != 'NotRunning':
        if running_instances >= normal_instance_count:
            status = PaastaColors.green("Healthy")
            instance_count = PaastaColors.green("(%d/%d)" % (running_instances, normal_instance_count))
        elif running_instances == 0:
            status = PaastaColors.yellow("Critical")
            instance_count = PaastaColors.red("(%d/%d)" % (running_instances, normal_instance_count))
        else:
            status = PaastaColors.yellow("Warning")
            instance_count = PaastaColors.yellow("(%d/%d)" % (running_instances, normal_instance_count))
        return "Marathon:   %s - up with %s instances. Status: %s%s" % (
            status, instance_count, deploy_status, stall_reason,
        )
    else:
        status = PaastaColors.yellow("Warning")
        return "Marathon:   %s - %s (app %s) is not configured in Marathon yet (waiting for bounce)%s" % (
            status, name, app_id, stall_reason,
        )


def marathon_app_deploy_status_human(status, backoff_seconds=None):
    status_string = marathon_tools.MarathonDeployStatus.tostring(status)

    if status == marathon_tools.MarathonDeployStatus.Waiting:
        deploy_status = "%s (new tasks waiting for capacity to become available)" % PaastaColors.red(status_string)
    elif status == marathon_tools.MarathonDeployStatus.Delayed:
        deploy_status = "%s (tasks are crashing, next won't launch for another %s seconds)" % (
                        PaastaColors.red(status_string), backoff_seconds,
        )
    elif status == marathon_tools.MarathonDeployStatus.Deploying:
        deploy_status = PaastaColors.yellow(status_string)
    elif status == marathon_tools.MarathonDeployStatus.Stopped:
        deploy_status = PaastaColors.grey(status_string)
    elif status == marathon_tools.MarathonDeployStatus.Running:
        deploy_status = PaastaColors.bold(status_string)
    else:
        deploy_status = status_string

    return deploy_status


def status_marathon_job(service, instance, app_id, normal_instance_count, client):
    status = marathon_tools.get_marathon_app_deploy_status(client, app_id)
    app_queue = marathon_tools.get_app_queue(client, app_id)
    unused_offers_summary = marathon_tools.summarize_unused_offers(app_queue)
    if status == marathon_tools.MarathonDeployStatus.Delayed:
        _, backoff_seconds = marathon_tools.get_app_queue_status_from_queue(app_queue)
        deploy_status_human = marathon_app_deploy_status_human(status, backoff_seconds)
    else:
        deploy_status_human = marathon_app_deploy_status_human(status)

    if status == marathon_tools.MarathonDeployStatus.NotRunning:
        running_instances = 0
    else:
        running_instances = client.get_app(app_id).tasks_running
    return status_marathon_job_human(
        service, instance, deploy_status_human, app_id,
        running_instances, normal_instance_count, unused_offers_summary,
    )


def get_marathon_dashboard(client, dashboards, app_id):
    if dashboards is not None:
        base_url = dashboards.get(client)
        if base_url:
            url = "{}/ui/#/apps/%2F{}".format(base_url.rstrip('/'), app_id.lstrip('/'))
            return "  Marathon dashboard: %s" % PaastaColors.blue(url)
    return "  Marathon app ID: %s" % PaastaColors.bold(app_id)


def get_verbose_status_of_marathon_app(marathon_client, app, service, instance, cluster, soa_dir, dashboards):
    """Takes a given marathon app object and returns the verbose details
    about the tasks, times, hosts, etc"""
    output = []
    create_datetime = datetime_from_utc_to_local(isodate.parse_datetime(app.version))
    output.append(get_marathon_dashboard(marathon_client, dashboards, app.id))
    output.append("    App created: %s (%s)" % (str(create_datetime), humanize.naturaltime(create_datetime)))

    output.append("    Tasks:")
    rows = [("Mesos Task ID", "Host deployed to", "Deployed at what localtime", "Health")]
    for task in app.tasks:
        local_deployed_datetime = datetime_from_utc_to_local(task.staged_at)
        if task.host is not None:
            hostname = "%s:%s" % (task.host.split(".")[0], task.ports[0])
        else:
            hostname = "Unknown"
        if not task.health_check_results:
            health_check_status = PaastaColors.grey("N/A")
        elif marathon_tools.is_task_healthy(task):
            health_check_status = PaastaColors.green("Healthy")
        else:
            health_check_status = PaastaColors.red("Unhealthy")

        rows.append((
            get_short_task_id(task.id),
            hostname,
            '%s (%s)' % (
                local_deployed_datetime.strftime("%Y-%m-%dT%H:%M"),
                humanize.naturaltime(local_deployed_datetime),
            ),
            health_check_status,
        ))
    output.append('\n'.join(["      %s" % line for line in format_table(rows)]))
    if len(app.tasks) == 0:
        output.append("      No tasks associated with this marathon app")
    return app.tasks, "\n".join(output)


def status_marathon_job_verbose(service, instance, clients, cluster, soa_dir, job_config, dashboards):
    """Returns detailed information about a marathon apps for a service
    and instance. Does not make assumptions about what the *exact*
    appid is, but instead does a fuzzy match on any marathon apps
    that match the given service.instance"""
    all_tasks = []
    all_output = []
    # For verbose mode, we want to see *any* matching app. As it may
    # not be the one that we think should be deployed. For example
    # during a bounce we want to see the old and new ones.

    relevant_clients = clients.get_all_clients_for_service(job_config)
    marathon_apps_with_clients = marathon_tools.get_marathon_apps_with_clients(relevant_clients, embed_tasks=True)

    autoscaling_info = get_autoscaling_info(relevant_clients, job_config)<commit_msg>Pass MarathonClients, not list of MarathonClient, to get_autoscaling_info in status_marathon_job_verbose.<commit_after>#!/usr/bin/env python
# Copyright 2015-2016 Yelp Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import datetime
import logging
from typing import Dict
from typing import List
from typing import Tuple

import humanize
import isodate
from marathon.models.app import MarathonTask
from requests.exceptions import ReadTimeout

from paasta_tools import marathon_tools
from paasta_tools.autoscaling.autoscaling_service_lib import get_autoscaling_info
from paasta_tools.autoscaling.autoscaling_service_lib import ServiceAutoscalingInfo
from paasta_tools.mesos_tools import get_all_slaves_for_blacklist_whitelist
from paasta_tools.mesos_tools import get_cached_list_of_running_tasks_from_frameworks
from paasta_tools.mesos_tools import get_mesos_slaves_grouped_by_attribute
from paasta_tools.mesos_tools import select_tasks_by_id
from paasta_tools.mesos_tools import status_mesos_tasks_verbose
from paasta_tools.smartstack_tools import backend_is_up
from paasta_tools.smartstack_tools import get_backends
from paasta_tools.smartstack_tools import match_backends_and_tasks
from paasta_tools.utils import _log
from paasta_tools.utils import calculate_tail_lines
from paasta_tools.utils import compose_job_id
from paasta_tools.utils import datetime_from_utc_to_local
from paasta_tools.utils import format_table
from paasta_tools.utils import is_under_replicated
from paasta_tools.utils import load_system_paasta_config
from paasta_tools.utils import NoDockerImageError
from paasta_tools.utils import paasta_print
from paasta_tools.utils import PaastaColors
from paasta_tools.utils import remove_ansi_escape_sequences

log = logging.getLogger(__name__)
log.addHandler(logging.NullHandler())


def restart_marathon_job(service, instance, app_id, client, cluster):
    name = PaastaColors.cyan(compose_job_id(service, instance))
    _log(
        service=service,
        line="EmergencyRestart: Scaling %s down to 0 instances, then letting them scale back up" % (name),
        component='deploy',
        level='event',
        cluster=cluster,
        instance=instance,
    )
    client.scale_app(app_id, instances=0, force=True)


def bouncing_status_human(app_count, bounce_method):
    if app_count == 0:
        return PaastaColors.red("Disabled")
    elif app_count == 1:
        return PaastaColors.green("Configured")
    elif app_count > 1:
        return PaastaColors.yellow("Bouncing (%s)" % bounce_method)
    else:
        return PaastaColors.red("Unknown (count: %s)" % app_count)


def get_bouncing_status(service, instance, client, job_config):
    apps = marathon_tools.get_matching_appids(service, instance, client)
    app_count = len(apps)
    bounce_method = job_config.get_bounce_method()
    return bouncing_status_human(app_count, bounce_method)


def desired_state_human(desired_state, instances):
    if desired_state == 'start' and instances != 0:
        return PaastaColors.bold('Started')
    elif desired_state == 'start' and instances == 0:
        return PaastaColors.bold('Stopped')
    elif desired_state == 'stop':
        return PaastaColors.red('Stopped')
    else:
        return PaastaColors.red('Unknown (desired_state: %s)' % desired_state)


def status_desired_state(service, instance, client, job_config):
    status = get_bouncing_status(service, instance, client, job_config)
    desired_state = desired_state_human(job_config.get_desired_state(), job_config.get_instances())
    return "Desired State:      %s and %s" % (status, desired_state)


def status_marathon_job_human(
    service, instance, deploy_status, app_id,
    running_instances, normal_instance_count, unused_offers_summary=None,
):
    name = PaastaColors.cyan(compose_job_id(service, instance))
    if unused_offers_summary is not None and len(unused_offers_summary) > 0:
        stalled_str = "\n    ".join(["%s: %s times" % (k, n) for k, n in unused_offers_summary.items()])
        stall_reason = "\n  Possibly stalled for:\n    %s" % stalled_str
    else:
        stall_reason = ""
    if deploy_status != 'NotRunning':
        if running_instances >= normal_instance_count:
            status = PaastaColors.green("Healthy")
            instance_count = PaastaColors.green("(%d/%d)" % (running_instances, normal_instance_count))
        elif running_instances == 0:
            status = PaastaColors.yellow("Critical")
            instance_count = PaastaColors.red("(%d/%d)" % (running_instances, normal_instance_count))
        else:
            status = PaastaColors.yellow("Warning")
            instance_count = PaastaColors.yellow("(%d/%d)" % (running_instances, normal_instance_count))
        return "Marathon:   %s - up with %s instances. Status: %s%s" % (
            status, instance_count, deploy_status, stall_reason,
        )
    else:
        status = PaastaColors.yellow("Warning")
        return "Marathon:   %s - %s (app %s) is not configured in Marathon yet (waiting for bounce)%s" % (
            status, name, app_id, stall_reason,
        )


def marathon_app_deploy_status_human(status, backoff_seconds=None):
    status_string = marathon_tools.MarathonDeployStatus.tostring(status)

    if status == marathon_tools.MarathonDeployStatus.Waiting:
        deploy_status = "%s (new tasks waiting for capacity to become available)" % PaastaColors.red(status_string)
    elif status == marathon_tools.MarathonDeployStatus.Delayed:
        deploy_status = "%s (tasks are crashing, next won't launch for another %s seconds)" % (
                        PaastaColors.red(status_string), backoff_seconds,
        )
    elif status == marathon_tools.MarathonDeployStatus.Deploying:
        deploy_status = PaastaColors.yellow(status_string)
    elif status == marathon_tools.MarathonDeployStatus.Stopped:
        deploy_status = PaastaColors.grey(status_string)
    elif status == marathon_tools.MarathonDeployStatus.Running:
        deploy_status = PaastaColors.bold(status_string)
    else:
        deploy_status = status_string

    return deploy_status


def status_marathon_job(service, instance, app_id, normal_instance_count, client):
    status = marathon_tools.get_marathon_app_deploy_status(client, app_id)
    app_queue = marathon_tools.get_app_queue(client, app_id)
    unused_offers_summary = marathon_tools.summarize_unused_offers(app_queue)
    if status == marathon_tools.MarathonDeployStatus.Delayed:
        _, backoff_seconds = marathon_tools.get_app_queue_status_from_queue(app_queue)
        deploy_status_human = marathon_app_deploy_status_human(status, backoff_seconds)
    else:
        deploy_status_human = marathon_app_deploy_status_human(status)

    if status == marathon_tools.MarathonDeployStatus.NotRunning:
        running_instances = 0
    else:
        running_instances = client.get_app(app_id).tasks_running
    return status_marathon_job_human(
        service, instance, deploy_status_human, app_id,
        running_instances, normal_instance_count, unused_offers_summary,
    )


def get_marathon_dashboard(client, dashboards, app_id):
    if dashboards is not None:
        base_url = dashboards.get(client)
        if base_url:
            url = "{}/ui/#/apps/%2F{}".format(base_url.rstrip('/'), app_id.lstrip('/'))
            return "  Marathon dashboard: %s" % PaastaColors.blue(url)
    return "  Marathon app ID: %s" % PaastaColors.bold(app_id)


def get_verbose_status_of_marathon_app(marathon_client, app, service, instance, cluster, soa_dir, dashboards):
    """Takes a given marathon app object and returns the verbose details
    about the tasks, times, hosts, etc"""
    output = []
    create_datetime = datetime_from_utc_to_local(isodate.parse_datetime(app.version))
    output.append(get_marathon_dashboard(marathon_client, dashboards, app.id))
    output.append("    App created: %s (%s)" % (str(create_datetime), humanize.naturaltime(create_datetime)))

    output.append("    Tasks:")
    rows = [("Mesos Task ID", "Host deployed to", "Deployed at what localtime", "Health")]
    for task in app.tasks:
        local_deployed_datetime = datetime_from_utc_to_local(task.staged_at)
        if task.host is not None:
            hostname = "%s:%s" % (task.host.split(".")[0], task.ports[0])
        else:
            hostname = "Unknown"
        if not task.health_check_results:
            health_check_status = PaastaColors.grey("N/A")
        elif marathon_tools.is_task_healthy(task):
            health_check_status = PaastaColors.green("Healthy")
        else:
            health_check_status = PaastaColors.red("Unhealthy")

        rows.append((
            get_short_task_id(task.id),
            hostname,
            '%s (%s)' % (
                local_deployed_datetime.strftime("%Y-%m-%dT%H:%M"),
                humanize.naturaltime(local_deployed_datetime),
            ),
            health_check_status,
        ))
    output.append('\n'.join(["      %s" % line for line in format_table(rows)]))
    if len(app.tasks) == 0:
        output.append("      No tasks associated with this marathon app")
    return app.tasks, "\n".join(output)


def status_marathon_job_verbose(
    service: str,
    instance: str,
    clients: marathon_tools.MarathonClients,
    cluster: str,
    soa_dir: str,
    job_config: marathon_tools.MarathonServiceConfig,
    dashboards: Dict[marathon_tools.MarathonClient, str],
) -> Tuple[List[MarathonTask], str]:
    """Returns detailed information about a marathon apps for a service
    and instance. Does not make assumptions about what the *exact*
    appid is, but instead does a fuzzy match on any marathon apps
    that match the given service.instance"""
    all_tasks: List[MarathonTask] = []
    all_output: List[str] = []
    # For verbose mode, we want to see *any* matching app. As it may
    # not be the one that we think should be deployed. For example
    # during a bounce we want to see the old and new ones.
    marathon_apps_with_clients = marathon_tools.get_marathon_apps_with_clients(
        clients=clients.get_all_clients_for_service(job_config),
        embed_tasks=True,
    )

    autoscaling_info = get_autoscaling_info(clients, job_config)
{'fraction_non_alphanumeric': 0.08713653452624566, 'fraction_numerical': 0.002474658544710417, 'mean_word_length': 3.972550875532418, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6523100', 'n_tokens_mistral': 6473, 'n_tokens_neox': 6297, 'n_words': 1524}