<commit_before>#!/usr/bin/env python
# Copyright 2015-2016 Yelp Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
import time
from math import ceil
from math import floor

import boto3
from botocore.exceptions import ClientError
from requests.exceptions import HTTPError

from paasta_tools.autoscaling.utils import _autoscaling_components
from paasta_tools.autoscaling.utils import register_autoscaling_component
from paasta_tools.mesos_maintenance import drain
from paasta_tools.mesos_maintenance import undrain
from paasta_tools.mesos_tools import get_mesos_master
from paasta_tools.mesos_tools import get_mesos_task_count_by_slave
from paasta_tools.mesos_tools import slave_pid_to_ip
from paasta_tools.paasta_maintenance import is_safe_to_kill
from paasta_tools.paasta_metastatus import get_resource_utilization_by_grouping
from paasta_tools.utils import load_system_paasta_config
from paasta_tools.utils import Timeout
from paasta_tools.utils import TimeoutError


CLUSTER_METRICS_PROVIDER_KEY = 'cluster_metrics_provider'
DEFAULT_TARGET_UTILIZATION = 0.8  # decimal fraction
DEFAULT_DRAIN_TIMEOUT = 600  # seconds
SCALER_KEY = 'scaler'

AWS_SPOT_MODIFY_TIMEOUT = 30
MISSING_SLAVE_PANIC_THRESHOLD = .3
MAX_CLUSTER_DELTA = .2

log = logging.getLogger(__name__)
log.addHandler(logging.NullHandler())


def get_cluster_metrics_provider(name):
    """
    Returns a cluster metrics provider matching the given name.
    """
    return _autoscaling_components[CLUSTER_METRICS_PROVIDER_KEY][name]


def get_scaler(name):
    """
    Returns a scaler matching the given name.
    """
    return _autoscaling_components[SCALER_KEY][name]


def get_spot_fleet_instances(spotfleet_request_id, region=None):
    ec2_client = boto3.client('ec2', region_name=region)
    spot_fleet_instances = ec2_client.describe_spot_fleet_instances(
        SpotFleetRequestId=spotfleet_request_id)['ActiveInstances']
    return spot_fleet_instances


def get_sfr_instance_ips(sfr, region=None):
    spot_fleet_instances = sfr['ActiveInstances']
    instance_descriptions = describe_instances([instance['InstanceId'] for instance in spot_fleet_instances],
                                               region=region)
    instance_ips = []
    for instance in instance_descriptions:
        try:
            instance_ips.append(instance['PrivateIpAddress'])
        except KeyError:
            log.warning("Instance {0} does not have an IP. This normally means it has been"
                        " terminated".format(instance['InstanceId']))
    return instance_ips


def get_sfr(spotfleet_request_id, region=None):
    ec2_client = boto3.client('ec2', region_name=region)
    try:
        sfrs = ec2_client.describe_spot_fleet_requests(SpotFleetRequestIds=[spotfleet_request_id])
    except ClientError as e:
        if e.response['Error']['Code'] == 'InvalidSpotFleetRequestId.NotFound':
            log.warn('Cannot find SFR {0}'.format(spotfleet_request_id))
            return None
        else:
            raise
    ret = sfrs['SpotFleetRequestConfigs'][0]
    return ret


def describe_instances(instance_ids, region=None, instance_filters=None):
    """This wraps ec2.describe_instances and catches instance not
    found errors. It returns a list of instance description
    dictionaries. It assumes one instance per reservation (which
    seems to be the case for SFRs) Optionally, a filter can be
    passed through to the ec2 call

    :param instance_ids: a list of instance ids, [] means all
    :param instance_filters: a list of ec2 filters
    :returns: a list of instance description dictionaries"""
    if not instance_filters:
        instance_filters = []
    ec2_client = boto3.client('ec2', region_name=region)
    try:
        instances = ec2_client.describe_instances(InstanceIds=instance_ids, Filters=instance_filters)
    except ClientError as e:
        if e.response['Error']['Code'] == 'InvalidInstanceID.NotFound':
            log.warn('Cannot find one or more instance from IDs {0}'.format(instance_ids))
            return None
        else:
            raise
    ret = [reservation['Instances'][0] for reservation in instances['Reservations']]
    return ret


@register_autoscaling_component('aws_spot_fleet_request', CLUSTER_METRICS_PROVIDER_KEY)
def spotfleet_metrics_provider(spotfleet_request_id, resource, pool_settings):
    mesos_state = get_mesos_master().state()
    sfr = get_sfr(spotfleet_request_id, region=resource['region'])
    if not sfr or not sfr['SpotFleetRequestState'] == 'active':
        log.error("Ignoring SFR {0} that does not exist or is not active.".format(spotfleet_request_id))
        return 0, 0
    sfr['ActiveInstances'] = get_spot_fleet_instances(spotfleet_request_id, region=resource['region'])
    resource['sfr'] = sfr
    desired_instances = len(sfr['ActiveInstances'])
    instance_ips = get_sfr_instance_ips(sfr, region=resource['region'])
    slaves = {
        slave['id']: slave for slave in mesos_state.get('slaves', [])<commit_msg>fix calling of state as function rather than property<commit_after>#!/usr/bin/env python
# Copyright 2015-2016 Yelp Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
import time
from math import ceil
from math import floor

import boto3
from botocore.exceptions import ClientError
from requests.exceptions import HTTPError

from paasta_tools.autoscaling.utils import _autoscaling_components
from paasta_tools.autoscaling.utils import register_autoscaling_component
from paasta_tools.mesos_maintenance import drain
from paasta_tools.mesos_maintenance import undrain
from paasta_tools.mesos_tools import get_mesos_master
from paasta_tools.mesos_tools import get_mesos_task_count_by_slave
from paasta_tools.mesos_tools import slave_pid_to_ip
from paasta_tools.paasta_maintenance import is_safe_to_kill
from paasta_tools.paasta_metastatus import get_resource_utilization_by_grouping
from paasta_tools.utils import load_system_paasta_config
from paasta_tools.utils import Timeout
from paasta_tools.utils import TimeoutError


CLUSTER_METRICS_PROVIDER_KEY = 'cluster_metrics_provider'
DEFAULT_TARGET_UTILIZATION = 0.8  # decimal fraction
DEFAULT_DRAIN_TIMEOUT = 600  # seconds
SCALER_KEY = 'scaler'

AWS_SPOT_MODIFY_TIMEOUT = 30
MISSING_SLAVE_PANIC_THRESHOLD = .3
MAX_CLUSTER_DELTA = .2

log = logging.getLogger(__name__)
log.addHandler(logging.NullHandler())


def get_cluster_metrics_provider(name):
    """
    Returns a cluster metrics provider matching the given name.
    """
    return _autoscaling_components[CLUSTER_METRICS_PROVIDER_KEY][name]


def get_scaler(name):
    """
    Returns a scaler matching the given name.
    """
    return _autoscaling_components[SCALER_KEY][name]


def get_spot_fleet_instances(spotfleet_request_id, region=None):
    ec2_client = boto3.client('ec2', region_name=region)
    spot_fleet_instances = ec2_client.describe_spot_fleet_instances(
        SpotFleetRequestId=spotfleet_request_id)['ActiveInstances']
    return spot_fleet_instances


def get_sfr_instance_ips(sfr, region=None):
    spot_fleet_instances = sfr['ActiveInstances']
    instance_descriptions = describe_instances([instance['InstanceId'] for instance in spot_fleet_instances],
                                               region=region)
    instance_ips = []
    for instance in instance_descriptions:
        try:
            instance_ips.append(instance['PrivateIpAddress'])
        except KeyError:
            log.warning("Instance {0} does not have an IP. This normally means it has been"
                        " terminated".format(instance['InstanceId']))
    return instance_ips


def get_sfr(spotfleet_request_id, region=None):
    ec2_client = boto3.client('ec2', region_name=region)
    try:
        sfrs = ec2_client.describe_spot_fleet_requests(SpotFleetRequestIds=[spotfleet_request_id])
    except ClientError as e:
        if e.response['Error']['Code'] == 'InvalidSpotFleetRequestId.NotFound':
            log.warn('Cannot find SFR {0}'.format(spotfleet_request_id))
            return None
        else:
            raise
    ret = sfrs['SpotFleetRequestConfigs'][0]
    return ret


def describe_instances(instance_ids, region=None, instance_filters=None):
    """This wraps ec2.describe_instances and catches instance not
    found errors. It returns a list of instance description
    dictionaries. It assumes one instance per reservation (which
    seems to be the case for SFRs) Optionally, a filter can be
    passed through to the ec2 call

    :param instance_ids: a list of instance ids, [] means all
    :param instance_filters: a list of ec2 filters
    :returns: a list of instance description dictionaries"""
    if not instance_filters:
        instance_filters = []
    ec2_client = boto3.client('ec2', region_name=region)
    try:
        instances = ec2_client.describe_instances(InstanceIds=instance_ids, Filters=instance_filters)
    except ClientError as e:
        if e.response['Error']['Code'] == 'InvalidInstanceID.NotFound':
            log.warn('Cannot find one or more instance from IDs {0}'.format(instance_ids))
            return None
        else:
            raise
    ret = [reservation['Instances'][0] for reservation in instances['Reservations']]
    return ret


@register_autoscaling_component('aws_spot_fleet_request', CLUSTER_METRICS_PROVIDER_KEY)
def spotfleet_metrics_provider(spotfleet_request_id, resource, pool_settings):
    mesos_state = get_mesos_master().state
    sfr = get_sfr(spotfleet_request_id, region=resource['region'])
    if not sfr or not sfr['SpotFleetRequestState'] == 'active':
        log.error("Ignoring SFR {0} that does not exist or is not active.".format(spotfleet_request_id))
        return 0, 0
    sfr['ActiveInstances'] = get_spot_fleet_instances(spotfleet_request_id, region=resource['region'])
    resource['sfr'] = sfr
    desired_instances = len(sfr['ActiveInstances'])
    instance_ips = get_sfr_instance_ips(sfr, region=resource['region'])
    slaves = {
        slave['id']: slave for slave in mesos_state.get('slaves', [])
{'fraction_non_alphanumeric': 0.07742402315484805, 'fraction_numerical': 0.008140376266280752, 'mean_word_length': 4.385776911836337, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5825672', 'n_tokens_mistral': 3424, 'n_tokens_neox': 3289, 'n_words': 870}