<commit_before># (C) Datadog, Inc. 2010-2017
# All rights reserved
# Licensed under Simplified BSD License (see LICENSE)

# stdlib
from datetime import datetime, timedelta
from urlparse import urljoin

# project
from checks import AgentCheck

# 3p
import re
import requests
import simplejson as json

SOURCE_TYPE = 'openstack'

V21_NOVA_API_VERSION = 'v2.1'

DEFAULT_KEYSTONE_API_VERSION = 'v3'
DEFAULT_NOVA_API_VERSION = V21_NOVA_API_VERSION
FALLBACK_NOVA_API_VERSION = 'v2'
DEFAULT_NEUTRON_API_VERSION = 'v2.0'

DEFAULT_API_REQUEST_TIMEOUT = 5 # seconds

NOVA_HYPERVISOR_METRICS = [
    'current_workload',
    'disk_available_least',
    'free_disk_gb',
    'free_ram_mb',
    'local_gb',
    'local_gb_used',
    'memory_mb',
    'memory_mb_used',
    'running_vms',
    'vcpus',
    'vcpus_used',
]

NOVA_SERVER_METRICS = [
    "hdd_errors",
    "hdd_read",
    "hdd_read_req",
    "hdd_write",
    "hdd_write_req",
    "memory",
    "memory-actual",
    "memory-rss",
    "cpu0_time",

    "vda_errors",
    "vda_read",
    "vda_read_req",
    "vda_write",
    "vda_write_req"
]

NOVA_SERVER_INTERFACE_SEGMENTS = ['_rx', '_tx']

PROJECT_METRICS = dict([
    ("maxImageMeta", "max_image_meta"),
    ("maxPersonality", "max_personality"),
    ("maxPersonalitySize", "max_personality_size"),
    ("maxSecurityGroupRules", "max_security_group_rules"),
    ("maxSecurityGroups", "max_security_groups"),
    ("maxServerMeta", "max_server_meta"),
    ("maxTotalCores", "max_total_cores"),
    ("maxTotalFloatingIps", "max_total_floating_ips"),
    ("maxTotalInstances", "max_total_instances"),
    ("maxTotalKeypairs", "max_total_keypairs"),
    ("maxTotalRAMSize", "max_total_ram_size"),

    ("totalImageMetaUsed", "total_image_meta_used"),
    ("totalPersonalityUsed", "total_personality_used"),
    ("totalPersonalitySizeUsed", "total_personality_size_used"),
    ("totalSecurityGroupRulesUsed", "total_security_group_rules_used"),
    ("totalSecurityGroupsUsed", "total_security_groups_used"),
    ("totalServerMetaUsed", "total_server_meta_used"),
    ("totalCoresUsed", "total_cores_used"),
    ("totalFloatingIpsUsed", "total_floating_ips_used"),
    ("totalInstancesUsed", "total_instances_used"),
    ("totalKeypairsUsed", "total_keypairs_used"),
    ("totalRAMUsed", "total_ram_used"),
])

DIAGNOSTICABLE_STATES = [
    'ACTIVE'
]

class OpenStackAuthFailure(Exception):
    pass

class InstancePowerOffFailure(Exception):
    pass

class IncompleteConfig(Exception):
    pass

class IncompleteAuthScope(IncompleteConfig):
    pass

class IncompleteIdentity(IncompleteConfig):
    pass

class MissingEndpoint(Exception):
    pass

class MissingNovaEndpoint(MissingEndpoint):
    pass

class MissingNeutronEndpoint(MissingEndpoint):
    pass

class KeystoneUnreachable(Exception):
    pass


class OpenStackProjectScope(object):
    """
    Container class for a single project's authorization scope
    Embeds the auth token to be included with API requests, and refreshes
    the token on expiry
    """
    def __init__(self, auth_token, auth_scope, service_catalog):
        self.auth_token = auth_token

        # Store some identifiers for this project
        self.project_name = auth_scope["project"].get("name")
        self.domain_id = auth_scope["project"].get("domain", {}).get("id")
        self.tenant_id = auth_scope["project"].get("id")
        self.service_catalog = service_catalog

    @classmethod
    def from_config(cls, init_config, instance_config, proxy_config=None):
        keystone_server_url = init_config.get("keystone_server_url")
        if not keystone_server_url:
            raise IncompleteConfig()

        ssl_verify = init_config.get("ssl_verify", False)
        nova_api_version = init_config.get("nova_api_version", DEFAULT_NOVA_API_VERSION)

        auth_scope = cls.get_auth_scope(instance_config)
        identity = cls.get_user_identity(instance_config)

        exception_msg = None
        try:
            auth_resp = cls.request_auth_token(auth_scope, identity, keystone_server_url, ssl_verify, proxy_config)
        except (requests.exceptions.HTTPError, requests.exceptions.Timeout, requests.exceptions.ConnectionError):
            exception_msg = "Failed keystone auth with identity:{id} scope:{scope} @{url}".format(
                id=identity,
                scope=auth_scope,
                url=keystone_server_url)

        if exception_msg:
            try:
                identity['password']['user']['domain']['name'] = identity['password']['user']['domain'].pop('id')

                if 'domain' in auth_scope['project']:
                    auth_scope['project']['domain']['name'] = auth_scope['project']['domain'].pop('id')
                else:
                    auth_scope['project']['name'] = auth_scope['project'].pop('id')
                auth_resp = cls.request_auth_token(auth_scope, identity, keystone_server_url, ssl_verify, proxy_config)
            except (requests.exceptions.HTTPError, requests.exceptions.Timeout, requests.exceptions.ConnectionError) as e:
                exception_msg = "{msg} and also failed kaystone auth with identity:{id} scope:{scope} @{url}: {ex}".format(
                    msg=exception_msg,
                    id=identity,
                    scope=auth_scope,
                    url=keystone_server_url,
                    ex=e)
                raise KeystoneUnreachable(exception_msg)

        auth_token = auth_resp.headers.get('X-Subject-Token')

        try:
            service_catalog = KeystoneCatalog.from_auth_response(
                auth_resp.json(), nova_api_version<commit_msg>Remove password in error message if keystone auth fails<commit_after># (C) Datadog, Inc. 2010-2017
# All rights reserved
# Licensed under Simplified BSD License (see LICENSE)

# stdlib
from datetime import datetime, timedelta
from urlparse import urljoin

# project
from checks import AgentCheck

# 3p
import re
import requests
import simplejson as json

SOURCE_TYPE = 'openstack'

V21_NOVA_API_VERSION = 'v2.1'

DEFAULT_KEYSTONE_API_VERSION = 'v3'
DEFAULT_NOVA_API_VERSION = V21_NOVA_API_VERSION
FALLBACK_NOVA_API_VERSION = 'v2'
DEFAULT_NEUTRON_API_VERSION = 'v2.0'

DEFAULT_API_REQUEST_TIMEOUT = 5 # seconds

NOVA_HYPERVISOR_METRICS = [
    'current_workload',
    'disk_available_least',
    'free_disk_gb',
    'free_ram_mb',
    'local_gb',
    'local_gb_used',
    'memory_mb',
    'memory_mb_used',
    'running_vms',
    'vcpus',
    'vcpus_used',
]

NOVA_SERVER_METRICS = [
    "hdd_errors",
    "hdd_read",
    "hdd_read_req",
    "hdd_write",
    "hdd_write_req",
    "memory",
    "memory-actual",
    "memory-rss",
    "cpu0_time",

    "vda_errors",
    "vda_read",
    "vda_read_req",
    "vda_write",
    "vda_write_req"
]

NOVA_SERVER_INTERFACE_SEGMENTS = ['_rx', '_tx']

PROJECT_METRICS = dict([
    ("maxImageMeta", "max_image_meta"),
    ("maxPersonality", "max_personality"),
    ("maxPersonalitySize", "max_personality_size"),
    ("maxSecurityGroupRules", "max_security_group_rules"),
    ("maxSecurityGroups", "max_security_groups"),
    ("maxServerMeta", "max_server_meta"),
    ("maxTotalCores", "max_total_cores"),
    ("maxTotalFloatingIps", "max_total_floating_ips"),
    ("maxTotalInstances", "max_total_instances"),
    ("maxTotalKeypairs", "max_total_keypairs"),
    ("maxTotalRAMSize", "max_total_ram_size"),

    ("totalImageMetaUsed", "total_image_meta_used"),
    ("totalPersonalityUsed", "total_personality_used"),
    ("totalPersonalitySizeUsed", "total_personality_size_used"),
    ("totalSecurityGroupRulesUsed", "total_security_group_rules_used"),
    ("totalSecurityGroupsUsed", "total_security_groups_used"),
    ("totalServerMetaUsed", "total_server_meta_used"),
    ("totalCoresUsed", "total_cores_used"),
    ("totalFloatingIpsUsed", "total_floating_ips_used"),
    ("totalInstancesUsed", "total_instances_used"),
    ("totalKeypairsUsed", "total_keypairs_used"),
    ("totalRAMUsed", "total_ram_used"),
])

DIAGNOSTICABLE_STATES = [
    'ACTIVE'
]

class OpenStackAuthFailure(Exception):
    pass

class InstancePowerOffFailure(Exception):
    pass

class IncompleteConfig(Exception):
    pass

class IncompleteAuthScope(IncompleteConfig):
    pass

class IncompleteIdentity(IncompleteConfig):
    pass

class MissingEndpoint(Exception):
    pass

class MissingNovaEndpoint(MissingEndpoint):
    pass

class MissingNeutronEndpoint(MissingEndpoint):
    pass

class KeystoneUnreachable(Exception):
    pass


class OpenStackProjectScope(object):
    """
    Container class for a single project's authorization scope
    Embeds the auth token to be included with API requests, and refreshes
    the token on expiry
    """
    def __init__(self, auth_token, auth_scope, service_catalog):
        self.auth_token = auth_token

        # Store some identifiers for this project
        self.project_name = auth_scope["project"].get("name")
        self.domain_id = auth_scope["project"].get("domain", {}).get("id")
        self.tenant_id = auth_scope["project"].get("id")
        self.service_catalog = service_catalog

    @classmethod
    def from_config(cls, init_config, instance_config, proxy_config=None):
        keystone_server_url = init_config.get("keystone_server_url")
        if not keystone_server_url:
            raise IncompleteConfig()

        ssl_verify = init_config.get("ssl_verify", False)
        nova_api_version = init_config.get("nova_api_version", DEFAULT_NOVA_API_VERSION)

        auth_scope = cls.get_auth_scope(instance_config)
        identity = cls.get_user_identity(instance_config)

        exception_msg = None
        try:
            auth_resp = cls.request_auth_token(auth_scope, identity, keystone_server_url, ssl_verify, proxy_config)
        except (requests.exceptions.HTTPError, requests.exceptions.Timeout, requests.exceptions.ConnectionError):
            exception_msg = "Failed keystone auth with user:{user} domain:{domain} scope:{scope} @{url}".format(
                user=identity['password']['user']['name'],
                domain=identity['password']['user']['domain']['id'],
                scope=auth_scope,
                url=keystone_server_url)

        if exception_msg:
            try:
                identity['password']['user']['domain']['name'] = identity['password']['user']['domain'].pop('id')

                if 'domain' in auth_scope['project']:
                    auth_scope['project']['domain']['name'] = auth_scope['project']['domain'].pop('id')
                else:
                    auth_scope['project']['name'] = auth_scope['project'].pop('id')
                auth_resp = cls.request_auth_token(auth_scope, identity, keystone_server_url, ssl_verify, proxy_config)
            except (requests.exceptions.HTTPError, requests.exceptions.Timeout, requests.exceptions.ConnectionError) as e:
                exception_msg = "{msg} and also failed keystone auth with identity:{id} domain:{domain} scope:{scope} @{url}: {ex}".format(
                    msg=exception_msg,
                    user=identity['password']['user']['name'],
                    domain = identity['password']['user']['domain']['name'],
                    scope=auth_scope,
                    url=keystone_server_url,
                    ex=e)
                raise KeystoneUnreachable(exception_msg)

        auth_token = auth_resp.headers.get('X-Subject-Token')

        try:
            service_catalog = KeystoneCatalog.from_auth_response(
                auth_resp.json(), nova_api_version
{'fraction_non_alphanumeric': 0.11657400972897845, 'fraction_numerical': 0.0036483669214732455, 'mean_word_length': 3.6668017835427644, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6337669', 'n_tokens_mistral': 3625, 'n_tokens_neox': 3521, 'n_words': 624}