<commit_before># Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import hashlib
import logging
import operator
import time

from cinderclient.v1 import client as cinder_client
import glanceclient
from ironicclient import client as ironic_client
from ironicclient import exceptions as ironic_exceptions
from keystoneclient import auth as ksc_auth
from keystoneclient import session as ksc_session
from novaclient import client as nova_client
from novaclient.v1_1 import floating_ips
from neutronclient.v2_0 import client as neutron_client
import os_client_config
import pbr.version
import swiftclient.client as swift_client
import swiftclient.exceptions as swift_exceptions
import troveclient.client as trove_client


from shade import meta

__version__ = pbr.version.VersionInfo('shade').version_string()
OBJECT_MD5_KEY = 'x-object-meta-x-shade-md5'
OBJECT_SHA256_KEY = 'x-object-meta-x-shade-sha256'
IMAGE_MD5_KEY = 'org.openstack.shade.md5'
IMAGE_SHA256_KEY = 'org.openstack.shade.sha256'


OBJECT_CONTAINER_ACLS = {
    'public': ".r:*,.rlistings",
    'private': '',
}


class OpenStackCloudException(Exception):
    pass


class OpenStackCloudTimeout(OpenStackCloudException):
    pass


def openstack_clouds(config=None):
    if not config:
        config = os_client_config.OpenStackConfig()
    return [OpenStackCloud(f.name, f.region, **f.config)
            for f in config.get_all_clouds()]


def openstack_cloud(debug=False, **kwargs):
    cloud_config = os_client_config.OpenStackConfig().get_one_cloud(
        **kwargs)
    return OpenStackCloud(
        cloud_config.name, cloud_config.region,
        debug=debug, **cloud_config.config)


def operator_cloud(**kwargs):
    cloud_config = os_client_config.OpenStackConfig().get_one_cloud(**kwargs)
    return OperatorCloud(
        cloud_config.name, cloud_config.region, **cloud_config.config)


def _get_service_values(kwargs, service_key):
    return {k[:-(len(service_key) + 1)]: kwargs[k]
            for k in kwargs.keys() if k.endswith(service_key)}


def _iterate_timeout(timeout, message):
    """Iterate and raise an exception on timeout.

    This is a generator that will continually yield and sleep for 2
    seconds, and if the timeout is reached, will raise an exception
    with <message>.

    """

    start = time.time()
    count = 0
    while (timeout is None) or (time.time() < start + timeout):
        count += 1
        yield count
        time.sleep(2)
    raise OpenStackCloudTimeout(message)


class OpenStackCloud(object):

    def __init__(self, cloud, region='',
                 auth_plugin='password',
                 insecure=False, verify=None, cacert=None, cert=None, key=None,
                 image_cache=None, flavor_cache=None, volume_cache=None,
                 debug=False, **kwargs):

        self.name = cloud
        self.region = region

        self.auth_plugin = auth_plugin
        self.auth = kwargs.get('auth')

        self.region_name = kwargs.get('region_name', region)
        self._auth_token = kwargs.get('auth_token', None)

        self.service_types = _get_service_values(kwargs, 'service_type')
        self.service_names = _get_service_values(kwargs, 'service_name')
        self.endpoints = _get_service_values(kwargs, 'endpoint')
        self.api_versions = _get_service_values(kwargs, 'api_version')

        self.endpoint_type = kwargs.get('endpoint_type', 'publicURL')
        self.private = kwargs.get('private', False)

        if verify is None:
            if insecure:
                verify = False
            else:
                verify = cacert or True
        self.verify = verify

        if cert and key:
            cert = (cert, key)
        self.cert = cert

        self._image_cache = image_cache
        self._flavor_cache = flavor_cache
        self._volume_cache = volume_cache
        self._container_cache = dict()
        self._file_hash_cache = dict()

        self.debug = debug

        self._keystone_session = None

        self._cinder_client = None
        self._glance_client = None
        self._glance_endpoint = None
        self._ironic_client = None
        self._neutron_client = None
        self._nova_client = None
        self._swift_client = None
        self._trove_client = None

        self.log = logging.getLogger('shade')
        log_level = logging.INFO
        if self.debug:
            log_level = logging.DEBUG
        self.log.setLevel(log_level)
        self.log.addHandler(logging.StreamHandler())

    def get_service_type(self, service):
        return self.service_types.get(service, service)

    def get_service_name(self, service):
        return self.service_names.get(service, None)

    def _get_nova_api_version(self):
        return self.api_versions['compute']

    @property
    def nova_client(self):
        if self._nova_client is None:

            # Make the connection
            try:
                self._nova_client = nova_client.Client(
                    self._get_nova_api_version(),
                    session=self.keystone_session,
                    service_name=self.get_service_name('compute'),
                    region_name=self.region_name)
            except Exception:
                self.log.debug("Couldn't construct nova object", exc_info=True)
                raise

            if self._nova_client is None:
                raise OpenStackCloudException(
                    "Failed to instantiate nova client."
                    " This could mean that your credentials are wrong.")

        return self._nova_client

    @property
    def keystone_session(self):
        if self._keystone_session is None:
            # keystoneclient does crazy things with logging that are
            # none of them interesting
            keystone_logging = logging.getLogger('keystoneclient')
            keystone_logging.addHandler(logging.NullHandler())

            try:
                auth_plugin = ksc_auth.get_plugin_class(self.auth_plugin)
            except Exception as e:
                self.log.debug("keystone auth plugin failure", exc_info=True)
                raise OpenStackCloudException(
                    "Could not find auth plugin: {plugin}".format(
                    plugin=self.auth_plugin))
            try:
                keystone_auth = auth_plugin(**self.auth)
            except Exception as e:
                self.log.debug(
                    "keystone couldn't construct plugin", exc_info=True)
                raise OpenStackCloudException(
                    "Error constructing auth plugin: {plugin}".format(
                        plugin=self.auth_plugin))

            try:
                self._keystone_session = ksc_session.Session(
                    auth=keystone_auth,
                    verify=self.verify,
                    cert=self.cert)
            except Exception as e:
                self.log.debug("keystone unknown issue", exc_info=True)
                raise OpenStackCloudException(
                    "Error authenticating to the keystone: %s " % e.message)
        return self._keystone_session

    @property
    def service_catalog(self):
        return self.keystone_session.auth.get_access(
            self.keystone_session).service_catalog.get_data()

    @property
    def auth_token(self):
        if not self._auth_token:
            self._auth_token = self.keystone_session.get_token()
        return self._auth_token

    def _get_glance_api_version(self):
        if 'image' in self.api_versions:
            return self.api_versions['image']
        # Yay. We get to guess ...
        # Get rid of trailing '/' if present
        endpoint = self._get_glance_endpoint()
        if endpoint.endswith('/'):
            endpoint = endpoint[:-1]
        url_bits = endpoint.split('/')
        if url_bits[-1].startswith('v'):
            return url_bits[-1][1]
        return '1'  # Who knows? Let's just try 1 ...

    def _get_glance_endpoint(self):
        if self._glance_endpoint is None:
            self._glance_endpoint = self.get_endpoint(
                service_type=self.get_service_type('image'))
        return self._glance_endpoint

    @property
    def glance_client(self):
        if self._glance_client is None:
            token = self.auth_token
            endpoint = self._get_glance_endpoint()
            glance_api_version = self._get_glance_api_version()
            try:
                self._glance_client = glanceclient.Client(
                    glance_api_version, endpoint, token=token,
                    session=self.keystone_session)
            except Exception as e:
                self.log.debug("glance unknown issue", exc_info=True)
                raise OpenStackCloudException(
                    "Error in connecting to glance: %s" % e.message)

            if not self._glance_client:
                raise OpenStackCloudException("Error connecting to glance")
        return self._glance_client

    @property
    def swift_client(self):
        if self._swift_client is None:
            token = self.auth_token
            endpoint = self.get_endpoint(
                service_type=self.get_service_type('object-store'))
            self._swift_client = swift_client.Connection(
                preauthurl=endpoint,
                preauthtoken=token,
                os_options=dict(region_name=self.region_name),
            )
        return self._swift_client

    @property
    def cinder_client(self):

        if self._cinder_client is None:
            # Make the connection
            self._cinder_client = cinder_client.Client(
                session=self.keystone_session,
                region_name=self.region_name)

            if self._cinder_client is None:
                raise OpenStackCloudException(
                    "Failed to instantiate cinder client."
                    " This could mean that your credentials are wrong.")

        return self._cinder_client

    def _get_trove_api_version(self, endpoint):
        if 'database' in self.api_versions:
            return self.api_versions['database']
        # Yay. We get to guess ...
        # Get rid of trailing '/' if present
        if endpoint.endswith('/'):
            endpoint = endpoint[:-1]
        url_bits = endpoint.split('/')
        for bit in url_bits:
            if bit.startswith('v'):
                return bit[1:]
        return '1.0'  # Who knows? Let's just try 1.0 ...

    @property
    def trove_client(self):
        if self._trove_client is None:
            endpoint = self.get_endpoint(
                service_type=self.get_service_type('database'))
            trove_api_version = self._get_trove_api_version(endpoint)
            # Make the connection - can't use keystone session until there
            # is one
            self._trove_client = trove_client.Client(
                trove_api_version,
                session=self.keystone_session,
                region_name=self.region_name,
                service_type=self.get_service_type('database'),
            )

            if self._trove_client is None:
                raise OpenStackCloudException(
                    "Failed to instantiate Trove client."
                    " This could mean that your credentials are wrong.")

        return self._trove_client

    @property
    def neutron_client(self):
        if self._neutron_client is None:
            self._neutron_client = neutron_client.Client(
                token=self.auth_token,
                session=self.keystone_session,
                region_name=self.region_name)
        return self._neutron_client

    def get_name(self):
        return self.name

    def get_region(self):
        return self.region_name

    @property
    def flavor_cache(self):
        if not self._flavor_cache:
            self._flavor_cache = {
                flavor.id: flavor
                for flavor in self.nova_client.flavors.list()}
        return self._flavor_cache

    def get_flavor_name(self, flavor_id):
        flavor = self.flavor_cache.get(flavor_id, None)
        if flavor:
            return flavor.name
        return None

    def get_flavor(self, name_or_id):
        for id, flavor in self.flavor_cache.items():
            if name_or_id in (id, flavor.name):
                return flavor
        return None

    def get_flavor_by_ram(self, ram, include=None):
        for flavor in sorted(
                self.flavor_cache.values(),
                key=operator.attrgetter('ram')):
            if (flavor.ram >= ram and
                    (not include or include in flavor.name)):
                return flavor
        raise OpenStackCloudException(
            "Cloud not find a flavor with {ram} and '{include}'".format(
                ram=ram, include=include))

    def get_endpoint(self, service_type):
        if service_type in self.endpoints:
            return self.endpoints[service_type]
        try:
            endpoint = self.keystone_session.get_endpoint(
                service_type=service_type,
                interface=self.endpoint_type,
                region_name=self.region_name)
        except Exception as e:
            self.log.debug("keystone cannot get endpoint", exc_info=True)
            raise OpenStackCloudException(
                "Error getting %s endpoint: %s" % (service_type, e.message))
        return endpoint

    def list_servers(self):
        return self.nova_client.servers.list()

    def list_server_dicts(self):
        return [self.get_openstack_vars(server)
                for server in self.list_servers()]

    def list_keypairs(self):
        return self.nova_client.keypairs.list()

    def list_keypair_dicts(self):
        return [meta.obj_to_dict(keypair)
                for keypair in self.list_keypairs()]

    def create_keypair(self, name, public_key):
        return self.nova_client.keypairs.create(name, public_key)

    def delete_keypair(self, name):
        return self.nova_client.keypairs.delete(name)

    def list_networks(self):
        return self.neutron_client.list_networks()['networks']

    def get_network(self, name_or_id):
        for network in self.list_networks():
            if name_or_id in (network['id'], network['name']):
                return network
        return None

    def _get_images_from_cloud(self, filter_deleted):
        # First, try to actually get images from glance, it's more efficient
        images = dict()
        try:
            # This can fail both because we don't have glanceclient installed
            # and because the cloud may not expose the glance API publically
            image_list = self.glance_client.images.list()
        except (OpenStackCloudException,
                glanceclient.exc.HTTPInternalServerError):
            # We didn't have glance, let's try nova
            # If this doesn't work - we just let the exception propagate
            image_list = self.nova_client.images.list()
        for image in image_list:
            # The cloud might return DELETED for invalid images.
            # While that's cute and all, that's an implementation detail.
            if not filter_deleted:
                images[image.id] = image
            elif image.status != 'DELETED':
                images[image.id] = image
        return images

    def list_images(self, filter_deleted=True):
        """Get available glance images.

        :param filter_deleted: Control whether deleted images are returned.
        :returns: A dictionary of glance images indexed by image UUID.
        """
        if self._image_cache is None:
            self._image_cache = self._get_images_from_cloud(filter_deleted)
        return self._image_cache

    def get_image_name(self, image_id, exclude=None):
        image = self.get_image(image_id, exclude)
        if image:
            return image.id
        self._image_cache[image_id] = None
        return None

    def get_image_id(self, image_name, exclude=None):
        image = self.get_image(image_name, exclude)
        if image:
            return image.id
        return None

    def get_image(self, name_or_id, exclude=None):
        for (image_id, image) in self.list_images().items():
            if image_id == name_or_id:
                return image
            if (name_or_id in image.name and (
                    not exclude or exclude not in image.name)):
                return image
        return None

    def get_image_dict(self, name_or_id, exclude=None):
        image = self.get_image(name_or_id, exclude)
        if not image:
            return image<commit_msg>Don't compare images when image is None<commit_after># Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import hashlib
import logging
import operator
import time

from cinderclient.v1 import client as cinder_client
import glanceclient
from ironicclient import client as ironic_client
from ironicclient import exceptions as ironic_exceptions
from keystoneclient import auth as ksc_auth
from keystoneclient import session as ksc_session
from novaclient import client as nova_client
from novaclient.v1_1 import floating_ips
from neutronclient.v2_0 import client as neutron_client
import os_client_config
import pbr.version
import swiftclient.client as swift_client
import swiftclient.exceptions as swift_exceptions
import troveclient.client as trove_client


from shade import meta

__version__ = pbr.version.VersionInfo('shade').version_string()
OBJECT_MD5_KEY = 'x-object-meta-x-shade-md5'
OBJECT_SHA256_KEY = 'x-object-meta-x-shade-sha256'
IMAGE_MD5_KEY = 'org.openstack.shade.md5'
IMAGE_SHA256_KEY = 'org.openstack.shade.sha256'


OBJECT_CONTAINER_ACLS = {
    'public': ".r:*,.rlistings",
    'private': '',
}


class OpenStackCloudException(Exception):
    pass


class OpenStackCloudTimeout(OpenStackCloudException):
    pass


def openstack_clouds(config=None):
    if not config:
        config = os_client_config.OpenStackConfig()
    return [OpenStackCloud(f.name, f.region, **f.config)
            for f in config.get_all_clouds()]


def openstack_cloud(debug=False, **kwargs):
    cloud_config = os_client_config.OpenStackConfig().get_one_cloud(
        **kwargs)
    return OpenStackCloud(
        cloud_config.name, cloud_config.region,
        debug=debug, **cloud_config.config)


def operator_cloud(**kwargs):
    cloud_config = os_client_config.OpenStackConfig().get_one_cloud(**kwargs)
    return OperatorCloud(
        cloud_config.name, cloud_config.region, **cloud_config.config)


def _get_service_values(kwargs, service_key):
    return {k[:-(len(service_key) + 1)]: kwargs[k]
            for k in kwargs.keys() if k.endswith(service_key)}


def _iterate_timeout(timeout, message):
    """Iterate and raise an exception on timeout.

    This is a generator that will continually yield and sleep for 2
    seconds, and if the timeout is reached, will raise an exception
    with <message>.

    """

    start = time.time()
    count = 0
    while (timeout is None) or (time.time() < start + timeout):
        count += 1
        yield count
        time.sleep(2)
    raise OpenStackCloudTimeout(message)


class OpenStackCloud(object):

    def __init__(self, cloud, region='',
                 auth_plugin='password',
                 insecure=False, verify=None, cacert=None, cert=None, key=None,
                 image_cache=None, flavor_cache=None, volume_cache=None,
                 debug=False, **kwargs):

        self.name = cloud
        self.region = region

        self.auth_plugin = auth_plugin
        self.auth = kwargs.get('auth')

        self.region_name = kwargs.get('region_name', region)
        self._auth_token = kwargs.get('auth_token', None)

        self.service_types = _get_service_values(kwargs, 'service_type')
        self.service_names = _get_service_values(kwargs, 'service_name')
        self.endpoints = _get_service_values(kwargs, 'endpoint')
        self.api_versions = _get_service_values(kwargs, 'api_version')

        self.endpoint_type = kwargs.get('endpoint_type', 'publicURL')
        self.private = kwargs.get('private', False)

        if verify is None:
            if insecure:
                verify = False
            else:
                verify = cacert or True
        self.verify = verify

        if cert and key:
            cert = (cert, key)
        self.cert = cert

        self._image_cache = image_cache
        self._flavor_cache = flavor_cache
        self._volume_cache = volume_cache
        self._container_cache = dict()
        self._file_hash_cache = dict()

        self.debug = debug

        self._keystone_session = None

        self._cinder_client = None
        self._glance_client = None
        self._glance_endpoint = None
        self._ironic_client = None
        self._neutron_client = None
        self._nova_client = None
        self._swift_client = None
        self._trove_client = None

        self.log = logging.getLogger('shade')
        log_level = logging.INFO
        if self.debug:
            log_level = logging.DEBUG
        self.log.setLevel(log_level)
        self.log.addHandler(logging.StreamHandler())

    def get_service_type(self, service):
        return self.service_types.get(service, service)

    def get_service_name(self, service):
        return self.service_names.get(service, None)

    def _get_nova_api_version(self):
        return self.api_versions['compute']

    @property
    def nova_client(self):
        if self._nova_client is None:

            # Make the connection
            try:
                self._nova_client = nova_client.Client(
                    self._get_nova_api_version(),
                    session=self.keystone_session,
                    service_name=self.get_service_name('compute'),
                    region_name=self.region_name)
            except Exception:
                self.log.debug("Couldn't construct nova object", exc_info=True)
                raise

            if self._nova_client is None:
                raise OpenStackCloudException(
                    "Failed to instantiate nova client."
                    " This could mean that your credentials are wrong.")

        return self._nova_client

    @property
    def keystone_session(self):
        if self._keystone_session is None:
            # keystoneclient does crazy things with logging that are
            # none of them interesting
            keystone_logging = logging.getLogger('keystoneclient')
            keystone_logging.addHandler(logging.NullHandler())

            try:
                auth_plugin = ksc_auth.get_plugin_class(self.auth_plugin)
            except Exception as e:
                self.log.debug("keystone auth plugin failure", exc_info=True)
                raise OpenStackCloudException(
                    "Could not find auth plugin: {plugin}".format(
                    plugin=self.auth_plugin))
            try:
                keystone_auth = auth_plugin(**self.auth)
            except Exception as e:
                self.log.debug(
                    "keystone couldn't construct plugin", exc_info=True)
                raise OpenStackCloudException(
                    "Error constructing auth plugin: {plugin}".format(
                        plugin=self.auth_plugin))

            try:
                self._keystone_session = ksc_session.Session(
                    auth=keystone_auth,
                    verify=self.verify,
                    cert=self.cert)
            except Exception as e:
                self.log.debug("keystone unknown issue", exc_info=True)
                raise OpenStackCloudException(
                    "Error authenticating to the keystone: %s " % e.message)
        return self._keystone_session

    @property
    def service_catalog(self):
        return self.keystone_session.auth.get_access(
            self.keystone_session).service_catalog.get_data()

    @property
    def auth_token(self):
        if not self._auth_token:
            self._auth_token = self.keystone_session.get_token()
        return self._auth_token

    def _get_glance_api_version(self):
        if 'image' in self.api_versions:
            return self.api_versions['image']
        # Yay. We get to guess ...
        # Get rid of trailing '/' if present
        endpoint = self._get_glance_endpoint()
        if endpoint.endswith('/'):
            endpoint = endpoint[:-1]
        url_bits = endpoint.split('/')
        if url_bits[-1].startswith('v'):
            return url_bits[-1][1]
        return '1'  # Who knows? Let's just try 1 ...

    def _get_glance_endpoint(self):
        if self._glance_endpoint is None:
            self._glance_endpoint = self.get_endpoint(
                service_type=self.get_service_type('image'))
        return self._glance_endpoint

    @property
    def glance_client(self):
        if self._glance_client is None:
            token = self.auth_token
            endpoint = self._get_glance_endpoint()
            glance_api_version = self._get_glance_api_version()
            try:
                self._glance_client = glanceclient.Client(
                    glance_api_version, endpoint, token=token,
                    session=self.keystone_session)
            except Exception as e:
                self.log.debug("glance unknown issue", exc_info=True)
                raise OpenStackCloudException(
                    "Error in connecting to glance: %s" % e.message)

            if not self._glance_client:
                raise OpenStackCloudException("Error connecting to glance")
        return self._glance_client

    @property
    def swift_client(self):
        if self._swift_client is None:
            token = self.auth_token
            endpoint = self.get_endpoint(
                service_type=self.get_service_type('object-store'))
            self._swift_client = swift_client.Connection(
                preauthurl=endpoint,
                preauthtoken=token,
                os_options=dict(region_name=self.region_name),
            )
        return self._swift_client

    @property
    def cinder_client(self):

        if self._cinder_client is None:
            # Make the connection
            self._cinder_client = cinder_client.Client(
                session=self.keystone_session,
                region_name=self.region_name)

            if self._cinder_client is None:
                raise OpenStackCloudException(
                    "Failed to instantiate cinder client."
                    " This could mean that your credentials are wrong.")

        return self._cinder_client

    def _get_trove_api_version(self, endpoint):
        if 'database' in self.api_versions:
            return self.api_versions['database']
        # Yay. We get to guess ...
        # Get rid of trailing '/' if present
        if endpoint.endswith('/'):
            endpoint = endpoint[:-1]
        url_bits = endpoint.split('/')
        for bit in url_bits:
            if bit.startswith('v'):
                return bit[1:]
        return '1.0'  # Who knows? Let's just try 1.0 ...

    @property
    def trove_client(self):
        if self._trove_client is None:
            endpoint = self.get_endpoint(
                service_type=self.get_service_type('database'))
            trove_api_version = self._get_trove_api_version(endpoint)
            # Make the connection - can't use keystone session until there
            # is one
            self._trove_client = trove_client.Client(
                trove_api_version,
                session=self.keystone_session,
                region_name=self.region_name,
                service_type=self.get_service_type('database'),
            )

            if self._trove_client is None:
                raise OpenStackCloudException(
                    "Failed to instantiate Trove client."
                    " This could mean that your credentials are wrong.")

        return self._trove_client

    @property
    def neutron_client(self):
        if self._neutron_client is None:
            self._neutron_client = neutron_client.Client(
                token=self.auth_token,
                session=self.keystone_session,
                region_name=self.region_name)
        return self._neutron_client

    def get_name(self):
        return self.name

    def get_region(self):
        return self.region_name

    @property
    def flavor_cache(self):
        if not self._flavor_cache:
            self._flavor_cache = {
                flavor.id: flavor
                for flavor in self.nova_client.flavors.list()}
        return self._flavor_cache

    def get_flavor_name(self, flavor_id):
        flavor = self.flavor_cache.get(flavor_id, None)
        if flavor:
            return flavor.name
        return None

    def get_flavor(self, name_or_id):
        for id, flavor in self.flavor_cache.items():
            if name_or_id in (id, flavor.name):
                return flavor
        return None

    def get_flavor_by_ram(self, ram, include=None):
        for flavor in sorted(
                self.flavor_cache.values(),
                key=operator.attrgetter('ram')):
            if (flavor.ram >= ram and
                    (not include or include in flavor.name)):
                return flavor
        raise OpenStackCloudException(
            "Cloud not find a flavor with {ram} and '{include}'".format(
                ram=ram, include=include))

    def get_endpoint(self, service_type):
        if service_type in self.endpoints:
            return self.endpoints[service_type]
        try:
            endpoint = self.keystone_session.get_endpoint(
                service_type=service_type,
                interface=self.endpoint_type,
                region_name=self.region_name)
        except Exception as e:
            self.log.debug("keystone cannot get endpoint", exc_info=True)
            raise OpenStackCloudException(
                "Error getting %s endpoint: %s" % (service_type, e.message))
        return endpoint

    def list_servers(self):
        return self.nova_client.servers.list()

    def list_server_dicts(self):
        return [self.get_openstack_vars(server)
                for server in self.list_servers()]

    def list_keypairs(self):
        return self.nova_client.keypairs.list()

    def list_keypair_dicts(self):
        return [meta.obj_to_dict(keypair)
                for keypair in self.list_keypairs()]

    def create_keypair(self, name, public_key):
        return self.nova_client.keypairs.create(name, public_key)

    def delete_keypair(self, name):
        return self.nova_client.keypairs.delete(name)

    def list_networks(self):
        return self.neutron_client.list_networks()['networks']

    def get_network(self, name_or_id):
        for network in self.list_networks():
            if name_or_id in (network['id'], network['name']):
                return network
        return None

    def _get_images_from_cloud(self, filter_deleted):
        # First, try to actually get images from glance, it's more efficient
        images = dict()
        try:
            # This can fail both because we don't have glanceclient installed
            # and because the cloud may not expose the glance API publically
            image_list = self.glance_client.images.list()
        except (OpenStackCloudException,
                glanceclient.exc.HTTPInternalServerError):
            # We didn't have glance, let's try nova
            # If this doesn't work - we just let the exception propagate
            image_list = self.nova_client.images.list()
        for image in image_list:
            # The cloud might return DELETED for invalid images.
            # While that's cute and all, that's an implementation detail.
            if not filter_deleted:
                images[image.id] = image
            elif image.status != 'DELETED':
                images[image.id] = image
        return images

    def list_images(self, filter_deleted=True):
        """Get available glance images.

        :param filter_deleted: Control whether deleted images are returned.
        :returns: A dictionary of glance images indexed by image UUID.
        """
        if self._image_cache is None:
            self._image_cache = self._get_images_from_cloud(filter_deleted)
        return self._image_cache

    def get_image_name(self, image_id, exclude=None):
        image = self.get_image(image_id, exclude)
        if image:
            return image.id
        self._image_cache[image_id] = None
        return None

    def get_image_id(self, image_name, exclude=None):
        image = self.get_image(image_name, exclude)
        if image:
            return image.id
        return None

    def get_image(self, name_or_id, exclude=None):
        for (image_id, image) in self.list_images().items():
            if image_id == name_or_id:
                return image
            if (image is not None and
                    name_or_id in image.name and (
                    not exclude or exclude not in image.name)):
                return image
        return None

    def get_image_dict(self, name_or_id, exclude=None):
        image = self.get_image(name_or_id, exclude)
        if not image:
            return image
{'fraction_non_alphanumeric': 0.07937569028657793, 'fraction_numerical': 0.0026739522176364586, 'mean_word_length': 2.1641530255655694, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3171408', 'n_tokens_mistral': 9687, 'n_tokens_neox': 9222, 'n_words': 2408}