<commit_before>import datetime
import dateutil.parser
import logging
import re
import time
import uuid

from django.conf import settings as django_settings
from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned
from django.db import transaction
from django.utils import six, dateparse, timezone
from requests import ConnectionError

from keystoneclient.auth.identity import v2
from keystoneclient.service_catalog import ServiceCatalog
from keystoneclient import session as keystone_session

from ceilometerclient import client as ceilometer_client
from cinderclient.v1 import client as cinder_client
from glanceclient.v1 import client as glance_client
from keystoneclient.v2_0 import client as keystone_client
from neutronclient.v2_0 import client as neutron_client
from novaclient.v1_1 import client as nova_client

from cinderclient import exceptions as cinder_exceptions
from glanceclient import exc as glance_exceptions
from keystoneclient import exceptions as keystone_exceptions
from neutronclient.client import exceptions as neutron_exceptions
from novaclient import exceptions as nova_exceptions

from nodeconductor.core.models import StateMixin
from nodeconductor.core.tasks import send_task
from nodeconductor.structure import ServiceBackend, ServiceBackendError, log_backend_action
from nodeconductor.structure.log import event_logger
from nodeconductor.openstack import models


logger = logging.getLogger(__name__)


class OpenStackBackendError(ServiceBackendError):
    pass


class OpenStackSession(dict):
    """ Serializable session """

    def __init__(self, ks_session=None, verify_ssl=False, **credentials):
        self.keystone_session = ks_session

        if not self.keystone_session:
            auth_plugin = v2.Password(**credentials)
            self.keystone_session = keystone_session.Session(auth=auth_plugin, verify=verify_ssl)

        try:
            # This will eagerly sign in throwing AuthorizationFailure on bad credentials
            self.keystone_session.get_token()
        except (keystone_exceptions.AuthorizationFailure, keystone_exceptions.ConnectionRefused) as e:
            six.reraise(OpenStackBackendError, e)

        for opt in ('auth_ref', 'auth_url', 'tenant_id', 'tenant_name'):
            self[opt] = getattr(self.auth, opt)

    def __getattr__(self, name):
        return getattr(self.keystone_session, name)

    @classmethod
    def recover(cls, session, verify_ssl=False):
        if not isinstance(session, dict) or not session.get('auth_ref'):
            raise OpenStackBackendError('Invalid OpenStack session')

        args = {'auth_url': session['auth_url'], 'token': session['auth_ref']['token']['id']}
        if session['tenant_id']:
            args['tenant_id'] = session['tenant_id']
        elif session['tenant_name']:
            args['tenant_name'] = session['tenant_name']

        ks_session = keystone_session.Session(auth=v2.Token(**args), verify=verify_ssl)
        return cls(
            ks_session=ks_session,
            tenant_id=session['tenant_id'],
            tenant_name=session['tenant_name'])

    def validate(self):
        expiresat = dateutil.parser.parse(self.auth.auth_ref['token']['expires'])
        if expiresat > timezone.now() + datetime.timedelta(minutes=10):
            return True

        raise OpenStackBackendError('OpenStack session is expired')

    def __str__(self):
        return str({k: v if k != 'password' else '***' for k, v in self})


class OpenStackClient(object):
    """ Generic OpenStack client. """

    def __init__(self, session=None, verify_ssl=False, **credentials):
        self.verify_ssl = verify_ssl
        if session:
            if isinstance(session, dict):
                logger.info('Trying to recover OpenStack session.')
                self.session = OpenStackSession.recover(session, verify_ssl=verify_ssl)
                self.session.validate()
            else:
                self.session = session
        else:
            try:
                self.session = OpenStackSession(verify_ssl=verify_ssl, **credentials)
            except AttributeError as e:
                logger.error('Failed to create OpenStack session.')
                six.reraise(OpenStackBackendError, e)

    @property
    def keystone(self):
        return keystone_client.Client(session=self.session.keystone_session)

    @property
    def nova(self):
        try:
            return nova_client.Client(session=self.session.keystone_session)
        except (nova_exceptions.ClientException, keystone_exceptions.ClientException) as e:
            logger.exception('Failed to create nova client: %s', e)
            six.reraise(OpenStackBackendError, e)

    @property
    def neutron(self):
        try:
            return neutron_client.Client(session=self.session.keystone_session)
        except (neutron_exceptions.NeutronClientException, keystone_exceptions.ClientException) as e:
            logger.exception('Failed to create neutron client: %s', e)
            six.reraise(OpenStackBackendError, e)

    @property
    def cinder(self):
        try:
            return cinder_client.Client(session=self.session.keystone_session)
        except (cinder_exceptions.ClientException, keystone_exceptions.ClientException) as e:
            logger.exception('Failed to create cinder client: %s', e)
            six.reraise(OpenStackBackendError, e)

    @property
    def glance(self):
        catalog = ServiceCatalog.factory(self.session.auth.auth_ref)
        endpoint = catalog.url_for(service_type='image')

        kwargs = {
            'token': self.session.get_token(),
            'insecure': not self.verify_ssl,
            'timeout': 600,
            'ssl_compression': True,
        }

        return glance_client.Client(endpoint, **kwargs)

    @property
    def ceilometer(self):
        catalog = ServiceCatalog.factory(self.session.auth.auth_ref)
        endpoint = catalog.url_for(service_type='metering')

        kwargs = {
            'token': lambda: self.session.get_token(),
            'endpoint': endpoint,
            'insecure': not self.verify_ssl,
            'timeout': 600,
            'ssl_compression': True,
        }

        return ceilometer_client.Client('2', **kwargs)


class OpenStackBackend(ServiceBackend):

    DEFAULT_TENANT = 'admin'

    def __init__(self, settings, tenant_id=None):
        self.settings = settings
        self.tenant_id = tenant_id

    def get_client(self, name=None, admin=False):
        credentials = {
            'auth_url': self.settings.backend_url,
            'username': self.settings.username,
            'password': self.settings.password,
        }

        if not admin:
            if not self.tenant_id:
                raise OpenStackBackendError(
                    "Can't create tenant session, please provide tenant ID")

            credentials['tenant_id'] = self.tenant_id
        elif self.settings.options:
            credentials['tenant_name'] = self.settings.options.get('tenant_name', self.DEFAULT_TENANT)
        else:
            credentials['tenant_name'] = self.DEFAULT_TENANT

        # Cache session in the object
        attr_name = 'admin_session' if admin else 'session'
        client = getattr(self, attr_name, None)
        if hasattr(self, attr_name):
            client = getattr(self, attr_name)
        else:
            client = OpenStackClient(**credentials)
            setattr(self, attr_name, client)

        if name:
            return getattr(client, name)
        else:
            return client

    def __getattr__(self, name):
        clients = 'keystone', 'nova', 'neutron', 'cinder', 'glance', 'ceilometer'
        for client in clients:
            if name == '{}_client'.format(client):
                return self.get_client(client, admin=False)

            if name == '{}_admin_client'.format(client):
                return self.get_client(client, admin=True)

        raise AttributeError(
            "'%s' object has no attribute '%s'" % (self.__class__.__name__, name))

    def ping(self, raise_exception=False):
        # Session validation occurs on class creation so assume it's active
        # TODO: Consider validating session depending on tenant permissions
        return True

    def ping_resource(self, instance):
        try:
            self.nova_client.servers.get(instance.backend_id)
        except (ConnectionError, nova_exceptions.ClientException):
            return False
        else:
            return True

    def sync(self):
        # Migration status:
        # [x] pull_flavors()
        # [x] pull_images()
        # [ ] pull_service_statistics() (TODO: NC-640)

        try:
            self.pull_flavors()
            self.pull_images()
        except (nova_exceptions.ClientException, glance_exceptions.ClientException) as e:
            logger.exception('Failed to synchronize OpenStack service %s', self.settings.backend_url)
            six.reraise(OpenStackBackendError, e)
        else:
            logger.info('Successfully synchronized OpenStack service %s', self.settings.backend_url)

    def provision(self, instance, flavor=None, image=None, ssh_key=None, **kwargs):
        if ssh_key:
            instance.key_name = self.get_key_name(ssh_key)
            instance.key_fingerprint = ssh_key.fingerprint
            kwargs['public_key'] = ssh_key.public_key

        instance.flavor_name = flavor.name
        instance.cores = flavor.cores
        instance.ram = flavor.ram
        instance.flavor_disk = flavor.disk
        instance.disk = instance.system_volume_size + instance.data_volume_size
        if image:
            instance.min_disk = image.min_disk
            instance.min_ram = image.min_ram
        instance.save()

        kwargs['backend_flavor_id'] = flavor.backend_id
        if image:
            kwargs['backend_image_id'] = image.backend_id

        send_task('openstack', 'provision')(instance.uuid.hex, **kwargs)

    def destroy(self, instance, force=False):
        instance.schedule_deletion()
        instance.save()
        send_task('openstack', 'destroy')(instance.uuid.hex, force=force)

    def start(self, instance):
        instance.schedule_starting()
        instance.save()
        send_task('openstack', 'start')(instance.uuid.hex)

    def stop(self, instance):
        instance.schedule_stopping()
        instance.save()
        send_task('openstack', 'stop')(instance.uuid.hex)

    def restart(self, instance):
        instance.schedule_restarting()
        instance.save()
        send_task('openstack', 'restart')(instance.uuid.hex)

    def get_key_name(self, public_key):
        # We want names to be human readable in backend.
        # OpenStack only allows latin letters, digits, dashes, underscores and spaces
        # as key names, thus we mangle the original name.

        safe_name = self.sanitize_key_name(public_key.name)
        key_name = '{0}-{1}'.format(public_key.uuid.hex, safe_name)
        return key_name

    def sanitize_key_name(self, key_name):
        # Safe key name length must be less than 17 chars due to limit of full key name to 50 chars.
        return re.sub(r'[^-a-zA-Z0-9 _]+', '_', key_name)[:17]

    def add_ssh_key(self, ssh_key, service_project_link):
        if service_project_link.tenant is not None:
            key_name = self.get_key_name(ssh_key)
            self.get_or_create_ssh_key_for_tenant(
                service_project_link.tenant, key_name, ssh_key.fingerprint, ssh_key.public_key)

    def get_or_create_ssh_key_for_tenant(self, tenant, key_name, fingerprint, public_key):
        nova = self.nova_client

        try:
            return nova.keypairs.find(fingerprint=fingerprint)
        except nova_exceptions.NotFound:
            # Fine, it's a new key, let's add it
            try:
                return nova.keypairs.create(name=key_name, public_key=public_key)
            except (nova_exceptions.ClientException, keystone_exceptions.ClientException) as e:
                six.reraise(OpenStackBackendError, e)
            else:
                logger.info('Propagating ssh public key %s to backend', key_name)
        except (nova_exceptions.ClientException, keystone_exceptions.ClientException) as e:
            six.reraise(OpenStackBackendError, e)
        else:
            # Found a key with the same fingerprint, skip adding
            logger.info('Skipped propagating ssh public key %s to backend', key_name)

    def remove_ssh_key(self, ssh_key, service_project_link):
        if service_project_link.tenant is not None:
            self.remove_ssh_key_from_tenant(service_project_link.tenant, ssh_key)

    @log_backend_action()
    def remove_ssh_key_from_tenant(self, tenant, key_name, fingerprint):
        nova = self.nova_client

        # There could be leftovers of key duplicates: remove them all
        keys = nova.keypairs.findall(fingerprint=fingerprint)
        for key in keys:
            # Remove only keys created with NC
            if key.name == key_name:
                nova.keypairs.delete(key)

        logger.info('Deleted ssh public key %s from backend', key_name)

    def _get_instance_state(self, instance):
        # See http://developer.openstack.org/api-ref-compute-v2.html
        nova_to_nodeconductor = {
            'ACTIVE': models.Instance.States.ONLINE,
            'BUILDING': models.Instance.States.PROVISIONING,
            # 'DELETED': models.Instance.States.DELETING,
            # 'SOFT_DELETED': models.Instance.States.DELETING,
            'ERROR': models.Instance.States.ERRED,
            'UNKNOWN': models.Instance.States.ERRED,

            'HARD_REBOOT': models.Instance.States.STOPPING,  # Or starting?
            'REBOOT': models.Instance.States.STOPPING,  # Or starting?
            'REBUILD': models.Instance.States.STARTING,  # Or stopping?

            'PASSWORD': models.Instance.States.ONLINE,
            'PAUSED': models.Instance.States.OFFLINE,

            'RESCUED': models.Instance.States.ONLINE,
            'RESIZED': models.Instance.States.OFFLINE,
            'REVERT_RESIZE': models.Instance.States.STOPPING,
            'SHUTOFF': models.Instance.States.OFFLINE,
            'STOPPED': models.Instance.States.OFFLINE,
            'SUSPENDED': models.Instance.States.OFFLINE,
            'VERIFY_RESIZE': models.Instance.States.OFFLINE,
        }
        return nova_to_nodeconductor.get(instance.status, models.Instance.States.ERRED)

    def _get_current_properties(self, model):
        return {p.backend_id: p for p in model.objects.filter(settings=self.settings)}

    def _are_rules_equal(self, backend_rule, nc_rule):
        if backend_rule['from_port'] != nc_rule.from_port:
            return False
        if backend_rule['to_port'] != nc_rule.to_port:
            return False
        if backend_rule['ip_protocol'] != nc_rule.protocol:
            return False
        if backend_rule['ip_range'].get('cidr', '') != nc_rule.cidr:
            return False
        return True

    def _are_security_groups_equal(self, backend_security_group, nc_security_group):
        if backend_security_group.name != nc_security_group.name:
            return False
        if len(backend_security_group.rules) != nc_security_group.rules.count():
            return False
        for backend_rule, nc_rule in zip(backend_security_group.rules, nc_security_group.rules.all()):
            if not self._are_rules_equal(backend_rule, nc_rule):
                return False
        return True

    def _normalize_security_group_rule(self, rule):
        if rule['ip_protocol'] is None:
            rule['ip_protocol'] = ''

        if 'cidr' not in rule['ip_range']:
            rule['ip_range']['cidr'] = '0.0.0.0/0'

        return rule

    def _wait_for_instance_status(self, server_id, nova, complete_status,
                                  error_status=None, retries=300, poll_interval=3):
        return self._wait_for_object_status(
            server_id, nova.servers.get, complete_status, error_status, retries, poll_interval)

    def _wait_for_volume_status(self, volume_id, cinder, complete_status,
                                error_status=None, retries=300, poll_interval=3):
        return self._wait_for_object_status(
            volume_id, cinder.volumes.get, complete_status, error_status, retries, poll_interval)

    def _wait_for_snapshot_status(self, snapshot_id, cinder, complete_status, error_status, retries=90, poll_interval=3):
        return self._wait_for_object_status(
            snapshot_id, cinder.volume_snapshots.get, complete_status, error_status, retries, poll_interval)

    def _wait_for_backup_status(self, backup, cinder, complete_status, error_status, retries=90, poll_interval=3):
        return self._wait_for_object_status(
            backup, cinder.backups.get, complete_status, error_status, retries, poll_interval)

    def _wait_for_object_status(self, obj_id, client_get_method, complete_status, error_status=None,
                                retries=30, poll_interval=3):
        complete_state_predicate = lambda o: o.status == complete_status
        if error_status is not None:
            error_state_predicate = lambda o: o.status == error_status
        else:
            error_state_predicate = lambda _: False

        for _ in range(retries):
            obj = client_get_method(obj_id)

            if complete_state_predicate(obj):
                return True

            if error_state_predicate(obj):
                return False

            time.sleep(poll_interval)
        else:
            return False

    def _wait_for_volume_deletion(self, volume_id, cinder, retries=90, poll_interval=3):
        try:
            for _ in range(retries):
                cinder.volumes.get(volume_id)
                time.sleep(poll_interval)

            return False
        except cinder_exceptions.NotFound:
            return True

    def _wait_for_snapshot_deletion(self, snapshot_id, cinder, retries=90, poll_interval=3):
        try:
            for _ in range(retries):
                cinder.volume_snapshots.get(snapshot_id)
                time.sleep(poll_interval)

            return False
        except (cinder_exceptions.NotFound, keystone_exceptions.NotFound):
            return True

    def _wait_for_instance_deletion(self, backend_instance_id, retries=90, poll_interval=3):
        nova = self.nova_client
        try:
            for _ in range(retries):
                nova.servers.get(backend_instance_id)
                time.sleep(poll_interval)

            return False
        except nova_exceptions.NotFound:
            return True

    def pull_flavors(self):
        nova = self.nova_admin_client
        with transaction.atomic():
            cur_flavors = self._get_current_properties(models.Flavor)
            for backend_flavor in nova.flavors.findall(is_public=True):
                cur_flavors.pop(backend_flavor.id, None)
                models.Flavor.objects.update_or_create(
                    settings=self.settings,
                    backend_id=backend_flavor.id,
                    defaults={
                        'name': backend_flavor.name,
                        'cores': backend_flavor.vcpus,
                        'ram': backend_flavor.ram,
                        'disk': self.gb2mb(backend_flavor.disk),
                    })

            models.Flavor.objects.filter(backend_id__in=cur_flavors.keys()).delete()

    def pull_images(self):
        glance = self.glance_admin_client
        with transaction.atomic():
            cur_images = self._get_current_properties(models.Image)
            for backend_image in glance.images.list():
                if backend_image.is_public and not backend_image.deleted:
                    cur_images.pop(backend_image.id, None)
                    models.Image.objects.update_or_create(
                        settings=self.settings,
                        backend_id=backend_image.id,
                        defaults={
                            'name': backend_image.name,
                            'min_ram': backend_image.min_ram,
                            'min_disk': self.gb2mb(backend_image.min_disk),
                        })

            models.Image.objects.filter(backend_id__in=cur_images.keys()).delete()

    @log_backend_action('push quotas for tenant')
    def push_tenant_quotas(self, tenant, quotas):
        if 'instances' in quotas:
            # convert instances quota to volumes and snapshots.
            quotas_ratios = django_settings.NODECONDUCTOR.get('OPENSTACK_QUOTAS_INSTANCE_RATIOS', {})
            volume_ratio = quotas_ratios.get('volumes', 4)
            snapshots_ratio = quotas_ratios.get('snapshots', 20)

            quotas['volumes'] = volume_ratio * quotas['instances']
            quotas['snapshots'] = snapshots_ratio * quotas['instances']

        cinder_quotas = {
            'gigabytes': self.mb2gb(quotas.get('storage')) if 'storage' in quotas else None,
            'volumes': quotas.get('volumes'),
            'snapshots': quotas.get('snapshots'),
        }
        cinder_quotas = {k: v for k, v in cinder_quotas.items() if v is not None}

        nova_quotas = {
            'instances': quotas.get('instances'),
            'cores': quotas.get('vcpu'),
        }
        nova_quotas = {k: v for k, v in nova_quotas.items() if v is not None}

        neutron_quotas = {
            'security_group': quotas.get('security_group_count'),
            'security_group_rule': quotas.get('security_group_rule_count'),
        }
        neutron_quotas = {k: v for k, v in neutron_quotas.items() if v is not None}

        try:
            if cinder_quotas:
                self.cinder_client.quotas.update(tenant.backend_id, **cinder_quotas)
            if nova_quotas:
                self.nova_client.quotas.update(tenant.backend_id, **nova_quotas)
            if neutron_quotas:
                self.neutron_client.update_quota(tenant.backend_id, {'quota': neutron_quotas})
        except Exception as e:
            # XXX: event logging should be moved executors level
            event_logger.service_project_link.warning(
                'Failed to push quotas to backend.',
                event_type='service_project_link_sync_failed',
                event_context={
                    'service_project_link': tenant.service_project_link,
                    'error_message': six.text_type(e),
                }
            )
            six.reraise(OpenStackBackendError, e)

    @log_backend_action('pull quotas for tenant')
    def pull_tenant_quotas(self, tenant):
        # XXX: backend quotas should be moved to tenant from SPL in future.
        nova = self.nova_client
        neutron = self.neutron_client
        cinder = self.cinder_client
        service_project_link = tenant.service_project_link

        try:
            nova_quotas = nova.quotas.get(tenant_id=tenant.backend_id)
            cinder_quotas = cinder.quotas.get(tenant_id=tenant.backend_id)
            neutron_quotas = neutron.show_quota(tenant_id=tenant.backend_id)['quota']
        except (nova_exceptions.ClientException,
                cinder_exceptions.ClientException,
                neutron_exceptions.NeutronClientException) as e:
            six.reraise(OpenStackBackendError, e)

        service_project_link.set_quota_limit('ram', nova_quotas.ram)
        service_project_link.set_quota_limit('vcpu', nova_quotas.cores)
        service_project_link.set_quota_limit('storage', self.gb2mb(cinder_quotas.gigabytes))
        service_project_link.set_quota_limit('instances', nova_quotas.instances)
        service_project_link.set_quota_limit('security_group_count', neutron_quotas['security_group'])
        service_project_link.set_quota_limit('security_group_rule_count', neutron_quotas['security_group_rule'])
        service_project_link.set_quota_limit('floating_ip_count', neutron_quotas['floatingip'])

        try:
            volumes = cinder.volumes.list()
            snapshots = cinder.volume_snapshots.list()
            instances = nova.servers.list()
            security_groups = nova.security_groups.list()
            floating_ips = neutron.list_floatingips(tenant_id=tenant.backend_id)['floatingips']

            flavors = {flavor.id: flavor for flavor in nova.flavors.list()}

            ram, vcpu = 0, 0
            for flavor_id in (instance.flavor['id'] for instance in instances):
                try:
                    flavor = flavors.get(flavor_id, nova.flavors.get(flavor_id))
                except nova_exceptions.NotFound:
                    logger.warning('Cannot find flavor with id %s', flavor_id)
                    continue

                ram += getattr(flavor, 'ram', 0)
                vcpu += getattr(flavor, 'vcpus', 0)

        except (nova_exceptions.ClientException,
                cinder_exceptions.ClientException,
                neutron_exceptions.NeutronClientException) as e:
            six.reraise(OpenStackBackendError, e)

        service_project_link.set_quota_usage('ram', ram)
        service_project_link.set_quota_usage('vcpu', vcpu)
        service_project_link.set_quota_usage('storage', sum(self.gb2mb(v.size) for v in volumes + snapshots))
        service_project_link.set_quota_usage('instances', len(instances), fail_silently=True)
        service_project_link.set_quota_usage('security_group_count', len(security_groups))
        service_project_link.set_quota_usage('security_group_rule_count', len(sum([sg.rules for sg in security_groups], [])))
        service_project_link.set_quota_usage('floating_ip_count', len(floating_ips))

    @log_backend_action('pull floating IPs for tenant')
    def pull_tenant_floating_ips(self, tenant):
        service_project_link = tenant.service_project_link
        neutron = self.neutron_client

        try:
            nc_floating_ips = {ip.backend_id: ip for ip in service_project_link.floating_ips.all()}
            try:
                backend_floating_ips = {
                    ip['id']: ip
                    for ip in neutron.list_floatingips(tenant_id=self.tenant_id)['floatingips']
                    if ip.get('floating_ip_address') and ip.get('status')
                }
            except neutron_exceptions.NeutronClientException as e:
                six.reraise(OpenStackBackendError, e)

            backend_ids = set(backend_floating_ips.keys())
            nc_ids = set(nc_floating_ips.keys())

            with transaction.atomic():
                for ip_id in nc_ids - backend_ids:
                    ip = nc_floating_ips[ip_id]
                    ip.delete()
                    logger.info('Deleted stale floating IP port %s in database', ip.uuid)

                for ip_id in backend_ids - nc_ids:
                    ip = backend_floating_ips[ip_id]
                    created_ip = service_project_link.floating_ips.create(
                        status=ip['status'],
                        backend_id=ip['id'],
                        address=ip['floating_ip_address'],
                        backend_network_id=ip['floating_network_id']
                    )
                    logger.info('Created new floating IP port %s in database', created_ip.uuid)

                for ip_id in nc_ids & backend_ids:
                    nc_ip = nc_floating_ips[ip_id]
                    backend_ip = backend_floating_ips[ip_id]
                    if nc_ip.status != backend_ip['status'] or nc_ip.address != backend_ip['floating_ip_address']\
                            or nc_ip.backend_network_id != backend_ip['floating_network_id']:
                        # If key is BOOKED by NodeConductor it can be still DOWN in OpenStack
                        if not (nc_ip.status == 'BOOKED' and backend_ip['status'] == 'DOWN'):
                            nc_ip.status = backend_ip['status']
                        nc_ip.address = backend_ip['floating_ip_address']
                        nc_ip.backend_network_id = backend_ip['floating_network_id']
                        nc_ip.save()
                        logger.info('Updated existing floating IP port %s in database', nc_ip.uuid)

        except Exception as e:
            event_logger.service_project_link.warning(
                'Failed to pull floating IPs from backend.',
                event_type='service_project_link_sync_failed',
                event_context={
                    'service_project_link': service_project_link,
                    'error_message': six.text_type(e),
                }
            )
            six.reraise(OpenStackBackendError, e)

    @log_backend_action('pull security groups for tenant')
    def pull_tenant_security_groups(self, tenant):
        nova = self.nova_client
        service_project_link = tenant.service_project_link

        try:
            try:
                backend_security_groups = nova.security_groups.list()
            except nova_exceptions.ClientException as e:
                six.reraise(OpenStackBackendError, e)

            # list of openstack security groups that do not exist in nc
            nonexistent_groups = []
            # list of openstack security groups that have wrong parameters in in nc
            unsynchronized_groups = []
            # list of nc security groups that do not exist in openstack

            extra_groups = service_project_link.security_groups.exclude(
                backend_id__in=[g.id for g in backend_security_groups],
            )

            with transaction.atomic():
                for backend_group in backend_security_groups:
                    try:
                        nc_group = service_project_link.security_groups.get(backend_id=backend_group.id)
                        if not self._are_security_groups_equal(backend_group, nc_group):
                            unsynchronized_groups.append(backend_group)
                    except models.SecurityGroup.DoesNotExist:
                        nonexistent_groups.append(backend_group)

                # deleting extra security groups
                extra_groups.delete()
                if extra_groups:
                    logger.debug('Deleted stale security group: %s.',
                                 ' ,'.join('%s (PK: %s)' % (sg.name, sg.pk) for sg in extra_groups))

                # synchronizing unsynchronized security groups
                for backend_group in unsynchronized_groups:
                    nc_security_group = service_project_link.security_groups.get(backend_id=backend_group.id)
                    if backend_group.name != nc_security_group.name:
                        nc_security_group.name = backend_group.name
                        nc_security_group.state = StateMixin.States.OK
                        nc_security_group.save()
                    self.pull_security_group_rules(nc_security_group)
                    logger.debug('Updated existing security group %s (PK: %s).',
                                 nc_security_group.name, nc_security_group.pk)

                # creating non-existed security groups
                for backend_group in nonexistent_groups:
                    nc_security_group = service_project_link.security_groups.create(
                        backend_id=backend_group.id,
                        name=backend_group.name,
                        state=StateMixin.States.OK
                    )
                    self.pull_security_group_rules(nc_security_group)
                    logger.debug('Created new security group %s (PK: %s).',
                                 nc_security_group.name, nc_security_group.pk)

        except Exception as e:
            event_logger.service_project_link.warning(
                'Failed to pull security groups from backend.',
                event_type='service_project_link_sync_failed',
                event_context={
                    'service_project_link': service_project_link,
                    'error_message': six.text_type(e),
                }
            )
            six.reraise(OpenStackBackendError, e)

    def pull_security_group_rules(self, security_group):
        nova = self.nova_client
        backend_security_group = nova.security_groups.get(group_id=security_group.backend_id)
        backend_rules = [
            self._normalize_security_group_rule(r)
            for r in backend_security_group.rules
        ]

        # list of openstack rules, that do not exist in nc
        nonexistent_rules = []
        # list of openstack rules, that have wrong parameters in in nc
        unsynchronized_rules = []
        # list of nc rules, that have do not exist in openstack
        extra_rules = security_group.rules.exclude(backend_id__in=[r['id'] for r in backend_rules])

        with transaction.atomic():<commit_msg>Do not emit events on minor failures<commit_after>import datetime
import dateutil.parser
import logging
import re
import time
import uuid

from django.conf import settings as django_settings
from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned
from django.db import transaction
from django.utils import six, dateparse, timezone
from requests import ConnectionError

from keystoneclient.auth.identity import v2
from keystoneclient.service_catalog import ServiceCatalog
from keystoneclient import session as keystone_session

from ceilometerclient import client as ceilometer_client
from cinderclient.v1 import client as cinder_client
from glanceclient.v1 import client as glance_client
from keystoneclient.v2_0 import client as keystone_client
from neutronclient.v2_0 import client as neutron_client
from novaclient.v1_1 import client as nova_client

from cinderclient import exceptions as cinder_exceptions
from glanceclient import exc as glance_exceptions
from keystoneclient import exceptions as keystone_exceptions
from neutronclient.client import exceptions as neutron_exceptions
from novaclient import exceptions as nova_exceptions

from nodeconductor.core.models import StateMixin
from nodeconductor.core.tasks import send_task
from nodeconductor.structure import ServiceBackend, ServiceBackendError, log_backend_action
from nodeconductor.structure.log import event_logger
from nodeconductor.openstack import models


logger = logging.getLogger(__name__)


class OpenStackBackendError(ServiceBackendError):
    pass


class OpenStackSession(dict):
    """ Serializable session """

    def __init__(self, ks_session=None, verify_ssl=False, **credentials):
        self.keystone_session = ks_session

        if not self.keystone_session:
            auth_plugin = v2.Password(**credentials)
            self.keystone_session = keystone_session.Session(auth=auth_plugin, verify=verify_ssl)

        try:
            # This will eagerly sign in throwing AuthorizationFailure on bad credentials
            self.keystone_session.get_token()
        except (keystone_exceptions.AuthorizationFailure, keystone_exceptions.ConnectionRefused) as e:
            six.reraise(OpenStackBackendError, e)

        for opt in ('auth_ref', 'auth_url', 'tenant_id', 'tenant_name'):
            self[opt] = getattr(self.auth, opt)

    def __getattr__(self, name):
        return getattr(self.keystone_session, name)

    @classmethod
    def recover(cls, session, verify_ssl=False):
        if not isinstance(session, dict) or not session.get('auth_ref'):
            raise OpenStackBackendError('Invalid OpenStack session')

        args = {'auth_url': session['auth_url'], 'token': session['auth_ref']['token']['id']}
        if session['tenant_id']:
            args['tenant_id'] = session['tenant_id']
        elif session['tenant_name']:
            args['tenant_name'] = session['tenant_name']

        ks_session = keystone_session.Session(auth=v2.Token(**args), verify=verify_ssl)
        return cls(
            ks_session=ks_session,
            tenant_id=session['tenant_id'],
            tenant_name=session['tenant_name'])

    def validate(self):
        expiresat = dateutil.parser.parse(self.auth.auth_ref['token']['expires'])
        if expiresat > timezone.now() + datetime.timedelta(minutes=10):
            return True

        raise OpenStackBackendError('OpenStack session is expired')

    def __str__(self):
        return str({k: v if k != 'password' else '***' for k, v in self})


class OpenStackClient(object):
    """ Generic OpenStack client. """

    def __init__(self, session=None, verify_ssl=False, **credentials):
        self.verify_ssl = verify_ssl
        if session:
            if isinstance(session, dict):
                logger.info('Trying to recover OpenStack session.')
                self.session = OpenStackSession.recover(session, verify_ssl=verify_ssl)
                self.session.validate()
            else:
                self.session = session
        else:
            try:
                self.session = OpenStackSession(verify_ssl=verify_ssl, **credentials)
            except AttributeError as e:
                logger.error('Failed to create OpenStack session.')
                six.reraise(OpenStackBackendError, e)

    @property
    def keystone(self):
        return keystone_client.Client(session=self.session.keystone_session)

    @property
    def nova(self):
        try:
            return nova_client.Client(session=self.session.keystone_session)
        except (nova_exceptions.ClientException, keystone_exceptions.ClientException) as e:
            logger.exception('Failed to create nova client: %s', e)
            six.reraise(OpenStackBackendError, e)

    @property
    def neutron(self):
        try:
            return neutron_client.Client(session=self.session.keystone_session)
        except (neutron_exceptions.NeutronClientException, keystone_exceptions.ClientException) as e:
            logger.exception('Failed to create neutron client: %s', e)
            six.reraise(OpenStackBackendError, e)

    @property
    def cinder(self):
        try:
            return cinder_client.Client(session=self.session.keystone_session)
        except (cinder_exceptions.ClientException, keystone_exceptions.ClientException) as e:
            logger.exception('Failed to create cinder client: %s', e)
            six.reraise(OpenStackBackendError, e)

    @property
    def glance(self):
        catalog = ServiceCatalog.factory(self.session.auth.auth_ref)
        endpoint = catalog.url_for(service_type='image')

        kwargs = {
            'token': self.session.get_token(),
            'insecure': not self.verify_ssl,
            'timeout': 600,
            'ssl_compression': True,
        }

        return glance_client.Client(endpoint, **kwargs)

    @property
    def ceilometer(self):
        catalog = ServiceCatalog.factory(self.session.auth.auth_ref)
        endpoint = catalog.url_for(service_type='metering')

        kwargs = {
            'token': lambda: self.session.get_token(),
            'endpoint': endpoint,
            'insecure': not self.verify_ssl,
            'timeout': 600,
            'ssl_compression': True,
        }

        return ceilometer_client.Client('2', **kwargs)


class OpenStackBackend(ServiceBackend):

    DEFAULT_TENANT = 'admin'

    def __init__(self, settings, tenant_id=None):
        self.settings = settings
        self.tenant_id = tenant_id

    def get_client(self, name=None, admin=False):
        credentials = {
            'auth_url': self.settings.backend_url,
            'username': self.settings.username,
            'password': self.settings.password,
        }

        if not admin:
            if not self.tenant_id:
                raise OpenStackBackendError(
                    "Can't create tenant session, please provide tenant ID")

            credentials['tenant_id'] = self.tenant_id
        elif self.settings.options:
            credentials['tenant_name'] = self.settings.options.get('tenant_name', self.DEFAULT_TENANT)
        else:
            credentials['tenant_name'] = self.DEFAULT_TENANT

        # Cache session in the object
        attr_name = 'admin_session' if admin else 'session'
        client = getattr(self, attr_name, None)
        if hasattr(self, attr_name):
            client = getattr(self, attr_name)
        else:
            client = OpenStackClient(**credentials)
            setattr(self, attr_name, client)

        if name:
            return getattr(client, name)
        else:
            return client

    def __getattr__(self, name):
        clients = 'keystone', 'nova', 'neutron', 'cinder', 'glance', 'ceilometer'
        for client in clients:
            if name == '{}_client'.format(client):
                return self.get_client(client, admin=False)

            if name == '{}_admin_client'.format(client):
                return self.get_client(client, admin=True)

        raise AttributeError(
            "'%s' object has no attribute '%s'" % (self.__class__.__name__, name))

    def ping(self, raise_exception=False):
        # Session validation occurs on class creation so assume it's active
        # TODO: Consider validating session depending on tenant permissions
        return True

    def ping_resource(self, instance):
        try:
            self.nova_client.servers.get(instance.backend_id)
        except (ConnectionError, nova_exceptions.ClientException):
            return False
        else:
            return True

    def sync(self):
        # Migration status:
        # [x] pull_flavors()
        # [x] pull_images()
        # [ ] pull_service_statistics() (TODO: NC-640)

        try:
            self.pull_flavors()
            self.pull_images()
        except (nova_exceptions.ClientException, glance_exceptions.ClientException) as e:
            logger.exception('Failed to synchronize OpenStack service %s', self.settings.backend_url)
            six.reraise(OpenStackBackendError, e)
        else:
            logger.info('Successfully synchronized OpenStack service %s', self.settings.backend_url)

    def provision(self, instance, flavor=None, image=None, ssh_key=None, **kwargs):
        if ssh_key:
            instance.key_name = self.get_key_name(ssh_key)
            instance.key_fingerprint = ssh_key.fingerprint
            kwargs['public_key'] = ssh_key.public_key

        instance.flavor_name = flavor.name
        instance.cores = flavor.cores
        instance.ram = flavor.ram
        instance.flavor_disk = flavor.disk
        instance.disk = instance.system_volume_size + instance.data_volume_size
        if image:
            instance.min_disk = image.min_disk
            instance.min_ram = image.min_ram
        instance.save()

        kwargs['backend_flavor_id'] = flavor.backend_id
        if image:
            kwargs['backend_image_id'] = image.backend_id

        send_task('openstack', 'provision')(instance.uuid.hex, **kwargs)

    def destroy(self, instance, force=False):
        instance.schedule_deletion()
        instance.save()
        send_task('openstack', 'destroy')(instance.uuid.hex, force=force)

    def start(self, instance):
        instance.schedule_starting()
        instance.save()
        send_task('openstack', 'start')(instance.uuid.hex)

    def stop(self, instance):
        instance.schedule_stopping()
        instance.save()
        send_task('openstack', 'stop')(instance.uuid.hex)

    def restart(self, instance):
        instance.schedule_restarting()
        instance.save()
        send_task('openstack', 'restart')(instance.uuid.hex)

    def get_key_name(self, public_key):
        # We want names to be human readable in backend.
        # OpenStack only allows latin letters, digits, dashes, underscores and spaces
        # as key names, thus we mangle the original name.

        safe_name = self.sanitize_key_name(public_key.name)
        key_name = '{0}-{1}'.format(public_key.uuid.hex, safe_name)
        return key_name

    def sanitize_key_name(self, key_name):
        # Safe key name length must be less than 17 chars due to limit of full key name to 50 chars.
        return re.sub(r'[^-a-zA-Z0-9 _]+', '_', key_name)[:17]

    def add_ssh_key(self, ssh_key, service_project_link):
        if service_project_link.tenant is not None:
            key_name = self.get_key_name(ssh_key)
            self.get_or_create_ssh_key_for_tenant(
                service_project_link.tenant, key_name, ssh_key.fingerprint, ssh_key.public_key)

    def get_or_create_ssh_key_for_tenant(self, tenant, key_name, fingerprint, public_key):
        nova = self.nova_client

        try:
            return nova.keypairs.find(fingerprint=fingerprint)
        except nova_exceptions.NotFound:
            # Fine, it's a new key, let's add it
            try:
                return nova.keypairs.create(name=key_name, public_key=public_key)
            except (nova_exceptions.ClientException, keystone_exceptions.ClientException) as e:
                six.reraise(OpenStackBackendError, e)
            else:
                logger.info('Propagating ssh public key %s to backend', key_name)
        except (nova_exceptions.ClientException, keystone_exceptions.ClientException) as e:
            six.reraise(OpenStackBackendError, e)
        else:
            # Found a key with the same fingerprint, skip adding
            logger.info('Skipped propagating ssh public key %s to backend', key_name)

    def remove_ssh_key(self, ssh_key, service_project_link):
        if service_project_link.tenant is not None:
            self.remove_ssh_key_from_tenant(service_project_link.tenant, ssh_key)

    @log_backend_action()
    def remove_ssh_key_from_tenant(self, tenant, key_name, fingerprint):
        nova = self.nova_client

        # There could be leftovers of key duplicates: remove them all
        keys = nova.keypairs.findall(fingerprint=fingerprint)
        for key in keys:
            # Remove only keys created with NC
            if key.name == key_name:
                nova.keypairs.delete(key)

        logger.info('Deleted ssh public key %s from backend', key_name)

    def _get_instance_state(self, instance):
        # See http://developer.openstack.org/api-ref-compute-v2.html
        nova_to_nodeconductor = {
            'ACTIVE': models.Instance.States.ONLINE,
            'BUILDING': models.Instance.States.PROVISIONING,
            # 'DELETED': models.Instance.States.DELETING,
            # 'SOFT_DELETED': models.Instance.States.DELETING,
            'ERROR': models.Instance.States.ERRED,
            'UNKNOWN': models.Instance.States.ERRED,

            'HARD_REBOOT': models.Instance.States.STOPPING,  # Or starting?
            'REBOOT': models.Instance.States.STOPPING,  # Or starting?
            'REBUILD': models.Instance.States.STARTING,  # Or stopping?

            'PASSWORD': models.Instance.States.ONLINE,
            'PAUSED': models.Instance.States.OFFLINE,

            'RESCUED': models.Instance.States.ONLINE,
            'RESIZED': models.Instance.States.OFFLINE,
            'REVERT_RESIZE': models.Instance.States.STOPPING,
            'SHUTOFF': models.Instance.States.OFFLINE,
            'STOPPED': models.Instance.States.OFFLINE,
            'SUSPENDED': models.Instance.States.OFFLINE,
            'VERIFY_RESIZE': models.Instance.States.OFFLINE,
        }
        return nova_to_nodeconductor.get(instance.status, models.Instance.States.ERRED)

    def _get_current_properties(self, model):
        return {p.backend_id: p for p in model.objects.filter(settings=self.settings)}

    def _are_rules_equal(self, backend_rule, nc_rule):
        if backend_rule['from_port'] != nc_rule.from_port:
            return False
        if backend_rule['to_port'] != nc_rule.to_port:
            return False
        if backend_rule['ip_protocol'] != nc_rule.protocol:
            return False
        if backend_rule['ip_range'].get('cidr', '') != nc_rule.cidr:
            return False
        return True

    def _are_security_groups_equal(self, backend_security_group, nc_security_group):
        if backend_security_group.name != nc_security_group.name:
            return False
        if len(backend_security_group.rules) != nc_security_group.rules.count():
            return False
        for backend_rule, nc_rule in zip(backend_security_group.rules, nc_security_group.rules.all()):
            if not self._are_rules_equal(backend_rule, nc_rule):
                return False
        return True

    def _normalize_security_group_rule(self, rule):
        if rule['ip_protocol'] is None:
            rule['ip_protocol'] = ''

        if 'cidr' not in rule['ip_range']:
            rule['ip_range']['cidr'] = '0.0.0.0/0'

        return rule

    def _wait_for_instance_status(self, server_id, nova, complete_status,
                                  error_status=None, retries=300, poll_interval=3):
        return self._wait_for_object_status(
            server_id, nova.servers.get, complete_status, error_status, retries, poll_interval)

    def _wait_for_volume_status(self, volume_id, cinder, complete_status,
                                error_status=None, retries=300, poll_interval=3):
        return self._wait_for_object_status(
            volume_id, cinder.volumes.get, complete_status, error_status, retries, poll_interval)

    def _wait_for_snapshot_status(self, snapshot_id, cinder, complete_status, error_status, retries=90, poll_interval=3):
        return self._wait_for_object_status(
            snapshot_id, cinder.volume_snapshots.get, complete_status, error_status, retries, poll_interval)

    def _wait_for_backup_status(self, backup, cinder, complete_status, error_status, retries=90, poll_interval=3):
        return self._wait_for_object_status(
            backup, cinder.backups.get, complete_status, error_status, retries, poll_interval)

    def _wait_for_object_status(self, obj_id, client_get_method, complete_status, error_status=None,
                                retries=30, poll_interval=3):
        complete_state_predicate = lambda o: o.status == complete_status
        if error_status is not None:
            error_state_predicate = lambda o: o.status == error_status
        else:
            error_state_predicate = lambda _: False

        for _ in range(retries):
            obj = client_get_method(obj_id)

            if complete_state_predicate(obj):
                return True

            if error_state_predicate(obj):
                return False

            time.sleep(poll_interval)
        else:
            return False

    def _wait_for_volume_deletion(self, volume_id, cinder, retries=90, poll_interval=3):
        try:
            for _ in range(retries):
                cinder.volumes.get(volume_id)
                time.sleep(poll_interval)

            return False
        except cinder_exceptions.NotFound:
            return True

    def _wait_for_snapshot_deletion(self, snapshot_id, cinder, retries=90, poll_interval=3):
        try:
            for _ in range(retries):
                cinder.volume_snapshots.get(snapshot_id)
                time.sleep(poll_interval)

            return False
        except (cinder_exceptions.NotFound, keystone_exceptions.NotFound):
            return True

    def _wait_for_instance_deletion(self, backend_instance_id, retries=90, poll_interval=3):
        nova = self.nova_client
        try:
            for _ in range(retries):
                nova.servers.get(backend_instance_id)
                time.sleep(poll_interval)

            return False
        except nova_exceptions.NotFound:
            return True

    def pull_flavors(self):
        nova = self.nova_admin_client
        with transaction.atomic():
            cur_flavors = self._get_current_properties(models.Flavor)
            for backend_flavor in nova.flavors.findall(is_public=True):
                cur_flavors.pop(backend_flavor.id, None)
                models.Flavor.objects.update_or_create(
                    settings=self.settings,
                    backend_id=backend_flavor.id,
                    defaults={
                        'name': backend_flavor.name,
                        'cores': backend_flavor.vcpus,
                        'ram': backend_flavor.ram,
                        'disk': self.gb2mb(backend_flavor.disk),
                    })

            models.Flavor.objects.filter(backend_id__in=cur_flavors.keys()).delete()

    def pull_images(self):
        glance = self.glance_admin_client
        with transaction.atomic():
            cur_images = self._get_current_properties(models.Image)
            for backend_image in glance.images.list():
                if backend_image.is_public and not backend_image.deleted:
                    cur_images.pop(backend_image.id, None)
                    models.Image.objects.update_or_create(
                        settings=self.settings,
                        backend_id=backend_image.id,
                        defaults={
                            'name': backend_image.name,
                            'min_ram': backend_image.min_ram,
                            'min_disk': self.gb2mb(backend_image.min_disk),
                        })

            models.Image.objects.filter(backend_id__in=cur_images.keys()).delete()

    @log_backend_action('push quotas for tenant')
    def push_tenant_quotas(self, tenant, quotas):
        if 'instances' in quotas:
            # convert instances quota to volumes and snapshots.
            quotas_ratios = django_settings.NODECONDUCTOR.get('OPENSTACK_QUOTAS_INSTANCE_RATIOS', {})
            volume_ratio = quotas_ratios.get('volumes', 4)
            snapshots_ratio = quotas_ratios.get('snapshots', 20)

            quotas['volumes'] = volume_ratio * quotas['instances']
            quotas['snapshots'] = snapshots_ratio * quotas['instances']

        cinder_quotas = {
            'gigabytes': self.mb2gb(quotas.get('storage')) if 'storage' in quotas else None,
            'volumes': quotas.get('volumes'),
            'snapshots': quotas.get('snapshots'),
        }
        cinder_quotas = {k: v for k, v in cinder_quotas.items() if v is not None}

        nova_quotas = {
            'instances': quotas.get('instances'),
            'cores': quotas.get('vcpu'),
        }
        nova_quotas = {k: v for k, v in nova_quotas.items() if v is not None}

        neutron_quotas = {
            'security_group': quotas.get('security_group_count'),
            'security_group_rule': quotas.get('security_group_rule_count'),
        }
        neutron_quotas = {k: v for k, v in neutron_quotas.items() if v is not None}

        try:
            if cinder_quotas:
                self.cinder_client.quotas.update(tenant.backend_id, **cinder_quotas)
            if nova_quotas:
                self.nova_client.quotas.update(tenant.backend_id, **nova_quotas)
            if neutron_quotas:
                self.neutron_client.update_quota(tenant.backend_id, {'quota': neutron_quotas})
        except Exception as e:
            six.reraise(OpenStackBackendError, e)

    @log_backend_action('pull quotas for tenant')
    def pull_tenant_quotas(self, tenant):
        # XXX: backend quotas should be moved to tenant from SPL in future.
        nova = self.nova_client
        neutron = self.neutron_client
        cinder = self.cinder_client
        service_project_link = tenant.service_project_link

        try:
            nova_quotas = nova.quotas.get(tenant_id=tenant.backend_id)
            cinder_quotas = cinder.quotas.get(tenant_id=tenant.backend_id)
            neutron_quotas = neutron.show_quota(tenant_id=tenant.backend_id)['quota']
        except (nova_exceptions.ClientException,
                cinder_exceptions.ClientException,
                neutron_exceptions.NeutronClientException) as e:
            six.reraise(OpenStackBackendError, e)

        service_project_link.set_quota_limit('ram', nova_quotas.ram)
        service_project_link.set_quota_limit('vcpu', nova_quotas.cores)
        service_project_link.set_quota_limit('storage', self.gb2mb(cinder_quotas.gigabytes))
        service_project_link.set_quota_limit('instances', nova_quotas.instances)
        service_project_link.set_quota_limit('security_group_count', neutron_quotas['security_group'])
        service_project_link.set_quota_limit('security_group_rule_count', neutron_quotas['security_group_rule'])
        service_project_link.set_quota_limit('floating_ip_count', neutron_quotas['floatingip'])

        try:
            volumes = cinder.volumes.list()
            snapshots = cinder.volume_snapshots.list()
            instances = nova.servers.list()
            security_groups = nova.security_groups.list()
            floating_ips = neutron.list_floatingips(tenant_id=tenant.backend_id)['floatingips']

            flavors = {flavor.id: flavor for flavor in nova.flavors.list()}

            ram, vcpu = 0, 0
            for flavor_id in (instance.flavor['id'] for instance in instances):
                try:
                    flavor = flavors.get(flavor_id, nova.flavors.get(flavor_id))
                except nova_exceptions.NotFound:
                    logger.warning('Cannot find flavor with id %s', flavor_id)
                    continue

                ram += getattr(flavor, 'ram', 0)
                vcpu += getattr(flavor, 'vcpus', 0)

        except (nova_exceptions.ClientException,
                cinder_exceptions.ClientException,
                neutron_exceptions.NeutronClientException) as e:
            six.reraise(OpenStackBackendError, e)

        service_project_link.set_quota_usage('ram', ram)
        service_project_link.set_quota_usage('vcpu', vcpu)
        service_project_link.set_quota_usage('storage', sum(self.gb2mb(v.size) for v in volumes + snapshots))
        service_project_link.set_quota_usage('instances', len(instances), fail_silently=True)
        service_project_link.set_quota_usage('security_group_count', len(security_groups))
        service_project_link.set_quota_usage('security_group_rule_count', len(sum([sg.rules for sg in security_groups], [])))
        service_project_link.set_quota_usage('floating_ip_count', len(floating_ips))

    @log_backend_action('pull floating IPs for tenant')
    def pull_tenant_floating_ips(self, tenant):
        service_project_link = tenant.service_project_link
        neutron = self.neutron_client

        try:
            nc_floating_ips = {ip.backend_id: ip for ip in service_project_link.floating_ips.all()}
            try:
                backend_floating_ips = {
                    ip['id']: ip
                    for ip in neutron.list_floatingips(tenant_id=self.tenant_id)['floatingips']
                    if ip.get('floating_ip_address') and ip.get('status')
                }
            except neutron_exceptions.NeutronClientException as e:
                six.reraise(OpenStackBackendError, e)

            backend_ids = set(backend_floating_ips.keys())
            nc_ids = set(nc_floating_ips.keys())

            with transaction.atomic():
                for ip_id in nc_ids - backend_ids:
                    ip = nc_floating_ips[ip_id]
                    ip.delete()
                    logger.info('Deleted stale floating IP port %s in database', ip.uuid)

                for ip_id in backend_ids - nc_ids:
                    ip = backend_floating_ips[ip_id]
                    created_ip = service_project_link.floating_ips.create(
                        status=ip['status'],
                        backend_id=ip['id'],
                        address=ip['floating_ip_address'],
                        backend_network_id=ip['floating_network_id']
                    )
                    logger.info('Created new floating IP port %s in database', created_ip.uuid)

                for ip_id in nc_ids & backend_ids:
                    nc_ip = nc_floating_ips[ip_id]
                    backend_ip = backend_floating_ips[ip_id]
                    if nc_ip.status != backend_ip['status'] or nc_ip.address != backend_ip['floating_ip_address']\
                            or nc_ip.backend_network_id != backend_ip['floating_network_id']:
                        # If key is BOOKED by NodeConductor it can be still DOWN in OpenStack
                        if not (nc_ip.status == 'BOOKED' and backend_ip['status'] == 'DOWN'):
                            nc_ip.status = backend_ip['status']
                        nc_ip.address = backend_ip['floating_ip_address']
                        nc_ip.backend_network_id = backend_ip['floating_network_id']
                        nc_ip.save()
                        logger.info('Updated existing floating IP port %s in database', nc_ip.uuid)

        except Exception as e:
            six.reraise(OpenStackBackendError, e)

    @log_backend_action('pull security groups for tenant')
    def pull_tenant_security_groups(self, tenant):
        nova = self.nova_client
        service_project_link = tenant.service_project_link

        try:
            try:
                backend_security_groups = nova.security_groups.list()
            except nova_exceptions.ClientException as e:
                six.reraise(OpenStackBackendError, e)

            # list of openstack security groups that do not exist in nc
            nonexistent_groups = []
            # list of openstack security groups that have wrong parameters in in nc
            unsynchronized_groups = []
            # list of nc security groups that do not exist in openstack

            extra_groups = service_project_link.security_groups.exclude(
                backend_id__in=[g.id for g in backend_security_groups],
            )

            with transaction.atomic():
                for backend_group in backend_security_groups:
                    try:
                        nc_group = service_project_link.security_groups.get(backend_id=backend_group.id)
                        if not self._are_security_groups_equal(backend_group, nc_group):
                            unsynchronized_groups.append(backend_group)
                    except models.SecurityGroup.DoesNotExist:
                        nonexistent_groups.append(backend_group)

                # deleting extra security groups
                extra_groups.delete()
                if extra_groups:
                    logger.debug('Deleted stale security group: %s.',
                                 ' ,'.join('%s (PK: %s)' % (sg.name, sg.pk) for sg in extra_groups))

                # synchronizing unsynchronized security groups
                for backend_group in unsynchronized_groups:
                    nc_security_group = service_project_link.security_groups.get(backend_id=backend_group.id)
                    if backend_group.name != nc_security_group.name:
                        nc_security_group.name = backend_group.name
                        nc_security_group.state = StateMixin.States.OK
                        nc_security_group.save()
                    self.pull_security_group_rules(nc_security_group)
                    logger.debug('Updated existing security group %s (PK: %s).',
                                 nc_security_group.name, nc_security_group.pk)

                # creating non-existed security groups
                for backend_group in nonexistent_groups:
                    nc_security_group = service_project_link.security_groups.create(
                        backend_id=backend_group.id,
                        name=backend_group.name,
                        state=StateMixin.States.OK
                    )
                    self.pull_security_group_rules(nc_security_group)
                    logger.debug('Created new security group %s (PK: %s).',
                                 nc_security_group.name, nc_security_group.pk)

        except Exception as e:
            six.reraise(OpenStackBackendError, e)

    def pull_security_group_rules(self, security_group):
        nova = self.nova_client
        backend_security_group = nova.security_groups.get(group_id=security_group.backend_id)
        backend_rules = [
            self._normalize_security_group_rule(r)
            for r in backend_security_group.rules
        ]

        # list of openstack rules, that do not exist in nc
        nonexistent_rules = []
        # list of openstack rules, that have wrong parameters in in nc
        unsynchronized_rules = []
        # list of nc rules, that have do not exist in openstack
        extra_rules = security_group.rules.exclude(backend_id__in=[r['id'] for r in backend_rules])

        with transaction.atomic():
{'fraction_non_alphanumeric': 0.08610061334489809, 'fraction_numerical': 0.0023852301592218574, 'mean_word_length': 2.3568160549027763, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2472136', 'n_tokens_mistral': 18650, 'n_tokens_neox': 17921, 'n_words': 3746}