<commit_before>import logging
import re
from itertools import groupby
from typing import Dict

from cinderclient import exceptions as cinder_exceptions
from django.db import IntegrityError, transaction
from django.utils import timezone
from django.utils.crypto import get_random_string
from django.utils.decorators import method_decorator
from keystoneclient import exceptions as keystone_exceptions
from neutronclient.client import exceptions as neutron_exceptions
from novaclient import exceptions as nova_exceptions

from waldur_core.core.utils import create_batch_fetcher
from waldur_core.structure import SupportedServices, log_backend_action
from waldur_core.structure.utils import (
    handle_resource_not_found,
    handle_resource_update_success,
    update_pulled_fields,
)
from waldur_openstack.openstack_base.backend import (
    BaseOpenStackBackend,
    OpenStackBackendError,
)

from . import models

logger = logging.getLogger(__name__)


class OpenStackBackend(BaseOpenStackBackend):
    DEFAULTS = {
        'tenant_name': 'admin',
    }

    def check_admin_tenant(self):
        try:
            self.keystone_admin_client
        except keystone_exceptions.AuthorizationFailure:
            return False
        except keystone_exceptions.ClientException as e:
            raise OpenStackBackendError(e)
        else:
            return True

    def pull_service_properties(self):
        self.pull_flavors()
        self.pull_images()
        self.pull_volume_types()
        self.pull_service_settings_quotas()

    def pull_resources(self):
        self.pull_tenants()

    def pull_subresources(self):
        self.pull_security_groups()
        self.pull_floating_ips()
        self.pull_networks()
        self.pull_subnets()
        self.pull_routers()

    def pull_tenants(self):
        keystone = self.keystone_admin_client

        try:
            backend_tenants = keystone.projects.list(domain=self._get_domain())
        except keystone_exceptions.ClientException as e:
            raise OpenStackBackendError(e)

        backend_tenants_mapping = {tenant.id: tenant for tenant in backend_tenants}

        tenants = models.Tenant.objects.filter(
            state__in=[models.Tenant.States.OK, models.Tenant.States.ERRED],
            service_project_link__service__settings=self.settings,
        )
        for tenant in tenants:
            backend_tenant = backend_tenants_mapping.get(tenant.backend_id)
            if backend_tenant is None:
                handle_resource_not_found(tenant)
                continue

            imported_backend_tenant = models.Tenant(
                name=backend_tenant.name,
                description=backend_tenant.description,
                backend_id=backend_tenant.id,
                state=models.Tenant.States.OK,
            )
            update_pulled_fields(
                tenant, imported_backend_tenant, models.Tenant.get_backend_fields()
            )
            handle_resource_update_success(tenant)

    def _get_domain(self):
        """ Get current domain """
        keystone = self.keystone_admin_client
        return keystone.domains.find(name=self.settings.domain or 'Default')

    def remove_ssh_key_from_tenant(self, tenant, key_name, fingerprint):
        nova = self.nova_client

        # There could be leftovers of key duplicates: remove them all
        keys = nova.keypairs.findall(fingerprint=fingerprint)
        for key in keys:
            # Remove only keys created with Waldur
            if key.name == key_name:
                nova.keypairs.delete(key)

        logger.info('Deleted ssh public key %s from backend', key_name)

    def _are_rules_equal(self, backend_rule, nc_rule):
        if backend_rule['port_range_min'] != nc_rule.from_port:
            return False
        if backend_rule['port_range_max'] != nc_rule.to_port:
            return False
        if backend_rule['protocol'] != nc_rule.protocol:
            return False
        if backend_rule['remote_ip_prefix'] != nc_rule.cidr:
            return False
        return True

    def pull_flavors(self):
        nova = self.nova_admin_client
        try:
            flavors = nova.flavors.findall(is_public=True)
        except nova_exceptions.ClientException as e:
            raise OpenStackBackendError(e)

        flavor_exclude_regex = self.settings.options.get('flavor_exclude_regex', '')
        name_pattern = (
            re.compile(flavor_exclude_regex) if flavor_exclude_regex else None
        )
        with transaction.atomic():
            cur_flavors = self._get_current_properties(models.Flavor)
            for backend_flavor in flavors:
                if (
                    name_pattern is not None
                    and name_pattern.match(backend_flavor.name) is not None
                ):
                    logger.debug(
                        'Skipping pull of %s flavor as it matches %s regex pattern.',
                        backend_flavor.name,
                        flavor_exclude_regex,
                    )
                    continue

                cur_flavors.pop(backend_flavor.id, None)
                models.Flavor.objects.update_or_create(
                    settings=self.settings,
                    backend_id=backend_flavor.id,
                    defaults={
                        'name': backend_flavor.name,
                        'cores': backend_flavor.vcpus,
                        'ram': backend_flavor.ram,
                        'disk': self.gb2mb(backend_flavor.disk),
                    },
                )

            models.Flavor.objects.filter(backend_id__in=cur_flavors.keys()).delete()

    def pull_images(self):
        self._pull_images(
            models.Image, lambda image: image['visibility'] == 'public', admin=True
        )

    def pull_volume_types(self):
        try:
            volume_types = self.cinder_admin_client.volume_types.list(is_public=True)
        except cinder_exceptions.ClientException as e:
            raise OpenStackBackendError(e)

        with transaction.atomic():
            cur_volume_types = self._get_current_properties(models.VolumeType)
            for backend_type in volume_types:
                cur_volume_types.pop(backend_type.id, None)
                models.VolumeType.objects.update_or_create(
                    settings=self.settings,
                    backend_id=backend_type.id,
                    defaults={
                        'name': backend_type.name,
                        'description': backend_type.description or '',
                    },
                )

            models.VolumeType.objects.filter(
                backend_id__in=cur_volume_types.keys(), settings=self.settings
            ).delete()

    @log_backend_action('push quotas for tenant')
    def push_tenant_quotas(self, tenant, quotas: Dict[str, int]):
        cinder_quotas = {
            'gigabytes': self.mb2gb(quotas.get('storage'))
            if 'storage' in quotas
            else None,
            'volumes': quotas.get('volumes'),
            'snapshots': quotas.get('snapshots'),
        }

        cinder_quotas = {k: v for k, v in cinder_quotas.items() if v is not None}

        # Filter volume-type quotas.
        volume_type_quotas = dict(
            (key, value)
            for (key, value) in quotas.items()
            if key.startswith('gigabytes_') and value is not None
        )

        if volume_type_quotas:
            cinder_quotas.update(volume_type_quotas)

        nova_quotas = {
            'instances': quotas.get('instances'),
            'cores': quotas.get('vcpu'),
            'ram': quotas.get('ram'),
        }
        nova_quotas = {k: v for k, v in nova_quotas.items() if v is not None}

        neutron_quotas = {
            'security_group': quotas.get('security_group_count'),
            'security_group_rule': quotas.get('security_group_rule_count'),
        }
        neutron_quotas = {k: v for k, v in neutron_quotas.items() if v is not None}

        try:
            if cinder_quotas:
                self.cinder_client.quotas.update(tenant.backend_id, **cinder_quotas)
            if nova_quotas:
                self.nova_client.quotas.update(tenant.backend_id, **nova_quotas)
            if neutron_quotas:
                self.neutron_client.update_quota(
                    tenant.backend_id, {'quota': neutron_quotas}
                )
        except Exception as e:
            raise OpenStackBackendError(e)

    @log_backend_action('pull quotas for tenant')
    def pull_tenant_quotas(self, tenant):
        self._pull_tenant_quotas(tenant.backend_id, tenant)

    def pull_quotas(self):
        for tenant in models.Tenant.objects.filter(
            state=models.Tenant.States.OK,
            service_project_link__service__settings=self.settings,
        ):
            self.pull_tenant_quotas(tenant)

    def pull_floating_ips(self, tenants=None):
        if tenants is None:
            tenants = models.Tenant.objects.filter(
                state=models.Tenant.States.OK,
                service_project_link__service__settings=self.settings,
            ).prefetch_related('floating_ips')
        tenant_mappings = {tenant.backend_id: tenant for tenant in tenants}
        if not tenant_mappings:
            return

        backend_floating_ips = self.list_floatingips(list(tenant_mappings.keys()))

        tenant_floating_ips = dict()
        for tenant_id, floating_ips in groupby(
            backend_floating_ips, lambda x: x['tenant_id']
        ):
            tenant_floating_ips[tenant_mappings[tenant_id]] = floating_ips

        with transaction.atomic():
            for tenant, floating_ips in tenant_floating_ips.items():
                self._update_tenant_floating_ips(tenant, floating_ips)

            self._remove_stale_floating_ips(tenants, backend_floating_ips)

    @method_decorator(create_batch_fetcher)
    def list_floatingips(self, tenants):
        neutron = self.neutron_admin_client

        try:
            return neutron.list_floatingips(tenant_id=tenants)['floatingips']
        except neutron_exceptions.NeutronClientException as e:
            raise OpenStackBackendError(e)

    @log_backend_action('pull floating IPs for tenant')
    def pull_tenant_floating_ips(self, tenant):
        neutron = self.neutron_client

        try:
            backend_floating_ips = neutron.list_floatingips(tenant_id=self.tenant_id)[
                'floatingips'
            ]
        except neutron_exceptions.NeutronClientException as e:
            raise OpenStackBackendError(e)

        with transaction.atomic():
            self._update_tenant_floating_ips(tenant, backend_floating_ips)
            self._remove_stale_floating_ips([tenant], backend_floating_ips)

    def _remove_stale_floating_ips(self, tenants, backend_floating_ips):
        remote_ids = {ip['id'] for ip in backend_floating_ips}
        stale_ips = models.FloatingIP.objects.filter(
            tenant__in=tenants,
            state__in=[models.FloatingIP.States.OK, models.FloatingIP.States.ERRED],
        ).exclude(backend_id__in=remote_ids)
        stale_ips.delete()

    def _update_tenant_floating_ips(self, tenant, backend_floating_ips):
        floating_ips = {
            ip.backend_id: ip
            for ip in tenant.floating_ips.filter(
                state__in=[models.FloatingIP.States.OK, models.FloatingIP.States.ERRED]
            )
        }

        for backend_ip in backend_floating_ips:
            imported_floating_ip = self._backend_floating_ip_to_floating_ip(
                backend_ip,
                tenant=tenant,
                service_project_link=tenant.service_project_link,
            )
            floating_ip = floating_ips.pop(imported_floating_ip.backend_id, None)
            if floating_ip is None:
                imported_floating_ip.save()
                continue

            # Don't update user defined name.
            if floating_ip.address != floating_ip.name:
                imported_floating_ip.name = floating_ip.name
            update_pulled_fields(
                floating_ip,
                imported_floating_ip,
                models.FloatingIP.get_backend_fields(),
            )
            handle_resource_update_success(floating_ip)

    def _backend_floating_ip_to_floating_ip(self, backend_floating_ip, **kwargs):
        floating_ip = models.FloatingIP(
            name=backend_floating_ip['floating_ip_address'],
            description=backend_floating_ip['description'],
            address=backend_floating_ip['floating_ip_address'],
            backend_network_id=backend_floating_ip['floating_network_id'],
            runtime_state=backend_floating_ip['status'],
            backend_id=backend_floating_ip['id'],
            state=models.FloatingIP.States.OK,
        )
        for field, value in kwargs.items():
            setattr(floating_ip, field, value)

        return floating_ip

    def pull_security_groups(self, tenants=None):

        if tenants is None:
            tenants = models.Tenant.objects.filter(
                state=models.Tenant.States.OK,
                service_project_link__service__settings=self.settings,
            ).prefetch_related('security_groups')
        tenant_mappings = {tenant.backend_id: tenant for tenant in tenants}
        if not tenant_mappings:
            return

        backend_security_groups = self.list_security_groups(
            list(tenant_mappings.keys())
        )

        tenant_security_groups = dict()
        for tenant_id, security_groups in groupby(
            backend_security_groups, lambda x: x['tenant_id']
        ):
            tenant_security_groups[tenant_mappings[tenant_id]] = security_groups

        with transaction.atomic():
            for tenant, security_groups in tenant_security_groups.items():
                self._update_tenant_security_groups(tenant, security_groups)
            self._remove_stale_security_groups(tenants, backend_security_groups)

    @method_decorator(create_batch_fetcher)
    def list_security_groups(self, tenants):
        neutron = self.neutron_admin_client

        try:
            return neutron.list_security_groups(tenant_id=tenants)['security_groups']
        except neutron_exceptions.NeutronClientException as e:
            raise OpenStackBackendError(e)

    @log_backend_action('pull security groups for tenant')
    def pull_tenant_security_groups(self, tenant):
        neutron = self.neutron_client
        try:
            backend_security_groups = neutron.list_security_groups(
                tenant_id=self.tenant_id
            )['security_groups']
        except neutron_exceptions.NeutronClientException as e:
            raise OpenStackBackendError(e)

        with transaction.atomic():
            self._update_tenant_security_groups(tenant, backend_security_groups)
            self._remove_stale_security_groups([tenant], backend_security_groups)

    def _remove_stale_security_groups(self, tenants, backend_security_groups):
        remote_ids = {ip['id'] for ip in backend_security_groups}
        stale_ips = models.SecurityGroup.objects.filter(
            tenant__in=tenants,
            state__in=[
                models.SecurityGroup.States.OK,
                models.SecurityGroup.States.ERRED,
            ],
        ).exclude(backend_id__in=remote_ids)
        stale_ips.delete()

    def _update_tenant_security_groups(self, tenant, backend_security_groups):
        security_group_uuids = []
        for backend_security_group in backend_security_groups:
            imported_security_group = self._backend_security_group_to_security_group(
                backend_security_group,
                tenant=tenant,
                service_project_link=tenant.service_project_link,
            )

            try:
                security_group = tenant.security_groups.get(
                    backend_id=imported_security_group.backend_id
                )
            except models.SecurityGroup.DoesNotExist:
                imported_security_group.save()
                security_group = imported_security_group
            else:
                update_pulled_fields(
                    security_group,
                    imported_security_group,
                    models.SecurityGroup.get_backend_fields(),
                )
                handle_resource_update_success(security_group)

            security_group_uuids.append(security_group.uuid)
            self._extract_security_group_rules(security_group, backend_security_group)

    def _backend_security_group_to_security_group(
        self, backend_security_group, **kwargs
    ):
        security_group = models.SecurityGroup(
            name=backend_security_group['name'],
            description=backend_security_group['description'],
            backend_id=backend_security_group['id'],
            state=models.SecurityGroup.States.OK,
        )

        for field, value in kwargs.items():
            setattr(security_group, field, value)

        return security_group

    def pull_routers(self):
        for tenant in models.Tenant.objects.filter(
            state=models.Tenant.States.OK,
            service_project_link__service__settings=self.settings,
        ):
            self.pull_tenant_routers(tenant)

    def pull_tenant_routers(self, tenant):
        neutron = self.neutron_admin_client

        try:
            backend_routers = neutron.list_routers(tenant_id=tenant.backend_id)[
                'routers'
            ]
        except neutron_exceptions.NeutronClientException as e:
            raise OpenStackBackendError(e)

        for backend_router in backend_routers:
            defaults = {
                'name': backend_router['name'],
                'description': backend_router['description'],
                'routes': backend_router['routes'],
                'service_project_link': tenant.service_project_link,
                'state': models.Router.States.OK,
            }
            backend_id = backend_router['id']
            try:
                models.Router.objects.update_or_create(
                    tenant=tenant, backend_id=backend_id, defaults=defaults
                )
            except IntegrityError:
                logger.warning(
                    'Could not create router with backend ID %s '
                    'and tenant %s due to concurrent update.',
                    backend_id,
                    tenant,
                )

        remote_ids = {ip['id'] for ip in backend_routers}
        stale_routers = models.Router.objects.filter(tenant=tenant).exclude(
            backend_id__in=remote_ids
        )
        stale_routers.delete()

    def pull_networks(self):
        tenants = (
            models.Tenant.objects.exclude(backend_id='')
            .filter(
                state__in=[models.Tenant.States.OK, models.Tenant.States.UPDATING],
                service_project_link__service__settings=self.settings,
            )
            .prefetch_related('networks')
        )

        self._pull_networks(tenants)

    def _pull_tenant_networks(self, tenant):
        return self._pull_networks([tenant])

    def _pull_networks(self, tenants):
        tenant_mappings = {tenant.backend_id: tenant for tenant in tenants}
        backend_networks = self.list_networks(list(tenant_mappings.keys()))

        networks = []
        with transaction.atomic():
            for backend_network in backend_networks:
                tenant = tenant_mappings.get(backend_network['tenant_id'])
                if not tenant:
                    logger.debug(
                        'Skipping network %s synchronization because its tenant %s is not available.',
                        backend_network['id'],
                        backend_network['tenant_id'],
                    )
                    continue

                imported_network = self._backend_network_to_network(
                    backend_network,
                    tenant=tenant,
                    service_project_link=tenant.service_project_link,
                )

                try:
                    network = tenant.networks.get(
                        backend_id=imported_network.backend_id
                    )
                except models.Network.DoesNotExist:
                    imported_network.save()
                    network = imported_network
                else:
                    update_pulled_fields(
                        network, imported_network, models.Network.get_backend_fields()
                    )
                    handle_resource_update_success(network)

                networks.append(network)

            networks_uuid = [network_item.uuid for network_item in networks]
            stale_networks = models.Network.objects.filter(
                state__in=[models.Network.States.OK, models.Network.States.ERRED],
                tenant__in=tenants,
            ).exclude(uuid__in=networks_uuid)
            stale_networks.delete()

        return networks

    @method_decorator(create_batch_fetcher)
    def list_networks(self, tenants):
        neutron = self.neutron_admin_client
        try:
            return neutron.list_networks(tenant_id=tenants)['networks']
        except neutron_exceptions.NeutronClientException as e:
            raise OpenStackBackendError(e)

    def _backend_network_to_network(self, backend_network, **kwargs):
        network = models.Network(
            name=backend_network['name'],
            description=backend_network['description'],
            is_external=backend_network['router:external'],
            runtime_state=backend_network['status'],
            mtu=backend_network.get('mtu'),
            backend_id=backend_network['id'],
            state=models.Network.States.OK,
        )
        if backend_network.get('provider:network_type'):
            network.type = backend_network['provider:network_type']
        if backend_network.get('provider:segmentation_id'):
            network.segmentation_id = backend_network['provider:segmentation_id']

        for field, value in kwargs.items():
            setattr(network, field, value)

        return network

    def pull_subnets(self, tenant=None, network=None):
        neutron = self.neutron_admin_client

        if tenant:
            networks = tenant.networks.all()
        elif network:
            networks = [network]
        else:
            networks = models.Network.objects.filter(
                state=models.Network.States.OK,
                service_project_link__service__settings=self.settings,
            )
        network_mappings = {network.backend_id: network for network in networks}
        if not network_mappings:
            return

        try:
            if tenant:
                backend_subnets = neutron.list_subnets(tenant_id=tenant.backend_id)[
                    'subnets'
                ]
            elif network:
                backend_subnets = neutron.list_subnets(network_id=network.backend_id)[
                    'subnets'
                ]
            else:
                # We can't filter subnets by network IDs because it exceeds maximum request length
                backend_subnets = neutron.list_subnets()['subnets']
        except neutron_exceptions.NeutronClientException as e:
            raise OpenStackBackendError(e)

        subnet_uuids = []
        with transaction.atomic():
            for backend_subnet in backend_subnets:
                network = network_mappings.get(backend_subnet['network_id'])

                if not network:
                    logger.debug(
                        'OpenStack network is not imported yet. Network ID: %s',
                        backend_subnet['network_id'],
                    )
                    continue

                imported_subnet = self._backend_subnet_to_subnet(
                    backend_subnet,
                    network=network,
                    service_project_link=network.service_project_link,
                )

                try:
                    subnet = network.subnets.get(backend_id=imported_subnet.backend_id)
                except models.SubNet.DoesNotExist:
                    imported_subnet.save()
                    subnet = imported_subnet
                else:
                    update_pulled_fields(
                        subnet, imported_subnet, models.SubNet.get_backend_fields()
                    )
                    handle_resource_update_success(subnet)

                subnet_uuids.append(subnet.uuid)

            stale_subnets = models.SubNet.objects.filter(
                state__in=[models.SubNet.States.OK, models.SubNet.States.ERRED],
                network__in=networks,
            ).exclude(uuid__in=subnet_uuids)
            stale_subnets.delete()

    @log_backend_action()
    def import_tenant_subnets(self, tenant):
        self.pull_subnets(tenant)

    def _backend_subnet_to_subnet(self, backend_subnet, **kwargs):
        subnet = models.SubNet(
            name=backend_subnet['name'],
            description=backend_subnet['description'],
            allocation_pools=backend_subnet['allocation_pools'],
            cidr=backend_subnet['cidr'],
            ip_version=backend_subnet['ip_version'],
            enable_dhcp=backend_subnet['enable_dhcp'],
            gateway_ip=backend_subnet.get('gateway_ip'),
            dns_nameservers=backend_subnet['dns_nameservers'],
            host_routes=backend_subnet['host_routes'],
            backend_id=backend_subnet['id'],
            state=models.SubNet.States.OK,
        )

        for field, value in kwargs.items():
            setattr(subnet, field, value)

        return subnet

    @log_backend_action()
    def create_tenant(self, tenant):
        keystone = self.keystone_admin_client
        try:
            backend_tenant = keystone.projects.create(
                name=tenant.name,
                description=tenant.description,
                domain=self._get_domain(),
            )
            tenant.backend_id = backend_tenant.id
            tenant.save(update_fields=['backend_id'])
        except keystone_exceptions.ClientException as e:
            raise OpenStackBackendError(e)

    @log_backend_action()
    def create_tenant_safe(self, tenant):<commit_msg>Fix tests.<commit_after>import logging
import re
from itertools import groupby
from typing import Dict

from cinderclient import exceptions as cinder_exceptions
from django.db import IntegrityError, transaction
from django.utils import timezone
from django.utils.crypto import get_random_string
from django.utils.decorators import method_decorator
from keystoneclient import exceptions as keystone_exceptions
from neutronclient.client import exceptions as neutron_exceptions
from novaclient import exceptions as nova_exceptions

from waldur_core.core.utils import create_batch_fetcher
from waldur_core.structure import SupportedServices, log_backend_action
from waldur_core.structure.utils import (
    handle_resource_not_found,
    handle_resource_update_success,
    update_pulled_fields,
)
from waldur_openstack.openstack_base.backend import (
    BaseOpenStackBackend,
    OpenStackBackendError,
)

from . import models

logger = logging.getLogger(__name__)


class OpenStackBackend(BaseOpenStackBackend):
    DEFAULTS = {
        'tenant_name': 'admin',
    }

    def check_admin_tenant(self):
        try:
            self.keystone_admin_client
        except keystone_exceptions.AuthorizationFailure:
            return False
        except keystone_exceptions.ClientException as e:
            raise OpenStackBackendError(e)
        else:
            return True

    def pull_service_properties(self):
        self.pull_flavors()
        self.pull_images()
        self.pull_volume_types()
        self.pull_service_settings_quotas()

    def pull_resources(self):
        self.pull_tenants()

    def pull_subresources(self):
        self.pull_security_groups()
        self.pull_floating_ips()
        self.pull_networks()
        self.pull_subnets()
        self.pull_routers()

    def pull_tenants(self):
        keystone = self.keystone_admin_client

        try:
            backend_tenants = keystone.projects.list(domain=self._get_domain())
        except keystone_exceptions.ClientException as e:
            raise OpenStackBackendError(e)

        backend_tenants_mapping = {tenant.id: tenant for tenant in backend_tenants}

        tenants = models.Tenant.objects.filter(
            state__in=[models.Tenant.States.OK, models.Tenant.States.ERRED],
            service_project_link__service__settings=self.settings,
        )
        for tenant in tenants:
            backend_tenant = backend_tenants_mapping.get(tenant.backend_id)
            if backend_tenant is None:
                handle_resource_not_found(tenant)
                continue

            imported_backend_tenant = models.Tenant(
                name=backend_tenant.name,
                description=backend_tenant.description,
                backend_id=backend_tenant.id,
                state=models.Tenant.States.OK,
            )
            update_pulled_fields(
                tenant, imported_backend_tenant, models.Tenant.get_backend_fields()
            )
            handle_resource_update_success(tenant)

    def _get_domain(self):
        """ Get current domain """
        keystone = self.keystone_admin_client
        return keystone.domains.find(name=self.settings.domain or 'Default')

    def remove_ssh_key_from_tenant(self, tenant, key_name, fingerprint):
        nova = self.nova_client

        # There could be leftovers of key duplicates: remove them all
        keys = nova.keypairs.findall(fingerprint=fingerprint)
        for key in keys:
            # Remove only keys created with Waldur
            if key.name == key_name:
                nova.keypairs.delete(key)

        logger.info('Deleted ssh public key %s from backend', key_name)

    def _are_rules_equal(self, backend_rule, nc_rule):
        if backend_rule['port_range_min'] != nc_rule.from_port:
            return False
        if backend_rule['port_range_max'] != nc_rule.to_port:
            return False
        if backend_rule['protocol'] != nc_rule.protocol:
            return False
        if backend_rule['remote_ip_prefix'] != nc_rule.cidr:
            return False
        return True

    def pull_flavors(self):
        nova = self.nova_admin_client
        try:
            flavors = nova.flavors.findall(is_public=True)
        except nova_exceptions.ClientException as e:
            raise OpenStackBackendError(e)

        flavor_exclude_regex = self.settings.options.get('flavor_exclude_regex', '')
        name_pattern = (
            re.compile(flavor_exclude_regex) if flavor_exclude_regex else None
        )
        with transaction.atomic():
            cur_flavors = self._get_current_properties(models.Flavor)
            for backend_flavor in flavors:
                if (
                    name_pattern is not None
                    and name_pattern.match(backend_flavor.name) is not None
                ):
                    logger.debug(
                        'Skipping pull of %s flavor as it matches %s regex pattern.',
                        backend_flavor.name,
                        flavor_exclude_regex,
                    )
                    continue

                cur_flavors.pop(backend_flavor.id, None)
                models.Flavor.objects.update_or_create(
                    settings=self.settings,
                    backend_id=backend_flavor.id,
                    defaults={
                        'name': backend_flavor.name,
                        'cores': backend_flavor.vcpus,
                        'ram': backend_flavor.ram,
                        'disk': self.gb2mb(backend_flavor.disk),
                    },
                )

            models.Flavor.objects.filter(backend_id__in=cur_flavors.keys()).delete()

    def pull_images(self):
        self._pull_images(
            models.Image, lambda image: image['visibility'] == 'public', admin=True
        )

    def pull_volume_types(self):
        try:
            volume_types = self.cinder_admin_client.volume_types.list(is_public=True)
        except cinder_exceptions.ClientException as e:
            raise OpenStackBackendError(e)

        with transaction.atomic():
            cur_volume_types = self._get_current_properties(models.VolumeType)
            for backend_type in volume_types:
                cur_volume_types.pop(backend_type.id, None)
                models.VolumeType.objects.update_or_create(
                    settings=self.settings,
                    backend_id=backend_type.id,
                    defaults={
                        'name': backend_type.name,
                        'description': backend_type.description or '',
                    },
                )

            models.VolumeType.objects.filter(
                backend_id__in=cur_volume_types.keys(), settings=self.settings
            ).delete()

    @log_backend_action('push quotas for tenant')
    def push_tenant_quotas(self, tenant, quotas: Dict[str, int]):
        cinder_quotas = {
            'gigabytes': self.mb2gb(quotas.get('storage'))
            if 'storage' in quotas
            else None,
            'volumes': quotas.get('volumes'),
            'snapshots': quotas.get('snapshots'),
        }

        cinder_quotas = {k: v for k, v in cinder_quotas.items() if v is not None}

        # Filter volume-type quotas.
        volume_type_quotas = dict(
            (key, value)
            for (key, value) in quotas.items()
            if key.startswith('gigabytes_') and value is not None
        )

        if volume_type_quotas:
            cinder_quotas.update(volume_type_quotas)

        nova_quotas = {
            'instances': quotas.get('instances'),
            'cores': quotas.get('vcpu'),
            'ram': quotas.get('ram'),
        }
        nova_quotas = {k: v for k, v in nova_quotas.items() if v is not None}

        neutron_quotas = {
            'security_group': quotas.get('security_group_count'),
            'security_group_rule': quotas.get('security_group_rule_count'),
        }
        neutron_quotas = {k: v for k, v in neutron_quotas.items() if v is not None}

        try:
            if cinder_quotas:
                self.cinder_client.quotas.update(tenant.backend_id, **cinder_quotas)
            if nova_quotas:
                self.nova_client.quotas.update(tenant.backend_id, **nova_quotas)
            if neutron_quotas:
                self.neutron_client.update_quota(
                    tenant.backend_id, {'quota': neutron_quotas}
                )
        except Exception as e:
            raise OpenStackBackendError(e)

    @log_backend_action('pull quotas for tenant')
    def pull_tenant_quotas(self, tenant):
        self._pull_tenant_quotas(tenant.backend_id, tenant)

    def pull_quotas(self):
        for tenant in models.Tenant.objects.filter(
            state=models.Tenant.States.OK,
            service_project_link__service__settings=self.settings,
        ):
            self.pull_tenant_quotas(tenant)

    def pull_floating_ips(self, tenants=None):
        if tenants is None:
            tenants = models.Tenant.objects.filter(
                state=models.Tenant.States.OK,
                service_project_link__service__settings=self.settings,
            ).prefetch_related('floating_ips')
        tenant_mappings = {tenant.backend_id: tenant for tenant in tenants}
        if not tenant_mappings:
            return

        backend_floating_ips = self.list_floatingips(list(tenant_mappings.keys()))

        tenant_floating_ips = dict()
        for tenant_id, floating_ips in groupby(
            backend_floating_ips, lambda x: x['tenant_id']
        ):
            tenant_floating_ips[tenant_mappings[tenant_id]] = floating_ips

        with transaction.atomic():
            for tenant, floating_ips in tenant_floating_ips.items():
                self._update_tenant_floating_ips(tenant, floating_ips)

            self._remove_stale_floating_ips(tenants, backend_floating_ips)

    @method_decorator(create_batch_fetcher)
    def list_floatingips(self, tenants):
        neutron = self.neutron_admin_client

        try:
            return neutron.list_floatingips(tenant_id=tenants)['floatingips']
        except neutron_exceptions.NeutronClientException as e:
            raise OpenStackBackendError(e)

    @log_backend_action('pull floating IPs for tenant')
    def pull_tenant_floating_ips(self, tenant):
        neutron = self.neutron_client

        try:
            backend_floating_ips = neutron.list_floatingips(tenant_id=self.tenant_id)[
                'floatingips'
            ]
        except neutron_exceptions.NeutronClientException as e:
            raise OpenStackBackendError(e)

        with transaction.atomic():
            self._update_tenant_floating_ips(tenant, backend_floating_ips)
            self._remove_stale_floating_ips([tenant], backend_floating_ips)

    def _remove_stale_floating_ips(self, tenants, backend_floating_ips):
        remote_ids = {ip['id'] for ip in backend_floating_ips}
        stale_ips = models.FloatingIP.objects.filter(
            tenant__in=tenants,
            state__in=[models.FloatingIP.States.OK, models.FloatingIP.States.ERRED],
        ).exclude(backend_id__in=remote_ids)
        stale_ips.delete()

    def _update_tenant_floating_ips(self, tenant, backend_floating_ips):
        floating_ips = {
            ip.backend_id: ip
            for ip in tenant.floating_ips.filter(
                state__in=[models.FloatingIP.States.OK, models.FloatingIP.States.ERRED]
            )
        }

        for backend_ip in backend_floating_ips:
            imported_floating_ip = self._backend_floating_ip_to_floating_ip(
                backend_ip,
                tenant=tenant,
                service_project_link=tenant.service_project_link,
            )
            floating_ip = floating_ips.pop(imported_floating_ip.backend_id, None)
            if floating_ip is None:
                imported_floating_ip.save()
                continue

            # Don't update user defined name.
            if floating_ip.address != floating_ip.name:
                imported_floating_ip.name = floating_ip.name
            update_pulled_fields(
                floating_ip,
                imported_floating_ip,
                models.FloatingIP.get_backend_fields(),
            )
            handle_resource_update_success(floating_ip)

    def _backend_floating_ip_to_floating_ip(self, backend_floating_ip, **kwargs):
        floating_ip = models.FloatingIP(
            name=backend_floating_ip['floating_ip_address'],
            description=backend_floating_ip['description'],
            address=backend_floating_ip['floating_ip_address'],
            backend_network_id=backend_floating_ip['floating_network_id'],
            runtime_state=backend_floating_ip['status'],
            backend_id=backend_floating_ip['id'],
            state=models.FloatingIP.States.OK,
        )
        for field, value in kwargs.items():
            setattr(floating_ip, field, value)

        return floating_ip

    def pull_security_groups(self, tenants=None):

        if tenants is None:
            tenants = models.Tenant.objects.filter(
                state=models.Tenant.States.OK,
                service_project_link__service__settings=self.settings,
            ).prefetch_related('security_groups')
        tenant_mappings = {tenant.backend_id: tenant for tenant in tenants}
        if not tenant_mappings:
            return

        backend_security_groups = self.list_security_groups(
            list(tenant_mappings.keys())
        )

        tenant_security_groups = dict()
        for tenant_id, security_groups in groupby(
            backend_security_groups, lambda x: x['tenant_id']
        ):
            tenant_security_groups[tenant_mappings[tenant_id]] = security_groups

        with transaction.atomic():
            for tenant, security_groups in tenant_security_groups.items():
                self._update_tenant_security_groups(tenant, security_groups)
            self._remove_stale_security_groups(tenants, backend_security_groups)

    @method_decorator(create_batch_fetcher)
    def list_security_groups(self, tenants):
        neutron = self.neutron_admin_client

        try:
            return neutron.list_security_groups(tenant_id=tenants)['security_groups']
        except neutron_exceptions.NeutronClientException as e:
            raise OpenStackBackendError(e)

    @log_backend_action('pull security groups for tenant')
    def pull_tenant_security_groups(self, tenant):
        neutron = self.neutron_client
        try:
            backend_security_groups = neutron.list_security_groups(
                tenant_id=self.tenant_id
            )['security_groups']
        except neutron_exceptions.NeutronClientException as e:
            raise OpenStackBackendError(e)

        with transaction.atomic():
            self._update_tenant_security_groups(tenant, backend_security_groups)
            self._remove_stale_security_groups([tenant], backend_security_groups)

    def _remove_stale_security_groups(self, tenants, backend_security_groups):
        remote_ids = {ip['id'] for ip in backend_security_groups}
        stale_ips = models.SecurityGroup.objects.filter(
            tenant__in=tenants,
            state__in=[
                models.SecurityGroup.States.OK,
                models.SecurityGroup.States.ERRED,
            ],
        ).exclude(backend_id__in=remote_ids)
        stale_ips.delete()

    def _update_tenant_security_groups(self, tenant, backend_security_groups):
        security_group_uuids = []
        for backend_security_group in backend_security_groups:
            imported_security_group = self._backend_security_group_to_security_group(
                backend_security_group,
                tenant=tenant,
                service_project_link=tenant.service_project_link,
            )

            try:
                security_group = tenant.security_groups.get(
                    backend_id=imported_security_group.backend_id
                )
            except models.SecurityGroup.DoesNotExist:
                imported_security_group.save()
                security_group = imported_security_group
            else:
                update_pulled_fields(
                    security_group,
                    imported_security_group,
                    models.SecurityGroup.get_backend_fields(),
                )
                handle_resource_update_success(security_group)

            security_group_uuids.append(security_group.uuid)
            self._extract_security_group_rules(security_group, backend_security_group)

    def _backend_security_group_to_security_group(
        self, backend_security_group, **kwargs
    ):
        security_group = models.SecurityGroup(
            name=backend_security_group['name'],
            description=backend_security_group['description'],
            backend_id=backend_security_group['id'],
            state=models.SecurityGroup.States.OK,
        )

        for field, value in kwargs.items():
            setattr(security_group, field, value)

        return security_group

    def pull_routers(self):
        for tenant in models.Tenant.objects.filter(
            state=models.Tenant.States.OK,
            service_project_link__service__settings=self.settings,
        ):
            self.pull_tenant_routers(tenant)

    def pull_tenant_routers(self, tenant):
        neutron = self.neutron_admin_client

        try:
            backend_routers = neutron.list_routers(tenant_id=tenant.backend_id)[
                'routers'
            ]
        except neutron_exceptions.NeutronClientException as e:
            raise OpenStackBackendError(e)

        for backend_router in backend_routers:
            defaults = {
                'name': backend_router['name'],
                'description': backend_router['description'],
                'routes': backend_router['routes'],
                'service_project_link': tenant.service_project_link,
                'state': models.Router.States.OK,
            }
            backend_id = backend_router['id']
            try:
                models.Router.objects.update_or_create(
                    tenant=tenant, backend_id=backend_id, defaults=defaults
                )
            except IntegrityError:
                logger.warning(
                    'Could not create router with backend ID %s '
                    'and tenant %s due to concurrent update.',
                    backend_id,
                    tenant,
                )

        remote_ids = {ip['id'] for ip in backend_routers}
        stale_routers = models.Router.objects.filter(tenant=tenant).exclude(
            backend_id__in=remote_ids
        )
        stale_routers.delete()

    def pull_networks(self):
        tenants = (
            models.Tenant.objects.exclude(backend_id='')
            .filter(
                state__in=[models.Tenant.States.OK, models.Tenant.States.UPDATING],
                service_project_link__service__settings=self.settings,
            )
            .prefetch_related('networks')
        )

        self._pull_networks(tenants)

    def _pull_tenant_networks(self, tenant):
        return self._pull_networks([tenant])

    def _pull_networks(self, tenants):
        tenant_mappings = {tenant.backend_id: tenant for tenant in tenants}
        backend_networks = self.list_networks(list(tenant_mappings.keys()))

        networks = []
        with transaction.atomic():
            for backend_network in backend_networks:
                tenant = tenant_mappings.get(backend_network['tenant_id'])
                if not tenant:
                    logger.debug(
                        'Skipping network %s synchronization because its tenant %s is not available.',
                        backend_network['id'],
                        backend_network['tenant_id'],
                    )
                    continue

                imported_network = self._backend_network_to_network(
                    backend_network,
                    tenant=tenant,
                    service_project_link=tenant.service_project_link,
                )

                try:
                    network = tenant.networks.get(
                        backend_id=imported_network.backend_id
                    )
                except models.Network.DoesNotExist:
                    imported_network.save()
                    network = imported_network
                else:
                    update_pulled_fields(
                        network, imported_network, models.Network.get_backend_fields()
                    )
                    handle_resource_update_success(network)

                networks.append(network)

            networks_uuid = [network_item.uuid for network_item in networks]
            stale_networks = models.Network.objects.filter(
                state__in=[models.Network.States.OK, models.Network.States.ERRED],
                tenant__in=tenants,
            ).exclude(uuid__in=networks_uuid)
            stale_networks.delete()

        return networks

    @method_decorator(create_batch_fetcher)
    def list_networks(self, tenants):
        neutron = self.neutron_admin_client
        try:
            return neutron.list_networks(tenant_id=tenants)['networks']
        except neutron_exceptions.NeutronClientException as e:
            raise OpenStackBackendError(e)

    def _backend_network_to_network(self, backend_network, **kwargs):
        network = models.Network(
            name=backend_network['name'],
            description=backend_network['description'],
            is_external=backend_network['router:external'],
            runtime_state=backend_network['status'],
            mtu=backend_network.get('mtu'),
            backend_id=backend_network['id'],
            state=models.Network.States.OK,
        )
        if backend_network.get('provider:network_type'):
            network.type = backend_network['provider:network_type']
        if backend_network.get('provider:segmentation_id'):
            network.segmentation_id = backend_network['provider:segmentation_id']

        for field, value in kwargs.items():
            setattr(network, field, value)

        return network

    def pull_subnets(self, tenant=None, network=None):
        neutron = self.neutron_admin_client

        if tenant:
            networks = tenant.networks.all()
        elif network:
            networks = [network]
        else:
            networks = models.Network.objects.filter(
                state=models.Network.States.OK,
                service_project_link__service__settings=self.settings,
            )
        network_mappings = {network.backend_id: network for network in networks}
        if not network_mappings:
            return

        try:
            if tenant:
                backend_subnets = neutron.list_subnets(tenant_id=tenant.backend_id)[
                    'subnets'
                ]
            elif network:
                backend_subnets = neutron.list_subnets(network_id=network.backend_id)[
                    'subnets'
                ]
            else:
                # We can't filter subnets by network IDs because it exceeds maximum request length
                backend_subnets = neutron.list_subnets()['subnets']
        except neutron_exceptions.NeutronClientException as e:
            raise OpenStackBackendError(e)

        subnet_uuids = []
        with transaction.atomic():
            for backend_subnet in backend_subnets:
                network = network_mappings.get(backend_subnet['network_id'])

                if not network:
                    logger.debug(
                        'OpenStack network is not imported yet. Network ID: %s',
                        backend_subnet['network_id'],
                    )
                    continue

                imported_subnet = self._backend_subnet_to_subnet(
                    backend_subnet,
                    network=network,
                    service_project_link=network.service_project_link,
                )

                try:
                    subnet = network.subnets.get(backend_id=imported_subnet.backend_id)
                except models.SubNet.DoesNotExist:
                    imported_subnet.save()
                    subnet = imported_subnet
                else:
                    update_pulled_fields(
                        subnet, imported_subnet, models.SubNet.get_backend_fields()
                    )
                    handle_resource_update_success(subnet)

                subnet_uuids.append(subnet.uuid)

            stale_subnets = models.SubNet.objects.filter(
                state__in=[models.SubNet.States.OK, models.SubNet.States.ERRED],
                network__in=networks,
            ).exclude(uuid__in=subnet_uuids)
            stale_subnets.delete()

    @log_backend_action()
    def import_tenant_subnets(self, tenant):
        self.pull_subnets(tenant)

    def _backend_subnet_to_subnet(self, backend_subnet, **kwargs):
        subnet = models.SubNet(
            name=backend_subnet['name'],
            description=backend_subnet['description'],
            allocation_pools=backend_subnet['allocation_pools'],
            cidr=backend_subnet['cidr'],
            ip_version=backend_subnet['ip_version'],
            enable_dhcp=backend_subnet['enable_dhcp'],
            gateway_ip=backend_subnet.get('gateway_ip'),
            dns_nameservers=backend_subnet['dns_nameservers'],
            host_routes=backend_subnet.get('host_routes', []),
            backend_id=backend_subnet['id'],
            state=models.SubNet.States.OK,
        )

        for field, value in kwargs.items():
            setattr(subnet, field, value)

        return subnet

    @log_backend_action()
    def create_tenant(self, tenant):
        keystone = self.keystone_admin_client
        try:
            backend_tenant = keystone.projects.create(
                name=tenant.name,
                description=tenant.description,
                domain=self._get_domain(),
            )
            tenant.backend_id = backend_tenant.id
            tenant.save(update_fields=['backend_id'])
        except keystone_exceptions.ClientException as e:
            raise OpenStackBackendError(e)

    @log_backend_action()
    def create_tenant_safe(self, tenant):
{'fraction_non_alphanumeric': 0.07605131995375898, 'fraction_numerical': 7.580495385373434e-05, 'mean_word_length': 2.0752374847019057, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4428941', 'n_tokens_mistral': 14654, 'n_tokens_neox': 14163, 'n_words': 2422}