<commit_before>import json
import logging
import re

from django.db import transaction
from django.utils import six, timezone, dateparse

from ceilometerclient import exc as ceilometer_exceptions
from cinderclient import exceptions as cinder_exceptions
from glanceclient import exc as glance_exceptions
from keystoneclient import exceptions as keystone_exceptions
from neutronclient.client import exceptions as neutron_exceptions
from novaclient import exceptions as nova_exceptions

from nodeconductor.structure import log_backend_action
from nodeconductor_openstack.openstack_base.backend import (
    BaseOpenStackBackend, OpenStackBackendError,
    update_pulled_fields, handle_resource_not_found, handle_resource_update_success)

from . import models


logger = logging.getLogger(__name__)


class OpenStackTenantBackend(BaseOpenStackBackend):

    def __init__(self, settings):
        super(OpenStackTenantBackend, self).__init__(settings, settings.options['tenant_id'])

    @property
    def external_network_id(self):
        return self.settings.options['external_network_id']

    def sync(self):
        # pull service properties
        self.pull_flavors()
        self.pull_images()
        self.pull_security_groups()
        self.pull_quotas()
        self.pull_networks()
        self.pull_subnets()
        self.pull_internal_ips()
        self.pull_floating_ips()

        # pull resources
        self.pull_volumes()
        self.pull_snapshots()
        self.pull_instances()

    def pull_volumes(self):
        backend_volumes = self.get_volumes()
        volumes = models.Volume.objects.filter(
            service_project_link__service__settings=self.settings,
            state__in=[models.Volume.States.OK, models.Volume.States.ERRED]
        )
        backend_volumes_map = {backend_volume.backend_id: backend_volume for backend_volume in backend_volumes}
        for volume in volumes:
            try:
                backend_volume = backend_volumes_map[volume.backend_id]
            except KeyError:
                handle_resource_not_found(volume)
            else:
                update_pulled_fields(volume, backend_volume, models.Volume.get_backend_fields())
                handle_resource_update_success(volume)

    def pull_snapshots(self):
        backend_snapshots = self.get_snapshots()
        snapshots = models.Snapshot.objects.filter(
            service_project_link__service__settings=self.settings,
            state__in=[models.Snapshot.States.OK, models.Snapshot.States.ERRED])
        backend_snapshots_map = {backend_snapshot.backend_id: backend_snapshot
                                 for backend_snapshot in backend_snapshots}
        for snapshot in snapshots:
            try:
                backend_snapshot = backend_snapshots_map[snapshot.backend_id]
            except KeyError:
                handle_resource_not_found(snapshot)
            else:
                update_pulled_fields(snapshot, backend_snapshot, models.Snapshot.get_backend_fields())
                handle_resource_update_success(snapshot)

    def pull_instances(self):
        backend_instances = self.get_instances()
        instances = models.Instance.objects.filter(
            service_project_link__service__settings=self.settings,
            state__in=[models.Instance.States.OK, models.Instance.States.ERRED],
        )
        backend_instances_map = {backend_instance.backend_id: backend_instance
                                 for backend_instance in backend_instances}
        for instance in instances:
            try:
                backend_instance = backend_instances_map[instance.backend_id]
            except KeyError:
                handle_resource_not_found(instance)
            else:
                update_pulled_fields(instance, backend_instance, models.Instance.get_backend_fields())
                # XXX: can be optimized after https://goo.gl/BZKo8Y will be resolved.
                self.pull_instance_security_groups(instance)
                handle_resource_update_success(instance)

    def pull_flavors(self):
        nova = self.nova_client
        try:
            flavors = nova.flavors.findall(is_public=True)
        except nova_exceptions.ClientException as e:
            six.reraise(OpenStackBackendError, e)

        flavor_exclude_regex = self.settings.options.get('flavor_exclude_regex', '')
        name_pattern = re.compile(flavor_exclude_regex) if flavor_exclude_regex else None
        with transaction.atomic():
            cur_flavors = self._get_current_properties(models.Flavor)
            for backend_flavor in flavors:
                if name_pattern is not None and name_pattern.match(backend_flavor.name) is not None:
                    logger.debug('Skipping pull of %s flavor as it matches %s regex pattern.',
                                 backend_flavor.name, flavor_exclude_regex)
                    continue

                cur_flavors.pop(backend_flavor.id, None)
                models.Flavor.objects.update_or_create(
                    settings=self.settings,
                    backend_id=backend_flavor.id,
                    defaults={
                        'name': backend_flavor.name,
                        'cores': backend_flavor.vcpus,
                        'ram': backend_flavor.ram,
                        'disk': self.gb2mb(backend_flavor.disk),
                    })

            models.Flavor.objects.filter(backend_id__in=cur_flavors.keys()).delete()

    def pull_images(self):
        glance = self.glance_client
        try:
            images = [image for image in glance.images.list() if image.is_public and not image.deleted]
        except glance_exceptions.ClientException as e:
            six.reraise(OpenStackBackendError, e)

        with transaction.atomic():
            cur_images = self._get_current_properties(models.Image)
            for backend_image in images:
                cur_images.pop(backend_image.id, None)
                models.Image.objects.update_or_create(
                    settings=self.settings,
                    backend_id=backend_image.id,
                    defaults={
                        'name': backend_image.name,
                        'min_ram': backend_image.min_ram,
                        'min_disk': self.gb2mb(backend_image.min_disk),
                    })

            models.Image.objects.filter(backend_id__in=cur_images.keys()).delete()

    def pull_floating_ips(self):
        # method assumes that instance internal IPs is up to date.
        neutron = self.neutron_client

        try:
            backend_floating_ips = neutron.list_floatingips(tenant_id=self.tenant_id)['floatingips']
        except neutron_exceptions.NeutronClientException as e:
            six.reraise(OpenStackBackendError, e)

        floating_ip_mappings = {ip.backend_id: ip for ip in models.FloatingIP.objects.filter(
            settings=self.settings, is_booked=False).exclude(backend_id='')}

        booked_ips = models.FloatingIP.objects.filter(
            settings=self.settings,
            is_booked=True).exclude(backend_id='').values_list('backend_id', flat=True)

        internal_ip_mappings = {ip.backend_id: ip for ip in models.InternalIP.objects.filter(
            instance__service_project_link__service__settings=self.settings).exclude(backend_id='')}

        with transaction.atomic():
            for backend_ip in backend_floating_ips:
                imported_floating_ip = self._backend_floating_ip_to_floating_ip(backend_ip)

                internal_ip = internal_ip_mappings.get(imported_floating_ip._internal_ip_backend_id)
                if imported_floating_ip._internal_ip_backend_id and internal_ip is None:
                    logger.warning('Failed to set internal_ip for Floating IP %s', imported_floating_ip.backend_id)
                    continue

                floating_ip = floating_ip_mappings.pop(imported_floating_ip.backend_id, None)
                # create Floating IP if it does not exist
                if floating_ip is None and backend_ip.backend_id not in booked_ips:
                    imported_floating_ip.internal_ip = internal_ip
                    imported_floating_ip.save()<commit_msg>Fix typo<commit_after>import json
import logging
import re

from django.db import transaction
from django.utils import six, timezone, dateparse

from ceilometerclient import exc as ceilometer_exceptions
from cinderclient import exceptions as cinder_exceptions
from glanceclient import exc as glance_exceptions
from keystoneclient import exceptions as keystone_exceptions
from neutronclient.client import exceptions as neutron_exceptions
from novaclient import exceptions as nova_exceptions

from nodeconductor.structure import log_backend_action
from nodeconductor_openstack.openstack_base.backend import (
    BaseOpenStackBackend, OpenStackBackendError,
    update_pulled_fields, handle_resource_not_found, handle_resource_update_success)

from . import models


logger = logging.getLogger(__name__)


class OpenStackTenantBackend(BaseOpenStackBackend):

    def __init__(self, settings):
        super(OpenStackTenantBackend, self).__init__(settings, settings.options['tenant_id'])

    @property
    def external_network_id(self):
        return self.settings.options['external_network_id']

    def sync(self):
        # pull service properties
        self.pull_flavors()
        self.pull_images()
        self.pull_security_groups()
        self.pull_quotas()
        self.pull_networks()
        self.pull_subnets()
        self.pull_internal_ips()
        self.pull_floating_ips()

        # pull resources
        self.pull_volumes()
        self.pull_snapshots()
        self.pull_instances()

    def pull_volumes(self):
        backend_volumes = self.get_volumes()
        volumes = models.Volume.objects.filter(
            service_project_link__service__settings=self.settings,
            state__in=[models.Volume.States.OK, models.Volume.States.ERRED]
        )
        backend_volumes_map = {backend_volume.backend_id: backend_volume for backend_volume in backend_volumes}
        for volume in volumes:
            try:
                backend_volume = backend_volumes_map[volume.backend_id]
            except KeyError:
                handle_resource_not_found(volume)
            else:
                update_pulled_fields(volume, backend_volume, models.Volume.get_backend_fields())
                handle_resource_update_success(volume)

    def pull_snapshots(self):
        backend_snapshots = self.get_snapshots()
        snapshots = models.Snapshot.objects.filter(
            service_project_link__service__settings=self.settings,
            state__in=[models.Snapshot.States.OK, models.Snapshot.States.ERRED])
        backend_snapshots_map = {backend_snapshot.backend_id: backend_snapshot
                                 for backend_snapshot in backend_snapshots}
        for snapshot in snapshots:
            try:
                backend_snapshot = backend_snapshots_map[snapshot.backend_id]
            except KeyError:
                handle_resource_not_found(snapshot)
            else:
                update_pulled_fields(snapshot, backend_snapshot, models.Snapshot.get_backend_fields())
                handle_resource_update_success(snapshot)

    def pull_instances(self):
        backend_instances = self.get_instances()
        instances = models.Instance.objects.filter(
            service_project_link__service__settings=self.settings,
            state__in=[models.Instance.States.OK, models.Instance.States.ERRED],
        )
        backend_instances_map = {backend_instance.backend_id: backend_instance
                                 for backend_instance in backend_instances}
        for instance in instances:
            try:
                backend_instance = backend_instances_map[instance.backend_id]
            except KeyError:
                handle_resource_not_found(instance)
            else:
                update_pulled_fields(instance, backend_instance, models.Instance.get_backend_fields())
                # XXX: can be optimized after https://goo.gl/BZKo8Y will be resolved.
                self.pull_instance_security_groups(instance)
                handle_resource_update_success(instance)

    def pull_flavors(self):
        nova = self.nova_client
        try:
            flavors = nova.flavors.findall(is_public=True)
        except nova_exceptions.ClientException as e:
            six.reraise(OpenStackBackendError, e)

        flavor_exclude_regex = self.settings.options.get('flavor_exclude_regex', '')
        name_pattern = re.compile(flavor_exclude_regex) if flavor_exclude_regex else None
        with transaction.atomic():
            cur_flavors = self._get_current_properties(models.Flavor)
            for backend_flavor in flavors:
                if name_pattern is not None and name_pattern.match(backend_flavor.name) is not None:
                    logger.debug('Skipping pull of %s flavor as it matches %s regex pattern.',
                                 backend_flavor.name, flavor_exclude_regex)
                    continue

                cur_flavors.pop(backend_flavor.id, None)
                models.Flavor.objects.update_or_create(
                    settings=self.settings,
                    backend_id=backend_flavor.id,
                    defaults={
                        'name': backend_flavor.name,
                        'cores': backend_flavor.vcpus,
                        'ram': backend_flavor.ram,
                        'disk': self.gb2mb(backend_flavor.disk),
                    })

            models.Flavor.objects.filter(backend_id__in=cur_flavors.keys()).delete()

    def pull_images(self):
        glance = self.glance_client
        try:
            images = [image for image in glance.images.list() if image.is_public and not image.deleted]
        except glance_exceptions.ClientException as e:
            six.reraise(OpenStackBackendError, e)

        with transaction.atomic():
            cur_images = self._get_current_properties(models.Image)
            for backend_image in images:
                cur_images.pop(backend_image.id, None)
                models.Image.objects.update_or_create(
                    settings=self.settings,
                    backend_id=backend_image.id,
                    defaults={
                        'name': backend_image.name,
                        'min_ram': backend_image.min_ram,
                        'min_disk': self.gb2mb(backend_image.min_disk),
                    })

            models.Image.objects.filter(backend_id__in=cur_images.keys()).delete()

    def pull_floating_ips(self):
        # method assumes that instance internal IPs is up to date.
        neutron = self.neutron_client

        try:
            backend_floating_ips = neutron.list_floatingips(tenant_id=self.tenant_id)['floatingips']
        except neutron_exceptions.NeutronClientException as e:
            six.reraise(OpenStackBackendError, e)

        floating_ip_mappings = {ip.backend_id: ip for ip in models.FloatingIP.objects.filter(
            settings=self.settings, is_booked=False).exclude(backend_id='')}

        booked_ips = models.FloatingIP.objects.filter(
            settings=self.settings,
            is_booked=True).exclude(backend_id='').values_list('backend_id', flat=True)

        internal_ip_mappings = {ip.backend_id: ip for ip in models.InternalIP.objects.filter(
            instance__service_project_link__service__settings=self.settings).exclude(backend_id='')}

        with transaction.atomic():
            for backend_ip in backend_floating_ips:
                imported_floating_ip = self._backend_floating_ip_to_floating_ip(backend_ip)

                internal_ip = internal_ip_mappings.get(imported_floating_ip._internal_ip_backend_id)
                if imported_floating_ip._internal_ip_backend_id and internal_ip is None:
                    logger.warning('Failed to set internal_ip for Floating IP %s', imported_floating_ip.backend_id)
                    continue

                floating_ip = floating_ip_mappings.pop(imported_floating_ip.backend_id, None)
                # create Floating IP if it does not exist
                if floating_ip is None and backend_ip not in booked_ips:
                    imported_floating_ip.internal_ip = internal_ip
                    imported_floating_ip.save()
{'fraction_non_alphanumeric': 0.06918124392614189, 'fraction_numerical': 0.00036443148688046647, 'mean_word_length': 2.4237887294655853, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '10688', 'n_tokens_mistral': 4507, 'n_tokens_neox': 4348, 'n_words': 816}