<commit_before>#
# Copyright (c) 2013 Juniper Networks, Inc. All rights reserved.
#

import sys
import json
import uuid
import gevent
import requests
import cgitb
import copy
import bottle
import logging
import logging.handlers
import Queue
import ConfigParser
import keystoneclient.v2_0.client as keystone

import cfgm_common
try:
    from cfgm_common import vnc_plugin_base
except ImportError:
    from common import vnc_plugin_base
from vnc_api import vnc_api
from vnc_api.gen.resource_xsd import *

from vnc_api.gen.resource_xsd import *
from vnc_api.gen.resource_common import *

import neutron_plugin_interface as npi

Q_CREATE = 'create'
Q_DELETE = 'delete'
Q_MAX_ITEMS = 1000


def fill_keystone_opts(obj, conf_sections):
    obj._auth_user = conf_sections.get('KEYSTONE', 'admin_user')
    obj._auth_passwd = conf_sections.get('KEYSTONE', '<PASSWORD>_password')
    obj._admin_token = conf_sections.get('KEYSTONE', 'admin_token')
    obj._admin_tenant = conf_sections.get('KEYSTONE', 'admin_tenant_name')
    try:
        obj._keystone_sync_on_demand = conf_sections.getboolean('KEYSTONE',
                                               'keystone_sync_on_demand')
    except ConfigParser.NoOptionError:
        obj._keystone_sync_on_demand = True

    try:
        obj._insecure = conf_sections.getboolean('KEYSTONE', 'insecure')
    except ConfigParser.NoOptionError:
        obj._insecure = True

    try:
        obj._auth_url = conf_sections.get('KEYSTONE', 'auth_url')
    except ConfigParser.NoOptionError:
        # deprecated knobs - for backward compat
        obj._auth_proto = conf_sections.get('KEYSTONE', 'auth_protocol')
        obj._auth_host = conf_sections.get('KEYSTONE', 'auth_host')
        obj._auth_port = conf_sections.get('KEYSTONE', 'auth_port')
        obj._auth_url = "%s://%s:%s/v2.0" % (obj._auth_proto, obj._auth_host,
                                             obj._auth_port)


openstack_driver = None
class OpenstackDriver(vnc_plugin_base.Resync):
    def __init__(self, api_server_ip, api_server_port, conf_sections):
        global openstack_driver
        openstack_driver = self
        if api_server_ip == '0.0.0.0':
            self._vnc_api_ip = '127.0.0.1'
        else:
            self._vnc_api_ip = api_server_ip

        self._vnc_api_port = api_server_port

        self._config_sections = conf_sections
        fill_keystone_opts(self, conf_sections)

        if 'v3' in self._auth_url.split('/')[-1]:
            self._get_keystone_conn = self._ksv3_get_conn
            self._ks_domains_list = self._ksv3_domains_list
            self._ks_domain_get = self._ksv3_domain_get
            self._ks_projects_list = self._ksv3_projects_list
            self._ks_project_get = self._ksv3_project_get
            self.sync_project_to_vnc = self._ksv3_sync_project_to_vnc
            self._add_project_to_vnc = self._ksv3_add_project_to_vnc
            self._del_project_from_vnc = self._ksv3_del_project_from_vnc
            self._vnc_default_domain_id = None
        else:
            self._get_keystone_conn = self._ksv2_get_conn
            self._ks_domains_list = None
            self._ks_domain_get = None
            self._ks_projects_list = self._ksv2_projects_list
            self._ks_project_get = self._ksv2_project_get
            self.sync_project_to_vnc = self._ksv2_sync_project_to_vnc
            self._add_project_to_vnc = self._ksv2_add_project_to_vnc
            self._del_project_from_vnc = self._ksv2_del_project_from_vnc
        
        self._resync_interval_secs = 2
        self._resync_number_workers = 10 #TODO(sahid) needs to be configured by conf.
        self._ks = None
        self._vnc_lib = None

        # resync failures, don't retry forever
        self._failed_domain_dels = set()
        self._failed_project_dels = set()

        # active domains/projects in contrail/vnc api server
        self._vnc_domain_ids = set()
        self._vnc_project_ids = set()

        # logging
        self._log_file_name = '/var/log/contrail/vnc_openstack.err'
        self._tmp_file_name = '/var/log/contrail/vnc_openstack.tmp'
        self._vnc_os_logger = logging.getLogger(__name__)
        self._vnc_os_logger.setLevel(logging.ERROR)
        # Add the log message handler to the logger
        try:
            handler = logging.handlers.RotatingFileHandler(self._log_file_name,
                                                           maxBytes=1024,
                                                           backupCount=5)
        except IOError:
            self._log_file_name = './vnc_openstack.err'
            self._tmp_file_name = './vnc_openstack.tmp'
            handler = logging.handlers.RotatingFileHandler(self._log_file_name,
                                                           maxBytes=1024,
                                                           backupCount=5)

        self._vnc_os_logger.addHandler(handler)
        self.q = Queue.Queue(maxsize=Q_MAX_ITEMS)
    #end __init__

    def __call__(self):
        pass
    #end __call__

    def _get_vnc_conn(self):
        if self._vnc_lib:
            return

        self._vnc_lib = vnc_api.VncApi(
            api_server_host=self._vnc_api_ip,
            api_server_port=self._vnc_api_port,
            username=self._auth_user,
            password=self._auth_<PASSWORD>,
            tenant_name=self._admin_tenant)
    # end _get_vnc_conn

    def _ksv2_get_conn(self):
        if not self._ks:
            if self._admin_token:
                self._ks = keystone.Client(token=self._admin_token,
                                           endpoint=self._auth_url,
                                           insecure=self._insecure)
            else:
                self._ks = keystone.Client(username=self._auth_user,
                                           password=<PASSWORD>,
                                           tenant_name=self._admin_tenant,
                                           auth_url=self._auth_url,
                                           insecure=self._insecure)
    # end _ksv2_get_conn

    def _ksv2_projects_list(self):
        return [{'id': tenant.id} for tenant in self._ks.tenants.list()]
    # end _ksv2_projects_list

    def _ksv2_project_get(self, id):
        return {'name': self._ks.tenants.get(id).name}
    # end _ksv2_project_get

    def _ksv2_sync_project_to_vnc(self, id=None):
        self._get_keystone_conn()
        self._get_vnc_conn()
        ks_project = self._ks_project_get(id=id.replace('-', ''))
        display_name = ks_project['name']

        # if earlier project exists with same name but diff id,
        # create with uniqified fq_name
        fq_name = ['default-domain', display_name]
        try:
            old_id = self._vnc_lib.fq_name_to_id('project', fq_name)
            proj_name = '%s-%s' %(display_name, str(uuid.uuid4()))
        except vnc_api.NoIdError:
            proj_name = display_name
<commit_msg>Fix potential deadlock in vnc_openstack<commit_after>#
# Copyright (c) 2013 Juniper Networks, Inc. All rights reserved.
#

import sys
import json
import uuid
import gevent
import requests
import cgitb
import copy
import bottle
import logging
import logging.handlers
import Queue
import ConfigParser
import keystoneclient.v2_0.client as keystone

import cfgm_common
try:
    from cfgm_common import vnc_plugin_base
except ImportError:
    from common import vnc_plugin_base
from vnc_api import vnc_api
from vnc_api.gen.resource_xsd import *

from vnc_api.gen.resource_xsd import *
from vnc_api.gen.resource_common import *

import neutron_plugin_interface as npi

Q_CREATE = 'create'
Q_DELETE = 'delete'
Q_MAX_ITEMS = 1000


def fill_keystone_opts(obj, conf_sections):
    obj._auth_user = conf_sections.get('KEYSTONE', 'admin_user')
    obj._auth_passwd = conf_sections.get('KEYSTONE', '<PASSWORD>')
    obj._admin_token = conf_sections.get('KEYSTONE', 'admin_token')
    obj._admin_tenant = conf_sections.get('KEYSTONE', 'admin_tenant_name')
    try:
        obj._keystone_sync_on_demand = conf_sections.getboolean('KEYSTONE',
                                               'keystone_sync_on_demand')
    except ConfigParser.NoOptionError:
        obj._keystone_sync_on_demand = True

    try:
        obj._insecure = conf_sections.getboolean('KEYSTONE', 'insecure')
    except ConfigParser.NoOptionError:
        obj._insecure = True

    try:
        obj._auth_url = conf_sections.get('KEYSTONE', 'auth_url')
    except ConfigParser.NoOptionError:
        # deprecated knobs - for backward compat
        obj._auth_proto = conf_sections.get('KEYSTONE', 'auth_protocol')
        obj._auth_host = conf_sections.get('KEYSTONE', 'auth_host')
        obj._auth_port = conf_sections.get('KEYSTONE', 'auth_port')
        obj._auth_url = "%s://%s:%s/v2.0" % (obj._auth_proto, obj._auth_host,
                                             obj._auth_port)


openstack_driver = None
class OpenstackDriver(vnc_plugin_base.Resync):
    def __init__(self, api_server_ip, api_server_port, conf_sections):
        global openstack_driver
        openstack_driver = self
        if api_server_ip == '0.0.0.0':
            self._vnc_api_ip = '127.0.0.1'
        else:
            self._vnc_api_ip = api_server_ip

        self._vnc_api_port = api_server_port

        self._config_sections = conf_sections
        fill_keystone_opts(self, conf_sections)

        if 'v3' in self._auth_url.split('/')[-1]:
            self._get_keystone_conn = self._ksv3_get_conn
            self._ks_domains_list = self._ksv3_domains_list
            self._ks_domain_get = self._ksv3_domain_get
            self._ks_projects_list = self._ksv3_projects_list
            self._ks_project_get = self._ksv3_project_get
            self.sync_project_to_vnc = self._ksv3_sync_project_to_vnc
            self._add_project_to_vnc = self._ksv3_add_project_to_vnc
            self._del_project_from_vnc = self._ksv3_del_project_from_vnc
            self._vnc_default_domain_id = None
        else:
            self._get_keystone_conn = self._ksv2_get_conn
            self._ks_domains_list = None
            self._ks_domain_get = None
            self._ks_projects_list = self._ksv2_projects_list
            self._ks_project_get = self._ksv2_project_get
            self.sync_project_to_vnc = self._ksv2_sync_project_to_vnc
            self._add_project_to_vnc = self._ksv2_add_project_to_vnc
            self._del_project_from_vnc = self._ksv2_del_project_from_vnc
        
        self._resync_interval_secs = 2
        self._resync_number_workers = 10 #TODO(sahid) needs to be configured by conf.
        self._ks = None
        self._vnc_lib = None

        # resync failures, don't retry forever
        self._failed_domain_dels = set()
        self._failed_project_dels = set()

        # active domains/projects in contrail/vnc api server
        self._vnc_domain_ids = set()
        self._vnc_project_ids = set()

        # logging
        self._log_file_name = '/var/log/contrail/vnc_openstack.err'
        self._tmp_file_name = '/var/log/contrail/vnc_openstack.tmp'
        self._vnc_os_logger = logging.getLogger(__name__)
        self._vnc_os_logger.setLevel(logging.ERROR)
        # Add the log message handler to the logger
        try:
            handler = logging.handlers.RotatingFileHandler(self._log_file_name,
                                                           maxBytes=1024,
                                                           backupCount=5)
        except IOError:
            self._log_file_name = './vnc_openstack.err'
            self._tmp_file_name = './vnc_openstack.tmp'
            handler = logging.handlers.RotatingFileHandler(self._log_file_name,
                                                           maxBytes=1024,
                                                           backupCount=5)

        self._vnc_os_logger.addHandler(handler)
        self.q = Queue.Queue(maxsize=Q_MAX_ITEMS)
    #end __init__

    def __call__(self):
        pass
    #end __call__

    def _get_vnc_conn(self):
        if self._vnc_lib:
            return

        self._vnc_lib = vnc_api.VncApi(
            api_server_host=self._vnc_api_ip,
            api_server_port=self._vnc_api_port,
            username=self._auth_user,
            password=<PASSWORD>,
            tenant_name=self._admin_tenant)
    # end _get_vnc_conn

    def _ksv2_get_conn(self):
        if not self._ks:
            if self._admin_token:
                self._ks = keystone.Client(token=self._admin_token,
                                           endpoint=self._auth_url,
                                           insecure=self._insecure)
            else:
                self._ks = keystone.Client(username=self._auth_user,
                                           password=self._auth_<PASSWORD>,
                                           tenant_name=self._admin_tenant,
                                           auth_url=self._auth_url,
                                           insecure=self._insecure)
    # end _ksv2_get_conn

    def _ksv2_projects_list(self):
        return [{'id': tenant.id} for tenant in self._ks.tenants.list()]
    # end _ksv2_projects_list

    def _ksv2_project_get(self, id):
        # Note: under certain circumstances (if it has been initailized
        # before endpoints are populated in keystone) keystoneclient may
        # be valid to list projects, but not to read them. As it won't
        # be reset by resync_all_projects, it is reseted on error here.
        try:
            return {'name': self._ks.tenants.get(id).name}
        except Exception as e:
            self._ks = None
            self._get_keystone_conn()
            return {'name': self._ks.tenants.get(id).name}
    # end _ksv2_project_get

    def _ksv2_sync_project_to_vnc(self, id=None):
        self._get_keystone_conn()
        self._get_vnc_conn()
        ks_project = self._ks_project_get(id=id.replace('-', ''))
        display_name = ks_project['name']

        # if earlier project exists with same name but diff id,
        # create with uniqified fq_name
        fq_name = ['default-domain', display_name]
        try:
            old_id = self._vnc_lib.fq_name_to_id('project', fq_name)
            proj_name = '%s-%s' %(display_name, str(uuid.uuid4()))
        except vnc_api.NoIdError:
            proj_name = display_name

{'fraction_non_alphanumeric': 0.07947756009448381, 'fraction_numerical': 0.008197860219535918, 'mean_word_length': 2.1368489867073435, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 48, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2885060', 'n_tokens_mistral': 4501, 'n_tokens_neox': 4370, 'n_words': 779}