<commit_before># -*- coding: utf-8 -*-

"""
    conpaas.core.controller
    =======================

    ConPaaS core: start/stop/list nodes.

    :copyright: (C) 2010-2013 by Contrail Consortium.
"""

from threading import Thread, Lock, Event

import os.path
import time
import json
import socket
import urlparse
from string import Template

from conpaas.core.log import create_logger
from conpaas.core import iaas
from conpaas.core import https

class Controller(object):
    """Implementation of the clouds controller. This class implements functions
       to easily work with the available cloud objects.

       So far, it provides the following functionalities/abstractions:
         - crediting system (also terminate service when user out of credits)
         - adding nodes (VMs)
         - removing nodes (VMs)
    """

    def __init__(self, config_parser, **kwargs):
        # Params for director callback
        self.__conpaas_creditUrl = config_parser.get('manager',
                                                'CREDIT_URL')
        self.__conpaas_terminateUrl = config_parser.get('manager',
                                                   'TERMINATE_URL')
        self.__conpaas_service_id = config_parser.get('manager',
                                                 'SERVICE_ID')
        self.__conpaas_user_id = config_parser.get('manager',
                                              'USER_ID')
        self.__conpaas_app_id = config_parser.get('manager',
                                              'APP_ID')
        self.__conpaas_caUrl = config_parser.get('manager',
                                            'CA_URL')

        # Set the CA URL as IPOP's base namespace
        self.__ipop_base_namespace = self.__conpaas_caUrl

        if config_parser.has_option('manager', 'IPOP_BASE_IP'):
            # Application-level network
            self.__ipop_base_ip = config_parser.get('manager', 'IPOP_BASE_IP')
        else:
            self.__ipop_base_ip = None

        if config_parser.has_option('manager', 'IPOP_NETMASK'):
            # Application-level netmask
            self.__ipop_netmask = config_parser.get('manager', 'IPOP_NETMASK')
        else:
            self.__ipop_netmask = None

        if config_parser.has_option('manager', 'IPOP_BOOTSTRAP_NODES'):
            # Application-level network's bootstrap nodes
            self.__ipop_bootstrap_nodes = config_parser.get('manager', 'IPOP_BOOTSTRAP_NODES')
        else:
            self.__ipop_bootstrap_nodes = None

        if config_parser.has_option('manager', 'IPOP_SUBNET'):
            # Only import from netaddr if IPOP has to be started
            from netaddr import IPNetwork

            # Subnet assigned to this service by the director
            self.__ipop_subnet = IPNetwork(
                config_parser.get('manager', 'IPOP_SUBNET'))
        else:
            self.__ipop_subnet = None

        # For crediting system
        self.__reservation_logger = create_logger('ReservationTimer')
        self.__reservation_map = {'manager': ReservationTimer(['manager'],
                                  55 * 60,  # 55mins
                                  self.__deduct_and_check_credit,
                                  self.__reservation_logger)}
        self.__reservation_map['manager'].start()
        self.__force_terminate_lock = Lock()

        self.config_parser = config_parser
        self.__created_nodes = []
        self.__partially_created_nodes = []
        self.__logger = create_logger(__name__)

        self.__available_clouds = []
        self.__default_cloud = None
        if config_parser.has_option('iaas', 'DRIVER'):
            self.__default_cloud = iaas.get_cloud_instance(
                'iaas',
                config_parser.get('iaas', 'DRIVER').lower(),
                config_parser)
            self.__available_clouds.append(self.__default_cloud)

        if config_parser.has_option('iaas', 'OTHER_CLOUDS'):
            self.__available_clouds.extend(iaas.get_clouds(config_parser))

        # Setting VM role
        self.role = 'agent'

    def get_available_ipop_address(self):
        """Return an unassigned IP address in this manager's VPN subnet"""
        # Network iterator
        network = self.__ipop_subnet.iter_hosts()
        
        # Currently running hosts
        running_hosts = [ str(node.ip) 
            for node in self.__created_nodes + self.__partially_created_nodes ] 

        self.__logger.debug("get_available_ipop_address: running nodes: %s" 
            % running_hosts)

        # The first address is used by IPOP internally
        network.next()
        # The second one is taken by manager 
        network.next()

        for host in network:
            host = str(host)

            if host not in running_hosts:
                self.__logger.debug("get_available_ipop_address: returning %s" 
                    % host)
                return host

    #=========================================================================#
    #               create_nodes(self, count, contextFile, test_agent)        #
    #=========================================================================#
    def create_nodes(self, count, test_agent, port, cloud=None, inst_type=None):
        """
        Creates the VMs associated with the list of nodes. It also tests
        if the agents started correctly.

        @param count The number of nodes to be created

        @param test_agent A callback function to test if the agent
                        started correctly in the newly created VM

        @param port The port on which the agent will listen

        @param cloud (Optional) If specified, this function will start new
                        nodes inside cloud, otherwise it will start new nodes
                        inside the default cloud or wherever the controller
                        wants (for now only the default cloud is used)

        @return A list of nodes of type node.ServiceNode

        """
        ready = []
        poll = []
        iteration = 0

        if cloud is None:
            cloud = self.__default_cloud

        if not self.deduct_credit(count):
            raise Exception('Could not add nodes. Not enough credits.')

        while len(ready) < count:
            iteration += 1
            msg = '[create_nodes] iter %d: creating %d nodes on cloud %s' % (
                iteration, count - len(ready), cloud.cloud_name)

            if inst_type:
                msg += ' of type %s' % inst_type

            self.__logger.debug(msg)

            try:
                self.__force_terminate_lock.acquire()<commit_msg>conpaas-core: do not fail if create_nodes is called with count=0<commit_after># -*- coding: utf-8 -*-

"""
    conpaas.core.controller
    =======================

    ConPaaS core: start/stop/list nodes.

    :copyright: (C) 2010-2013 by Contrail Consortium.
"""

from threading import Thread, Lock, Event

import os.path
import time
import json
import socket
import urlparse
from string import Template

from conpaas.core.log import create_logger
from conpaas.core import iaas
from conpaas.core import https

class Controller(object):
    """Implementation of the clouds controller. This class implements functions
       to easily work with the available cloud objects.

       So far, it provides the following functionalities/abstractions:
         - crediting system (also terminate service when user out of credits)
         - adding nodes (VMs)
         - removing nodes (VMs)
    """

    def __init__(self, config_parser, **kwargs):
        # Params for director callback
        self.__conpaas_creditUrl = config_parser.get('manager',
                                                'CREDIT_URL')
        self.__conpaas_terminateUrl = config_parser.get('manager',
                                                   'TERMINATE_URL')
        self.__conpaas_service_id = config_parser.get('manager',
                                                 'SERVICE_ID')
        self.__conpaas_user_id = config_parser.get('manager',
                                              'USER_ID')
        self.__conpaas_app_id = config_parser.get('manager',
                                              'APP_ID')
        self.__conpaas_caUrl = config_parser.get('manager',
                                            'CA_URL')

        # Set the CA URL as IPOP's base namespace
        self.__ipop_base_namespace = self.__conpaas_caUrl

        if config_parser.has_option('manager', 'IPOP_BASE_IP'):
            # Application-level network
            self.__ipop_base_ip = config_parser.get('manager', 'IPOP_BASE_IP')
        else:
            self.__ipop_base_ip = None

        if config_parser.has_option('manager', 'IPOP_NETMASK'):
            # Application-level netmask
            self.__ipop_netmask = config_parser.get('manager', 'IPOP_NETMASK')
        else:
            self.__ipop_netmask = None

        if config_parser.has_option('manager', 'IPOP_BOOTSTRAP_NODES'):
            # Application-level network's bootstrap nodes
            self.__ipop_bootstrap_nodes = config_parser.get('manager', 'IPOP_BOOTSTRAP_NODES')
        else:
            self.__ipop_bootstrap_nodes = None

        if config_parser.has_option('manager', 'IPOP_SUBNET'):
            # Only import from netaddr if IPOP has to be started
            from netaddr import IPNetwork

            # Subnet assigned to this service by the director
            self.__ipop_subnet = IPNetwork(
                config_parser.get('manager', 'IPOP_SUBNET'))
        else:
            self.__ipop_subnet = None

        # For crediting system
        self.__reservation_logger = create_logger('ReservationTimer')
        self.__reservation_map = {'manager': ReservationTimer(['manager'],
                                  55 * 60,  # 55mins
                                  self.__deduct_and_check_credit,
                                  self.__reservation_logger)}
        self.__reservation_map['manager'].start()
        self.__force_terminate_lock = Lock()

        self.config_parser = config_parser
        self.__created_nodes = []
        self.__partially_created_nodes = []
        self.__logger = create_logger(__name__)

        self.__available_clouds = []
        self.__default_cloud = None
        if config_parser.has_option('iaas', 'DRIVER'):
            self.__default_cloud = iaas.get_cloud_instance(
                'iaas',
                config_parser.get('iaas', 'DRIVER').lower(),
                config_parser)
            self.__available_clouds.append(self.__default_cloud)

        if config_parser.has_option('iaas', 'OTHER_CLOUDS'):
            self.__available_clouds.extend(iaas.get_clouds(config_parser))

        # Setting VM role
        self.role = 'agent'

    def get_available_ipop_address(self):
        """Return an unassigned IP address in this manager's VPN subnet"""
        # Network iterator
        network = self.__ipop_subnet.iter_hosts()
        
        # Currently running hosts
        running_hosts = [ str(node.ip) 
            for node in self.__created_nodes + self.__partially_created_nodes ] 

        self.__logger.debug("get_available_ipop_address: running nodes: %s" 
            % running_hosts)

        # The first address is used by IPOP internally
        network.next()
        # The second one is taken by manager 
        network.next()

        for host in network:
            host = str(host)

            if host not in running_hosts:
                self.__logger.debug("get_available_ipop_address: returning %s" 
                    % host)
                return host

    #=========================================================================#
    #               create_nodes(self, count, contextFile, test_agent)        #
    #=========================================================================#
    def create_nodes(self, count, test_agent, port, cloud=None, inst_type=None):
        """
        Creates the VMs associated with the list of nodes. It also tests
        if the agents started correctly.

        @param count The number of nodes to be created

        @param test_agent A callback function to test if the agent
                        started correctly in the newly created VM

        @param port The port on which the agent will listen

        @param cloud (Optional) If specified, this function will start new
                        nodes inside cloud, otherwise it will start new nodes
                        inside the default cloud or wherever the controller
                        wants (for now only the default cloud is used)

        @return A list of nodes of type node.ServiceNode

        """
        ready = []
        poll = []
        iteration = 0

        if count == 0:
            return []

        if cloud is None:
            cloud = self.__default_cloud

        if not self.deduct_credit(count):
            raise Exception('Could not add nodes. Not enough credits.')

        while len(ready) < count:
            iteration += 1
            msg = '[create_nodes] iter %d: creating %d nodes on cloud %s' % (
                iteration, count - len(ready), cloud.cloud_name)

            if inst_type:
                msg += ' of type %s' % inst_type

            self.__logger.debug(msg)

            try:
                self.__force_terminate_lock.acquire()
{'fraction_non_alphanumeric': 0.10090036461046209, 'fraction_numerical': 0.0026787707418706747, 'mean_word_length': 1.9564452265728112, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 32, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7550244', 'n_tokens_mistral': 3512, 'n_tokens_neox': 3323, 'n_words': 928}