<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
Cloud Controller: Implementation of EC2 REST API calls, which are
dispatched to other nodes via AMQP RPC. State is via distributed
datastore.
"""

import base64
import logging
import os
import time

from twisted.internet import defer

from nova import datastore
from nova import exception
from nova import flags
from nova import rpc
from nova import utils
from nova.auth import rbac
from nova.auth import manager
from nova.compute import model
from nova.compute.instance_types import INSTANCE_TYPES
from nova.endpoint import images
from nova.network import service as network_service
from nova.network import model as network_model
from nova.volume import service


FLAGS = flags.FLAGS


def _gen_key(user_id, key_name):
    """ Tuck this into AuthManager """
    try:
        mgr = manager.AuthManager()
        private_key, fingerprint = mgr.generate_key_pair(user_id, key_name)
    except Exception as ex:
        return {'exception': ex}
    return {'private_key': private_key, 'fingerprint': fingerprint}


class CloudController(object):
    """ CloudController provides the critical dispatch between
 inbound API calls through the endpoint and messages
 sent to the other nodes.
"""
    def __init__(self):
        self.instdir = model.InstanceDirectory()
        self.setup()

    @property
    def instances(self):
        """ All instances in the system, as dicts """
        return self.instdir.all

    @property
    def volumes(self):
        """ returns a list of all volumes """
        for volume_id in datastore.Redis.instance().smembers("volumes"):
            volume = service.get_volume(volume_id)
            yield volume

    def __str__(self):
        return 'CloudController'

    def setup(self):
        """ Ensure the keychains and folders exist. """
        # Create keys folder, if it doesn't exist
        if not os.path.exists(FLAGS.keys_path):
            os.makedirs(FLAGS.keys_path)
        # Gen root CA, if we don't have one
        root_ca_path = os.path.join(FLAGS.ca_path, FLAGS.ca_file)
        if not os.path.exists(root_ca_path):
            start = os.getcwd()
            os.chdir(FLAGS.ca_path)
            utils.runthis("Generating root CA: %s", "sh genrootca.sh")
            os.chdir(start)
            # TODO: Do this with M2Crypto instead

    def get_instance_by_ip(self, ip):
        return self.instdir.by_ip(ip)

    def _get_mpi_data(self, project_id):
        result = {}
        for instance in self.instdir.all:
            if instance['project_id'] == project_id:
                line = '%s slots=%d' % (instance['private_dns_name'],
                    INSTANCE_TYPES[instance['instance_type']]['vcpus'])
                if instance['key_name'] in result:
                    result[instance['key_name']].append(line)
                else:
                    result[instance['key_name']] = [line]
        return result

    def get_metadata(self, ipaddress):
        i = self.get_instance_by_ip(ipaddress)
        if i is None:
            return None
        mpi = self._get_mpi_data(i['project_id'])
        if i['key_name']:
            keys = {
                '0': {
                    '_name': i['key_name'],
                    'openssh-key': i['key_data']
                }
            }
        else:
            keys = ''

        address_record = network_model.FixedIp(i['private_dns_name'])
        if address_record:
            hostname = address_record['hostname']
        else:
            hostname = 'ip-%s' % i['private_dns_name'].replace('.', '-')
        data = {
            'user-data': base64.b64decode(i['user_data']),
            'meta-data': {
                'ami-id': i['image_id'],
                'ami-launch-index': i['ami_launch_index'],
                'ami-manifest-path': 'FIXME', # image property
                'block-device-mapping': { # TODO: replace with real data
                    'ami': 'sda1',
                    'ephemeral0': 'sda2',
                    'root': '/dev/sda1',
                    'swap': 'sda3'
                },
                'hostname': hostname,
                'instance-action': 'none',
                'instance-id': i['instance_id'],
                'instance-type': i.get('instance_type', ''),
                'local-hostname': hostname,
                'local-ipv4': i['private_dns_name'], # TODO: switch to IP
                'kernel-id': i.get('kernel_id', ''),
                'placement': {
                    'availaibility-zone': i.get('availability_zone', 'nova'),
                },
                'public-hostname': hostname,
                'public-ipv4': i.get('dns_name', ''), # TODO: switch to IP
                'public-keys': keys,
                'ramdisk-id': i.get('ramdisk_id', ''),
                'reservation-id': i['reservation_id'],
                'security-groups': i.get('groups', ''),
                'mpi': mpi
            }
        }
        if False: # TODO: store ancestor ids
            data['ancestor-ami-ids'] = []
        if i.get('product_codes', None):
            data['product-codes'] = i['product_codes']
        return data

    @rbac.allow('all')
    def describe_availability_zones(self, context, **kwargs):
        return {'availabilityZoneInfo': [{'zoneName': 'nova',
                                          'zoneState': 'available'}]}

    @rbac.allow('all')
    def describe_regions(self, context, region_name=None, **kwargs):
        if FLAGS.region_list:
            regions = []
            for region in FLAGS.region_list:
                name, _sep, url = region.partition(',')
                regions.append({'regionName': name,
                                'regionUrl': url})
        else:
            regions = [{'regionName': 'nova',
                        'regionUrl': FLAGS.ec2_url}]
        if region_name:
            regions = [r for r in regions if r['regionName'] in region_name]
        return {'regionInfo': regions }

    @rbac.allow('all')
    def describe_snapshots(self,
                           context,
                           snapshot_id=None,
                           owner=None,
                           restorable_by=None,
                           **kwargs):
        return {'snapshotSet': [{'snapshotId': 'fixme',
                                 'volumeId': 'fixme',
                                 'status': 'fixme',
                                 'startTime': 'fixme',
                                 'progress': 'fixme',
                                 'ownerId': 'fixme',
                                 'volumeSize': 0,
                                 'description': 'fixme'}]}

    @rbac.allow('all')
    def describe_key_pairs(self, context, key_name=None, **kwargs):
        key_pairs = context.user.get_key_pairs()
        if not key_name is None:
            key_pairs = [x for x in key_pairs if x.name in key_name]
<commit_msg>it is called regionEndpoint, and use pipe as a separator<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
Cloud Controller: Implementation of EC2 REST API calls, which are
dispatched to other nodes via AMQP RPC. State is via distributed
datastore.
"""

import base64
import logging
import os
import time

from twisted.internet import defer

from nova import datastore
from nova import exception
from nova import flags
from nova import rpc
from nova import utils
from nova.auth import rbac
from nova.auth import manager
from nova.compute import model
from nova.compute.instance_types import INSTANCE_TYPES
from nova.endpoint import images
from nova.network import service as network_service
from nova.network import model as network_model
from nova.volume import service


FLAGS = flags.FLAGS


def _gen_key(user_id, key_name):
    """ Tuck this into AuthManager """
    try:
        mgr = manager.AuthManager()
        private_key, fingerprint = mgr.generate_key_pair(user_id, key_name)
    except Exception as ex:
        return {'exception': ex}
    return {'private_key': private_key, 'fingerprint': fingerprint}


class CloudController(object):
    """ CloudController provides the critical dispatch between
 inbound API calls through the endpoint and messages
 sent to the other nodes.
"""
    def __init__(self):
        self.instdir = model.InstanceDirectory()
        self.setup()

    @property
    def instances(self):
        """ All instances in the system, as dicts """
        return self.instdir.all

    @property
    def volumes(self):
        """ returns a list of all volumes """
        for volume_id in datastore.Redis.instance().smembers("volumes"):
            volume = service.get_volume(volume_id)
            yield volume

    def __str__(self):
        return 'CloudController'

    def setup(self):
        """ Ensure the keychains and folders exist. """
        # Create keys folder, if it doesn't exist
        if not os.path.exists(FLAGS.keys_path):
            os.makedirs(FLAGS.keys_path)
        # Gen root CA, if we don't have one
        root_ca_path = os.path.join(FLAGS.ca_path, FLAGS.ca_file)
        if not os.path.exists(root_ca_path):
            start = os.getcwd()
            os.chdir(FLAGS.ca_path)
            utils.runthis("Generating root CA: %s", "sh genrootca.sh")
            os.chdir(start)
            # TODO: Do this with M2Crypto instead

    def get_instance_by_ip(self, ip):
        return self.instdir.by_ip(ip)

    def _get_mpi_data(self, project_id):
        result = {}
        for instance in self.instdir.all:
            if instance['project_id'] == project_id:
                line = '%s slots=%d' % (instance['private_dns_name'],
                    INSTANCE_TYPES[instance['instance_type']]['vcpus'])
                if instance['key_name'] in result:
                    result[instance['key_name']].append(line)
                else:
                    result[instance['key_name']] = [line]
        return result

    def get_metadata(self, ipaddress):
        i = self.get_instance_by_ip(ipaddress)
        if i is None:
            return None
        mpi = self._get_mpi_data(i['project_id'])
        if i['key_name']:
            keys = {
                '0': {
                    '_name': i['key_name'],
                    'openssh-key': i['key_data']
                }
            }
        else:
            keys = ''

        address_record = network_model.FixedIp(i['private_dns_name'])
        if address_record:
            hostname = address_record['hostname']
        else:
            hostname = 'ip-%s' % i['private_dns_name'].replace('.', '-')
        data = {
            'user-data': base64.b64decode(i['user_data']),
            'meta-data': {
                'ami-id': i['image_id'],
                'ami-launch-index': i['ami_launch_index'],
                'ami-manifest-path': 'FIXME', # image property
                'block-device-mapping': { # TODO: replace with real data
                    'ami': 'sda1',
                    'ephemeral0': 'sda2',
                    'root': '/dev/sda1',
                    'swap': 'sda3'
                },
                'hostname': hostname,
                'instance-action': 'none',
                'instance-id': i['instance_id'],
                'instance-type': i.get('instance_type', ''),
                'local-hostname': hostname,
                'local-ipv4': i['private_dns_name'], # TODO: switch to IP
                'kernel-id': i.get('kernel_id', ''),
                'placement': {
                    'availaibility-zone': i.get('availability_zone', 'nova'),
                },
                'public-hostname': hostname,
                'public-ipv4': i.get('dns_name', ''), # TODO: switch to IP
                'public-keys': keys,
                'ramdisk-id': i.get('ramdisk_id', ''),
                'reservation-id': i['reservation_id'],
                'security-groups': i.get('groups', ''),
                'mpi': mpi
            }
        }
        if False: # TODO: store ancestor ids
            data['ancestor-ami-ids'] = []
        if i.get('product_codes', None):
            data['product-codes'] = i['product_codes']
        return data

    @rbac.allow('all')
    def describe_availability_zones(self, context, **kwargs):
        return {'availabilityZoneInfo': [{'zoneName': 'nova',
                                          'zoneState': 'available'}]}

    @rbac.allow('all')
    def describe_regions(self, context, region_name=None, **kwargs):
        if FLAGS.region_list:
            regions = []
            for region in FLAGS.region_list:
                name, _sep, url = region.partition('|')
                regions.append({'regionName': name,
                                'regionEndpoint': url})
        else:
            regions = [{'regionName': 'nova',
                        'regionEndpoint': FLAGS.ec2_url}]
        if region_name:
            regions = [r for r in regions if r['regionName'] in region_name]
        return {'regionInfo': regions }

    @rbac.allow('all')
    def describe_snapshots(self,
                           context,
                           snapshot_id=None,
                           owner=None,
                           restorable_by=None,
                           **kwargs):
        return {'snapshotSet': [{'snapshotId': 'fixme',
                                 'volumeId': 'fixme',
                                 'status': 'fixme',
                                 'startTime': 'fixme',
                                 'progress': 'fixme',
                                 'ownerId': 'fixme',
                                 'volumeSize': 0,
                                 'description': 'fixme'}]}

    @rbac.allow('all')
    def describe_key_pairs(self, context, key_name=None, **kwargs):
        key_pairs = context.user.get_key_pairs()
        if not key_name is None:
            key_pairs = [x for x in key_pairs if x.name in key_name]

{'fraction_non_alphanumeric': 0.11054852320675106, 'fraction_numerical': 0.0037650113599480687, 'mean_word_length': 1.8817807706696597, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2188928', 'n_tokens_mistral': 4355, 'n_tokens_neox': 4217, 'n_words': 1159}