<commit_before># Copyright 2014 OpenCore LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import boto
import boto.cloudformation
import boto.ec2
import boto.vpc
from boto.ec2.address import *
from boto.ec2.networkinterface import *
from boto.exception import BotoServerError, EC2ResponseError
from boto.ec2.blockdevicemapping import BlockDeviceType, EBSBlockDeviceType, BlockDeviceMapping
import copy
import ferry.install
import json
import logging
import math
import os
from pymongo import MongoClient
import sys
import time
import yaml

class AWSLauncher(object):
    """
    Launches new instances/networks on an AWS
    cluster and initializes the instances to run Ferry processes. 
    """
    def __init__(self, controller):
        self.name = "AWS launcher"
        self.docker_registry = None
        self.docker_user = None
        
        self.controller = controller
        self.subnets = []
        self.stacks = {}
        self.num_network_hosts = 1024
        self.num_subnet_hosts = 256
        self.vpc_cidr = None

        self._init_app_db()
        self._init_aws_stack()

    def support_proxy(self):
        """
        The AWS does not support proxy mode at the moment, since that makes 
        it difficult to communicate with nodes in a private VPC subnet. 
        """
        return False

    def _init_app_db(self):
        self.mongo = MongoClient(os.environ['MONGODB'], 27017, connectTimeoutMS=6000)
        self.apps = self.mongo['cloud']['aws']

    def _init_aws_stack(self):
        conf = ferry.install.read_ferry_config()
        provider = conf['system']['provider']

        params = conf[provider]['params']
        self.default_dc = params['dc']
        self.default_zone = params['zone']


        # Figure out what disk volume to use for the
        # data storage. If one is not listed, then the
        # system will use EBS. 
        if 'volume' in params:
            self.data_volume = params['volume']
        else:
            self.data_volume = "ebs:8"

        # This gives us information about the image to use
        # for the supplied provider. 
        deploy = conf[provider]['deploy']

        self.default_image = deploy['image']
        self.default_personality = deploy['personality']
        self.default_user = deploy['default-user']
        self.ssh_key = deploy['ssh']
        self.ssh_user = deploy['ssh-user']

        # Make sure that the ssh key is actually present. 
        keypath = self._get_host_key()
        if not os.path.exists(keypath):
            logging.error("could not find ssh key (%s)" % self.ssh_key)
            raise ValueError("Missing ssh keys")

        # Get user credentials
        self.aws_user = deploy['user']
        self.aws_access_key = deploy['access']
        self.aws_secret_key = deploy['secret']
        
        # Check for an existing VPC ID. If there isn't
        # one, we'll just go ahead and allocate one. 
        if 'vpc' in deploy:
            self.vpc_id = deploy['vpc']
        else:
            self.vpc_id = None
        # The user can also supply specific subnets for both
        # data/compute and connectors. If they aren't supplied,
        # then Ferry will just create them automatically. 
        if 'data_subnet' in deploy:
            self.data_subnet = deploy['data_subnet']
        else:
            self.data_subnet = None
        if 'manage_subnet' in deploy:
            self.manage_subnet = deploy['manage_subnet']
        else:
            self.manage_subnet = None

        # Figure out if we want to place the storage/compute in a public 
        # or private subnet. By default it is a private subnet. Note 
        # that the actual NIC used by the container does not get a 
        # public IP, so it's only for the primary management NIC.
        if 'public' in deploy:
            self.public_data = deploy['public']
            logging.warning("PUBLIC STATUS: " + str(self.public_data))
        else:
            self.public_data = False

        # The NAT image enables private subnets to communicate
        # with the outside world. The user can supply their own
        # image ID, although they probably shouldn't. 
        if 'nat_image' in deploy:
            self.nat_image = deploy['nat_image']
        else:
            # The default image is the official Amazon VPC NAT
            # with HVM (HVM is compatible with more instance types). 
            self.nat_image = "ami-8b6912bb"

        # Initialize the AWS clients.
        self._init_aws_clients()

    def _init_aws_clients(self):
        # We will need to use both EC2 and VPC. 
        self.ec2 = boto.ec2.connect_to_region(self.default_dc,
                                              aws_access_key_id = self.aws_access_key,
                                              aws_secret_access_key = self.aws_secret_key)
        self.vpc = boto.vpc.VPCConnection(aws_access_key_id = self.aws_access_key,
                                          aws_secret_access_key = self.aws_secret_key)
        self.cf = boto.cloudformation.connect_to_region(self.default_dc, 
                                                        aws_access_key_id = self.aws_access_key,
                                                        aws_secret_access_key = self.aws_secret_key)

    def _get_host_key(self):
        """
        Get the location of the private ssh key. 
        """
        p = self.ssh_key.split("/")
        if len(p) == 1:
            return "/ferry/keys/" + self.ssh_key + ".pem"
        else:
            return self.ssh_key + ".pem"

    def _define_address_range(self, num_hosts, starting_address):
        """
        Choose a private address range. 
        """

        # First determine the cidr block. 
        exp = 32 - math.log(num_hosts, 2)

        # Now figure out where to start counting. As a note
        # we reserve one of the networks for management. 
        addr = map(int, starting_address.split("."))
        if num_hosts <= 256:
            slot = 2
        elif num_hosts <= 65536:
            slot = 1
        addr[slot] = len(self.subnets) + 1

        # Now set the gateway IP and address.
        gw = "%d.%d.%d.%d" % (addr[0], addr[1], addr[2], 1)
        cidr = "%d.%d.%d.%d/%d" % (addr[0], addr[1], addr[2], addr[3], exp)

        # Define the allocation pool. 
        start_pool = "%d.%d.%d.%d" % (addr[0], addr[1], addr[2], 2)
        end_pool = "%d.%d.%d.%d" % (addr[0], addr[1], addr[2], num_hosts - 2)

        return cidr, gw, start_pool, end_pool

    def _create_vpc(self, name):
        """
        Create a network (equivalent to VPC). 
        """
        cidr, _, _, _ = self._define_address_range(self.num_network_hosts,
                                                            "10.0.0.0")        
        self.vpc_cidr = cidr.split("/")[0]
        desc = { name : { "Type" : "AWS::EC2::VPC",
                          "Properties" : { "CidrBlock" : cidr }}}
        return desc

    def _create_subnet(self, name, network):
        """
        Create a subnet. The subnet is attached to a network, but
        doesn't have a router associated with it. 
        """

        # Define the subnet range and store the information. 
        cidr, gateway, pool_start, pool_end = self._define_address_range(self.num_subnet_hosts, self.vpc_cidr)
        self.subnets.append( { name : { 'cidr' : cidr,
                                        'gw' : gateway }} )

        plan = { name : { "Type" : "AWS::EC2::Subnet",
                          "Properties" : { "CidrBlock" : cidr,
                                           "AvailabilityZone" : self.default_zone, 
                                           "VpcId" : network}}}

        desc = { name : { "type" : "AWS::EC2::Subnet",
                          "cidr" : cidr }}
        return plan, desc

    def _create_routetable(self, name, subnet, vpc):
        plan = { name : { "Type" : "AWS::EC2::RouteTable",
                          "Properties" : { "VpcId" : vpc}}}
        desc = { name : { "type" : "AWS::EC2::RouteTable",
                          "vpc" : vpc }}
        return plan, desc

    def _create_routeassoc(self, name, table, subnet):
        plan = { name : { "Type" : "AWS::EC2::SubnetRouteTableAssociation",
                          "Properties" : { "SubnetId" : { "Ref" : subnet},
                                           "RouteTableId" : { "Ref" : table } }}}
        desc = { name : { "type" : "AWS::EC2::SubnetRouteTableAssociation",
                          "subnet" : subnet,
                          "table" : table }}
        return plan, desc

    def _create_security_group(self, group_name, network, is_ref, ports, internal, outbound):
        """
        Create and assign a security group to the supplied server. 
        """

        # Determine whether we're creating a VPC that we've just created
        # or one that already exists. 
        if is_ref:
            vpc = { "Ref" : network }
        else:
            vpc = network

        # Create the basic security group. 
        # This only includes SSH. We can later update the group
        # to include additional ports. 
        desc = { group_name : { "Type" : "AWS::EC2::SecurityGroup",
                                "Properties" : { "GroupDescription" : "Ferry firewall rules", 
                                                 "VpcId" : vpc, 
                                                 "SecurityGroupIngress" : [ { "IpProtocol" : "tcp",
                                                                              "CidrIp": "0.0.0.0/0",
                                                                              "FromPort" : "22", 
                                                                              "ToPort" : "22" }]}}}
        # Ports for internal connections. 
        for p in ports:
            min_port = p[0]
            max_port = p[1]
            desc[group_name]["Properties"]["SecurityGroupIngress"].append({ "IpProtocol" : "tcp",
                                                                            "CidrIp": "0.0.0.0/0",
                                                                            "FromPort" : min_port, 
                                                                            "ToPort" : max_port })

        # Additional ports for the security group. These
        # port values can only be accessed from within the same network. 
        for p in internal:
            min_port = p[0]
            max_port = p[1]
            desc[group_name]["Properties"]["SecurityGroupIngress"].append({ "IpProtocol" : "tcp",
                                                                            "CidrIp": self.subnet["cidr"],
                                                                            "FromPort" : min_port,
                                                                            "ToPort" : max_port })

        # Outbound ports. This is not required. If the user
        # doesn't supply egress rules, then all outgoing requests are allowed. 
        if len(outbound) > 0:
            desc[group_name]["Properties"]["SecurityGroupEgress"] = []
        for p in outbound:
            min_port = p[0]
            max_port = p[1]
            desc[group_name]["Properties"]["SecurityGroupEgress"].append({ "IpProtocol" : "tcp",
                                                                           "CidrIp": "0.0.0.0/0",
                                                                           "FromPort" : min_port,
                                                                           "ToPort" : max_port })
        return desc
        
    def _create_server_init(self):
        """
        Create the server init process. These commands are run on the
        host after the host has booted up. 
        """

        # "ip route add default via %s dev eth1 tab 2\n" % data_ip
        # "ip rule add from %s/32 tab 2 priority 600\n" % data_ip

        user_data = {
            "Fn::Base64": {
              "Fn::Join": [
                "",
                  [
                    "#!/bin/bash -v\n",
                    "parted --script /dev/xvdb mklabel gpt\n", 
                    "parted --script /dev/xvdb mkpart primary xfs 0% 100%\n",
                    "mkfs.xfs /dev/xvdb1\n", 
                    "mkdir /ferry/data\n",
                    "mkdir /ferry/keys\n",
                    "mkdir /ferry/containers\n",
                    "mount -o noatime /dev/xvdb1 /ferry/data\n",
                    "export FERRY_SCRATCH=/ferry/data\n", 
                    "export FERRY_DIR=/ferry/master\n",
                    "echo export FERRY_SCRATCH=/ferry/data >> /etc/profile\n", 
                    "echo export FERRY_DIR=/ferry/master >> /etc/profile\n",
                    "export HOME=/root\n",
                    "export USER=root\n",
                    "mkdir /home/ferry/.ssh\n",
                    "cp /home/%s/.ssh/authorized_keys /home/ferry/.ssh/\n" % self.default_user,
                    "cp /home/%s/.ssh/authorized_keys /root/.ssh/\n" % self.default_user,
                    "chown -R ferry:ferry /home/ferry/.ssh\n",
                    "chown -R ferry:ferry /ferry/data\n",
                    "chown -R ferry:ferry /ferry/keys\n",
                    "chown -R ferry:ferry /ferry/containers\n",
                    "ferry server -n\n",
                    "sleep 3\n"
                  ]
              ]
          }}
        return user_data

    def _create_instance(self, name, subnet, image, size, sec_group, user_data):
        """
        Create a new instance
        """

        v = self.data_volume.split(":")
        if v[0] == "ephemeral":
            # The user wants us to use an ephemeral storage.
            # These storage types are potentially faster but
            # do not survive restarts. 
            volume_description = { "DeviceName" : "/dev/sdb",
                                   "VirtualName" : "ephemeral0" }
        else:
            # The user wants us to use EBS storage. These storage
            # types are persistant across restarts and more configurable,
            # but come at a cost of network communication.
            volume_description = { "DeviceName" : "/dev/sdb",
                                   "Ebs" : { "VolumeSize" : "%s" % v[1] } }

        # # Create a secondary NIC for the actual container. 
        # # That way we can still interact with the host.
        # data_nic_name = name + "NIC"
        # data_nic_resource = {data_nic_name : { "Type" : "AWS::EC2::NetworkInterface",
        #                                        "Properties" : {
        #                                            "GroupSet" : [ { "Ref" : sec_group } ],
        #                                            "SourceDestCheck" : False,
        #                                            "SubnetId" : subnet
        #                                        }
        #                   }}
        # attach_name = name + "NICAttach"
        # attach_resource = { attach_name : { "Type" : "AWS::EC2::NetworkInterfaceAttachment",
        #                                     "Properties" : {
        #                                         "DeviceIndex": "1",
        #                                         "InstanceId": { "Ref" : name },
        #                                         "NetworkInterfaceId": { "Ref" : data_nic_name },
        #                                     }
        #                }}

        # Specify the actual instance. 
        instance_resource = { name : { "Type" : "AWS::EC2::Instance",
                                       "Properties" : { "Tags" : [{ "Key" : "Name", "Value" : name }],
                                                        "ImageId" : image,
                                                        "InstanceType" : size,
                                                        "BlockDeviceMappings" : [ volume_description ], 
                                                        "KeyName" : self.ssh_key, 
                                                        "AvailabilityZone" : self.default_zone, 
                                                        "SubnetId" : subnet, 
                                                        "SecurityGroupIds" : [ { "Ref" : sec_group } ] }}}
        desc = { name : { "type" : "AWS::EC2::Instance",
                          "name" : name, 
                          "data_nic" : data_nic_name,
                          "nics" : [] }}

        # Now add the user script. This script is executed after the
        # VM boots up.
        if user_data:
            instance_resource[name]["Properties"]["UserData"] = user_data

        # plan = dict(instance_resource.items() + 
        #             data_nic_resource.items() + 
        #             attach_resource.items())
        plan = instance_resource
        return plan, desc

    def _create_floatingip_plan(self, cluster_uuid, instances):
        """
        Assign an elastic IP to the secondary device of each instance. 
        """
        plan = { "AWSTemplateFormatVersion" : "2010-09-09",
                 "Description" : "Ferry generated CloudFormation plan",
                 "Resources" :  {} }
        desc = {}
        for instance in instances:
            eip_name = "FerryEIP" + instance["name"]
            assoc_name = "FerryEIPAssoc" + instance["name"]
            eip_resource = {
                "Type" : "AWS::EC2::EIP",
                "Properties" : {
                    "Domain" : "vpc"<commit_msg>rm data_nic name<commit_after># Copyright 2014 OpenCore LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import boto
import boto.cloudformation
import boto.ec2
import boto.vpc
from boto.ec2.address import *
from boto.ec2.networkinterface import *
from boto.exception import BotoServerError, EC2ResponseError
from boto.ec2.blockdevicemapping import BlockDeviceType, EBSBlockDeviceType, BlockDeviceMapping
import copy
import ferry.install
import json
import logging
import math
import os
from pymongo import MongoClient
import sys
import time
import yaml

class AWSLauncher(object):
    """
    Launches new instances/networks on an AWS
    cluster and initializes the instances to run Ferry processes. 
    """
    def __init__(self, controller):
        self.name = "AWS launcher"
        self.docker_registry = None
        self.docker_user = None
        
        self.controller = controller
        self.subnets = []
        self.stacks = {}
        self.num_network_hosts = 1024
        self.num_subnet_hosts = 256
        self.vpc_cidr = None

        self._init_app_db()
        self._init_aws_stack()

    def support_proxy(self):
        """
        The AWS does not support proxy mode at the moment, since that makes 
        it difficult to communicate with nodes in a private VPC subnet. 
        """
        return False

    def _init_app_db(self):
        self.mongo = MongoClient(os.environ['MONGODB'], 27017, connectTimeoutMS=6000)
        self.apps = self.mongo['cloud']['aws']

    def _init_aws_stack(self):
        conf = ferry.install.read_ferry_config()
        provider = conf['system']['provider']

        params = conf[provider]['params']
        self.default_dc = params['dc']
        self.default_zone = params['zone']


        # Figure out what disk volume to use for the
        # data storage. If one is not listed, then the
        # system will use EBS. 
        if 'volume' in params:
            self.data_volume = params['volume']
        else:
            self.data_volume = "ebs:8"

        # This gives us information about the image to use
        # for the supplied provider. 
        deploy = conf[provider]['deploy']

        self.default_image = deploy['image']
        self.default_personality = deploy['personality']
        self.default_user = deploy['default-user']
        self.ssh_key = deploy['ssh']
        self.ssh_user = deploy['ssh-user']

        # Make sure that the ssh key is actually present. 
        keypath = self._get_host_key()
        if not os.path.exists(keypath):
            logging.error("could not find ssh key (%s)" % self.ssh_key)
            raise ValueError("Missing ssh keys")

        # Get user credentials
        self.aws_user = deploy['user']
        self.aws_access_key = deploy['access']
        self.aws_secret_key = deploy['secret']
        
        # Check for an existing VPC ID. If there isn't
        # one, we'll just go ahead and allocate one. 
        if 'vpc' in deploy:
            self.vpc_id = deploy['vpc']
        else:
            self.vpc_id = None
        # The user can also supply specific subnets for both
        # data/compute and connectors. If they aren't supplied,
        # then Ferry will just create them automatically. 
        if 'data_subnet' in deploy:
            self.data_subnet = deploy['data_subnet']
        else:
            self.data_subnet = None
        if 'manage_subnet' in deploy:
            self.manage_subnet = deploy['manage_subnet']
        else:
            self.manage_subnet = None

        # Figure out if we want to place the storage/compute in a public 
        # or private subnet. By default it is a private subnet. Note 
        # that the actual NIC used by the container does not get a 
        # public IP, so it's only for the primary management NIC.
        if 'public' in deploy:
            self.public_data = deploy['public']
            logging.warning("PUBLIC STATUS: " + str(self.public_data))
        else:
            self.public_data = False

        # The NAT image enables private subnets to communicate
        # with the outside world. The user can supply their own
        # image ID, although they probably shouldn't. 
        if 'nat_image' in deploy:
            self.nat_image = deploy['nat_image']
        else:
            # The default image is the official Amazon VPC NAT
            # with HVM (HVM is compatible with more instance types). 
            self.nat_image = "ami-8b6912bb"

        # Initialize the AWS clients.
        self._init_aws_clients()

    def _init_aws_clients(self):
        # We will need to use both EC2 and VPC. 
        self.ec2 = boto.ec2.connect_to_region(self.default_dc,
                                              aws_access_key_id = self.aws_access_key,
                                              aws_secret_access_key = self.aws_secret_key)
        self.vpc = boto.vpc.VPCConnection(aws_access_key_id = self.aws_access_key,
                                          aws_secret_access_key = self.aws_secret_key)
        self.cf = boto.cloudformation.connect_to_region(self.default_dc, 
                                                        aws_access_key_id = self.aws_access_key,
                                                        aws_secret_access_key = self.aws_secret_key)

    def _get_host_key(self):
        """
        Get the location of the private ssh key. 
        """
        p = self.ssh_key.split("/")
        if len(p) == 1:
            return "/ferry/keys/" + self.ssh_key + ".pem"
        else:
            return self.ssh_key + ".pem"

    def _define_address_range(self, num_hosts, starting_address):
        """
        Choose a private address range. 
        """

        # First determine the cidr block. 
        exp = 32 - math.log(num_hosts, 2)

        # Now figure out where to start counting. As a note
        # we reserve one of the networks for management. 
        addr = map(int, starting_address.split("."))
        if num_hosts <= 256:
            slot = 2
        elif num_hosts <= 65536:
            slot = 1
        addr[slot] = len(self.subnets) + 1

        # Now set the gateway IP and address.
        gw = "%d.%d.%d.%d" % (addr[0], addr[1], addr[2], 1)
        cidr = "%d.%d.%d.%d/%d" % (addr[0], addr[1], addr[2], addr[3], exp)

        # Define the allocation pool. 
        start_pool = "%d.%d.%d.%d" % (addr[0], addr[1], addr[2], 2)
        end_pool = "%d.%d.%d.%d" % (addr[0], addr[1], addr[2], num_hosts - 2)

        return cidr, gw, start_pool, end_pool

    def _create_vpc(self, name):
        """
        Create a network (equivalent to VPC). 
        """
        cidr, _, _, _ = self._define_address_range(self.num_network_hosts,
                                                            "10.0.0.0")        
        self.vpc_cidr = cidr.split("/")[0]
        desc = { name : { "Type" : "AWS::EC2::VPC",
                          "Properties" : { "CidrBlock" : cidr }}}
        return desc

    def _create_subnet(self, name, network):
        """
        Create a subnet. The subnet is attached to a network, but
        doesn't have a router associated with it. 
        """

        # Define the subnet range and store the information. 
        cidr, gateway, pool_start, pool_end = self._define_address_range(self.num_subnet_hosts, self.vpc_cidr)
        self.subnets.append( { name : { 'cidr' : cidr,
                                        'gw' : gateway }} )

        plan = { name : { "Type" : "AWS::EC2::Subnet",
                          "Properties" : { "CidrBlock" : cidr,
                                           "AvailabilityZone" : self.default_zone, 
                                           "VpcId" : network}}}

        desc = { name : { "type" : "AWS::EC2::Subnet",
                          "cidr" : cidr }}
        return plan, desc

    def _create_routetable(self, name, subnet, vpc):
        plan = { name : { "Type" : "AWS::EC2::RouteTable",
                          "Properties" : { "VpcId" : vpc}}}
        desc = { name : { "type" : "AWS::EC2::RouteTable",
                          "vpc" : vpc }}
        return plan, desc

    def _create_routeassoc(self, name, table, subnet):
        plan = { name : { "Type" : "AWS::EC2::SubnetRouteTableAssociation",
                          "Properties" : { "SubnetId" : { "Ref" : subnet},
                                           "RouteTableId" : { "Ref" : table } }}}
        desc = { name : { "type" : "AWS::EC2::SubnetRouteTableAssociation",
                          "subnet" : subnet,
                          "table" : table }}
        return plan, desc

    def _create_security_group(self, group_name, network, is_ref, ports, internal, outbound):
        """
        Create and assign a security group to the supplied server. 
        """

        # Determine whether we're creating a VPC that we've just created
        # or one that already exists. 
        if is_ref:
            vpc = { "Ref" : network }
        else:
            vpc = network

        # Create the basic security group. 
        # This only includes SSH. We can later update the group
        # to include additional ports. 
        desc = { group_name : { "Type" : "AWS::EC2::SecurityGroup",
                                "Properties" : { "GroupDescription" : "Ferry firewall rules", 
                                                 "VpcId" : vpc, 
                                                 "SecurityGroupIngress" : [ { "IpProtocol" : "tcp",
                                                                              "CidrIp": "0.0.0.0/0",
                                                                              "FromPort" : "22", 
                                                                              "ToPort" : "22" }]}}}
        # Ports for internal connections. 
        for p in ports:
            min_port = p[0]
            max_port = p[1]
            desc[group_name]["Properties"]["SecurityGroupIngress"].append({ "IpProtocol" : "tcp",
                                                                            "CidrIp": "0.0.0.0/0",
                                                                            "FromPort" : min_port, 
                                                                            "ToPort" : max_port })

        # Additional ports for the security group. These
        # port values can only be accessed from within the same network. 
        for p in internal:
            min_port = p[0]
            max_port = p[1]
            desc[group_name]["Properties"]["SecurityGroupIngress"].append({ "IpProtocol" : "tcp",
                                                                            "CidrIp": self.subnet["cidr"],
                                                                            "FromPort" : min_port,
                                                                            "ToPort" : max_port })

        # Outbound ports. This is not required. If the user
        # doesn't supply egress rules, then all outgoing requests are allowed. 
        if len(outbound) > 0:
            desc[group_name]["Properties"]["SecurityGroupEgress"] = []
        for p in outbound:
            min_port = p[0]
            max_port = p[1]
            desc[group_name]["Properties"]["SecurityGroupEgress"].append({ "IpProtocol" : "tcp",
                                                                           "CidrIp": "0.0.0.0/0",
                                                                           "FromPort" : min_port,
                                                                           "ToPort" : max_port })
        return desc
        
    def _create_server_init(self):
        """
        Create the server init process. These commands are run on the
        host after the host has booted up. 
        """

        # "ip route add default via %s dev eth1 tab 2\n" % data_ip
        # "ip rule add from %s/32 tab 2 priority 600\n" % data_ip

        user_data = {
            "Fn::Base64": {
              "Fn::Join": [
                "",
                  [
                    "#!/bin/bash -v\n",
                    "parted --script /dev/xvdb mklabel gpt\n", 
                    "parted --script /dev/xvdb mkpart primary xfs 0% 100%\n",
                    "mkfs.xfs /dev/xvdb1\n", 
                    "mkdir /ferry/data\n",
                    "mkdir /ferry/keys\n",
                    "mkdir /ferry/containers\n",
                    "mount -o noatime /dev/xvdb1 /ferry/data\n",
                    "export FERRY_SCRATCH=/ferry/data\n", 
                    "export FERRY_DIR=/ferry/master\n",
                    "echo export FERRY_SCRATCH=/ferry/data >> /etc/profile\n", 
                    "echo export FERRY_DIR=/ferry/master >> /etc/profile\n",
                    "export HOME=/root\n",
                    "export USER=root\n",
                    "mkdir /home/ferry/.ssh\n",
                    "cp /home/%s/.ssh/authorized_keys /home/ferry/.ssh/\n" % self.default_user,
                    "cp /home/%s/.ssh/authorized_keys /root/.ssh/\n" % self.default_user,
                    "chown -R ferry:ferry /home/ferry/.ssh\n",
                    "chown -R ferry:ferry /ferry/data\n",
                    "chown -R ferry:ferry /ferry/keys\n",
                    "chown -R ferry:ferry /ferry/containers\n",
                    "ferry server -n\n",
                    "sleep 3\n"
                  ]
              ]
          }}
        return user_data

    def _create_instance(self, name, subnet, image, size, sec_group, user_data):
        """
        Create a new instance
        """

        v = self.data_volume.split(":")
        if v[0] == "ephemeral":
            # The user wants us to use an ephemeral storage.
            # These storage types are potentially faster but
            # do not survive restarts. 
            volume_description = { "DeviceName" : "/dev/sdb",
                                   "VirtualName" : "ephemeral0" }
        else:
            # The user wants us to use EBS storage. These storage
            # types are persistant across restarts and more configurable,
            # but come at a cost of network communication.
            volume_description = { "DeviceName" : "/dev/sdb",
                                   "Ebs" : { "VolumeSize" : "%s" % v[1] } }

        # # Create a secondary NIC for the actual container. 
        # # That way we can still interact with the host.
        # data_nic_name = name + "NIC"
        # data_nic_resource = {data_nic_name : { "Type" : "AWS::EC2::NetworkInterface",
        #                                        "Properties" : {
        #                                            "GroupSet" : [ { "Ref" : sec_group } ],
        #                                            "SourceDestCheck" : False,
        #                                            "SubnetId" : subnet
        #                                        }
        #                   }}
        # attach_name = name + "NICAttach"
        # attach_resource = { attach_name : { "Type" : "AWS::EC2::NetworkInterfaceAttachment",
        #                                     "Properties" : {
        #                                         "DeviceIndex": "1",
        #                                         "InstanceId": { "Ref" : name },
        #                                         "NetworkInterfaceId": { "Ref" : data_nic_name },
        #                                     }
        #                }}

        # Specify the actual instance. 
        instance_resource = { name : { "Type" : "AWS::EC2::Instance",
                                       "Properties" : { "Tags" : [{ "Key" : "Name", "Value" : name }],
                                                        "ImageId" : image,
                                                        "InstanceType" : size,
                                                        "BlockDeviceMappings" : [ volume_description ], 
                                                        "KeyName" : self.ssh_key, 
                                                        "AvailabilityZone" : self.default_zone, 
                                                        "SubnetId" : subnet, 
                                                        "SecurityGroupIds" : [ { "Ref" : sec_group } ] }}}
        desc = { name : { "type" : "AWS::EC2::Instance",
                          "name" : name, 
                          "nics" : [] }}

        # Now add the user script. This script is executed after the
        # VM boots up.
        if user_data:
            instance_resource[name]["Properties"]["UserData"] = user_data

        # plan = dict(instance_resource.items() + 
        #             data_nic_resource.items() + 
        #             attach_resource.items())
        plan = instance_resource
        return plan, desc

    def _create_floatingip_plan(self, cluster_uuid, instances):
        """
        Assign an elastic IP to the secondary device of each instance. 
        """
        plan = { "AWSTemplateFormatVersion" : "2010-09-09",
                 "Description" : "Ferry generated CloudFormation plan",
                 "Resources" :  {} }
        desc = {}
        for instance in instances:
            eip_name = "FerryEIP" + instance["name"]
            assoc_name = "FerryEIPAssoc" + instance["name"]
            eip_resource = {
                "Type" : "AWS::EC2::EIP",
                "Properties" : {
                    "Domain" : "vpc"
{'fraction_non_alphanumeric': 0.10062266500622664, 'fraction_numerical': 0.007914764079147641, 'mean_word_length': 1.3474080810705469, 'pattern_counts': {'":': 22, '<': 7, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 8, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 194, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6251328', 'n_tokens_mistral': 9856, 'n_tokens_neox': 9319, 'n_words': 2587}