<commit_before>"""
trop.py is a module that uses the Troposphere library to build up
a AWS Cloudformation (CFN) template dynamically, using values from
the projects file and a bunch of sensible defaults.

It's job is to return the correct CFN JSON given a dictionary of
data called a `context`.

`cfngen.py` is in charge of constructing this data struct and writing
it to the correct file etc."""

from collections import OrderedDict
from os.path import join
from . import config, utils, bvars, aws
from .config import ConfigurationError
from troposphere import GetAtt, Output, Ref, Template, ec2, rds, sns, sqs, Base64, route53, Parameter, Tags
from troposphere import s3, cloudfront, elasticloadbalancing as elb, elasticache
from functools import partial
from .utils import ensure, subdict, lmap, isstr, deepcopy
import logging

LOG = logging.getLogger(__name__)

# TODO: embarassing code. some of these constants should be pulled form project config or given better names.

SECURITY_GROUP_TITLE = "StackSecurityGroup"
SECURITY_GROUP_ELB_TITLE = "ELBSecurityGroup"
EC2_TITLE = 'EC2Instance1'
EC2_TITLE_NODE = 'EC2Instance%d'
ELB_TITLE = 'ElasticLoadBalancer'
RDS_TITLE = "AttachedDB"
RDS_SG_ID = "DBSecurityGroup"
RDS_DB_PG = "RDSDBParameterGroup"
DBSUBNETGROUP_TITLE = 'AttachedDBSubnet'
EXT_TITLE = "ExtraStorage%s"
EXT_MP_TITLE = "MountPoint%s"
R53_EXT_TITLE = "ExtDNS"
R53_EXT_TITLE_NODE = "ExtDNS%s"
R53_INT_TITLE = "IntDNS"
R53_INT_TITLE_NODE = "IntDNS%s"
R53_CDN_TITLE = "CloudFrontCDNDNS%s"
R53_CNAME_TITLE = "CnameDNS%s"
R53_FASTLY_TITLE = "FastlyDNS%s"
CLOUDFRONT_TITLE = 'CloudFrontCDN'
# from http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-route53-aliastarget.html
CLOUDFRONT_HOSTED_ZONE_ID = 'Z2FDTNDATAQYW2'
CLOUDFRONT_ERROR_ORIGIN_ID = 'ErrorsOrigin'
ELASTICACHE_TITLE = 'ElastiCache%s'
ELASTICACHE_SECURITY_GROUP_TITLE = 'ElastiCacheSecurityGroup'
ELASTICACHE_SUBNET_GROUP_TITLE = 'ElastiCacheSubnetGroup'
ELASTICACHE_PARAMETER_GROUP_TITLE = 'ElastiCacheParameterGroup'

KEYPAIR = "KeyName"

def _read_script(script_filename):
    path = join(config.SCRIPTS_PATH, script_filename)
    with open(path, 'r') as fp:
        return fp.read()

def _convert_ports_to_dictionary(ports):
    if isinstance(ports, list):
        ports_map = OrderedDict()
        for p in ports:
            if isinstance(p, int):
                ports_map[p] = {}
            elif isinstance(p, dict):
                ensure(len(p) == 1, "Single port definition cannot contain more than one value")
                from_port = list(p.keys())[0]
                configuration = list(p.values())[0]
                ports_map[from_port] = configuration
            else:
                raise ValueError("Invalid port definition: %s" % (p,))
    elif isinstance(ports, dict):
        ports_map = OrderedDict()
        for p, configuration in ports.items():
            if isinstance(configuration, bool):
                # temporary
                ports_map[p] = {}
            elif isinstance(configuration, int):
                ports_map[p] = {'guest': configuration}
            elif isinstance(configuration, OrderedDict):
                ports_map[p] = configuration
            else:
                raise ValueError("Invalid port definition: %s => %s" % (p, configuration))
    else:
        raise ValueError("Invalid ports definition: %s" % ports)

    return ports_map

def merge_ports(ports, another):
    ports = OrderedDict(ports)
    ports.update(another)
    return ports

def convert_ports_dict_to_troposphere(ports):
    def _port_to_dict(port, configuration):
        return ec2.SecurityGroupRule(**{
            'FromPort': port,
            'ToPort': configuration.get('guest', port),
            'IpProtocol': configuration.get('protocol', 'tcp'),
            'CidrIp': configuration.get('cidr-ip', '0.0.0.0/0'),
        })
    return [_port_to_dict(port, configuration) for port, configuration in ports.items()]

def security_group(group_id, vpc_id, ingress_data, description=""):
    return ec2.SecurityGroup(group_id, **{
        'GroupDescription': description or 'security group',
        'VpcId': vpc_id,
        'SecurityGroupIngress': convert_ports_dict_to_troposphere(ingress_data),
    })

def ec2_security(context):
    ec2_port_data = context['ec2']['ports']
    ec2_ports = _convert_ports_to_dictionary(ec2_port_data)
    security_group_data = context['ec2']['security-group'].get('ports', {})
    security_group_ports = _convert_ports_to_dictionary(security_group_data)
    ingress = merge_ports(ec2_ports, security_group_ports)

    return security_group(
        SECURITY_GROUP_TITLE,
        context['aws']['vpc-id'],
        ingress
    )

def rds_security(context):
    """returns a security group for the rds instance.

    this security group only allows access within the subnet, not because of the ip address range but because this is dealt with in the subnet configuration"""
    engine_ports = {
        'postgres': 5432,
        'mysql': 3306
    }
    ingress_data = [engine_ports[context['rds']['engine'].lower()]]
    ingress_ports = _convert_ports_to_dictionary(ingress_data)
    return security_group("VPCSecurityGroup",
                          context['aws']['vpc-id'],
                          ingress_ports,
                          "RDS DB security group")

#
#
#

def instance_tags(context, node=None):
    # NOTE: RDS and Elasticache instances also call this function
    tags = aws.generic_tags(context)
    if node:
        # this instance is part of a cluster
        tags.update({
            'Name': '%s--%d' % (context['stackname'], node), # "journal--prod--1"
            'Node': node, # "1"
        })
    return [ec2.Tag(key, str(value)) for key, value in tags.items()]

def elb_tags(context):
    tags = aws.generic_tags(context)
    tags.update({
        'Name': '%s--elb' % context['stackname'], # "journal--prod--elb"
    })
    return [ec2.Tag(key, value) for key, value in tags.items()]

def mkoutput(title, desc, val):
    if isinstance(val, tuple):
        val = GetAtt(val[0], val[1])
    return Output(title, Description=desc, Value=val)

#
#
#

def build_vars(context, node):
    """returns a subset of given context data with some extra node information
    that will be encoded and stored on the ec2 instance at /etc/build-vars.json.b64"""
    buildvars = deepcopy(context)

    # preseve some of the project data. all of it is too much
    keepers = [
        'formula-repo',
        'formula-dependencies'
    ]
    buildvars['project'] = subdict(buildvars['project'], keepers)

    buildvars['node'] = node
    buildvars['nodename'] = "%s--%s" % (context['stackname'], node) # "journal--prod--1"

    return buildvars

def ec2instance(context, node):
    lu = partial(utils.lu, context)
    buildvars = build_vars(context, node)
    buildvars_serialization = bvars.encode_bvars(buildvars)

    odd = node % 2 == 1
    subnet_id = lu('aws.subnet-id') if odd else lu('aws.redundant-subnet-id')
    clean_server = _read_script('.clean-server.sh.fragment')
    project_ec2 = {
        "ImageId": lu('ec2.ami'),
        "InstanceType": lu('ec2.type'), # "t2.small", "m1.medium", etc
        "KeyName": Ref(KEYPAIR),
        "SecurityGroupIds": [Ref(SECURITY_GROUP_TITLE)],
        "SubnetId": subnet_id, # "subnet-1d4eb46a"
        "Tags": instance_tags(context, node),

        # https://alestic.com/2010/12/ec2-user-data-output/
        "UserData": Base64("""#!/bin/bash
set -x
exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1
echo %s > /etc/build-vars.json.b64

%s""" % (buildvars_serialization, clean_server)),
    }

    if lu('ec2.cpu-credits') != 'standard':
        project_ec2["CreditSpecification"] = ec2.CreditSpecification(
            CPUCredits=lu('ec2.cpu-credits'),
        )

    # TODO: 'root' is undefined in the project definition
    # TODO: extract in private method?
    if context['ec2'].get('root'):
        project_ec2['BlockDeviceMappings'] = [{
            'DeviceName': '/dev/sda1',
            'Ebs': {
                'VolumeSize': context['ec2']['root']['size'],
                'VolumeType': context['ec2']['root'].get('type', 'standard'),<commit_msg>Note down we can't give a name to root volumes<commit_after>"""
trop.py is a module that uses the Troposphere library to build up
a AWS Cloudformation (CFN) template dynamically, using values from
the projects file and a bunch of sensible defaults.

It's job is to return the correct CFN JSON given a dictionary of
data called a `context`.

`cfngen.py` is in charge of constructing this data struct and writing
it to the correct file etc."""

from collections import OrderedDict
from os.path import join
from . import config, utils, bvars, aws
from .config import ConfigurationError
from troposphere import GetAtt, Output, Ref, Template, ec2, rds, sns, sqs, Base64, route53, Parameter, Tags
from troposphere import s3, cloudfront, elasticloadbalancing as elb, elasticache
from functools import partial
from .utils import ensure, subdict, lmap, isstr, deepcopy
import logging

LOG = logging.getLogger(__name__)

# TODO: embarassing code. some of these constants should be pulled form project config or given better names.

SECURITY_GROUP_TITLE = "StackSecurityGroup"
SECURITY_GROUP_ELB_TITLE = "ELBSecurityGroup"
EC2_TITLE = 'EC2Instance1'
EC2_TITLE_NODE = 'EC2Instance%d'
ELB_TITLE = 'ElasticLoadBalancer'
RDS_TITLE = "AttachedDB"
RDS_SG_ID = "DBSecurityGroup"
RDS_DB_PG = "RDSDBParameterGroup"
DBSUBNETGROUP_TITLE = 'AttachedDBSubnet'
EXT_TITLE = "ExtraStorage%s"
EXT_MP_TITLE = "MountPoint%s"
R53_EXT_TITLE = "ExtDNS"
R53_EXT_TITLE_NODE = "ExtDNS%s"
R53_INT_TITLE = "IntDNS"
R53_INT_TITLE_NODE = "IntDNS%s"
R53_CDN_TITLE = "CloudFrontCDNDNS%s"
R53_CNAME_TITLE = "CnameDNS%s"
R53_FASTLY_TITLE = "FastlyDNS%s"
CLOUDFRONT_TITLE = 'CloudFrontCDN'
# from http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-route53-aliastarget.html
CLOUDFRONT_HOSTED_ZONE_ID = 'Z2FDTNDATAQYW2'
CLOUDFRONT_ERROR_ORIGIN_ID = 'ErrorsOrigin'
ELASTICACHE_TITLE = 'ElastiCache%s'
ELASTICACHE_SECURITY_GROUP_TITLE = 'ElastiCacheSecurityGroup'
ELASTICACHE_SUBNET_GROUP_TITLE = 'ElastiCacheSubnetGroup'
ELASTICACHE_PARAMETER_GROUP_TITLE = 'ElastiCacheParameterGroup'

KEYPAIR = "KeyName"

def _read_script(script_filename):
    path = join(config.SCRIPTS_PATH, script_filename)
    with open(path, 'r') as fp:
        return fp.read()

def _convert_ports_to_dictionary(ports):
    if isinstance(ports, list):
        ports_map = OrderedDict()
        for p in ports:
            if isinstance(p, int):
                ports_map[p] = {}
            elif isinstance(p, dict):
                ensure(len(p) == 1, "Single port definition cannot contain more than one value")
                from_port = list(p.keys())[0]
                configuration = list(p.values())[0]
                ports_map[from_port] = configuration
            else:
                raise ValueError("Invalid port definition: %s" % (p,))
    elif isinstance(ports, dict):
        ports_map = OrderedDict()
        for p, configuration in ports.items():
            if isinstance(configuration, bool):
                # temporary
                ports_map[p] = {}
            elif isinstance(configuration, int):
                ports_map[p] = {'guest': configuration}
            elif isinstance(configuration, OrderedDict):
                ports_map[p] = configuration
            else:
                raise ValueError("Invalid port definition: %s => %s" % (p, configuration))
    else:
        raise ValueError("Invalid ports definition: %s" % ports)

    return ports_map

def merge_ports(ports, another):
    ports = OrderedDict(ports)
    ports.update(another)
    return ports

def convert_ports_dict_to_troposphere(ports):
    def _port_to_dict(port, configuration):
        return ec2.SecurityGroupRule(**{
            'FromPort': port,
            'ToPort': configuration.get('guest', port),
            'IpProtocol': configuration.get('protocol', 'tcp'),
            'CidrIp': configuration.get('cidr-ip', '0.0.0.0/0'),
        })
    return [_port_to_dict(port, configuration) for port, configuration in ports.items()]

def security_group(group_id, vpc_id, ingress_data, description=""):
    return ec2.SecurityGroup(group_id, **{
        'GroupDescription': description or 'security group',
        'VpcId': vpc_id,
        'SecurityGroupIngress': convert_ports_dict_to_troposphere(ingress_data),
    })

def ec2_security(context):
    ec2_port_data = context['ec2']['ports']
    ec2_ports = _convert_ports_to_dictionary(ec2_port_data)
    security_group_data = context['ec2']['security-group'].get('ports', {})
    security_group_ports = _convert_ports_to_dictionary(security_group_data)
    ingress = merge_ports(ec2_ports, security_group_ports)

    return security_group(
        SECURITY_GROUP_TITLE,
        context['aws']['vpc-id'],
        ingress
    )

def rds_security(context):
    """returns a security group for the rds instance.

    this security group only allows access within the subnet, not because of the ip address range but because this is dealt with in the subnet configuration"""
    engine_ports = {
        'postgres': 5432,
        'mysql': 3306
    }
    ingress_data = [engine_ports[context['rds']['engine'].lower()]]
    ingress_ports = _convert_ports_to_dictionary(ingress_data)
    return security_group("VPCSecurityGroup",
                          context['aws']['vpc-id'],
                          ingress_ports,
                          "RDS DB security group")

#
#
#

def instance_tags(context, node=None):
    # NOTE: RDS and Elasticache instances also call this function
    tags = aws.generic_tags(context)
    if node:
        # this instance is part of a cluster
        tags.update({
            'Name': '%s--%d' % (context['stackname'], node), # "journal--prod--1"
            'Node': node, # "1"
        })
    return [ec2.Tag(key, str(value)) for key, value in tags.items()]

def elb_tags(context):
    tags = aws.generic_tags(context)
    tags.update({
        'Name': '%s--elb' % context['stackname'], # "journal--prod--elb"
    })
    return [ec2.Tag(key, value) for key, value in tags.items()]

def mkoutput(title, desc, val):
    if isinstance(val, tuple):
        val = GetAtt(val[0], val[1])
    return Output(title, Description=desc, Value=val)

#
#
#

def build_vars(context, node):
    """returns a subset of given context data with some extra node information
    that will be encoded and stored on the ec2 instance at /etc/build-vars.json.b64"""
    buildvars = deepcopy(context)

    # preseve some of the project data. all of it is too much
    keepers = [
        'formula-repo',
        'formula-dependencies'
    ]
    buildvars['project'] = subdict(buildvars['project'], keepers)

    buildvars['node'] = node
    buildvars['nodename'] = "%s--%s" % (context['stackname'], node) # "journal--prod--1"

    return buildvars

def ec2instance(context, node):
    lu = partial(utils.lu, context)
    buildvars = build_vars(context, node)
    buildvars_serialization = bvars.encode_bvars(buildvars)

    odd = node % 2 == 1
    subnet_id = lu('aws.subnet-id') if odd else lu('aws.redundant-subnet-id')
    clean_server = _read_script('.clean-server.sh.fragment')
    project_ec2 = {
        "ImageId": lu('ec2.ami'),
        "InstanceType": lu('ec2.type'), # "t2.small", "m1.medium", etc
        "KeyName": Ref(KEYPAIR),
        "SecurityGroupIds": [Ref(SECURITY_GROUP_TITLE)],
        "SubnetId": subnet_id, # "subnet-1d4eb46a"
        "Tags": instance_tags(context, node),

        # https://alestic.com/2010/12/ec2-user-data-output/
        "UserData": Base64("""#!/bin/bash
set -x
exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1
echo %s > /etc/build-vars.json.b64

%s""" % (buildvars_serialization, clean_server)),
    }

    if lu('ec2.cpu-credits') != 'standard':
        project_ec2["CreditSpecification"] = ec2.CreditSpecification(
            CPUCredits=lu('ec2.cpu-credits'),
        )

    # TODO: 'root' is undefined in the project definition
    # TODO: extract in private method?
    if context['ec2'].get('root'):
        project_ec2['BlockDeviceMappings'] = [{
            'DeviceName': '/dev/sda1',
            'Ebs': {
                'VolumeSize': context['ec2']['root']['size'],
                'VolumeType': context['ec2']['root'].get('type', 'standard'),
                # unfortunately root volumes do not support Tags:
                # https://blog.cloudability.com/two-solutions-for-tagging-elusive-aws-ebs-volumes/
{'fraction_non_alphanumeric': 0.1198749549116268, 'fraction_numerical': 0.011903330527834556, 'mean_word_length': 3.306238674605229, 'pattern_counts': {'":': 14, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '479816', 'n_tokens_mistral': 5457, 'n_tokens_neox': 5311, 'n_words': 1278}