<commit_before>from __future__ import absolute_import, division, print_function, unicode_literals

import os, sys, json, io, gzip, time, socket, hashlib, uuid
import requests
from warnings import warn
from datetime import datetime, timedelta
from typing import List, Dict, Any

import boto3, botocore.session
from botocore.exceptions import ClientError
from botocore.utils import parse_to_aware_datetime

from ... import logger
from .. import paginate
from ..exceptions import AegeaException
from ..compat import str
from . import clients, resources

def get_ssm_parameter(name):
    return clients.ssm.get_parameter(Name=name)["Parameter"]["Value"]

def locate_ami(distribution, release, architecture):
    """
    Examples::
        locate_ami(distribution="Ubuntu", release="20.04", architecture="amd64")
        locate_ami(distribution="Amazon Linux", release="2", architecture="arm64")
    """
    if distribution == "Amazon Linux" and release == "2":
        if architecture == "amd64":
            architecture = "x86_64"
        ssm_param_name = "/aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-{}-gp2".format(architecture)
        ami_id = clients.ssm.get_parameters(Names=[ssm_param_name])["Parameters"][0]["Value"]
        logger.info("Found %s for %s %s %s", ami_id, distribution, release, architecture)
        return ami_id
    elif distribution == "Ubuntu":
        if architecture == "x86_64":
            architecture = "amd64"
        ssm_param_name = "/aws/service/canonical/ubuntu/{product}/{release}/stable/current/{arch}/hvm/ebs-gp2/ami-id"
        ssm_param_name = ssm_param_name.format(product="server", release=release, arch=architecture)
        ami_id = clients.ssm.get_parameters(Names=[ssm_param_name])["Parameters"][0]["Value"]
        logger.info("Found %s for %s %s %s", ami_id, distribution, release, architecture)
        return ami_id
    raise AegeaException("No AMI found for {} {} {}".format(distribution, release, architecture))

def ensure_vpc():
    """
    If a default VPC exists in the current account/region, return it; otherwise, return the first VPC managed by aegea,
    creating a new one (with an associated IGW, route table, and subnet per AZ) if it does not exist.
    """
    for vpc in resources.ec2.vpcs.filter(Filters=[dict(Name="isDefault", Values=["true"])]):
        break
    else:
        for vpc in resources.ec2.vpcs.filter(Filters=[dict(Name="tag:managedBy", Values=["aegea"])]):
            break
        else:
            from ... import config
            logger.info("Creating VPC with CIDR %s", config.vpc.cidr[ARN.get_region()])
            tags = dict(Name="aegea-vpc", managedBy="aegea")
            tag_spec = dict(ResourceType="vpc", Tags=encode_tags(tags))
            vpc = resources.ec2.create_vpc(CidrBlock=config.vpc.cidr[ARN.get_region()], TagSpecifications=[tag_spec])
            clients.ec2.get_waiter("vpc_available").wait(VpcIds=[vpc.id])
            vpc.modify_attribute(EnableDnsSupport=dict(Value=config.vpc.enable_dns_support))
            vpc.modify_attribute(EnableDnsHostnames=dict(Value=config.vpc.enable_dns_hostnames))
            clients.ec2.associate_vpc_cidr_block(VpcId=vpc.id, AmazonProvidedIpv6CidrBlock=True)

            tags = dict(Name="aegea-igw", managedBy="aegea")
            tag_spec = dict(ResourceType="internet-gateway", Tags=encode_tags(tags))
            internet_gateway = resources.ec2.create_internet_gateway(TagSpecifications=[tag_spec])
            vpc.attach_internet_gateway(InternetGatewayId=internet_gateway.id)
            for route_table in vpc.route_tables.all():
                route_table.create_route(DestinationCidrBlock="0.0.0.0/0", GatewayId=internet_gateway.id)

            tags = dict(Name="aegea-eigw", managedBy="aegea")
            tag_spec = dict(ResourceType="egress-only-internet-gateway", Tags=encode_tags(tags))
            res = clients.ec2.create_egress_only_internet_gateway(VpcId=vpc.id, TagSpecifications=[tag_spec])
            eigw_id = res["EgressOnlyInternetGateway"]["EgressOnlyInternetGatewayId"]
            for route_table in vpc.route_tables.all():
                route_table.create_route(DestinationIpv6CidrBlock="::/0", EgressOnlyInternetGatewayId=eigw_id)

            ensure_subnet(vpc)
    return vpc

def availability_zones():
    for az in clients.ec2.describe_availability_zones()["AvailabilityZones"]:
        yield az["ZoneName"]

def ensure_subnet(vpc, availability_zone=None):
    """
    Find and return a subnet in the given VPC, or create one subnet per AZ and return one of them if no suitable subnet
    is found.
    """
    # FIXME: if config.vpc.map_public_ip_on_launch is set, ignore subnets that don't have that attribute set<commit_msg>Route ipv6 traffic in aegea VPCs<commit_after>from __future__ import absolute_import, division, print_function, unicode_literals

import os, sys, json, io, gzip, time, socket, hashlib, uuid, ipaddress
import requests
from warnings import warn
from datetime import datetime, timedelta
from typing import List, Dict, Any

import boto3, botocore.session
from botocore.exceptions import ClientError
from botocore.utils import parse_to_aware_datetime

from ... import logger
from .. import paginate
from ..exceptions import AegeaException
from ..compat import str
from . import clients, resources

def get_ssm_parameter(name):
    return clients.ssm.get_parameter(Name=name)["Parameter"]["Value"]

def locate_ami(distribution, release, architecture):
    """
    Examples::
        locate_ami(distribution="Ubuntu", release="20.04", architecture="amd64")
        locate_ami(distribution="Amazon Linux", release="2", architecture="arm64")
    """
    if distribution == "Amazon Linux" and release == "2":
        if architecture == "amd64":
            architecture = "x86_64"
        ssm_param_name = "/aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-{}-gp2".format(architecture)
        ami_id = clients.ssm.get_parameters(Names=[ssm_param_name])["Parameters"][0]["Value"]
        logger.info("Found %s for %s %s %s", ami_id, distribution, release, architecture)
        return ami_id
    elif distribution == "Ubuntu":
        if architecture == "x86_64":
            architecture = "amd64"
        ssm_param_name = "/aws/service/canonical/ubuntu/{product}/{release}/stable/current/{arch}/hvm/ebs-gp2/ami-id"
        ssm_param_name = ssm_param_name.format(product="server", release=release, arch=architecture)
        ami_id = clients.ssm.get_parameters(Names=[ssm_param_name])["Parameters"][0]["Value"]
        logger.info("Found %s for %s %s %s", ami_id, distribution, release, architecture)
        return ami_id
    raise AegeaException("No AMI found for {} {} {}".format(distribution, release, architecture))

def ensure_vpc():
    """
    If a default VPC exists in the current account/region, return it; otherwise, return the first VPC managed by aegea,
    creating a new one (with an associated IGW, route table, and subnet per AZ) if it does not exist.
    """
    for vpc in resources.ec2.vpcs.filter(Filters=[dict(Name="isDefault", Values=["true"])]):
        break
    else:
        for vpc in resources.ec2.vpcs.filter(Filters=[dict(Name="tag:managedBy", Values=["aegea"])]):
            break
        else:
            from ... import config
            logger.info("Creating VPC with CIDR %s", config.vpc.cidr[ARN.get_region()])
            tags = dict(Name="aegea-vpc", managedBy="aegea")
            tag_spec = dict(ResourceType="vpc", Tags=encode_tags(tags))
            vpc = resources.ec2.create_vpc(CidrBlock=config.vpc.cidr[ARN.get_region()], TagSpecifications=[tag_spec])
            clients.ec2.get_waiter("vpc_available").wait(VpcIds=[vpc.id])
            vpc.modify_attribute(EnableDnsSupport=dict(Value=config.vpc.enable_dns_support))
            vpc.modify_attribute(EnableDnsHostnames=dict(Value=config.vpc.enable_dns_hostnames))
            clients.ec2.associate_vpc_cidr_block(VpcId=vpc.id, AmazonProvidedIpv6CidrBlock=True)

            ensure_subnet(vpc)
            vpc_cidr_block = vpc.ipv6_cidr_block_association_set[0]["Ipv6CidrBlock"]
            subnet_cidr_blocks = list(ipaddress.ip_network(vpc_cidr_block).subnets(new_prefix=64))
            for i, subnet in enumerate(vpc.subnets.all()):
                logger.info("Assigning IPv6 CIDR block %s to %s", subnet_cidr_blocks[i], subnet)
                clients.ec2.associate_subnet_cidr_block(SubnetId=subnet.id, Ipv6CidrBlock=str(subnet_cidr_blocks[i]))

            tags = dict(Name="aegea-igw", managedBy="aegea")
            tag_spec = dict(ResourceType="internet-gateway", Tags=encode_tags(tags))
            internet_gateway = resources.ec2.create_internet_gateway(TagSpecifications=[tag_spec])
            vpc.attach_internet_gateway(InternetGatewayId=internet_gateway.id)
            for route_table in vpc.route_tables.all():
                route_table.create_route(DestinationCidrBlock="0.0.0.0/0", GatewayId=internet_gateway.id)

            tags = dict(Name="aegea-eigw", managedBy="aegea")
            tag_spec = dict(ResourceType="egress-only-internet-gateway", Tags=encode_tags(tags))
            res = clients.ec2.create_egress_only_internet_gateway(VpcId=vpc.id, TagSpecifications=[tag_spec])
            eigw_id = res["EgressOnlyInternetGateway"]["EgressOnlyInternetGatewayId"]
            for route_table in vpc.route_tables.all():
                route_table.create_route(DestinationIpv6CidrBlock="::/0", EgressOnlyInternetGatewayId=eigw_id)
    return vpc

def availability_zones():
    for az in clients.ec2.describe_availability_zones()["AvailabilityZones"]:
        yield az["ZoneName"]

def ensure_subnet(vpc, availability_zone=None):
    """
    Find and return a subnet in the given VPC, or create one subnet per AZ and return one of them if no suitable subnet
    is found.
    """
    # FIXME: if config.vpc.map_public_ip_on_launch is set, ignore subnets that don't have that attribute set
{'fraction_non_alphanumeric': 0.11659283125251711, 'fraction_numerical': 0.009766411598872331, 'mean_word_length': 3.770893371757925, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '209564', 'n_tokens_mistral': 3233, 'n_tokens_neox': 3171, 'n_words': 635}