<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import commands
import os
import shutil

import boto
import boto.ec2
from boto.ec2.connection import EC2Connection

from config import *
import instances as inst
import util
import time


def _connect():
    """Return connection to AWS"""

    return boto.vpc.connect_to_region(AWS_REGION)



def _create_vpc(conn):
    """Create VPC in AWS"""

    print 'Creating VPC: {0} ...'.format(VPC_TAG)

    vpc = conn.create_vpc(VPC_CIDR)
    conn.modify_vpc_attribute(vpc.id, enable_dns_support=True)
    conn.modify_vpc_attribute(vpc.id, enable_dns_hostnames=True)
    vpc.add_tag('Name', VPC_TAG)

    return vpc


def delete_vpc(user_vpc):
    """Delete AWS VPC"""

    print "Deleting VPC: {0} ...".format(TRAINER + '-{0}'.format(VPC))
    conn = boto.vpc.connect_to_region(AWS_REGION)

    subnets = conn.get_all_subnets(filters = {'vpc-id': user_vpc.id})
    igws = conn.get_all_internet_gateways(filters={'attachment.vpc-id': user_vpc.id})

    if subnets:
        for subnet in subnets:
            conn.delete_subnet(subnet.id)
    if igws:
        for igw in igws:
            conn.detach_internet_gateway(igw.id, user_vpc.id)
            conn.delete_internet_gateway(igw.id)

    conn.delete_vpc(user_vpc.id)


def _create_gateway(conn, vpc):
    """Create AWS IGW"""

    print 'Creating gateway: {0} ...'.format(IGW)

    gateway = conn.create_internet_gateway()
    # state polling copied from the following
    # http://stackoverflow.com/questions/22263300/aws-boto-how-to-refresh-subnet-state-after-creating-it-its-stuck-in-pending
    # while gateway.state == 'pending':
    #     """Waiting for AWS gateway creation to complete"""
    #     gateways = conn.get_all_internet_gateways()
    #     for item in gateways:
    #         """Getting AWS gateway status"""
    #         if item.id == gateway.id:
    #             gateway.state = item.state
    #             time.sleep(5)

    # add an artificial delay until we can poll an available state
    time.sleep(5)

    gateway.add_tag('Name', IGW)
    conn.attach_internet_gateway(gateway.id, vpc.id)

    return gateway


def _create_route_table(conn, vpc, gateway):
    """Create AWS route table"""

    print 'Creating route table: {0} ...'.format(ROUTE_TABLE)

    route_table = conn.get_all_route_tables(filters = {'vpc-id':vpc.id})
    route = conn.create_route(route_table[0].id, '0.0.0.0/0', gateway.id)
    route_table[0].add_tag('Name', ROUTE_TABLE)

    return route_table
<commit_msg>for potential subnet and gateway creation races conditions add polling before continuing<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import commands
import os
import shutil

import boto
import boto.ec2
from boto.ec2.connection import EC2Connection

from config import *
import instances as inst
import util
import time


def _connect():
    """Return connection to AWS"""

    return boto.vpc.connect_to_region(AWS_REGION)



def _create_vpc(conn):
    """Create VPC in AWS"""

    print 'Creating VPC: {0} ...'.format(VPC_TAG)

    vpc = conn.create_vpc(VPC_CIDR)
    conn.modify_vpc_attribute(vpc.id, enable_dns_support=True)
    conn.modify_vpc_attribute(vpc.id, enable_dns_hostnames=True)
    vpc.add_tag('Name', VPC_TAG)

    return vpc


def delete_vpc(user_vpc):
    """Delete AWS VPC"""

    print "Deleting VPC: {0} ...".format(TRAINER + '-{0}'.format(VPC))
    conn = boto.vpc.connect_to_region(AWS_REGION)

    subnets = conn.get_all_subnets(filters = {'vpc-id': user_vpc.id})
    igws = conn.get_all_internet_gateways(filters={'attachment.vpc-id': user_vpc.id})

    if subnets:
        for subnet in subnets:
            conn.delete_subnet(subnet.id)
    if igws:
        for igw in igws:
            conn.detach_internet_gateway(igw.id, user_vpc.id)
            conn.delete_internet_gateway(igw.id)

    conn.delete_vpc(user_vpc.id)


def _create_gateway(conn, vpc):
    """Create AWS IGW"""

    print 'Creating gateway: {0} ...'.format(IGW)

    gateway = conn.create_internet_gateway()
    gateway.add_tag('Name', IGW)

    attempts=0
    while True:
        try:
            # workaround for race where the gateway may not be ready yet
            attempts += 1
            conn.attach_internet_gateway(gateway.id, vpc.id)
            break
        except boto.exception.EC2ResponseError:
            time.sleep(1)
            if attempts > 5:
                raise

    return gateway


def _create_route_table(conn, vpc, gateway):
    """Create AWS route table"""

    print 'Creating route table: {0} ...'.format(ROUTE_TABLE)

    route_table = conn.get_all_route_tables(filters = {'vpc-id':vpc.id})
    route = conn.create_route(route_table[0].id, '0.0.0.0/0', gateway.id)
    route_table[0].add_tag('Name', ROUTE_TABLE)

    return route_table

{'fraction_non_alphanumeric': 0.12193091968372867, 'fraction_numerical': 0.009779442363712027, 'mean_word_length': 3.0702794242167655, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5476704', 'n_tokens_mistral': 1769, 'n_tokens_neox': 1687, 'n_words': 359}