<commit_before>from boto.ec2 import connect_to_region
from ConfigParser import SafeConfigParser
import os, time

USERNAME = 'ec2-user'
AMI_ID = 'ami-c7c99982'
INSTANCE_TYPE = 't1.micro'

EC2_REGION = 'us-west-1'
EC2_SSH_KEY_NAME = 'us-west'
EC2_SSH_KEY_PATH = os.path.join(os.getenv('HOME'), '.aws', 'us-west.pem')

INSTANCES_FILE = os.path.join(os.getenv('HOME'), '.aws', 'aws_instances.ini')
S3_DROP_BOX_FILE = os.path.join(os.getenv('HOME'), '.s3dropbox')

class Node:
    def __init__(self, public_dns_name):
        self.hostname = public_dns_name
        self.ssh_key_file = EC2_SSH_KEY_PATH
        self.ssh_user = USERNAME

def provision_with_boto(name):
    config = read_config()
    if config.has_section(name):
        return Node(config.get(name, 'public_dns_name'))
    else:
        return Node(run_instance(name))

def connect():
    properties = {}
    with open(S3_DROP_BOX_FILE) as prop_file:
        for line in prop_file:
            text = line.strip()
            if len(text) > 0 and not text.startswith('#'):
                key, sep, val = text.partition('=')
                properties[key.strip()] = val.strip()
    access_key = properties['AMAZON_ACCESS_KEY_ID']
    secret_key = properties['AMAZON_SECRET_ACCESS_KEY']
    return connect_to_region(EC2_REGION, aws_access_key_id=access_key, aws_secret_access_key=secret_key)

def run_instance(name):
    conn = connect()
    res = conn.run_instances(AMI_ID, key_name=EC2_SSH_KEY_NAME, instance_type=INSTANCE_TYPE)
    instance = res.instances[0]

    print "Waiting for", name, "to start ..."
    time.sleep(10)
    instance.update()

    while instance.state != 'running':
        time.sleep(10)
        instance.update()

    conn.create_tags([instance.id], {'Name': name})

    config = read_config()
    config.add_section(name)
    config.set(name, 'instance_id', instance.id)
    config.set(name, 'public_dns_name', instance.public_dns_name)
    write_config(config)

    return instance.public_dns_name

def terminate_instance(name):
    config = read_config()
    instance_id = config.get(name, 'instance_id')

    conn = connect()
    conn.terminate_instances([instance_id])

    config.remove_section(name)
    write_config(config)

def terminate_all_instances():
    conn = connect()
    for reservation in conn.get_all_instances():
        for instance in reservation.instances:
            instance.terminate()

def write_config(config):
    with open(INSTANCES_FILE, 'w') as fp:
        config.write(fp)

def read_config():
    config = SafeConfigParser()
    if os.path.isfile(INSTANCES_FILE):
        with open(INSTANCES_FILE) as fp:
            config.readfp(fp)
    return config<commit_msg>terminate_all_instances should remove the instances config file<commit_after>from boto.ec2 import connect_to_region
from ConfigParser import SafeConfigParser
import os, time

USERNAME = 'ec2-user'
AMI_ID = 'ami-c7c99982'
INSTANCE_TYPE = 't1.micro'

EC2_REGION = 'us-west-1'
EC2_SSH_KEY_NAME = 'us-west'
EC2_SSH_KEY_PATH = os.path.join(os.getenv('HOME'), '.aws', 'us-west.pem')

INSTANCES_FILE = os.path.join(os.getenv('HOME'), '.aws', 'aws_instances.ini')
S3_DROP_BOX_FILE = os.path.join(os.getenv('HOME'), '.s3dropbox')

class Node:
    def __init__(self, public_dns_name):
        self.hostname = public_dns_name
        self.ssh_key_file = EC2_SSH_KEY_PATH
        self.ssh_user = USERNAME

def provision_with_boto(name):
    config = read_config()
    if config.has_section(name):
        return Node(config.get(name, 'public_dns_name'))
    else:
        return Node(run_instance(name))

def connect():
    properties = {}
    with open(S3_DROP_BOX_FILE) as prop_file:
        for line in prop_file:
            text = line.strip()
            if len(text) > 0 and not text.startswith('#'):
                key, sep, val = text.partition('=')
                properties[key.strip()] = val.strip()
    access_key = properties['AMAZON_ACCESS_KEY_ID']
    secret_key = properties['AMAZON_SECRET_ACCESS_KEY']
    return connect_to_region(EC2_REGION, aws_access_key_id=access_key, aws_secret_access_key=secret_key)

def run_instance(name):
    conn = connect()
    res = conn.run_instances(AMI_ID, key_name=EC2_SSH_KEY_NAME, instance_type=INSTANCE_TYPE)
    instance = res.instances[0]

    print "Waiting for", name, "to start ..."
    time.sleep(10)
    instance.update()

    while instance.state != 'running':
        time.sleep(10)
        instance.update()

    conn.create_tags([instance.id], {'Name': name})

    config = read_config()
    config.add_section(name)
    config.set(name, 'instance_id', instance.id)
    config.set(name, 'public_dns_name', instance.public_dns_name)
    write_config(config)

    return instance.public_dns_name

def terminate_instance(name):
    config = read_config()
    instance_id = config.get(name, 'instance_id')

    conn = connect()
    conn.terminate_instances([instance_id])

    config.remove_section(name)
    write_config(config)

def terminate_all_instances():
    conn = connect()
    for reservation in conn.get_all_instances():
        for instance in reservation.instances:
            instance.terminate()
    os.remove(INSTANCES_FILE)

def write_config(config):
    with open(INSTANCES_FILE, 'w') as fp:
        config.write(fp)

def read_config():
    config = SafeConfigParser()
    if os.path.isfile(INSTANCES_FILE):
        with open(INSTANCES_FILE) as fp:
            config.readfp(fp)
    return config
{'fraction_non_alphanumeric': 0.11826182618261827, 'fraction_numerical': 0.009167583425009167, 'mean_word_length': 3.5878889823380993, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7602782', 'n_tokens_mistral': 1914, 'n_tokens_neox': 1870, 'n_words': 350}