<commit_before>from os import environ
from logging import getLogger
from os.path import join, dirname
from argparse import ArgumentParser
from time import time, sleep

from . import jobs
from boto.ec2 import EC2Connection
from boto.ec2.blockdevicemapping import BlockDeviceMapping, BlockDeviceType
from requests import get

parser = ArgumentParser(description='Run some source files.')

parser.add_argument('bucketname',
                    help='Required S3 bucket name.')

parser.add_argument('-a', '--access-key', default=environ.get('AWS_ACCESS_KEY_ID', None),
                    help='Optional AWS access key name for writing to S3. Defaults to value of AWS_ACCESS_KEY_ID environment variable.')

parser.add_argument('-s', '--secret-key', default=environ.get('AWS_SECRET_ACCESS_KEY', None),
                    help='Optional AWS secret key name for writing to S3. Defaults to value of AWS_SECRET_ACCESS_KEY environment variable.')

parser.add_argument('--ec2-access-key',
                    help='Optional AWS access key name for setting up EC2; distinct from access key for populating S3 bucket. Defaults to value of EC2_ACCESS_KEY_ID environment variable or S3 access key.')

parser.add_argument('--ec2-secret-key',
                    help='Optional AWS secret key name for setting up EC2; distinct from secret key for populating S3 bucket. Defaults to value of EC2_SECRET_ACCESS_KEY environment variable or S3 secret key.')

parser.add_argument('--instance-type', default='m3.xlarge',
                    help='EC2 instance type, defaults to m3.xlarge.')

parser.add_argument('--machine-image', default='ami-4ae27e22',
                    help='AMI identifier, defaults to Alestic Ubuntu 14.04 (ami-4ae27e22).')

def main():
    args = parser.parse_args()
    jobs.setup_logger(None)
    
    #
    # Prepare init script for new EC2 instance to run.
    #
    with open(join(dirname(__file__), 'templates', 'user-data.sh')) as file:
        user_data = file.read().format(**args.__dict__)
    
    getLogger('openaddr').info('Prepared {} bytes of instance user data'.format(len(user_data)))

    #
    # Figure out how much we're willing to bid on a spot instance.
    #
    ec2_access_key = args.ec2_access_key or environ.get('EC2_ACCESS_KEY_ID', args.access_key)
    ec2_secret_key = args.ec2_secret_key or environ.get('EC2_SECRET_ACCESS_KEY', args.secret_key)
    ec2 = EC2Connection(ec2_access_key, ec2_secret_key)

    history = ec2.get_spot_price_history(instance_type=args.instance_type)
    median = sorted([h.price for h in history])[len(history)/2]
    bid = median + .01

    getLogger('openaddr').info('Bidding ${:.4f}/hour for {} instance'.format(bid, args.instance_type))
    
    #
    # Request a spot instance with 99GB storage.
    #
    device_sda1 = BlockDeviceType(size=99, delete_on_termination=True)
    device_map = BlockDeviceMapping(); device_map['/dev/sda1'] = device_sda1
    
    spot_args = dict(instance_type=args.instance_type, user_data=user_data,
                     key_name='cfa-keypair-2013', security_groups=['default'],
                     block_device_map=device_map)

    spot_req = ec2.request_spot_instances(bid, args.machine_image, **spot_args)[0]

    getLogger('openaddr').info('https://console.aws.amazon.com/ec2/v2/home?region=us-east-1#SpotInstances:search={}'.format(spot_req.id))
    
    #
    # Wait while EC2 does its thing, unless the user interrupts.
    #
    try:
        wait_it_out(spot_req, time() + 6 * 60 * 60)

    finally:<commit_msg>Moved SSH key pair and EC2 security group to options<commit_after>from os import environ
from logging import getLogger
from os.path import join, dirname
from argparse import ArgumentParser
from time import time, sleep

from . import jobs
from boto.ec2 import EC2Connection
from boto.ec2.blockdevicemapping import BlockDeviceMapping, BlockDeviceType
from requests import get

parser = ArgumentParser(description='Run some source files.')

parser.add_argument('bucketname',
                    help='Required S3 bucket name.')

parser.add_argument('-a', '--access-key', default=environ.get('AWS_ACCESS_KEY_ID', None),
                    help='Optional AWS access key name for writing to S3. Defaults to value of AWS_ACCESS_KEY_ID environment variable.')

parser.add_argument('-s', '--secret-key', default=environ.get('AWS_SECRET_ACCESS_KEY', None),
                    help='Optional AWS secret key name for writing to S3. Defaults to value of AWS_SECRET_ACCESS_KEY environment variable.')

parser.add_argument('--ec2-access-key',
                    help='Optional AWS access key name for setting up EC2; distinct from access key for populating S3 bucket. Defaults to value of EC2_ACCESS_KEY_ID environment variable or S3 access key.')

parser.add_argument('--ec2-secret-key',
                    help='Optional AWS secret key name for setting up EC2; distinct from secret key for populating S3 bucket. Defaults to value of EC2_SECRET_ACCESS_KEY environment variable or S3 secret key.')

parser.add_argument('--instance-type', default='m3.xlarge',
                    help='EC2 instance type, defaults to m3.xlarge.')

parser.add_argument('--ssh-keypair', default='oa-keypair',
                    help='SSH key pair name, defaults to "oa-keypair".')

parser.add_argument('--security-group', default='default',
                    help='EC2 security group name, defaults to "default".')

parser.add_argument('--machine-image', default='ami-4ae27e22',
                    help='AMI identifier, defaults to Alestic Ubuntu 14.04 (ami-4ae27e22).')

def main():
    args = parser.parse_args()
    jobs.setup_logger(None)
    
    #
    # Prepare init script for new EC2 instance to run.
    #
    with open(join(dirname(__file__), 'templates', 'user-data.sh')) as file:
        user_data = file.read().format(**args.__dict__)
    
    getLogger('openaddr').info('Prepared {} bytes of instance user data'.format(len(user_data)))

    #
    # Figure out how much we're willing to bid on a spot instance.
    #
    ec2_access_key = args.ec2_access_key or environ.get('EC2_ACCESS_KEY_ID', args.access_key)
    ec2_secret_key = args.ec2_secret_key or environ.get('EC2_SECRET_ACCESS_KEY', args.secret_key)
    ec2 = EC2Connection(ec2_access_key, ec2_secret_key)

    history = ec2.get_spot_price_history(instance_type=args.instance_type)
    median = sorted([h.price for h in history])[len(history)/2]
    bid = median + .01

    getLogger('openaddr').info('Bidding ${:.4f}/hour for {} instance'.format(bid, args.instance_type))
    
    #
    # Request a spot instance with 99GB storage.
    #
    device_sda1 = BlockDeviceType(size=99, delete_on_termination=True)
    device_map = BlockDeviceMapping(); device_map['/dev/sda1'] = device_sda1
    
    spot_args = dict(instance_type=args.instance_type, user_data=user_data,
                     key_name=args.ssh_keypair, block_device_map=device_map,
                     security_groups=[args.security_group])

    spot_req = ec2.request_spot_instances(bid, args.machine_image, **spot_args)[0]

    getLogger('openaddr').info('https://console.aws.amazon.com/ec2/v2/home?region=us-east-1#SpotInstances:search={}'.format(spot_req.id))
    
    #
    # Wait while EC2 does its thing, unless the user interrupts.
    #
    try:
        wait_it_out(spot_req, time() + 6 * 60 * 60)

    finally:
{'fraction_non_alphanumeric': 0.10814491964042496, 'fraction_numerical': 0.019068373740125308, 'mean_word_length': 3.53831891223733, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3573528', 'n_tokens_mistral': 2316, 'n_tokens_neox': 2238, 'n_words': 576}