<commit_before>#!/usr/bin/python

import argparse
import boto
import os, time, sys, socket
import h2o_cmd
import h2o
import h2o_hosts
import json
import commands
import traceback

'''
    Simple EC2 utility:
     
       * to setup clooud of 5 nodes: ./ec2_cmd.py create --instances 5
       * to terminated the cloud   : ./ec2_cmd.py terminate --hosts <host file returned by previous command>
'''

DEFAULT_NUMBER_OF_INSTANCES = 4
DEFAULT_HOSTS_FILENAME = 'ec2-config-{0}.json'
DEFAULT_REGION = 'us-east-1'
DEFAULT_INSTANCE_NAME='node_{0}'.format(os.getenv('USER'))
ADVANCED_SSH_OPTIONS='-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o GlobalKnownHostsFile=/dev/null'

'''
Default EC2 instance setup
'''
DEFAULT_EC2_INSTANCE_CONFIGS = {
  'us-east-1':{
              'image_id'        : 'ami-cf5132a6', #'ami-30c6a059', #'ami-b85cc4d1', # 'ami-cd9a11a4',
              'security_groups' : [ 'MrJenkinsTest' ],
              'key_name'        : 'mrjenkins_test',
              'instance_type'   : 'm1.xlarge',
              'region'          : 'us-east-1',
              'pem'             : '~/.ec2/keys/mrjenkins_test.pem',
              'username'        : '0xdiag',      
              'aws_credentials' : '~/.ec2/AwsCredentials.properties',
              'hdfs_config'     : '~/.ec2/core-site.xml',
             },
  'us-west-1':{
              'image_id'        : 'ami-a6cbe6e3', # 'ami-cd9a11a4',
              'security_groups' : [ 'MrJenkinsTest' ],
              'key_name'        : 'mrjenkins_test',
              'instance_type'   : 'm1.xlarge',
              'pem'             : '~/.ec2/keys/mrjenkins_test.pem',
              'username'        : '0xdiag',      
              'aws_credentials' : '~/.ec2/AwsCredentials.properties',
              'hdfs_config'     : '~/.ec2/core-site.xml',
              'region'          : 'us-west-1',
             },
}

''' Memory mappings for instance kinds '''
MEMORY_MAPPING = {
    'm1.large'   : { 'xmx' : 5  },  # $0.24/hr
    'm1.xlarge'  : { 'xmx' : 11 },  # $0.48/hr
    'm2.xlarge'  : { 'xmx' : 13 },  # $0.41/hr
    'm2.2xlarge' : { 'xmx' : 30 },  # $0.82/hr
    'm2.4xlarge' : { 'xmx' : 60 },  # $1.64/hr
    'm3.xlarge'  : { 'xmx' : 11 },  # $0.50/hr
    'm3.2xlarge' : { 'xmx' : 24 },  # $1.00/hr
    'c1.xlarge'  : { 'xmx' : 6  },  # $0.58/hr
    'hi1.4xlarge'  : { 'xmx' : 56},  # $3.10/hr 60.5GB dram. 8 cores/8 threads. 2TB SSD. 10GE
    'cr1.8xlarge'  : { 'xmx' : 230},  # $4.60/hr 244GB dram. 2 E5-2670. 240GB SSD. 10GE
}

''' EC2 API default configuration. The corresponding values are replaces by EC2 user config. '''
EC2_API_RUN_INSTANCE = {
'image_id'        :None,
'min_count'       :1,
'max_count'       :1,
'key_name'        :None,
'security_groups' :None,
'user_data'       :None,
'addressing_type' :None,
'instance_type'   :None,
'placement'       :None,
'monitoring_enabled':False,
'subnet_id'       :None,
'block_device_map':None,
'disable_api_termination':False,
'instance_initiated_shutdown_behavior':None
}

def inheritparams(parent, kid):
    newkid = {}
    for k,v in kid.items():
        if parent.has_key(k):
            newkid[k] = parent[k]

    return newkid

def find_file(base):
    f = base
    if not os.path.exists(f): f = os.path.expanduser(base)
    if not os.path.exists(f): f = os.path.expanduser("~")+ '/' + base
    if not os.path.exists(f):
        return None
    return f

''' Returns a boto connection to given region ''' 
def ec2_connect(region):
    check_required_env_variables()
    import boto.ec2
    conn = boto.ec2.connect_to_region(region)
    if not conn:
        raise Exception("\033[91m[ec2] Cannot create EC2 connection into {0} region!\033[0m".format(region))

    return conn

def check_required_env_variables():
    ok = True
    if not os.environ['AWS_ACCESS_KEY_ID']: 
        warn("AWS_ACCESS_KEY_ID need to be defined!")
        ok = False
    if not os.environ['AWS_SECRET_ACCESS_KEY']:
        warn("AWS_SECRET_ACCESS_KEY need to be defined!")
        ok = False

    if not ok: raise Exception("\033[91m[ec2] Missing AWS environment variables!\033[0m")

''' Run number of EC2 instance.
Waits forthem and optionaly waits for ssh service.
'''
def run_instances(count, ec2_config, region, waitForSSH=True, tags=None):
    '''Create a new reservation for count instances'''

    ec2params = inheritparams(ec2_config, EC2_API_RUN_INSTANCE)
    ec2params.setdefault('min_count', count)
    ec2params.setdefault('max_count', count)

    reservation = None
    conn = ec2_connect(region)
    try:
        reservation = conn.run_instances(**ec2params)
        log('Reservation: {0}'.format(reservation.id))
        log('Waiting for {0} EC2 instances {1} to come up, this can take 1-2 minutes.'.format(len(reservation.instances), reservation.instances))
        start = time.time()
        time.sleep(1)
        for instance in reservation.instances:
            while instance.update() == 'pending':
               time.sleep(1)
               h2o_cmd.dot()

            if not instance.state == 'running':
                raise Exception('\033[91m[ec2] Error waiting for running state. Instance is in state {0}.\033[0m'.format(instance.state))

        log('Instances started in {0} seconds'.format(time.time() - start))
        log('Instances: ')
        for inst in reservation.instances: log("   {0} ({1}) : public ip: {2}, private ip: {3}".format(inst.public_dns_name, inst.id, inst.ip_address, inst.private_ip_address))
        
        if waitForSSH:
            wait_for_ssh([ i.ip_address for i in reservation.instances ])

        # Tag instances
        try:<commit_msg>use private ip addresses for the wait_for_ssh test after creating instances was using public ip address (would pass, even though you were in wrong domain for using private addresses..I think we don't support cross-domain python/h2o?<commit_after>#!/usr/bin/python

import argparse
import boto
import os, time, sys, socket
import h2o_cmd
import h2o
import h2o_hosts
import json
import commands
import traceback

'''
    Simple EC2 utility:
     
       * to setup clooud of 5 nodes: ./ec2_cmd.py create --instances 5
       * to terminated the cloud   : ./ec2_cmd.py terminate --hosts <host file returned by previous command>
'''

DEFAULT_NUMBER_OF_INSTANCES = 4
DEFAULT_HOSTS_FILENAME = 'ec2-config-{0}.json'
DEFAULT_REGION = 'us-east-1'
DEFAULT_INSTANCE_NAME='node_{0}'.format(os.getenv('USER'))
ADVANCED_SSH_OPTIONS='-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o GlobalKnownHostsFile=/dev/null'

'''
Default EC2 instance setup
'''
DEFAULT_EC2_INSTANCE_CONFIGS = {
  'us-east-1':{
              'image_id'        : 'ami-cf5132a6', #'ami-30c6a059', #'ami-b85cc4d1', # 'ami-cd9a11a4',
              'security_groups' : [ 'MrJenkinsTest' ],
              'key_name'        : 'mrjenkins_test',
              'instance_type'   : 'm1.xlarge',
              'region'          : 'us-east-1',
              'pem'             : '~/.ec2/keys/mrjenkins_test.pem',
              'username'        : '0xdiag',      
              'aws_credentials' : '~/.ec2/AwsCredentials.properties',
              'hdfs_config'     : '~/.ec2/core-site.xml',
             },
  'us-west-1':{
              'image_id'        : 'ami-a6cbe6e3', # 'ami-cd9a11a4',
              'security_groups' : [ 'MrJenkinsTest' ],
              'key_name'        : 'mrjenkins_test',
              'instance_type'   : 'm1.xlarge',
              'pem'             : '~/.ec2/keys/mrjenkins_test.pem',
              'username'        : '0xdiag',      
              'aws_credentials' : '~/.ec2/AwsCredentials.properties',
              'hdfs_config'     : '~/.ec2/core-site.xml',
              'region'          : 'us-west-1',
             },
}

''' Memory mappings for instance kinds '''
MEMORY_MAPPING = {
    'm1.large'   : { 'xmx' : 5  },  # $0.24/hr
    'm1.xlarge'  : { 'xmx' : 11 },  # $0.48/hr
    'm2.xlarge'  : { 'xmx' : 13 },  # $0.41/hr
    'm2.2xlarge' : { 'xmx' : 30 },  # $0.82/hr
    'm2.4xlarge' : { 'xmx' : 60 },  # $1.64/hr
    'm3.xlarge'  : { 'xmx' : 11 },  # $0.50/hr
    'm3.2xlarge' : { 'xmx' : 24 },  # $1.00/hr
    'c1.xlarge'  : { 'xmx' : 6  },  # $0.58/hr
    'hi1.4xlarge'  : { 'xmx' : 56},  # $3.10/hr 60.5GB dram. 8 cores/8 threads. 2TB SSD. 10GE
    'cr1.8xlarge'  : { 'xmx' : 230},  # $4.60/hr 244GB dram. 2 E5-2670. 240GB SSD. 10GE
}

''' EC2 API default configuration. The corresponding values are replaces by EC2 user config. '''
EC2_API_RUN_INSTANCE = {
'image_id'        :None,
'min_count'       :1,
'max_count'       :1,
'key_name'        :None,
'security_groups' :None,
'user_data'       :None,
'addressing_type' :None,
'instance_type'   :None,
'placement'       :None,
'monitoring_enabled':False,
'subnet_id'       :None,
'block_device_map':None,
'disable_api_termination':False,
'instance_initiated_shutdown_behavior':None
}

def inheritparams(parent, kid):
    newkid = {}
    for k,v in kid.items():
        if parent.has_key(k):
            newkid[k] = parent[k]

    return newkid

def find_file(base):
    f = base
    if not os.path.exists(f): f = os.path.expanduser(base)
    if not os.path.exists(f): f = os.path.expanduser("~")+ '/' + base
    if not os.path.exists(f):
        return None
    return f

''' Returns a boto connection to given region ''' 
def ec2_connect(region):
    check_required_env_variables()
    import boto.ec2
    conn = boto.ec2.connect_to_region(region)
    if not conn:
        raise Exception("\033[91m[ec2] Cannot create EC2 connection into {0} region!\033[0m".format(region))

    return conn

def check_required_env_variables():
    ok = True
    if not os.environ['AWS_ACCESS_KEY_ID']: 
        warn("AWS_ACCESS_KEY_ID need to be defined!")
        ok = False
    if not os.environ['AWS_SECRET_ACCESS_KEY']:
        warn("AWS_SECRET_ACCESS_KEY need to be defined!")
        ok = False

    if not ok: raise Exception("\033[91m[ec2] Missing AWS environment variables!\033[0m")

''' Run number of EC2 instance.
Waits forthem and optionaly waits for ssh service.
'''
def run_instances(count, ec2_config, region, waitForSSH=True, tags=None):
    '''Create a new reservation for count instances'''

    ec2params = inheritparams(ec2_config, EC2_API_RUN_INSTANCE)
    ec2params.setdefault('min_count', count)
    ec2params.setdefault('max_count', count)

    reservation = None
    conn = ec2_connect(region)
    try:
        reservation = conn.run_instances(**ec2params)
        log('Reservation: {0}'.format(reservation.id))
        log('Waiting for {0} EC2 instances {1} to come up, this can take 1-2 minutes.'.format(len(reservation.instances), reservation.instances))
        start = time.time()
        time.sleep(1)
        for instance in reservation.instances:
            while instance.update() == 'pending':
               time.sleep(1)
               h2o_cmd.dot()

            if not instance.state == 'running':
                raise Exception('\033[91m[ec2] Error waiting for running state. Instance is in state {0}.\033[0m'.format(instance.state))

        log('Instances started in {0} seconds'.format(time.time() - start))
        log('Instances: ')
        for inst in reservation.instances: log("   {0} ({1}) : public ip: {2}, private ip: {3}".format(inst.public_dns_name, inst.id, inst.ip_address, inst.private_ip_address))
        
        if waitForSSH:
            # kbn: changing to private address, so it should fail if not in right domain
            # used to have the public ip address
            wait_for_ssh([ i.ip_private_address for i in reservation.instances ])

        # Tag instances
        try:
{'fraction_non_alphanumeric': 0.14124342502371304, 'fraction_numerical': 0.035440200051737515, 'mean_word_length': 2.499698249849125, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1621840', 'n_tokens_mistral': 4284, 'n_tokens_neox': 4054, 'n_words': 902}