<commit_before>#!/usr/bin/env python
"""

This file is part of the mumax-ec2 package.

Copyright (c) 2014-2015 <NAME>, <NAME>

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

"""

__version__ = 1.0

__TAG__ = u'mumax-ec2'
__STATUS__ = u'status'
__READY__ = u'1'
PORT = 35367
__ON_AWS__ = "==================== AWS INSTANCE ===================="


import argparse
import boto.ec2
import paramiko
import sys, os
from time import sleep
import select
from sshtunnel import SSHTunnelForwarder


# Start the config parser
import ConfigParser
config = ConfigParser.ConfigParser()
config.read(os.path.dirname(os.path.realpath(__file__))+"/config.ini")

# Connect to Amazon Web Services (AWS)
conn = boto.ec2.connect_to_region(config.get('EC2', 'Region'),
            aws_access_key_id=config.get('EC2', 'AccessID'),
            aws_secret_access_key=config.get('EC2', 'SecretKey'))
            
up_condition = lambda i: (
    i.state == u'running'
)

mumax_ec2_condition = lambda i: (
    __TAG__ in i.tags
)

ready_condition = lambda i: (
    __STATUS__ in i.tags and
    i.tags[__STATUS__] == __READY__
)

instances = conn.get_only_instances()
mumax_ec2_instances = [i for i in instances if mumax_ec2_condition(i)]
ready_instances = [i for i in instances if (
    mumax_ec2_condition(i) and ready_condition(i)
)]


def has_id(id, running=True):
    """ Returns True if the ID is a valid mumax-ec2 instance
    and if it is running or not
    """
    for instance in mumax_ec2_instances:
        if instance.id == id:
            if up_condition(instance) == running:
                return True
            else:
                return False
    return False


def launch_instance():
    """ Launch a new AWS instance """
    print "Creating a new instance of %s" % config.get('EC2', 'Image')
    reservation = conn.run_instances(
        config.get('EC2', 'Image'),
        key_name=config.get('EC2', 'PrivateKeyName'),
        instance_type=config.get('EC2', 'InstanceType'),
        security_groups=config.get('EC2', 'SecurityGroups').split(',')
    )
    instance = reservation.instances[0]
    instance.add_tag(__TAG__, __version__)
    instance.add_tag(__STATUS__, __READY__)
    return instance


def wait_for_instance(instance, delay=10):
    """ Waits for an instance to boot up
    """
    print "Waiting for instance %s to boot up..." % instance.id
    while instance.state != u'running':
        sleep(delay)
        instance.update()
    sleep(delay)
    print "Instance %s is ready" % instance.id


def get_ready_instance():
    """ Returns an instance from the ready list or launches 
    a new instance upon prompt
    """
    if len(ready_instances) == 0:
        print "There are no instances waiting to be used."
        answer = raw_input("Create a new instance for this job? [Yn]: ")
        if len(answer) == 0 or answer.startswith(("Y", "y")):
            instance = launch_instance()
            wait_for_instance()
            return instance
        else:
            print "No instance will be launched"
            return None
    else:
        instance = ready_instances[0] # Select the 1st ready instance
        print "Instance %s is ready" % instance.id
        return instance


def run(args):
    """Run the mumax input file on a ready instance
    """
    local_input_file = os.path.realpath(args.filename[0])

    instance = get_ready_instance()
    if instance == None: return None<commit_msg>Added missing argument to wait_for_instace() call.<commit_after>#!/usr/bin/env python
"""

This file is part of the mumax-ec2 package.

Copyright (c) 2014-2015 <NAME>, <NAME>

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

"""

__version__ = 1.0

__TAG__ = u'mumax-ec2'
__STATUS__ = u'status'
__READY__ = u'1'
PORT = 35367
__ON_AWS__ = "==================== AWS INSTANCE ===================="


import argparse
import boto.ec2
import paramiko
import sys, os
from time import sleep
import select
from sshtunnel import SSHTunnelForwarder


# Start the config parser
import ConfigParser
config = ConfigParser.ConfigParser()
config.read(os.path.dirname(os.path.realpath(__file__))+"/config.ini")

# Connect to Amazon Web Services (AWS)
conn = boto.ec2.connect_to_region(config.get('EC2', 'Region'),
            aws_access_key_id=config.get('EC2', 'AccessID'),
            aws_secret_access_key=config.get('EC2', 'SecretKey'))
            
up_condition = lambda i: (
    i.state == u'running'
)

mumax_ec2_condition = lambda i: (
    __TAG__ in i.tags
)

ready_condition = lambda i: (
    __STATUS__ in i.tags and
    i.tags[__STATUS__] == __READY__
)

instances = conn.get_only_instances()
mumax_ec2_instances = [i for i in instances if mumax_ec2_condition(i)]
ready_instances = [i for i in instances if (
    mumax_ec2_condition(i) and ready_condition(i)
)]


def has_id(id, running=True):
    """ Returns True if the ID is a valid mumax-ec2 instance
    and if it is running or not
    """
    for instance in mumax_ec2_instances:
        if instance.id == id:
            if up_condition(instance) == running:
                return True
            else:
                return False
    return False


def launch_instance():
    """ Launch a new AWS instance """
    print "Creating a new instance of %s" % config.get('EC2', 'Image')
    reservation = conn.run_instances(
        config.get('EC2', 'Image'),
        key_name=config.get('EC2', 'PrivateKeyName'),
        instance_type=config.get('EC2', 'InstanceType'),
        security_groups=config.get('EC2', 'SecurityGroups').split(',')
    )
    instance = reservation.instances[0]
    instance.add_tag(__TAG__, __version__)
    instance.add_tag(__STATUS__, __READY__)
    return instance


def wait_for_instance(instance, delay=10):
    """ Waits for an instance to boot up
    """
    print "Waiting for instance %s to boot up..." % instance.id
    while instance.state != u'running':
        sleep(delay)
        instance.update()
    sleep(delay)
    print "Instance %s is ready" % instance.id


def get_ready_instance():
    """ Returns an instance from the ready list or launches 
    a new instance upon prompt
    """
    if len(ready_instances) == 0:
        print "There are no instances waiting to be used."
        answer = raw_input("Create a new instance for this job? [Yn]: ")
        if len(answer) == 0 or answer.startswith(("Y", "y")):
            instance = launch_instance()
            wait_for_instance(instance)
            return instance
        else:
            print "No instance will be launched"
            return None
    else:
        instance = ready_instances[0] # Select the 1st ready instance
        print "Instance %s is ready" % instance.id
        return instance


def run(args):
    """Run the mumax input file on a ready instance
    """
    local_input_file = os.path.realpath(args.filename[0])

    instance = get_ready_instance()
    if instance == None: return None
{'fraction_non_alphanumeric': 0.099152063312606, 'fraction_numerical': 0.009496890898812888, 'mean_word_length': 3.368395061728395, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1488299', 'n_tokens_mistral': 2630, 'n_tokens_neox': 2535, 'n_words': 936}