<commit_before>
#---------written by <NAME> (FSO)-----------------
#-DESCRIPTION:
# This is based on fabfile from Raincloud Project (simplified)
#
#-Last modified:  Thu Jul 09, 2015  13:10
#@author <NAME>
#-----------------------------------------------------------
from __future__ import with_statement, print_function
from fabric.api import *
import boto.ec2
from boto.ec2.blockdevicemapping import EBSBlockDeviceType, BlockDeviceMapping
import os
import time
import sys
import socket
import datetime
import math
from collections import defaultdict

#-----------------------------------------------------------
# SHORT DOCU
#-----------------------------------------------------------

# -------- SETUP BOTO and Fabric-----------------
# Virtualenv/Generic pip:
# pip install boto fabric
#
# Conda:
# conda install boto fabric
#
# Debian/Ubuntu:
# apt-get install fabric python-boto
#
# install all other missing modules from list above (start eg. ipython an copy all imports above and see what's missing;
# all modules should be available via pip or easy_install)
#
# Create credentials file: ~/.boto and fill it with the info given by admin (most likely Ben)
# (replace XXXX with what you want to use in fabfile)
#
# [profile XXXX]
# aws_access_key_id = YOUR Access Key ID HERE
# aws_secret_access_key = YOUR Secret Access Key HERE
#
# If you don't want to be prompted to accept ssh keys with every new instance, place these lines into the ~/.ssh/config file:
#
# Host *amazonaws.com
#     User root
#     StrictHostKeyChecking no
#     UserKnownHostsFile /dev/null
#
#
# ------------RUNNING-------------
# look at fabfile.py
#
# to list all possible task of fabfile:
# fab -l
#
# A few first steps:
#     1. Go through setup below and adjust at least: ec2Profile, def_logfile
#     2. Create instance with
#         fab cloud_make
#        If you are using spot instances and require your instances to be in the same region
#         fab instance_start
#        This will use the region configured in def_default_avz.
#     3. Takes between 5 - 10 minutes (if still using spot as def_default_requesttype)
#     4. Use
#         fab install_node_software
#        to setup a virtualenv ready for OGGM.
#
#        If you already setup a virtualenv on your user volume
#         fab install_node_apt
#        to install only required system components.
#     5. Use
#         fab connect
#        to ssh into instance
#     6. play around with the instance, install software etc
#     7. look at current costs with
#         fab calc_approx_costs_running
#        or list all instances with
#         fab cloud_list
#     8. Once you have enough, shut down your instance via
#         fab terminate_one
#        Or terminate all running instances if you are sure they all belong to you
#         fab cloud_terminate
#        you can also delete volumes with:
#         fab terminate_perm_user_vol:name='your_volume_name'


#-----------------------------------------------------------
# SETUP
#-----------------------------------------------------------
env.disable_known_hosts=True
env.user = 'ubuntu'

# FSO--- default name used in tags and instance names:
# set this eg. to your name
def_cn = 'AWS'

# Change to a string identifying yourself
user_identifier = None

# FSO--- ssh and credentials setup
# FSO---the name of the amazon keypair (will be created if it does not exist)
keyn=(user_identifier or 'None') + '_oggm'
# FSO--- the same name as you used in boto setup XXXX (see Readme)
ec2Profile = 'OGGM'
def_key_dir=os.path.expanduser('~/.ssh')

# FSO--- Amazon AWS region setup
def_regions = ['us-east-1','eu-west-1'] #regions for spot search
def_default_avz = 'us-east-1b' #Default availability zone if ondemand is used

# FSO--- type of instance pricing, either:
# ondemand: faster availability, more expensive
# spot: cheaper, takes longer to start up, might be shutdown without warning
def_default_requesttype = 'spot'
# def_default_requesttype = 'ondemand'

# FSO--- the AMI to use
def_ami = dict()
def_ami['eu-west-1'] = 'ami-abc579d8' #eu Ubuntu 14.04 LTS
def_ami['us-east-1'] = 'ami-415f6d2b' #us Ubuntu 14.04 LTS

# Size of the rootfs of created instances
rootfs_size_gb = 50

# Name and size of the persistent /work file system
home_volume_ebs_name = "ebs_" + (user_identifier or 'None') # Set to None to disable home volume
new_homefs_size_gb = 50 # GiB, only applies to newly created volumes

# FSO---log file with timestamps to analyse cloud performance
# look at it with tail -f cloudexecution.log
def_logfile = os.path.expanduser('~/cloudexecution.log')

# Default instance type, index into instance_infos array below
def_inst_type = 1


#-----------------------------------------------------------
# SETUP END
#-----------------------------------------------------------
fabfile_dir = os.path.dirname(os.path.abspath(__file__))

if user_identifier is None:
    raise RuntimeError('user identifier must be set')

instance_infos = [
    {
        'type': 't2.micro',
        'vcpus': 1,
        'price': 0.014,
    },
    {
        'type': 'm4.xlarge',
        'vcpus': 4,
        'price': 0.264,
    },
    {
        'type': 'c4.2xlarge',
        'vcpus': 8,
        'price': 0.477,
    },
    {
        'type': 'c4.8xlarge',
        'vcpus': 36,
        'price': 1.906,
    },
]

def_price = instance_infos[def_inst_type]['price']


def update_key_filename(avz):
    key_name = get_keypair_name()
    key_dir = os.path.expanduser(def_key_dir)
    key_dir = os.path.expandvars(key_dir)
    env.key_filename = os.path.join(key_dir, key_name + '.pem')
    print('Current key filename: %s' % env.key_filename)


def find_inst_info(inst_type):
    for info in instance_infos:
        if info['type'] == inst_type:
            return info
    return None


@task
def cloud_make(cn=def_cn):
    """
    Start and prepare instance  -THIS IS THE MAIN ACTIVITY-
    """
    # t = time.time()
    log_with_ts("fabric started ------------------------------")
    log_with_ts("Instance: " + instance_infos[def_inst_type]['type'] + "(" + str(instance_infos[def_inst_type]['vcpus']) + " CPUs)")

    # FSO---set best avz
    if def_default_requesttype == 'spot':
        best_avz,request_type = get_cheapest_availability_zone(def_price)
    else:
        best_avz = def_default_avz
        request_type = 'ondemand'

    print(best_avz, request_type)
    log_with_ts('avz: ' + best_avz)
    log_with_ts('request_type: ' + request_type)

    # FSO--- start instances
    instance_start(cn=cn,avz=best_avz,rt=request_type)
    print("Done setting up instance")
    log_with_ts("instance ready")

    # t_init = time.time()

    # # FSO--- run workflow and get cost of nodes back
    # this is an example, adjust as needed
    # tf = run_workflow(cn=cn,avz=best_avz)

    # # FSO--- get costs of and log
    # costs = calc_approx_costs_running(cn=cn')
    # log_with_ts('Ondemand costs: '+str(costs['ondemand'])+'USD')
    # log_with_ts('Actual costs: '+str(costs['running'])+'USD')

    # # FSO--- terminate instances
    # uncomment if you want to terminate your instances automatically
    # cloud_terminate(cn=cn)
    # log_with_ts("all instances terminated")

    # t_end = time.time()

    # print "Time needed for init (min)", (t_init - t)/60.
    # print "Time needed for workflow and terminate", (t_end - t_init)/60.
    # log_with_ts("fabric end")

@task
def list_ubuntu_amis(regions=def_regions):
    """
    List all available ubuntu 14.04 AMIs in all configured regions
    """
    for region in regions:
        print("Region:", region)
        cloud = boto.ec2.connect_to_region(region,profile_name=ec2Profile)
        imgs = cloud.get_all_images(owners=['099720109477'], filters={'architecture': 'x86_64', 'name': 'ubuntu/images/hvm-ssd/ubuntu-trusty-14.04-amd64-server-*'})
        for img in sorted(imgs, key=lambda v: v.name):
            print(img.id,':',img.name)
        print()


@task
def instance_start(cn=def_cn,
        avz=def_default_avz,
        rt=def_default_requesttype):
    """
    Start and prepare instances
    """
    # FSO---find already existing nodes
    cloud = boto.ec2.connect_to_region(avz[:-1],profile_name=ec2Profile)
    filters = {'tag:type': cn+'node'}
    insta = cloud.get_all_instances(filters=filters)

    # FSO---install each new node
    print("Requesting new instance")
    log_with_ts("Requesting new instance")

    nodenumber = len(insta) + 1

    node_install(cn=cn, avz=avz, rt=rt, idn=nodenumber)

    log_with_ts('Finished installing instance')

    cloud_list()


def print_instance(inst):
    if inst.state != 'terminated':
        cu_time = datetime.datetime.utcnow()
        it =  datetime.datetime.strptime(inst.launch_time,'%Y-%m-%dT%H:%M:%S.000Z')
    else:
        try:
            cu_time = datetime.datetime.strptime(inst.tags.get('terminate_time'),'%Y-%m-%dT%H:%M:%S.%f')
        except:
            cu_time = datetime.datetime.utcnow()

        it = datetime.datetime.strptime(inst.launch_time,'%Y-%m-%dT%H:%M:%S.000Z')

    time_taken = cu_time - it
    hours, rest = divmod(time_taken.total_seconds(),3600)
    minutes, seconds = divmod(rest, 60)

    print(inst.id, inst.instance_type, \
            inst.tags.get('Name'), \
            inst.tags.get('type'), \
            inst.state, \
            inst.dns_name, \
            inst.private_ip_address, \
            inst.private_dns_name, \
            inst.tags.get('current_price'), \
            inst.tags.get('billable_hours'), \
            inst.tags.get('terminate_time'), \
            inst.placement, \
            'Owner:%s' % inst.tags.get('node-owner'))
    print("running for: ", hours,'h', minutes, "min")


def print_volume(vol):
    info = ""
    if 'vol-lifetime' in vol.tags:
        info += '\tLifetime: ' + vol.tags['vol-lifetime']
    if 'vol-user-name' in vol.tags:
        info += '\tUservolume Name: ' + vol.tags['vol-user-name']
    if 'vol-owner' in vol.tags:
        info += '\tOwner: ' + vol.tags['vol-owner']

    print(vol.id, "\t", vol.zone, "\t", vol.status, '\t', vol.size, info)


@task
def cloud_list(cn=def_cn,itype='all',regions=def_regions):
    """
    List all ec2 instances.
    """
    for region in regions:
        cloud = boto.ec2.connect_to_region(region,profile_name=ec2Profile)
        instances = cloud.get_all_instances()
        vols = cloud.get_all_volumes()
        print()
        print("-------CURRENT RUNNING-----------")
        print("       REGION:", region)
        print()
        print("Instances:")
        print()

        update_costs(cn=cn,regions=regions,itype=itype)

        for reservation in instances:
            for inst in reservation.instances:
                print_instance(inst)
                print()

        print()
        print("Volumes:")
        print()

        for vol in vols:
            print_volume(vol)


def check_keypair(cloud, keynames):
    # Check to see if specified keypair already exists.
    # If we get an InvalidKeyPair.NotFound error back from EC2,
    # it means that it doesn't exist and we need to create it.
    key_dir = def_key_dir
    try:
        cloud.get_all_key_pairs(keynames=[keynames])[0]
    except cloud.ResponseError as e:
        if e.code == 'InvalidKeyPair.NotFound':
            print('Creating keypair: %s' % keynames)
            # Create an SSH key to use when logging into instances.
            key = cloud.create_key_pair(keynames)
            # Make sure the specified key_dir actually exists.
            # If not, create it.
            key_dir = os.path.expanduser(key_dir)
            key_dir = os.path.expandvars(key_dir)
            # if not os.path.isdir(key_dir):
            #     os.mkdir(key_dir, 0700)
            #
            # AWS will store the public key but the private key is
            # generated and returned and needs to be stored locally.
            # The save method will also chmod the file to protect
            # your private key.
            key.save(key_dir)
        else:
            raise


def get_keypair_name():
    key_dir = def_key_dir
    key_dir = os.path.expanduser(key_dir)
    key_dir = os.path.expandvars(key_dir)

    un_file = os.path.join(key_dir, '%s_unique.txt' % keyn)

    if os.path.exists(un_file):
        with open(un_file, 'r') as un:
            unique_part = un.read().strip()
    else:
        import uuid
        unique_part = str(uuid.uuid4().get_hex().upper()[0:8])
        with open(un_file, 'w') as un:
            un.write(unique_part)

    return keyn + '_' + unique_part


def get_user_persist_ebs(cloud, avz):
    if home_volume_ebs_name is None:
        return None

    vols = cloud.get_all_volumes(filters={'tag:vol-user-name':home_volume_ebs_name, 'availability-zone': avz})

    if len(vols) == 0:
        print("Creating new EBS volume for user volume %s" % home_volume_ebs_name)
        vol = cloud.create_volume(new_homefs_size_gb, avz)
        vol.add_tag('vol-user-name', home_volume_ebs_name)
        vol.add_tag('vol-lifetime', 'perm')
        vol.add_tag('vol-owner', user_identifier)
    else:
        vol = vols[0]
        print("Found existing volume %s for user volume %s!" % (vol.id, home_volume_ebs_name))

        if vol.status != 'available':
            print("But it's not available...")
            return None

    return vol


@task
def node_install(cn=def_cn,inst_type_idx=def_inst_type,idn=0,
        avz=def_default_avz,rt=def_default_requesttype,
        group_name='oggmssh',
        ssh_port=22,
        cidr='0.0.0.0/0'):
    """
    Request and prepare single instance
    """
    # FSO---connect
    cloud = boto.ec2.connect_to_region(avz[:-1],profile_name=ec2Profile)
    aminfo = cloud.get_image(def_ami[avz[:-1]])

    # FSO---check if node with same name already exists
    if node_exists(cn + '_node' + str(idn)):
        print("Node already exists")
        sys.exit()

    # Check if ssh keypair exists
    key_name = get_keypair_name()
    check_keypair(cloud, key_name)

    # FSO---create a bigger root device
    dev_sda1 = EBSBlockDeviceType()
    dev_sda1.size = rootfs_size_gb
    dev_sda1.delete_on_termination = True
    bdm = BlockDeviceMapping()
    bdm['/dev/sda1'] = dev_sda1

    dev_sdf_vol = get_user_persist_ebs(cloud, avz)

    # Check to see if specified security group already exists.
    # If we get an InvalidGroup.NotFound error back from EC2,
    # it means that it doesn't exist and we need to create it.
    try:
        group = cloud.get_all_security_groups(groupnames=[group_name])[0]
    except cloud.ResponseError as e:
        if e.code == 'InvalidGroup.NotFound':
            print('Creating Security Group: %s' % group_name)
            # Create a security group to control access to instance via SSH.
            group = cloud.create_security_group(group_name, 'A group that allows SSH access')
        else:
            raise

    # Add a rule to the security group to authorize SSH traffic
    # on the specified port.
    try:
        group.authorize('tcp', ssh_port, ssh_port, cidr)
    except cloud.ResponseError as e:
        if e.code == 'InvalidPermission.Duplicate':
            print('Security Group: %s already authorized' % group_name)
        else:
            raise

    log_with_ts("request node "+str(idn))
    print('Reserving instance for node', aminfo.id, instance_infos[inst_type_idx]['type'], aminfo.name, aminfo.region)

    if rt == 'spot':
        print("placing node in ",avz)
        requests = cloud.request_spot_instances(def_price,
                      def_ami[avz[:-1]],
                      count=1,
                      type='one-time',
                      security_groups=[group_name],
                      key_name=key_name,
                      placement=avz,
                      instance_type=instance_infos[inst_type_idx]['type'],
                      block_device_map=bdm)
        req_ids = [request.id for request in requests]
        instance_ids = wait_for_fulfillment(cloud,req_ids)
        instances = cloud.get_only_instances(instance_ids=instance_ids)
        node = instances[0]
        log_with_ts("fullfilled spot node "+str(idn))
    else:
        print("placing node in ",avz)
        reservation = cloud.run_instances(image_id=def_ami[avz[:-1]],
                key_name=key_name,
                placement = avz,
                security_groups=[group_name],
                instance_type=instance_infos[inst_type_idx]['type'],
                block_device_map= bdm)
        node = reservation.instances[0]
        log_with_ts("fullfilled ondemand node "+str(idn))

    time.sleep(2)
    while not node.update() == 'running':
        print('waiting for', cn, 'node', idn, 'to boot...')
        time.sleep(5)

    log_with_ts("booted node "+str(idn))

    if dev_sdf_vol is not None:
        cloud.attach_volume(dev_sdf_vol.id, node.id, "/dev/sdf")

    node.add_tag('Name', cn+'_node'+str(idn))
    node.add_tag('type', cn+'node')
    node.add_tag('node-owner', user_identifier)

    # FSO---set delete on termination flag to true for ebs block device
    node.modify_attribute('blockDeviceMapping', { '/dev/sda1' : True })

    # FSO--- test socket connect to ssh service
    ssh_test(node)
    log_with_ts("reachable node "+str(idn))

    update_key_filename(node.region.name)

    # Mount potential user volume
    if dev_sdf_vol is not None:
        use_user_volume(node.dns_name)

    log_with_ts("finished node "+str(idn))


@task
def install_node_software(nn=''):
    """
    Setup ready-for-use virtualenv for OGGM on instance
    """
    inst = select_instance(nn)
    install_node_apt('', inst)
    install_node_pip('', inst)

    run('sudo shutdown -r now')


@task
def install_node_pip(nn='', inst=None):
    """
    Install oggm dependencies via pip
    """
    if inst is None:
        inst = select_instance(nn)
    update_key_filename(inst.region.name)
    env.host_string = inst.dns_name
    env.user = 'ubuntu'

    run("""
    source ~/.virtenvrc &&
    workon oggm_env &&
    pip install --upgrade pip &&
    pip install numpy &&
    pip install scipy &&
    pip install pandas shapely cython &&
    pip install matplotlib &&
    pip install gdal==1.10.0 --install-option="build_ext" --install-option="--include-dirs=/usr/include/gdal" &&
    pip install fiona --install-option="build_ext" --install-option="--include-dirs=/usr/include/gdal" &&
    pip install pyproj rasterio Pillow geopandas netcdf4 scikit-image configobj joblib xarray nose &&
    pip install git+https://github.com/fmaussion/motionless.git &&
    pip install git+https://github.com/fmaussion/salem.git &&
    pip install git+https://github.com/fmaussion/cleo.git &&
    sed -i 's/^backend.*/backend      : Agg/' "${WORKON_HOME}"/oggm_env/lib/python?.?/site-packages/matplotlib/mpl-data/matplotlibrc
    """, pty=False)


@task
def install_node_apt(nn='', inst=None):
    """
    Install required OGGM apt dependencies
    """
    if inst is None:
        inst = select_instance(nn)
    update_key_filename(inst.region.name)
    env.host_string = inst.dns_name
    env.user = 'ubuntu'

    run("""
    sudo apt-get update &&
    sudo apt-get -y dist-upgrade &&
    sudo apt-get install -y build-essential liblapack-dev gfortran libproj-dev gdal-bin libgdal-dev netcdf-bin ncview python-netcdf tk-dev python3-tk python3-dev ttf-bitstream-vera python-pip git awscli virtualenvwrapper
    """, pty=False)

    aws_file = os.path.expanduser('~/.aws/config')
    if os.path.exists(aws_file):
        run('mkdir -p ~/.aws')
        put(aws_file, '~/.aws/config')

    s_file = os.path.expanduser('~/.screenrc')
    if os.path.exists(s_file):
        put(s_file, '~/.screenrc')

    run("""<commit_msg>fabfile: Refactor initial file uploading, add gitconfig.<commit_after>
#---------written by <NAME> (FSO)-----------------
#-DESCRIPTION:
# This is based on fabfile from Raincloud Project (simplified)
#
#-Last modified:  Thu Jul 09, 2015  13:10
#@author <NAME>
#-----------------------------------------------------------
from __future__ import with_statement, print_function
from fabric.api import *
import boto.ec2
from boto.ec2.blockdevicemapping import EBSBlockDeviceType, BlockDeviceMapping
import os
import time
import sys
import socket
import datetime
import math
from collections import defaultdict

#-----------------------------------------------------------
# SHORT DOCU
#-----------------------------------------------------------

# -------- SETUP BOTO and Fabric-----------------
# Virtualenv/Generic pip:
# pip install boto fabric
#
# Conda:
# conda install boto fabric
#
# Debian/Ubuntu:
# apt-get install fabric python-boto
#
# install all other missing modules from list above (start eg. ipython an copy all imports above and see what's missing;
# all modules should be available via pip or easy_install)
#
# Create credentials file: ~/.boto and fill it with the info given by admin (most likely Ben)
# (replace XXXX with what you want to use in fabfile)
#
# [profile XXXX]
# aws_access_key_id = YOUR Access Key ID HERE
# aws_secret_access_key = YOUR Secret Access Key HERE
#
# If you don't want to be prompted to accept ssh keys with every new instance, place these lines into the ~/.ssh/config file:
#
# Host *amazonaws.com
#     User root
#     StrictHostKeyChecking no
#     UserKnownHostsFile /dev/null
#
#
# ------------RUNNING-------------
# look at fabfile.py
#
# to list all possible task of fabfile:
# fab -l
#
# A few first steps:
#     1. Go through setup below and adjust at least: ec2Profile, def_logfile
#     2. Create instance with
#         fab cloud_make
#        If you are using spot instances and require your instances to be in the same region
#         fab instance_start
#        This will use the region configured in def_default_avz.
#     3. Takes between 5 - 10 minutes (if still using spot as def_default_requesttype)
#     4. Use
#         fab install_node_software
#        to setup a virtualenv ready for OGGM.
#
#        If you already setup a virtualenv on your user volume
#         fab install_node_apt
#        to install only required system components.
#     5. Use
#         fab connect
#        to ssh into instance
#     6. play around with the instance, install software etc
#     7. look at current costs with
#         fab calc_approx_costs_running
#        or list all instances with
#         fab cloud_list
#     8. Once you have enough, shut down your instance via
#         fab terminate_one
#        Or terminate all running instances if you are sure they all belong to you
#         fab cloud_terminate
#        you can also delete volumes with:
#         fab terminate_perm_user_vol:name='your_volume_name'


#-----------------------------------------------------------
# SETUP
#-----------------------------------------------------------
env.disable_known_hosts=True
env.user = 'ubuntu'

# FSO--- default name used in tags and instance names:
# set this eg. to your name
def_cn = 'AWS'

# Change to a string identifying yourself
user_identifier = None

# FSO--- ssh and credentials setup
# FSO---the name of the amazon keypair (will be created if it does not exist)
keyn=(user_identifier or 'None') + '_oggm'
# FSO--- the same name as you used in boto setup XXXX (see Readme)
ec2Profile = 'OGGM'
def_key_dir=os.path.expanduser('~/.ssh')

# FSO--- Amazon AWS region setup
def_regions = ['us-east-1','eu-west-1'] #regions for spot search
def_default_avz = 'us-east-1b' #Default availability zone if ondemand is used

# FSO--- type of instance pricing, either:
# ondemand: faster availability, more expensive
# spot: cheaper, takes longer to start up, might be shutdown without warning
def_default_requesttype = 'spot'
# def_default_requesttype = 'ondemand'

# FSO--- the AMI to use
def_ami = dict()
def_ami['eu-west-1'] = 'ami-abc579d8' #eu Ubuntu 14.04 LTS
def_ami['us-east-1'] = 'ami-415f6d2b' #us Ubuntu 14.04 LTS

# Size of the rootfs of created instances
rootfs_size_gb = 50

# Name and size of the persistent /work file system
home_volume_ebs_name = "ebs_" + (user_identifier or 'None') # Set to None to disable home volume
new_homefs_size_gb = 50 # GiB, only applies to newly created volumes

# FSO---log file with timestamps to analyse cloud performance
# look at it with tail -f cloudexecution.log
def_logfile = os.path.expanduser('~/cloudexecution.log')

# Default instance type, index into instance_infos array below
def_inst_type = 1


#-----------------------------------------------------------
# SETUP END
#-----------------------------------------------------------
fabfile_dir = os.path.dirname(os.path.abspath(__file__))

if user_identifier is None:
    raise RuntimeError('user identifier must be set')

instance_infos = [
    {
        'type': 't2.micro',
        'vcpus': 1,
        'price': 0.014,
    },
    {
        'type': 'm4.xlarge',
        'vcpus': 4,
        'price': 0.264,
    },
    {
        'type': 'c4.2xlarge',
        'vcpus': 8,
        'price': 0.477,
    },
    {
        'type': 'c4.8xlarge',
        'vcpus': 36,
        'price': 1.906,
    },
]

def_price = instance_infos[def_inst_type]['price']


def update_key_filename(avz):
    key_name = get_keypair_name()
    key_dir = os.path.expanduser(def_key_dir)
    key_dir = os.path.expandvars(key_dir)
    env.key_filename = os.path.join(key_dir, key_name + '.pem')
    print('Current key filename: %s' % env.key_filename)


def find_inst_info(inst_type):
    for info in instance_infos:
        if info['type'] == inst_type:
            return info
    return None


@task
def cloud_make(cn=def_cn):
    """
    Start and prepare instance  -THIS IS THE MAIN ACTIVITY-
    """
    # t = time.time()
    log_with_ts("fabric started ------------------------------")
    log_with_ts("Instance: " + instance_infos[def_inst_type]['type'] + "(" + str(instance_infos[def_inst_type]['vcpus']) + " CPUs)")

    # FSO---set best avz
    if def_default_requesttype == 'spot':
        best_avz,request_type = get_cheapest_availability_zone(def_price)
    else:
        best_avz = def_default_avz
        request_type = 'ondemand'

    print(best_avz, request_type)
    log_with_ts('avz: ' + best_avz)
    log_with_ts('request_type: ' + request_type)

    # FSO--- start instances
    instance_start(cn=cn,avz=best_avz,rt=request_type)
    print("Done setting up instance")
    log_with_ts("instance ready")

    # t_init = time.time()

    # # FSO--- run workflow and get cost of nodes back
    # this is an example, adjust as needed
    # tf = run_workflow(cn=cn,avz=best_avz)

    # # FSO--- get costs of and log
    # costs = calc_approx_costs_running(cn=cn')
    # log_with_ts('Ondemand costs: '+str(costs['ondemand'])+'USD')
    # log_with_ts('Actual costs: '+str(costs['running'])+'USD')

    # # FSO--- terminate instances
    # uncomment if you want to terminate your instances automatically
    # cloud_terminate(cn=cn)
    # log_with_ts("all instances terminated")

    # t_end = time.time()

    # print "Time needed for init (min)", (t_init - t)/60.
    # print "Time needed for workflow and terminate", (t_end - t_init)/60.
    # log_with_ts("fabric end")

@task
def list_ubuntu_amis(regions=def_regions):
    """
    List all available ubuntu 14.04 AMIs in all configured regions
    """
    for region in regions:
        print("Region:", region)
        cloud = boto.ec2.connect_to_region(region,profile_name=ec2Profile)
        imgs = cloud.get_all_images(owners=['099720109477'], filters={'architecture': 'x86_64', 'name': 'ubuntu/images/hvm-ssd/ubuntu-trusty-14.04-amd64-server-*'})
        for img in sorted(imgs, key=lambda v: v.name):
            print(img.id,':',img.name)
        print()


@task
def instance_start(cn=def_cn,
        avz=def_default_avz,
        rt=def_default_requesttype):
    """
    Start and prepare instances
    """
    # FSO---find already existing nodes
    cloud = boto.ec2.connect_to_region(avz[:-1],profile_name=ec2Profile)
    filters = {'tag:type': cn+'node'}
    insta = cloud.get_all_instances(filters=filters)

    # FSO---install each new node
    print("Requesting new instance")
    log_with_ts("Requesting new instance")

    nodenumber = len(insta) + 1

    node_install(cn=cn, avz=avz, rt=rt, idn=nodenumber)

    log_with_ts('Finished installing instance')

    cloud_list()


def print_instance(inst):
    if inst.state != 'terminated':
        cu_time = datetime.datetime.utcnow()
        it =  datetime.datetime.strptime(inst.launch_time,'%Y-%m-%dT%H:%M:%S.000Z')
    else:
        try:
            cu_time = datetime.datetime.strptime(inst.tags.get('terminate_time'),'%Y-%m-%dT%H:%M:%S.%f')
        except:
            cu_time = datetime.datetime.utcnow()

        it = datetime.datetime.strptime(inst.launch_time,'%Y-%m-%dT%H:%M:%S.000Z')

    time_taken = cu_time - it
    hours, rest = divmod(time_taken.total_seconds(),3600)
    minutes, seconds = divmod(rest, 60)

    print(inst.id, inst.instance_type, \
            inst.tags.get('Name'), \
            inst.tags.get('type'), \
            inst.state, \
            inst.dns_name, \
            inst.private_ip_address, \
            inst.private_dns_name, \
            inst.tags.get('current_price'), \
            inst.tags.get('billable_hours'), \
            inst.tags.get('terminate_time'), \
            inst.placement, \
            'Owner:%s' % inst.tags.get('node-owner'))
    print("running for: ", hours,'h', minutes, "min")


def print_volume(vol):
    info = ""
    if 'vol-lifetime' in vol.tags:
        info += '\tLifetime: ' + vol.tags['vol-lifetime']
    if 'vol-user-name' in vol.tags:
        info += '\tUservolume Name: ' + vol.tags['vol-user-name']
    if 'vol-owner' in vol.tags:
        info += '\tOwner: ' + vol.tags['vol-owner']

    print(vol.id, "\t", vol.zone, "\t", vol.status, '\t', vol.size, info)


@task
def cloud_list(cn=def_cn,itype='all',regions=def_regions):
    """
    List all ec2 instances.
    """
    for region in regions:
        cloud = boto.ec2.connect_to_region(region,profile_name=ec2Profile)
        instances = cloud.get_all_instances()
        vols = cloud.get_all_volumes()
        print()
        print("-------CURRENT RUNNING-----------")
        print("       REGION:", region)
        print()
        print("Instances:")
        print()

        update_costs(cn=cn,regions=regions,itype=itype)

        for reservation in instances:
            for inst in reservation.instances:
                print_instance(inst)
                print()

        print()
        print("Volumes:")
        print()

        for vol in vols:
            print_volume(vol)


def check_keypair(cloud, keynames):
    # Check to see if specified keypair already exists.
    # If we get an InvalidKeyPair.NotFound error back from EC2,
    # it means that it doesn't exist and we need to create it.
    key_dir = def_key_dir
    try:
        cloud.get_all_key_pairs(keynames=[keynames])[0]
    except cloud.ResponseError as e:
        if e.code == 'InvalidKeyPair.NotFound':
            print('Creating keypair: %s' % keynames)
            # Create an SSH key to use when logging into instances.
            key = cloud.create_key_pair(keynames)
            # Make sure the specified key_dir actually exists.
            # If not, create it.
            key_dir = os.path.expanduser(key_dir)
            key_dir = os.path.expandvars(key_dir)
            # if not os.path.isdir(key_dir):
            #     os.mkdir(key_dir, 0700)
            #
            # AWS will store the public key but the private key is
            # generated and returned and needs to be stored locally.
            # The save method will also chmod the file to protect
            # your private key.
            key.save(key_dir)
        else:
            raise


def get_keypair_name():
    key_dir = def_key_dir
    key_dir = os.path.expanduser(key_dir)
    key_dir = os.path.expandvars(key_dir)

    un_file = os.path.join(key_dir, '%s_unique.txt' % keyn)

    if os.path.exists(un_file):
        with open(un_file, 'r') as un:
            unique_part = un.read().strip()
    else:
        import uuid
        unique_part = str(uuid.uuid4().get_hex().upper()[0:8])
        with open(un_file, 'w') as un:
            un.write(unique_part)

    return keyn + '_' + unique_part


def get_user_persist_ebs(cloud, avz):
    if home_volume_ebs_name is None:
        return None

    vols = cloud.get_all_volumes(filters={'tag:vol-user-name':home_volume_ebs_name, 'availability-zone': avz})

    if len(vols) == 0:
        print("Creating new EBS volume for user volume %s" % home_volume_ebs_name)
        vol = cloud.create_volume(new_homefs_size_gb, avz)
        vol.add_tag('vol-user-name', home_volume_ebs_name)
        vol.add_tag('vol-lifetime', 'perm')
        vol.add_tag('vol-owner', user_identifier)
    else:
        vol = vols[0]
        print("Found existing volume %s for user volume %s!" % (vol.id, home_volume_ebs_name))

        if vol.status != 'available':
            print("But it's not available...")
            return None

    return vol


@task
def node_install(cn=def_cn,inst_type_idx=def_inst_type,idn=0,
        avz=def_default_avz,rt=def_default_requesttype,
        group_name='oggmssh',
        ssh_port=22,
        cidr='0.0.0.0/0'):
    """
    Request and prepare single instance
    """
    # FSO---connect
    cloud = boto.ec2.connect_to_region(avz[:-1],profile_name=ec2Profile)
    aminfo = cloud.get_image(def_ami[avz[:-1]])

    # FSO---check if node with same name already exists
    if node_exists(cn + '_node' + str(idn)):
        print("Node already exists")
        sys.exit()

    # Check if ssh keypair exists
    key_name = get_keypair_name()
    check_keypair(cloud, key_name)

    # FSO---create a bigger root device
    dev_sda1 = EBSBlockDeviceType()
    dev_sda1.size = rootfs_size_gb
    dev_sda1.delete_on_termination = True
    bdm = BlockDeviceMapping()
    bdm['/dev/sda1'] = dev_sda1

    dev_sdf_vol = get_user_persist_ebs(cloud, avz)

    # Check to see if specified security group already exists.
    # If we get an InvalidGroup.NotFound error back from EC2,
    # it means that it doesn't exist and we need to create it.
    try:
        group = cloud.get_all_security_groups(groupnames=[group_name])[0]
    except cloud.ResponseError as e:
        if e.code == 'InvalidGroup.NotFound':
            print('Creating Security Group: %s' % group_name)
            # Create a security group to control access to instance via SSH.
            group = cloud.create_security_group(group_name, 'A group that allows SSH access')
        else:
            raise

    # Add a rule to the security group to authorize SSH traffic
    # on the specified port.
    try:
        group.authorize('tcp', ssh_port, ssh_port, cidr)
    except cloud.ResponseError as e:
        if e.code == 'InvalidPermission.Duplicate':
            print('Security Group: %s already authorized' % group_name)
        else:
            raise

    log_with_ts("request node "+str(idn))
    print('Reserving instance for node', aminfo.id, instance_infos[inst_type_idx]['type'], aminfo.name, aminfo.region)

    if rt == 'spot':
        print("placing node in ",avz)
        requests = cloud.request_spot_instances(def_price,
                      def_ami[avz[:-1]],
                      count=1,
                      type='one-time',
                      security_groups=[group_name],
                      key_name=key_name,
                      placement=avz,
                      instance_type=instance_infos[inst_type_idx]['type'],
                      block_device_map=bdm)
        req_ids = [request.id for request in requests]
        instance_ids = wait_for_fulfillment(cloud,req_ids)
        instances = cloud.get_only_instances(instance_ids=instance_ids)
        node = instances[0]
        log_with_ts("fullfilled spot node "+str(idn))
    else:
        print("placing node in ",avz)
        reservation = cloud.run_instances(image_id=def_ami[avz[:-1]],
                key_name=key_name,
                placement = avz,
                security_groups=[group_name],
                instance_type=instance_infos[inst_type_idx]['type'],
                block_device_map= bdm)
        node = reservation.instances[0]
        log_with_ts("fullfilled ondemand node "+str(idn))

    time.sleep(2)
    while not node.update() == 'running':
        print('waiting for', cn, 'node', idn, 'to boot...')
        time.sleep(5)

    log_with_ts("booted node "+str(idn))

    if dev_sdf_vol is not None:
        cloud.attach_volume(dev_sdf_vol.id, node.id, "/dev/sdf")

    node.add_tag('Name', cn+'_node'+str(idn))
    node.add_tag('type', cn+'node')
    node.add_tag('node-owner', user_identifier)

    # FSO---set delete on termination flag to true for ebs block device
    node.modify_attribute('blockDeviceMapping', { '/dev/sda1' : True })

    # FSO--- test socket connect to ssh service
    ssh_test(node)
    log_with_ts("reachable node "+str(idn))

    update_key_filename(node.region.name)

    # Mount potential user volume
    if dev_sdf_vol is not None:
        use_user_volume(node.dns_name)

    log_with_ts("finished node "+str(idn))


@task
def install_node_software(nn=''):
    """
    Setup ready-for-use virtualenv for OGGM on instance
    """
    inst = select_instance(nn)
    install_node_apt('', inst)
    install_node_pip('', inst)

    run('sudo shutdown -r now')


@task
def install_node_pip(nn='', inst=None):
    """
    Install oggm dependencies via pip
    """
    if inst is None:
        inst = select_instance(nn)
    update_key_filename(inst.region.name)
    env.host_string = inst.dns_name
    env.user = 'ubuntu'

    run("""
    source ~/.virtenvrc &&
    workon oggm_env &&
    pip install --upgrade pip &&
    pip install numpy &&
    pip install scipy &&
    pip install pandas shapely cython &&
    pip install matplotlib &&
    pip install gdal==1.10.0 --install-option="build_ext" --install-option="--include-dirs=/usr/include/gdal" &&
    pip install fiona --install-option="build_ext" --install-option="--include-dirs=/usr/include/gdal" &&
    pip install pyproj rasterio Pillow geopandas netcdf4 scikit-image configobj joblib xarray nose &&
    pip install git+https://github.com/fmaussion/motionless.git &&
    pip install git+https://github.com/fmaussion/salem.git &&
    pip install git+https://github.com/fmaussion/cleo.git &&
    sed -i 's/^backend.*/backend      : Agg/' "${WORKON_HOME}"/oggm_env/lib/python?.?/site-packages/matplotlib/mpl-data/matplotlibrc
    """, pty=False)


@task
def install_node_apt(nn='', inst=None):
    """
    Install required OGGM apt dependencies
    """
    if inst is None:
        inst = select_instance(nn)
    update_key_filename(inst.region.name)
    env.host_string = inst.dns_name
    env.user = 'ubuntu'

    run("""
    sudo apt-get update &&
    sudo apt-get -y dist-upgrade &&
    sudo apt-get install -y build-essential liblapack-dev gfortran libproj-dev gdal-bin libgdal-dev netcdf-bin ncview python-netcdf tk-dev python3-tk python3-dev ttf-bitstream-vera python-pip git awscli virtualenvwrapper
    """, pty=False)

    copy_files = ['~/.aws/config', '~/.screenrc', '~/.gitconfig']

    for cf in copy_files:
        if not os.path.exists(os.path.expanduser(cf)):
            continue
        run('mkdir -p %s' % os.path.dirname(cf))
        put(cf, cf)

    run("""
{'fraction_non_alphanumeric': 0.1393244720054896, 'fraction_numerical': 0.00889521437466643, 'mean_word_length': 2.4976, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2602731', 'n_tokens_mistral': 13011, 'n_tokens_neox': 12427, 'n_words': 3220}