<commit_before>"""Create an Intel ICEL stack on AWS.
"""
from __future__ import print_function

import argparse
import getpass
import json
import os
import re
import socket
import struct
import sys
import time

import boto.cloudformation
import boto.ec2
import boto.s3
import requests

from bcbiovm.aws import common


ICEL_TEMPLATES = {
    'ap-northeast-1': 'http://s3-ap-northeast-1.amazonaws.com/hpdd-templates-ap-northeast-1/gs-hvm/1.0.1/hpdd-gs-hvm-ha-c3-small-1.0.1.template',
    'ap-southeast-1': 'http://s3-ap-southeast-1.amazonaws.com/hpdd-templates-ap-southeast-1/gs-hvm/1.0.1/hpdd-gs-hvm-ha-c3-small-1.0.1.template',
    'ap-southeast-2': 'http://s3-ap-southeast-2.amazonaws.com/hpdd-templates-ap-southeast-2/gs-hvm/1.0.1/hpdd-gs-hvm-ha-c3-small-1.0.1.template',
    'eu-west-1': 'http://s3-eu-west-1.amazonaws.com/hpdd-templates-eu-west-1/gs-hvm/1.0.1/hpdd-gs-hvm-ha-c3-small-1.0.1.template',
    'sa-east-1': 'http://s3-sa-east-1.amazonaws.com/hpdd-templates-sa-east-1/gs-hvm/1.0.1/hpdd-gs-hvm-ha-c3-small-1.0.1.template',
    'us-east-1': 'http://s3.amazonaws.com/hpdd-templates-us-east-1/gs-hvm/1.0.1/hpdd-gs-hvm-ha-c3-small-1.0.1.template',
    'us-west-1': 'http://s3-us-west-1.amazonaws.com/hpdd-templates-us-west-1/gs-hvm/1.0.1/hpdd-gs-hvm-ha-c3-small-1.0.1.template',
    'us-west-2': 'http://s3-us-west-2.amazonaws.com/hpdd-templates-us-west-2/gs-hvm/1.0.1/hpdd-gs-hvm-ha-c3-small-1.0.1.template',
}


def setup_cmd(awsparser):
    parser_c = awsparser.add_parser("icel",
                                    help="Create scratch filesystem using Intel Cloud Edition for Lustre")
    icel_parser = parser_c.add_subparsers(title="[icel create]")

    # ## Create

    parser = icel_parser.add_parser("create",
                                    help="Create scratch filesystem using Intel Cloud Edition for Lustre",
                                    formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument("--econfig", help="Elasticluster bcbio configuration file",
                        default=common.DEFAULT_EC_CONFIG)
    parser.add_argument("--recreate", action="store_true", default=False,
                        help="Remove and recreate the stack, "
                             "destroying all data stored on it")
    parser.add_argument("-c", "--cluster", default="bcbio",
                        help="elasticluster cluster name")

    parser.add_argument("-s", "--size", type=int, default="2048",
                        help="Size of the Lustre filesystem, in gigabytes")
    parser.add_argument("-o", "--oss-count", type=int, default="4",
                        help="Number of OSS nodes")
    parser.add_argument("-l", "--lun-count", type=int, default="4",
                        help="Number of EBS LUNs per OSS")

    parser.add_argument("-n", "--network", metavar="NETWORK", dest="network",
                        help="Network (in CIDR notation, a.b.c.d/e) to "
                             "place Lustre servers in")

    parser.add_argument("-b", "--bucket", default="bcbio-lustre-%s" % getpass.getuser(),
                        help="bucket to store generated ICEL template for CloudFormation")
    parser.add_argument(metavar="STACK_NAME", dest="stack_name", nargs="?",
                        default="bcbiolustre",
                        help="CloudFormation name for the new stack")
    parser.set_defaults(func=create)

    # ## Spec

    parser = icel_parser.add_parser("fs_spec",
                                    help="Get the filesystem spec for a running filesystem",
                                    formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument("--econfig", help="Elasticluster bcbio configuration file",
                        default=common.DEFAULT_EC_CONFIG)
    parser.add_argument("-c", "--cluster", default="bcbio",
                        help="elasticluster cluster name")
    parser.add_argument(metavar="STACK_NAME", dest="stack_name", nargs="?",
                        default="bcbiolustre",
                        help="CloudFormation name for the stack")
    parser.set_defaults(func=fs_spec)

    # ## Mount

    parser = icel_parser.add_parser("mount",
                                    help="Mount Lustre filesystem on all cluster nodes",
                                    formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument("--econfig", help="Elasticluster bcbio configuration file",
                        default=common.DEFAULT_EC_CONFIG)
    parser.add_argument("-c", "--cluster", default="bcbio",
                        help="elasticluster cluster name")
    parser.add_argument("-v", "--verbose", action="count", default=0,
                        help="Emit verbose output when running "
                             "Ansible playbooks")
    parser.add_argument(metavar="STACK_NAME", dest="stack_name", nargs="?",
                        default="bcbiolustre",
                        help="CloudFormation name for the new stack")
    parser.set_defaults(func=mount)

    # ## Stop

    parser = icel_parser.add_parser("stop",
                                    help="Stop the running Lustre filesystem and clean up resources",
                                    formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument("--econfig", help="Elasticluster bcbio configuration file",
                        default=common.DEFAULT_EC_CONFIG)
    parser.add_argument("-c", "--cluster", default="bcbio",
                        help="elasticluster cluster name")
    parser.add_argument(metavar="STACK_NAME", dest="stack_name", nargs="?",
                        default="bcbiolustre",
                        help="CloudFormation name for the new stack")
    parser.set_defaults(func=stop)


def create(args):
    if args.network:
        cidr_regex = r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/\d{1,2}$'
        if not re.search(cidr_regex, args.network):
            sys.stderr.write(
                'Network {} is not in CIDR (a.b.c.d/e) format.\n'.format(
                    args.network))
            sys.exit(1)

    cluster_config = common.ecluster_config(args.cluster, args.econfig)

    icel_param = {
        'oss_count': args.oss_count,
        'ost_vol_size': args.size / args.oss_count / args.lun_count,
        'ost_vol_count': args.lun_count,
    }
    template_url = _upload_icel_cf_template(
        icel_param, args.bucket, cluster_config['cloud'])

    _create_icel_stack(
        args.stack_name, template_url, args.network,
        args.cluster, cluster_config, args.recreate)
    try:
        sys.stdout.write('Waiting for stack to launch (this will take '
                         'a few minutes)')
        sys.stdout.flush()
        _wait_for_stack(args.stack_name, 'CREATE_COMPLETE',
                        15 * 60, cluster_config['cloud'])
    except Exception as e:
        sys.stderr.write('{}\n'.format(str(e)))
        sys.exit(1)


def fs_spec(args):
    cluster_config = common.ecluster_config(args.cluster, args.econfig)
    print(_get_fs_spec(args.stack_name, cluster_config['cloud']))


def mount(args):
    playbook_path = os.path.join(sys.prefix, "share", "bcbio-vm", "ansible",
                                 "roles", "lustre_client", "tasks", "main.yml")
    def get_lustre_vars(args, cluster_config):
        return {'lustre_fs_spec': _get_fs_spec(
            args.stack_name, cluster_config['cloud'])}
    common.run_ansible_pb(playbook_path, args, get_lustre_vars)

def _template_param(tree, param):
    return [
        (i, name)
        for i, name
         in enumerate(tree)
         if type(name) in (str, unicode) and
            name.startswith(param)
    ][0]

def _upload_icel_cf_template(param, bucket_name, aws_config):
    url = ICEL_TEMPLATES[aws_config['ec2_region']]
    source_template = requests.get(url)
    tree = json.loads(source_template.text)
    tree['Description'].replace(
        '4 Object Storage Servers',
        '{} Object Storage Servers'.format(param['oss_count']))
    if aws_config["ec2_region"] == "us-east-1":
        tree["Parameters"]["NATInstanceType"]["AllowedValues"].append("m3.medium")
        tree["Mappings"]["AWSNATAMI"]["us-east-1"]["AMI"] = "ami-184dc970"
    resources = tree['Resources']

    # We don't need the demo Lustre client instance.
    del resources['ClientInstanceProfile']
    del resources['ClientLaunchConfig']<commit_msg>actually update the icel CF template's description<commit_after>"""Create an Intel ICEL stack on AWS.
"""
from __future__ import print_function

import argparse
import getpass
import json
import os
import re
import socket
import struct
import sys
import time

import boto.cloudformation
import boto.ec2
import boto.s3
import requests

from bcbiovm.aws import common


ICEL_TEMPLATES = {
    'ap-northeast-1': 'http://s3-ap-northeast-1.amazonaws.com/hpdd-templates-ap-northeast-1/gs-hvm/1.0.1/hpdd-gs-hvm-ha-c3-small-1.0.1.template',
    'ap-southeast-1': 'http://s3-ap-southeast-1.amazonaws.com/hpdd-templates-ap-southeast-1/gs-hvm/1.0.1/hpdd-gs-hvm-ha-c3-small-1.0.1.template',
    'ap-southeast-2': 'http://s3-ap-southeast-2.amazonaws.com/hpdd-templates-ap-southeast-2/gs-hvm/1.0.1/hpdd-gs-hvm-ha-c3-small-1.0.1.template',
    'eu-west-1': 'http://s3-eu-west-1.amazonaws.com/hpdd-templates-eu-west-1/gs-hvm/1.0.1/hpdd-gs-hvm-ha-c3-small-1.0.1.template',
    'sa-east-1': 'http://s3-sa-east-1.amazonaws.com/hpdd-templates-sa-east-1/gs-hvm/1.0.1/hpdd-gs-hvm-ha-c3-small-1.0.1.template',
    'us-east-1': 'http://s3.amazonaws.com/hpdd-templates-us-east-1/gs-hvm/1.0.1/hpdd-gs-hvm-ha-c3-small-1.0.1.template',
    'us-west-1': 'http://s3-us-west-1.amazonaws.com/hpdd-templates-us-west-1/gs-hvm/1.0.1/hpdd-gs-hvm-ha-c3-small-1.0.1.template',
    'us-west-2': 'http://s3-us-west-2.amazonaws.com/hpdd-templates-us-west-2/gs-hvm/1.0.1/hpdd-gs-hvm-ha-c3-small-1.0.1.template',
}


def setup_cmd(awsparser):
    parser_c = awsparser.add_parser("icel",
                                    help="Create scratch filesystem using Intel Cloud Edition for Lustre")
    icel_parser = parser_c.add_subparsers(title="[icel create]")

    # ## Create

    parser = icel_parser.add_parser("create",
                                    help="Create scratch filesystem using Intel Cloud Edition for Lustre",
                                    formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument("--econfig", help="Elasticluster bcbio configuration file",
                        default=common.DEFAULT_EC_CONFIG)
    parser.add_argument("--recreate", action="store_true", default=False,
                        help="Remove and recreate the stack, "
                             "destroying all data stored on it")
    parser.add_argument("-c", "--cluster", default="bcbio",
                        help="elasticluster cluster name")

    parser.add_argument("-s", "--size", type=int, default="2048",
                        help="Size of the Lustre filesystem, in gigabytes")
    parser.add_argument("-o", "--oss-count", type=int, default="4",
                        help="Number of OSS nodes")
    parser.add_argument("-l", "--lun-count", type=int, default="4",
                        help="Number of EBS LUNs per OSS")

    parser.add_argument("-n", "--network", metavar="NETWORK", dest="network",
                        help="Network (in CIDR notation, a.b.c.d/e) to "
                             "place Lustre servers in")

    parser.add_argument("-b", "--bucket", default="bcbio-lustre-%s" % getpass.getuser(),
                        help="bucket to store generated ICEL template for CloudFormation")
    parser.add_argument(metavar="STACK_NAME", dest="stack_name", nargs="?",
                        default="bcbiolustre",
                        help="CloudFormation name for the new stack")
    parser.set_defaults(func=create)

    # ## Spec

    parser = icel_parser.add_parser("fs_spec",
                                    help="Get the filesystem spec for a running filesystem",
                                    formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument("--econfig", help="Elasticluster bcbio configuration file",
                        default=common.DEFAULT_EC_CONFIG)
    parser.add_argument("-c", "--cluster", default="bcbio",
                        help="elasticluster cluster name")
    parser.add_argument(metavar="STACK_NAME", dest="stack_name", nargs="?",
                        default="bcbiolustre",
                        help="CloudFormation name for the stack")
    parser.set_defaults(func=fs_spec)

    # ## Mount

    parser = icel_parser.add_parser("mount",
                                    help="Mount Lustre filesystem on all cluster nodes",
                                    formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument("--econfig", help="Elasticluster bcbio configuration file",
                        default=common.DEFAULT_EC_CONFIG)
    parser.add_argument("-c", "--cluster", default="bcbio",
                        help="elasticluster cluster name")
    parser.add_argument("-v", "--verbose", action="count", default=0,
                        help="Emit verbose output when running "
                             "Ansible playbooks")
    parser.add_argument(metavar="STACK_NAME", dest="stack_name", nargs="?",
                        default="bcbiolustre",
                        help="CloudFormation name for the new stack")
    parser.set_defaults(func=mount)

    # ## Stop

    parser = icel_parser.add_parser("stop",
                                    help="Stop the running Lustre filesystem and clean up resources",
                                    formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument("--econfig", help="Elasticluster bcbio configuration file",
                        default=common.DEFAULT_EC_CONFIG)
    parser.add_argument("-c", "--cluster", default="bcbio",
                        help="elasticluster cluster name")
    parser.add_argument(metavar="STACK_NAME", dest="stack_name", nargs="?",
                        default="bcbiolustre",
                        help="CloudFormation name for the new stack")
    parser.set_defaults(func=stop)


def create(args):
    if args.network:
        cidr_regex = r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/\d{1,2}$'
        if not re.search(cidr_regex, args.network):
            sys.stderr.write(
                'Network {} is not in CIDR (a.b.c.d/e) format.\n'.format(
                    args.network))
            sys.exit(1)

    cluster_config = common.ecluster_config(args.cluster, args.econfig)

    icel_param = {
        'oss_count': args.oss_count,
        'ost_vol_size': args.size / args.oss_count / args.lun_count,
        'ost_vol_count': args.lun_count,
    }
    template_url = _upload_icel_cf_template(
        icel_param, args.bucket, cluster_config['cloud'])

    _create_icel_stack(
        args.stack_name, template_url, args.network,
        args.cluster, cluster_config, args.recreate)
    try:
        sys.stdout.write('Waiting for stack to launch (this will take '
                         'a few minutes)')
        sys.stdout.flush()
        _wait_for_stack(args.stack_name, 'CREATE_COMPLETE',
                        15 * 60, cluster_config['cloud'])
    except Exception as e:
        sys.stderr.write('{}\n'.format(str(e)))
        sys.exit(1)


def fs_spec(args):
    cluster_config = common.ecluster_config(args.cluster, args.econfig)
    print(_get_fs_spec(args.stack_name, cluster_config['cloud']))


def mount(args):
    playbook_path = os.path.join(sys.prefix, "share", "bcbio-vm", "ansible",
                                 "roles", "lustre_client", "tasks", "main.yml")
    def get_lustre_vars(args, cluster_config):
        return {'lustre_fs_spec': _get_fs_spec(
            args.stack_name, cluster_config['cloud'])}
    common.run_ansible_pb(playbook_path, args, get_lustre_vars)

def _template_param(tree, param):
    return [
        (i, name)
        for i, name
         in enumerate(tree)
         if type(name) in (str, unicode) and
            name.startswith(param)
    ][0]

def _upload_icel_cf_template(param, bucket_name, aws_config):
    url = ICEL_TEMPLATES[aws_config['ec2_region']]
    source_template = requests.get(url)
    tree = json.loads(source_template.text)
    tree['Description'] = tree['Description'].replace(
        '4 Object Storage Servers',
        '{} Object Storage Servers'.format(param['oss_count']))
    if aws_config["ec2_region"] == "us-east-1":
        tree["Parameters"]["NATInstanceType"]["AllowedValues"].append("m3.medium")
        tree["Mappings"]["AWSNATAMI"]["us-east-1"]["AMI"] = "ami-184dc970"
    resources = tree['Resources']

    # We don't need the demo Lustre client instance.
    del resources['ClientInstanceProfile']
    del resources['ClientLaunchConfig']
{'fraction_non_alphanumeric': 0.13756863671252287, 'fraction_numerical': 0.014760583338253528, 'mean_word_length': 2.403254972875226, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7205536', 'n_tokens_mistral': 5494, 'n_tokens_neox': 5329, 'n_words': 902}