<commit_before># -*- coding: utf-8 -*-

from __future__ import with_statement

import codecs, logging, os, pprint, sys, re, time, subprocess
from random import choice
from functools import wraps
import argparse
import virtualenv
from libcloud.drivers import ec2, rackspace, linode
# Add as you test support for more providers

# http://wiki.apache.org/incubator/LibcloudSSL
CA_CERTS_PATH = '/etc/ssl/certs/' # Debian ca-certificates package CA_CERTS path
if os.path.exists(CA_CERTS_PATH):
    import libcloud.security
    libcloud.security.VERIFY_SSL_CERT = True
    libcloud.security.CA_CERTS_PATH.append(CA_CERTS_PATH)

import kraftwerk
from kraftwerk.project import Project
from kraftwerk.node import Node
from kraftwerk.exc import ConfigError
from kraftwerk.config import Config, path as config_path
from kraftwerk.cli.parser import subparsers
from kraftwerk.cli.utils import confirm
from kraftwerk.compat import relpath
from kraftwerk import etchosts
from kraftwerk import services

IP_RE = re.compile(r'(?:[\d]{1,3})\.(?:[\d]{1,3})\.(?:[\d]{1,3})\.(?:[\d]{1,3})')

def command(function):
    """Decorator/wrapper to declare a function as a kraftwerk CLI task."""
    
    cmd_name = function.__name__.replace('_', '-')
    help = (function.__doc__ or '').rstrip('.') or None
    parser = subparsers.add_parser(cmd_name, help=help)
    
    @wraps(function)
    def wrapper(config, args):
        logging.getLogger('kraftwerk').debug('Running kraftwerk.%s' % cmd_name)
        return function(config, args)
    wrapper.parser = parser
    
    return wrapper

class ProjectAction(argparse.Action):
    def __call__(self, parser, namespace, value, option_string=None):
        if value is None:
            value = os.getcwd()
        try:
            proj = Project(os.path.abspath(value))
        except IOError:
            sys.exit("kraftwerk.yaml is missing or project path is " \
                     "not a kraftwerk project directory. \n")
        try:
            proj.clean()
        except ConfigError, e:
            print "Project config did not validate: %s" % e
            sys.exit()
        setattr(namespace, self.dest, proj)

class NodeAction(argparse.Action):
    def __call__(self, parser, namespace, value, option_string=None):
        if value is None:
            config = Config.for_file(namespace.config)
            if "default_node" in config:
                value = config["default_node"]
                print "Using default_node (%s)" % value
        if value:
            setattr(namespace, self.dest, Node(value))
        else:
            sys.exit("You must specify a 'node' value")
            

## Utilities

@command
def show_config(config, args):
    """Pretty-print the current kraftwerk configuration."""
    pprint.pprint(config)

def _copy_project(config, title, project_root):
    """`init` helper to push the project skeleton through templates 
    and copy over."""
    
    secret = ''.join([choice('abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)') for i in range(50)])
    start = os.path.join(kraftwerk.templates_root, 'project')
    
    for dirpath, dirnames, filenames in os.walk(start):
        # Copy template skeleton
        for name in dirnames + filenames:
            
            path = os.path.join(dirpath, name)
            rel = relpath(path, start)
            
            project_rel = rel
            if "project" in rel:
                project_rel = rel.replace("project", title)

            dest_path = os.path.join(project_root, project_rel)

            if not os.path.exists(dest_path):
                if os.path.isdir(path):
                    os.makedirs(dest_path)
                else:
                    with codecs.open(dest_path, 'w', encoding='utf-8') as fp:
                        tpl = config.templates.get_template(os.path.join('project', rel))
                        fp.write(tpl.render({'project_title': title, 'secret': secret}))

@command
def init(config, args):
    """Create a new WSGI project skeleton and config."""
    
    log = logging.getLogger('kraftwerk.init')
    
    # project_root/src_root
    project_root = os.path.abspath(os.path.join(os.getcwd(), args.title))
    src_root = os.path.join(project_root, args.title)
    
    if os.path.exists(project_root) and os.listdir(project_root):
        init.parser.error("Directory exists and isn't empty")
    elif not os.path.exists(project_root):
        log.debug('makedirs %s' % project_root)
        os.makedirs(project_root)
    elif not os.path.isdir(project_root):
        init.parser.error("A file with this name already exists here and isn't a directory")
    
    log.debug('virtualenv %s/' % args.title)
    logger = virtualenv.Logger([
                 (virtualenv.Logger.level_for_integer(2), sys.stdout)])
    virtualenv.logger = logger
    virtualenv.create_environment(args.title, site_packages=True)
    
    log.debug('mkdir %s/service/' % project_root)
    os.makedirs(os.path.join(args.title, 'service'))
    log.debug('mkdir %s/static/' % project_root)
    os.makedirs(os.path.join(args.title, 'static'))
    log.debug('mkdir %s/' % src_root)
    os.makedirs(src_root)
    
    _copy_project(config, args.title, project_root)
    print "Your new project is at: %s" % project_root

init.parser.add_argument('title', default=None,
    help="Create kraftwerk project (if omitted, defaults to current directory)")


@command
def create_node(config, args):
    """Commissions a new server node."""
    log = logging.getLogger('kraftwerk.create-node')

    if 'pubkey' in config:
        pubkey_paths = [config["pubkey"]]
    else:
        pubkey_paths = [os.path.join(os.environ['HOME'], '.ssh', f) for f \
            in ['id_rsa.pub', 'id_dsa.pub']]

    for path in pubkey_paths:
        if os.path.exists(path):
            print 'SSH public key: %s' % path
            with open(path) as fp:
                pubkey = fp.read().strip()
            break
    else:
        pubkey = raw_input('Your SSH public key (for root login): ')
    
    if not re.search(r'^[a-z0-9.-]+$', args.hostname):
        raise CommandError(
            "Invalid hostname (must contain only letters, numbers, ., and -): %r"
            % args.hostname)
    
    # Query driver for size, image, and location
    
    image_id = getattr(args, 'image-id', config["image_id"])
    for i in config.driver.list_images():
        if str(i.id) == image_id:
            image = i
            break
    else:
        sys.exit("Image %s not found for this provider. Aborting." % image_id)
            
    size_id = getattr(args, 'size-id', config["size_id"])
    for s in config.driver.list_sizes():
        if str(s.id) == size_id:
            size = s
            break
    else:
        sys.exit("Size %s not found for this provider. Aborting." % size_id)
    
    location_id = getattr(args, 'location-id', config.get("location_id", "0"))
    for l in config.driver.list_locations():
        if str(l.id) == location_id:
            location = l
            break
    else:
        sys.exit("Location %s not found for this provider. Aborting." % location_id)
    
    if isinstance(config.driver, ec2.EC2NodeDriver):
        extra = dict(ex_userdata="""#!/bin/bash
echo '%s' > /root/.ssh/authorized_keys""" % pubkey)
        if not "keyname" in config:<commit_msg>Making sure that the YAML parsed stuff is a string to avoid failing in the next if comparison<commit_after># -*- coding: utf-8 -*-

from __future__ import with_statement

import codecs, logging, os, pprint, sys, re, time, subprocess
from random import choice
from functools import wraps
import argparse
import virtualenv
from libcloud.drivers import ec2, rackspace, linode
# Add as you test support for more providers

# http://wiki.apache.org/incubator/LibcloudSSL
CA_CERTS_PATH = '/etc/ssl/certs/' # Debian ca-certificates package CA_CERTS path
if os.path.exists(CA_CERTS_PATH):
    import libcloud.security
    libcloud.security.VERIFY_SSL_CERT = True
    libcloud.security.CA_CERTS_PATH.append(CA_CERTS_PATH)

import kraftwerk
from kraftwerk.project import Project
from kraftwerk.node import Node
from kraftwerk.exc import ConfigError
from kraftwerk.config import Config, path as config_path
from kraftwerk.cli.parser import subparsers
from kraftwerk.cli.utils import confirm
from kraftwerk.compat import relpath
from kraftwerk import etchosts
from kraftwerk import services

IP_RE = re.compile(r'(?:[\d]{1,3})\.(?:[\d]{1,3})\.(?:[\d]{1,3})\.(?:[\d]{1,3})')

def command(function):
    """Decorator/wrapper to declare a function as a kraftwerk CLI task."""
    
    cmd_name = function.__name__.replace('_', '-')
    help = (function.__doc__ or '').rstrip('.') or None
    parser = subparsers.add_parser(cmd_name, help=help)
    
    @wraps(function)
    def wrapper(config, args):
        logging.getLogger('kraftwerk').debug('Running kraftwerk.%s' % cmd_name)
        return function(config, args)
    wrapper.parser = parser
    
    return wrapper

class ProjectAction(argparse.Action):
    def __call__(self, parser, namespace, value, option_string=None):
        if value is None:
            value = os.getcwd()
        try:
            proj = Project(os.path.abspath(value))
        except IOError:
            sys.exit("kraftwerk.yaml is missing or project path is " \
                     "not a kraftwerk project directory. \n")
        try:
            proj.clean()
        except ConfigError, e:
            print "Project config did not validate: %s" % e
            sys.exit()
        setattr(namespace, self.dest, proj)

class NodeAction(argparse.Action):
    def __call__(self, parser, namespace, value, option_string=None):
        if value is None:
            config = Config.for_file(namespace.config)
            if "default_node" in config:
                value = config["default_node"]
                print "Using default_node (%s)" % value
        if value:
            setattr(namespace, self.dest, Node(value))
        else:
            sys.exit("You must specify a 'node' value")
            

## Utilities

@command
def show_config(config, args):
    """Pretty-print the current kraftwerk configuration."""
    pprint.pprint(config)

def _copy_project(config, title, project_root):
    """`init` helper to push the project skeleton through templates 
    and copy over."""
    
    secret = ''.join([choice('abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)') for i in range(50)])
    start = os.path.join(kraftwerk.templates_root, 'project')
    
    for dirpath, dirnames, filenames in os.walk(start):
        # Copy template skeleton
        for name in dirnames + filenames:
            
            path = os.path.join(dirpath, name)
            rel = relpath(path, start)
            
            project_rel = rel
            if "project" in rel:
                project_rel = rel.replace("project", title)

            dest_path = os.path.join(project_root, project_rel)

            if not os.path.exists(dest_path):
                if os.path.isdir(path):
                    os.makedirs(dest_path)
                else:
                    with codecs.open(dest_path, 'w', encoding='utf-8') as fp:
                        tpl = config.templates.get_template(os.path.join('project', rel))
                        fp.write(tpl.render({'project_title': title, 'secret': secret}))

@command
def init(config, args):
    """Create a new WSGI project skeleton and config."""
    
    log = logging.getLogger('kraftwerk.init')
    
    # project_root/src_root
    project_root = os.path.abspath(os.path.join(os.getcwd(), args.title))
    src_root = os.path.join(project_root, args.title)
    
    if os.path.exists(project_root) and os.listdir(project_root):
        init.parser.error("Directory exists and isn't empty")
    elif not os.path.exists(project_root):
        log.debug('makedirs %s' % project_root)
        os.makedirs(project_root)
    elif not os.path.isdir(project_root):
        init.parser.error("A file with this name already exists here and isn't a directory")
    
    log.debug('virtualenv %s/' % args.title)
    logger = virtualenv.Logger([
                 (virtualenv.Logger.level_for_integer(2), sys.stdout)])
    virtualenv.logger = logger
    virtualenv.create_environment(args.title, site_packages=True)
    
    log.debug('mkdir %s/service/' % project_root)
    os.makedirs(os.path.join(args.title, 'service'))
    log.debug('mkdir %s/static/' % project_root)
    os.makedirs(os.path.join(args.title, 'static'))
    log.debug('mkdir %s/' % src_root)
    os.makedirs(src_root)
    
    _copy_project(config, args.title, project_root)
    print "Your new project is at: %s" % project_root

init.parser.add_argument('title', default=None,
    help="Create kraftwerk project (if omitted, defaults to current directory)")


@command
def create_node(config, args):
    """Commissions a new server node."""
    log = logging.getLogger('kraftwerk.create-node')

    if 'pubkey' in config:
        pubkey_paths = [config["pubkey"]]
    else:
        pubkey_paths = [os.path.join(os.environ['HOME'], '.ssh', f) for f \
            in ['id_rsa.pub', 'id_dsa.pub']]

    for path in pubkey_paths:
        if os.path.exists(path):
            print 'SSH public key: %s' % path
            with open(path) as fp:
                pubkey = fp.read().strip()
            break
    else:
        pubkey = raw_input('Your SSH public key (for root login): ')
    
    if not re.search(r'^[a-z0-9.-]+$', args.hostname):
        raise CommandError(
            "Invalid hostname (must contain only letters, numbers, ., and -): %r"
            % args.hostname)
    
    # Query driver for size, image, and location
    
    image_id = str(getattr(args, 'image-id', config["image_id"]))
    for i in config.driver.list_images():
        if str(i.id) == image_id:
            image = i
            break
    else:
        sys.exit("Image %s not found for this provider. Aborting." % image_id)
            
    size_id = str(getattr(args, 'size-id', config["size_id"]))
    for s in config.driver.list_sizes():
        if str(s.id) == size_id:
            size = s
            break
    else:
        sys.exit("Size %s not found for this provider. Aborting." % size_id)
    
    location_id = str(getattr(args, 'location-id', config.get("location_id", "0")))
    for l in config.driver.list_locations():
        if str(l.id) == location_id:
            location = l
            break
    else:
        sys.exit("Location %s not found for this provider. Aborting." % location_id)
    
    if isinstance(config.driver, ec2.EC2NodeDriver):
        extra = dict(ex_userdata="""#!/bin/bash
echo '%s' > /root/.ssh/authorized_keys""" % pubkey)
        if not "keyname" in config:
{'fraction_non_alphanumeric': 0.12175093315235833, 'fraction_numerical': 0.003936206311503224, 'mean_word_length': 2.587147030185005, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2234082', 'n_tokens_mistral': 4636, 'n_tokens_neox': 4573, 'n_words': 1166}