<commit_before>'''
Cloudformation related functions
'''

import sys
import time
import yaml
import json
import jinja2
import hashlib
import boto

from jinja2 import meta
from fnmatch import fnmatch
from tabulate import tabulate
from boto.exception import BotoServerError

from stacks.aws import get_stack_tag

YES = ['y', 'Y', 'yes', 'YES', 'Yes']
FAILED_STACK_STATES = [
    'CREATE_FAILED',
    'ROLLBACK_FAILED',
    'DELETE_FAILED',
    'UPDATE_ROLLBACK_FAILED'
]
COMPLETE_STACK_STATES = [
    'CREATE_COMPLETE',
    'UPDATE_COMPLETE',
]
ROLLBACK_STACK_STATES = [
    'ROLLBACK_COMPLETE',
    'UPDATE_ROLLBACK_COMPLETE',
]
IN_PROGRESS_STACK_STATES = [
    'CREATE_IN_PROGRESS',
    'ROLLBACK_IN_PROGRESS',
    'DELETE_IN_PROGRESS',
    'UPDATE_IN_PROGRESS',
    'UPDATE_COMPLETE_CLEANUP_IN_PROGRESS',
    'UPDATE_ROLLBACK_IN_PROGRESS',
    'UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS',
]


def gen_template(template, config, pretty=False):
    '''Return generated CloudFormation template string'''
    tpl_str = template.read()

    _check_missing_vars(tpl_str, config)

    tpl = jinja2.Template(tpl_str)
    out = tpl.render(config)
    docs = list(yaml.load_all(out))
    indent = 2 if pretty else None

    if len(docs) == 2:
        return (json.dumps(docs[1], indent=indent), docs[0])
    else:
        return (json.dumps(docs[0], indent=indent), None)


def _check_missing_vars(tpl_str, config):
    '''Check for missing variables in a template string'''
    env = jinja2.Environment()
    ast = env.parse(tpl_str)
    required_properties = meta.find_undeclared_variables(ast)
    missing_properties = required_properties - config.keys()

    if len(missing_properties) > 0:
        print('Requred properties not set: {}'.format(','.join(missing_properties)))
        sys.exit(1)


# TODO(vaijab): fix 'S3ResponseError: 301 Moved Permanently', this happens when
# a connection to S3 is being made from a different region than the one a bucket
# was created in.
def upload_template(conn, config, tpl, stack_name):
    '''Upload a template to S3 bucket and returns S3 key url'''
    bn = config.get('templates_bucket_name', '{}-stacks-{}'.format(config['env'], config['region']))

    try:
        b = config['s3_conn'].get_bucket(bn)
    except boto.exception.S3ResponseError as err:
        if err.code == 'NoSuchBucket':
            print('Bucket {} does not exist.'.format(bn))
        else:
            print(err)
        sys.exit(1)

    h = hashlib.md5(tpl.encode('utf-8')).hexdigest()<commit_msg>feature: add support for snippets<commit_after>'''
Cloudformation related functions
'''

import sys
import time
import yaml
import json
import jinja2
import hashlib
import boto

from os import path
from jinja2 import meta
from fnmatch import fnmatch
from tabulate import tabulate
from boto.exception import BotoServerError

from stacks.aws import get_stack_tag

YES = ['y', 'Y', 'yes', 'YES', 'Yes']
FAILED_STACK_STATES = [
    'CREATE_FAILED',
    'ROLLBACK_FAILED',
    'DELETE_FAILED',
    'UPDATE_ROLLBACK_FAILED'
]
COMPLETE_STACK_STATES = [
    'CREATE_COMPLETE',
    'UPDATE_COMPLETE',
]
ROLLBACK_STACK_STATES = [
    'ROLLBACK_COMPLETE',
    'UPDATE_ROLLBACK_COMPLETE',
]
IN_PROGRESS_STACK_STATES = [
    'CREATE_IN_PROGRESS',
    'ROLLBACK_IN_PROGRESS',
    'DELETE_IN_PROGRESS',
    'UPDATE_IN_PROGRESS',
    'UPDATE_COMPLETE_CLEANUP_IN_PROGRESS',
    'UPDATE_ROLLBACK_IN_PROGRESS',
    'UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS',
]


def gen_template(tpl_file, config, pretty=False):
    '''Return generated CloudFormation template string'''
    tpl_path, tpl_fname = path.split(tpl_file.name)
    env = _new_jinja_env(tpl_path)

    _check_missing_vars(env, tpl_file, config)

    tpl = env.get_template(tpl_fname)
    rendered = tpl.render(config)
    docs = list(yaml.load_all(rendered))
    indent = 2 if pretty else None

    if len(docs) == 2:
        return (json.dumps(docs[1], indent=indent), docs[0])
    else:
        return (json.dumps(docs[0], indent=indent), None)


def _check_missing_vars(env, tpl_file, config):
    '''Check for missing variables in a template string'''
    tpl_str = tpl_file.read()
    ast = env.parse(tpl_str)
    required_properties = meta.find_undeclared_variables(ast)
    missing_properties = required_properties - config.keys()

    if len(missing_properties) > 0:
        print('Requred properties not set: {}'.format(','.join(missing_properties)))
        sys.exit(1)


def _new_jinja_env(tpl_path):
    loader = jinja2.loaders.FileSystemLoader(tpl_path)
    env = jinja2.Environment(loader=loader)
    return env


# TODO(vaijab): fix 'S3ResponseError: 301 Moved Permanently', this happens when
# a connection to S3 is being made from a different region than the one a bucket
# was created in.
def upload_template(conn, config, tpl, stack_name):
    '''Upload a template to S3 bucket and returns S3 key url'''
    bn = config.get('templates_bucket_name', '{}-stacks-{}'.format(config['env'], config['region']))

    try:
        b = config['s3_conn'].get_bucket(bn)
    except boto.exception.S3ResponseError as err:
        if err.code == 'NoSuchBucket':
            print('Bucket {} does not exist.'.format(bn))
        else:
            print(err)
        sys.exit(1)

    h = hashlib.md5(tpl.encode('utf-8')).hexdigest()
{'fraction_non_alphanumeric': 0.1145931659429866, 'fraction_numerical': 0.008684160845761752, 'mean_word_length': 4.0122989593188265, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3544455', 'n_tokens_mistral': 1828, 'n_tokens_neox': 1807, 'n_words': 412}