<commit_before>import glob
import json
import os
import sys

import logging
import yaml
from jinja2 import Environment, FileSystemLoader
from jinja2.exceptions import TemplateSyntaxError, TemplateNotFound
import arrow

from .exceptions import FormicaArgumentException

from . import yaml_tags
# To silence pyflakes warning of unused import
assert yaml_tags

logger = logging.getLogger(__name__)

FILE_TYPES = ['yml', 'yaml', 'json']

RESOURCES_KEY = "Resources"
MODULE_KEY = "From"

try:
    basestring
except NameError:
    basestring = str

ALLOWED_ATTRIBUTES = {
    "AWSTemplateFormatVersion": basestring,
    "Description": basestring,
    "Metadata": dict,
    "Parameters": dict,
    "Mappings": dict,
    "Conditions": dict,
    "Transform": [basestring, list],
    RESOURCES_KEY: dict,
    "Outputs": dict,
}


def code_escape(source):
    return source.replace('\n', '\\n').replace('"', '\\"')


def mandatory(a):
    from jinja2.runtime import Undefined
    if isinstance(a, Undefined) or a is None:
        raise FormicaArgumentException('Mandatory variable not set.')
    return a


def resource(name):
    if name is None:
        return ''
    else:
        return ''.join(e for e in name.title() if e.isalnum())


def novalue(variable):
    return variable or '{"Ref": "AWS::NoValue"}'


class Loader(object):
    def __init__(self, path='.', file='*', variables=None):
        if variables is None:
            variables = {}
        self.cftemplate = {}
        self.path = path
        self.file = file
        self.env = Environment(loader=FileSystemLoader('./', followlinks=True))
        self.env.filters.update({
            'code_escape': code_escape,
            'mandatory': mandatory,
            'resource': resource,
            'novalue': novalue
        })
        self.variables = variables

    def include_file(self, filename, **args):
        source = self.render(filename, **args)
        value = code_escape(source)
        return value

    def render(self, filename, **args):
        template_path = os.path.normpath("{}/{}".format(self.path, filename))
        return self.env.get_template(template_path).render(code=self.include_file,
                                                           now=arrow.now,
                                                           utcnow=arrow.utcnow,
                                                           ** args)

    def template(self, indent=4, sort_keys=True, separators=(',', ':'), dumper=None):
        if dumper is not None:
            return dumper(self.cftemplate)
        return json.dumps(self.cftemplate, indent=indent, sort_keys=sort_keys, separators=separators)

    def template_dictionary(self):
        return self.cftemplate

    def merge(self, template, file):
        if template:
            for key in template.keys():
                new = template[key]
                new_type = type(new)
                types = ALLOWED_ATTRIBUTES.get(key)
                if key in ALLOWED_ATTRIBUTES.keys() and (new_type == types or new_type in types):
                    if new_type == basestring or new_type == list:
                        self.cftemplate[key] = new
                    elif new_type == dict:
                        for element_key, element_value in template[key].items():
                            if key == RESOURCES_KEY and isinstance(element_value, dict) and MODULE_KEY in element_value:
                                self.load_module(element_value[MODULE_KEY], element_key, element_value)
                            else:
                                self.cftemplate.setdefault(key, {})[element_key] = element_value
                else:
                    logger.info("Key '{}' in file {} is not valid".format(key, file))
                    sys.exit(1)<commit_msg>Fix tests for python 2.7<commit_after>import glob
import json
import os
import sys

import logging
import yaml
from jinja2 import Environment, FileSystemLoader
from jinja2.exceptions import TemplateSyntaxError, TemplateNotFound
import arrow

from .exceptions import FormicaArgumentException

from . import yaml_tags
# To silence pyflakes warning of unused import
assert yaml_tags

logger = logging.getLogger(__name__)

FILE_TYPES = ['yml', 'yaml', 'json']

RESOURCES_KEY = "Resources"
MODULE_KEY = "From"

try:
    basestring
except NameError:
    basestring = str

ALLOWED_ATTRIBUTES = {
    "AWSTemplateFormatVersion": basestring,
    "Description": [basestring, str],
    "Metadata": dict,
    "Parameters": dict,
    "Mappings": dict,
    "Conditions": dict,
    "Transform": [str, basestring, list],
    RESOURCES_KEY: dict,
    "Outputs": dict,
}


def code_escape(source):
    return source.replace('\n', '\\n').replace('"', '\\"')


def mandatory(a):
    from jinja2.runtime import Undefined
    if isinstance(a, Undefined) or a is None:
        raise FormicaArgumentException('Mandatory variable not set.')
    return a


def resource(name):
    if name is None:
        return ''
    else:
        return ''.join(e for e in name.title() if e.isalnum())


def novalue(variable):
    return variable or '{"Ref": "AWS::NoValue"}'


class Loader(object):
    def __init__(self, path='.', file='*', variables=None):
        if variables is None:
            variables = {}
        self.cftemplate = {}
        self.path = path
        self.file = file
        self.env = Environment(loader=FileSystemLoader('./', followlinks=True))
        self.env.filters.update({
            'code_escape': code_escape,
            'mandatory': mandatory,
            'resource': resource,
            'novalue': novalue
        })
        self.variables = variables

    def include_file(self, filename, **args):
        source = self.render(filename, **args)
        value = code_escape(source)
        return value

    def render(self, filename, **args):
        template_path = os.path.normpath("{}/{}".format(self.path, filename))
        return self.env.get_template(template_path).render(code=self.include_file,
                                                           now=arrow.now,
                                                           utcnow=arrow.utcnow,
                                                           ** args)

    def template(self, indent=4, sort_keys=True, separators=(',', ':'), dumper=None):
        if dumper is not None:
            return dumper(self.cftemplate)
        return json.dumps(self.cftemplate, indent=indent, sort_keys=sort_keys, separators=separators)

    def template_dictionary(self):
        return self.cftemplate

    def merge(self, template, file):
        if template:
            for key in template.keys():
                new = template[key]
                new_type = type(new)
                types = ALLOWED_ATTRIBUTES.get(key)
                if type(types) != list:
                    types = [types]
                if key in ALLOWED_ATTRIBUTES.keys() and new_type in types:
                    if new_type == basestring or new_type == str or new_type == list:
                        self.cftemplate[key] = new
                    elif new_type == dict:
                        for element_key, element_value in template[key].items():
                            if key == RESOURCES_KEY and isinstance(element_value, dict) and MODULE_KEY in element_value:
                                self.load_module(element_value[MODULE_KEY], element_key, element_value)
                            else:
                                self.cftemplate.setdefault(key, {})[element_key] = element_value
                else:
                    logger.info("Key '{}' in file {} is not valid".format(key, file))
                    sys.exit(1)
{'fraction_non_alphanumeric': 0.10490055894969452, 'fraction_numerical': 0.0015598596126348628, 'mean_word_length': 2.069006781013163, 'pattern_counts': {'":': 18, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3466176', 'n_tokens_mistral': 2157, 'n_tokens_neox': 2090, 'n_words': 548}