<commit_before>#!/usr/bin/env python
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import argparse
import jinja2
import os
import shutil
import six
import sys
import yaml


def parse_opts(argv):
    parser = argparse.ArgumentParser(
        description='Configure host network interfaces using a JSON'
        ' config file format.')
    parser.add_argument('-p', '--base_path', metavar='BASE_PATH',
                        help="""base path of templates to process.""",
                        default='.')
    parser.add_argument('-r', '--roles-data', metavar='ROLES_DATA',
                        help="""relative path to the roles_data.yaml file.""",
                        default='roles_data.yaml')
    parser.add_argument('--safe',
                        action='store_true',
                        help="""Enable safe mode (do not overwrite files).""",
                        default=False)
    parser.add_argument('-o', '--output-dir', metavar='OUTPUT_DIR',
                        help="""Output dir for all the templates""",
                        default='')
    opts = parser.parse_args(argv[1:])

    return opts


def _j2_render_to_file(j2_template, j2_data, outfile_name=None,
                       overwrite=True):
    yaml_f = outfile_name or j2_template.replace('.j2.yaml', '.yaml')
    print('rendering j2 template to file: %s' % outfile_name)

    if not overwrite and os.path.exists(outfile_name):
        print('ERROR: path already exists for file: %s' % outfile_name)
        sys.exit(1)

    try:
        # Render the j2 template
        template = jinja2.Environment().from_string(j2_template)
        r_template = template.render(**j2_data)
    except jinja2.exceptions.TemplateError as ex:
        error_msg = ("Error rendering template %s : %s"
                     % (yaml_f, six.text_type(ex)))
        print(error_msg)
        raise Exception(error_msg)
    with open(outfile_name, 'w') as out_f:
        out_f.write(r_template)


def process_templates(template_path, role_data_path, output_dir, overwrite):

    with open(role_data_path) as role_data_file:
        role_data = yaml.safe_load(role_data_file)

    j2_excludes_path = os.path.join(template_path, 'j2_excludes.yaml')
    with open(j2_excludes_path) as role_data_file:
        j2_excludes = yaml.safe_load(role_data_file)

    if output_dir and not os.path.isdir(output_dir):
        if os.path.exists(output_dir):
            raise RuntimeError('Output dir %s is not a directory' % output_dir)<commit_msg>Add support for Jinja2 includes<commit_after>#!/usr/bin/env python
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import argparse
import jinja2
import os
import shutil
import six
import sys
import yaml

__tht_root_dir = os.path.dirname(os.path.dirname(__file__))


def parse_opts(argv):
    parser = argparse.ArgumentParser(
        description='Configure host network interfaces using a JSON'
        ' config file format.')
    parser.add_argument('-p', '--base_path', metavar='BASE_PATH',
                        help="""base path of templates to process.""",
                        default='.')
    parser.add_argument('-r', '--roles-data', metavar='ROLES_DATA',
                        help="""relative path to the roles_data.yaml file.""",
                        default='roles_data.yaml')
    parser.add_argument('--safe',
                        action='store_true',
                        help="""Enable safe mode (do not overwrite files).""",
                        default=False)
    parser.add_argument('-o', '--output-dir', metavar='OUTPUT_DIR',
                        help="""Output dir for all the templates""",
                        default='')
    opts = parser.parse_args(argv[1:])

    return opts


def _j2_render_to_file(j2_template, j2_data, outfile_name=None,
                       overwrite=True):
    yaml_f = outfile_name or j2_template.replace('.j2.yaml', '.yaml')
    print('rendering j2 template to file: %s' % outfile_name)

    if not overwrite and os.path.exists(outfile_name):
        print('ERROR: path already exists for file: %s' % outfile_name)
        sys.exit(1)

    # Search for templates relative to the current template path first
    template_base = os.path.dirname(yaml_f)
    j2_loader = jinja2.loaders.FileSystemLoader([template_base, __tht_root_dir])

    try:
        # Render the j2 template
        template = jinja2.Environment(loader=j2_loader).from_string(
            j2_template)
        r_template = template.render(**j2_data)
    except jinja2.exceptions.TemplateError as ex:
        error_msg = ("Error rendering template %s : %s"
                     % (yaml_f, six.text_type(ex)))
        print(error_msg)
        raise Exception(error_msg)
    with open(outfile_name, 'w') as out_f:
        out_f.write(r_template)


def process_templates(template_path, role_data_path, output_dir, overwrite):

    with open(role_data_path) as role_data_file:
        role_data = yaml.safe_load(role_data_file)

    j2_excludes_path = os.path.join(template_path, 'j2_excludes.yaml')
    with open(j2_excludes_path) as role_data_file:
        j2_excludes = yaml.safe_load(role_data_file)

    if output_dir and not os.path.isdir(output_dir):
        if os.path.exists(output_dir):
            raise RuntimeError('Output dir %s is not a directory' % output_dir)
{'fraction_non_alphanumeric': 0.10073748627020242, 'fraction_numerical': 0.007531774674407657, 'mean_word_length': 2.67801500288517, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4376233', 'n_tokens_mistral': 1924, 'n_tokens_neox': 1850, 'n_words': 519}