<commit_before>#!/usr/bin/python

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

import ast
import logging
import logging.config
import os

from jinja2 import Environment, FileSystemLoader

import constants
from configparserutil import ConfigParserUtil


PATH = os.path.dirname(os.path.abspath(__file__))

logging.config.fileConfig(os.path.join(PATH, 'conf', 'logging_config.ini'))
log = logging.getLogger(__name__)

TEMPLATE_ENVIRONMENT = Environment(
    autoescape=False,
    loader=FileSystemLoader(os.path.join(PATH)),
    trim_blocks=False)
PACK_LOCATION = None


def render_template(template_filename, context):
    """
    parse the xml file and return the content as a text

    :param template_filename: template filename path
    :param context: dictionary containing configurations
    :return: xml as a string
    """
    log.info("Rendering template: %s", template_filename)
    return TEMPLATE_ENVIRONMENT.get_template(template_filename).render(context)


def create_output_xml(template_path, output_path, context):
    """
    This method write the output generated from jinja file to xml file

    :param template_path: Path to the template file
    :param output_path: Path to the output xml file
    :param context: Dictionary containing values to be used by jinja engine
    :return: None
    """
    directory = os.path.dirname(output_path)
    if not os.path.exists(directory):
        os.makedirs(directory)
    with open(output_path, 'w') as xml_file:
        content = render_template(template_path, context)
        xml_file.write(content)
        log.info("Creating output file :" + output_path)
        log.debug("Creating content: " + content)


def generate_context(config_file_path):
    """
    Read the config.ini and generate context based on settings

    :param config_file_path: location of the config.ini file
    :return: dictionary containing configurations for jinja engine
    """
    # Read configuration file
    config_parser = ConfigParserUtil()
    config_parser.optionxform = str
    config_parser.read(os.path.join(PATH, config_file_path))
    configurations = config_parser.as_dictionary()

    # Reading the default values
    context = configurations[constants.CONFIG_PARAMS]
    settings = configurations[constants.CONFIG_SETTINGS]
    global PACK_LOCATION
    PACK_LOCATION = settings["REPOSITORY_CONF_DIRECTORY"]



    # if read_env_variables is true context will be generated from environment variables
    # if read_env_variables is not true context will be read from config.ini
    if settings["READ_FROM_ENVIRONMENT"] == "true":
        log.info("Reading from environment variables")
        for key, value in context.iteritems():
            # check if value exists for given key; use default if not exists
            context[key] = os.environ.get(key, context[key])

    # Converting Members to dictionary
    # if "CONFIG_PARAM_WKA_MEMBERS" in context:
    #     context["CONFIG_PARAM_WKA_MEMBERS"] = ConfigParserUtil.convert_properties_to_dictionary(
    #         context['CONFIG_PARAM_WKA_MEMBERS'])
    #
    # if "STRATOS_HTTP_PORT_MAPPING" in context:
    #     context["STRATOS_HTTP_PORT_MAPPING"] = ConfigParserUtil.convert_properties_to_dictionary(
    #         context["STRATOS_HTTP_PORT_MAPPING"])
    #
    # if "STRATOS_HTTPS_PORT_MAPPING" in context:
    #     context["STRATOS_HTTPS_PORT_MAPPING"] = ConfigParserUtil.convert_properties_to_dictionary(
    #         context["STRATOS_HTTPS_PORT_MAPPING"])

    ConfigParserUtil.get_multivalued_attributes_as_dictionary(context)

    log.info("Context generated: %s", context)
    return context


def traverse(root_dir, context):
    """
    traverse through the folder structure and generate xml files
<commit_msg>changing the comment<commit_after>#!/usr/bin/python

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

import ast
import logging
import logging.config
import os

from jinja2 import Environment, FileSystemLoader

import constants
from configparserutil import ConfigParserUtil


PATH = os.path.dirname(os.path.abspath(__file__))

logging.config.fileConfig(os.path.join(PATH, 'conf', 'logging_config.ini'))
log = logging.getLogger(__name__)

TEMPLATE_ENVIRONMENT = Environment(
    autoescape=False,
    loader=FileSystemLoader(os.path.join(PATH)),
    trim_blocks=False)
PACK_LOCATION = None


def render_template(template_filename, context):
    """
    parse the xml file and return the content as a text

    :param template_filename: template filename path
    :param context: dictionary containing configurations
    :return: xml as a string
    """
    log.info("Rendering template: %s", template_filename)
    return TEMPLATE_ENVIRONMENT.get_template(template_filename).render(context)


def create_output_xml(template_path, output_path, context):
    """
    This method write the output generated from jinja file to xml file

    :param template_path: Path to the template file
    :param output_path: Path to the output xml file
    :param context: Dictionary containing values to be used by jinja engine
    :return: None
    """
    directory = os.path.dirname(output_path)
    if not os.path.exists(directory):
        os.makedirs(directory)
    with open(output_path, 'w') as xml_file:
        content = render_template(template_path, context)
        xml_file.write(content)
        log.info("Creating output file :" + output_path)
        log.debug("Creating content: " + content)


def generate_context(config_file_path):
    """
    Read the config.ini and generate context based on settings

    :param config_file_path: location of the config.ini file
    :return: dictionary containing configurations for jinja engine
    """
    # Read configuration file
    config_parser = ConfigParserUtil()
    config_parser.optionxform = str
    config_parser.read(os.path.join(PATH, config_file_path))
    configurations = config_parser.as_dictionary()

    # Reading the default values
    context = configurations[constants.CONFIG_PARAMS]
    settings = configurations[constants.CONFIG_SETTINGS]
    global PACK_LOCATION
    PACK_LOCATION = settings["REPOSITORY_CONF_DIRECTORY"]



    # if read_env_variables is true context will be generated from environment variables
    # if read_env_variables is not true context will be read from config.ini
    if settings["READ_FROM_ENVIRONMENT"] == "true":
        log.info("Reading from environment variables")
        for key, value in context.iteritems():
            # check if value exists for given key; use default if not exists
            context[key] = os.environ.get(key, context[key])

    #Converting multi-valued params to dictionary
    ConfigParserUtil.get_multivalued_attributes_as_dictionary(context)

    log.info("Context generated: %s", context)
    return context


def traverse(root_dir, context):
    """
    traverse through the folder structure and generate xml files

{'fraction_non_alphanumeric': 0.0752212389380531, 'fraction_numerical': 0.001195886151638364, 'mean_word_length': 4.090079123554474, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 16}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7324638', 'n_tokens_mistral': 2309, 'n_tokens_neox': 2252, 'n_words': 758}