<commit_before>import collections
import io
import jinja2
import logging
import os
import socket
import time
import yaml


logger = logging.getLogger()


def _get_ip_address():
    """Returns the localhost ip address as a string (eg. "192.168.0.6")."""

    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    s.connect(("8.8.8.8", 80))
    return s.getsockname()[0]


def _parse_jinja_template(jinja_template_path, template_variables):
    with io.open(jinja_template_path, encoding="UTF-8") as jinja_template_file:
        try:
            jinja_template_contents = jinja_template_file.read()
            yaml_string = jinja2.Template(jinja_template_contents).render(template_variables)
        except TypeError as e:
            raise ValueError('unable to render file: %(e)s' % locals())

    return yaml_string


def load_settings(settings_file_paths, settings=None):
    """Parses yaml settings file(s) and returns a dictionary of settings.
    These yaml files are treated as Jinja templates.
    If a settings dictionary is also provided as an argument, it will be used as context for Jinja template processing.

    Args:
        settings_file_paths (list): a list of yaml settings file paths to load
        settings (dict): optional dictionary of settings values

    Return:
        dict: settings file containing all settings parsed from the given settings file(s)
    """

    if settings is None:
        settings = collections.OrderedDict()

    # add generic global options
    settings["HOME"] = os.path.expanduser("~")
    settings["TIMESTAMP"] = time.strftime("%Y%m%d_%H%M%S")
    settings["HOST_MACHINE_IP"] = _get_ip_address()

    # process settings_file_paths<commit_msg>remove unsused settings<commit_after>import collections
import io
import jinja2
import logging
import os
import time
import yaml


logger = logging.getLogger()


def _parse_jinja_template(jinja_template_path, template_variables):
    with io.open(jinja_template_path, encoding="UTF-8") as jinja_template_file:
        try:
            jinja_template_contents = jinja_template_file.read()
            yaml_string = jinja2.Template(jinja_template_contents).render(template_variables)
        except TypeError as e:
            raise ValueError('unable to render file: %(e)s' % locals())

    return yaml_string


def load_settings(settings_file_paths, settings=None):
    """Parses yaml settings file(s) and returns a dictionary of settings.
    These yaml files are treated as Jinja templates.
    If a settings dictionary is also provided as an argument, it will be used as context for Jinja template processing.

    Args:
        settings_file_paths (list): a list of yaml settings file paths to load
        settings (dict): optional dictionary of settings values

    Return:
        dict: settings file containing all settings parsed from the given settings file(s)
    """

    if settings is None:
        settings = collections.OrderedDict()

    # add generic global options
    settings["TIMESTAMP"] = time.strftime("%Y%m%d_%H%M%S")

    # process settings_file_paths
{'fraction_non_alphanumeric': 0.08124796880077997, 'fraction_numerical': 0.0068248293792655184, 'mean_word_length': 3.684931506849315, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '517062', 'n_tokens_mistral': 910, 'n_tokens_neox': 850, 'n_words': 284}