<commit_before>import glob
import os
import sys
import logging

logger = logging.getLogger(__name__)

import yaml

def load_config(config_file='config.yaml'):
    stack = []
    root = os.path.split(os.path.abspath(config_file))[0]
    def recursive_preprocess(filename):
        stack.append(os.path.abspath(filename))
        c = []
        with open(filename) as fd:
            for lineno, line in enumerate(fd):
                line = line.strip()
                if line.startswith('!include '):
                    path = line.split(None, 1)[1]
                    # if the include doesn't have a fully qualified path then
                    # assume the relative path is based off the directory of
                    # the initial config file
                    if not path.startswith('/'):
                        path = os.path.join(root, path)
                    files = glob.glob(path)
                    if not files:
                        logger.warning("Include statement '%s' at %s:%d did "
                                "not match any files.  Skipping." % (line,
                                        filename, lineno))
                        continue
                    for f in files:
                        f = os.path.abspath(f)
                        if f in stack:
                            logger.warning("Already parsed %s, skipping "
                                    "(%s:%d) to avoid infinite loop." % (f,
                                            filename, lineno))
                            continue
                        if os.path.isfile(f):
                            logger.info("Parsing include (%s:%d): %s" % (
                                    filename, lineno, f))
                            c.extend(recursive_preprocess(f))
                        else:
                            logger.warning("%s is not a regular file, "
                                    "skipping (%s:%d)." % (f, filename,
                                            lineno))
                    continue
                c.append(line)
        return c
    logger.info("Loading config file: %s" % (config_file))
    config = recursive_preprocess(config_file)
    return yaml.safe_load(os.linesep.join(config))
<commit_msg>Only strip whitespace off the right side<commit_after>import glob
import os
import sys
import logging

logger = logging.getLogger(__name__)

import yaml

def load_config(config_file='config.yaml'):
    stack = []
    root = os.path.split(os.path.abspath(config_file))[0]
    def recursive_preprocess(filename):
        stack.append(os.path.abspath(filename))
        c = []
        with open(filename) as fd:
            for lineno, line in enumerate(fd):
                line = line.rstrip()
                if line.startswith('!include '):
                    path = line.split(None, 1)[1]
                    # if the include doesn't have a fully qualified path then
                    # assume the relative path is based off the directory of
                    # the initial config file
                    if not path.startswith('/'):
                        path = os.path.join(root, path)
                    files = glob.glob(path)
                    if not files:
                        logger.warning("Include statement '%s' at %s:%d did "
                                "not match any files.  Skipping." % (line,
                                        filename, lineno))
                        continue
                    for f in files:
                        f = os.path.abspath(f)
                        if f in stack:
                            logger.warning("Already parsed %s, skipping "
                                    "(%s:%d) to avoid infinite loop." % (f,
                                            filename, lineno))
                            continue
                        if os.path.isfile(f):
                            logger.info("Parsing include (%s:%d): %s" % (
                                    filename, lineno, f))
                            c.extend(recursive_preprocess(f))
                        else:
                            logger.warning("%s is not a regular file, "
                                    "skipping (%s:%d)." % (f, filename,
                                            lineno))
                    continue
                c.append(line)
        return c
    logger.info("Loading config file: %s" % (config_file))
    config = recursive_preprocess(config_file)
    return yaml.safe_load(os.linesep.join(config))

{'fraction_non_alphanumeric': 0.09035087719298246, 'fraction_numerical': 0.0013157894736842105, 'mean_word_length': 1.1028123559243892, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4201753', 'n_tokens_mistral': 1125, 'n_tokens_neox': 1048, 'n_words': 296}