<commit_before>import os
import binascii
import importlib
import functools as ft
import glob
from string import Template


def generate_secret():  # pragma: no cover
    return binascii.hexlify(os.urandom(24)).decode()


def find_commands(commands_dir):  # pragma: no cover
    command_modules = os.listdir(commands_dir)
    commands = []
    for command_module in command_modules:
        if not command_module.startswith("_"):
            command, _ = os.path.splitext(command_module)
            commands.append(command)
    return commands


TEMPLATE_DIR = os.path.join(
    os.path.dirname(os.path.abspath(__file__)), "templates")


def load_template(path):
    with open(path) as template_file:
        template = Template(template_file.read())
    return template


def find_template(template_dir, template_name):
    _, ext = os.path.splitext(template_name)
    pattern = os.path.join(template_dir, "**", "*" + ext)
    for path in glob.iglob(pattern, recursive=True):
        if path.endswith(template_name):
            return load_template(path)
    raise FileNotFoundError("{0} checked"
        " {1}".format(template_name, template_dir))


load_core_template = ft.partial(find_template, TEMPLATE_DIR)<commit_msg>py < 3.5 globbing fix<commit_after>import os
import sys
import binascii
import importlib
import functools as ft
import glob
from string import Template

if sys.version_info.minor >= 5:
    glob.iglob = ft.partial(glob.iglob, recursive=True)


def generate_secret():  # pragma: no cover
    return binascii.hexlify(os.urandom(24)).decode()


def find_commands(commands_dir):  # pragma: no cover
    command_modules = os.listdir(commands_dir)
    commands = []
    for command_module in command_modules:
        if not command_module.startswith("_"):
            command, _ = os.path.splitext(command_module)
            commands.append(command)
    return commands


TEMPLATE_DIR = os.path.join(
    os.path.dirname(os.path.abspath(__file__)), "templates")


def load_template(path):
    with open(path) as template_file:
        template = Template(template_file.read())
    return template


def find_template(template_dir, template_name):
    _, ext = os.path.splitext(template_name)
    pattern = os.path.join(template_dir, "**", "*" + ext)
    for path in glob.iglob(pattern):
        if path.endswith(template_name):
            return load_template(path)
    raise FileNotFoundError("{0} checked"
        " {1}".format(template_name, template_dir))


load_core_template = ft.partial(find_template, TEMPLATE_DIR)
{'fraction_non_alphanumeric': 0.10660896931549961, 'fraction_numerical': 0.004327301337529505, 'mean_word_length': 3.825426944971537, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '80654', 'n_tokens_mistral': 819, 'n_tokens_neox': 786, 'n_words': 173}