<commit_before>#!/usr/bin/env python2.7

from collections import defaultdict
from itertools import chain, imap
import os
import os.path
import stat
import subprocess
import sys
import yaml

MAIN_CONFIG_FILE = "skipjack.yaml"

MODULE_CONFIG_FILE = "module.yaml"

ENFORCER_PATH = "modules/truth/manifests/enforcer.pp"


class ConfigError(ValueError):
    pass


def find_internal_modules():
    for name in os.listdir("modules"):
        if name[0] == ".":
            continue
        path = os.path.join("modules", name)
        config_file = os.path.join(path, MODULE_CONFIG_FILE)
        if os.path.isdir(path) and os.path.isfile(config_file):
            yield path


def find_external_modules(root):
    for dirpath, dirnames, filenames in os.walk(root):
        # Remove hidden directories
        for dirname in dirnames:
            if dirname[0] == ".":
                dirnames.remove(dirname)
        # See if we are in a module directory
        if MODULE_CONFIG_FILE in filenames:
            yield dirpath
            dirnames[:] = []


def get_mod_name(mod_dir):
    return os.path.basename(mod_dir)


def load_yaml(path):
    with open(path, "r") as handle:
        return yaml.safe_load(handle)


def read_roles(roles, mod_name):
    path = os.path.join("modules", mod_name, MODULE_CONFIG_FILE)
    doc = load_yaml(path)
    doc_roles = doc and doc["roles"]
    if doc_roles:
        if type(doc_roles) is not list:
            raise ConfigError("'roles' must be a list")
        for role in doc_roles:
            if type(role) is not str:
                raise ConfigError("each role must be a string")
            if role in roles:
                roles[role].append(mod_name)
            else:
                roles[role] = [mod_name]


def write_truth(roles):
    with open(ENFORCER_PATH, "w") as out:
        out.write("""
      class truth::enforcer {
        require truth::universal
    """)
        for role in roles:
            out.write("if has_role(\"%s\") {\n" % (role,))
            for mod in roles[role]:
                out.write("include %s\n" % (mod,))
            out.write("notice(\"I am a %s\")\n}\n" % (role,))
        out.write("}\n")


def goto_repo_root():
    os.chdir(subprocess.check_output(
             ["git", "rev-parse", "--show-toplevel"]).rstrip())


def find_config():
    for dirpath, dirnames, filenames in os.walk("."):
        if MAIN_CONFIG_FILE in filenames:
            return os.path.join(dirpath, MAIN_CONFIG_FILE)


def setup():
    config = load_yaml(find_config())
    module_path = config["module_path"]
    roles = defaultdict(list)
    for mod_dir in chain(*imap(find_external_modules, module_path)):
        mod_name = get_mod_name(mod_dir)
        os.symlink(os.path.abspath(mod_dir), os.path.join("modules", mod_name))
        read_roles(roles, mod_name)
    for mod_dir in find_internal_modules():
        read_roles(roles, get_mod_name(mod_dir))
    write_truth(roles)


def clean():
    if os.path.isfile(ENFORCER_PATH):<commit_msg>Allow non-existence of skipjack.yaml<commit_after>#!/usr/bin/env python2.7

from collections import defaultdict
from itertools import chain, imap
import os
import os.path
import stat
import subprocess
import sys
import yaml

MAIN_CONFIG_FILE = "skipjack.yaml"

MODULE_CONFIG_FILE = "module.yaml"

ENFORCER_PATH = "modules/truth/manifests/enforcer.pp"


class ConfigError(ValueError):
    pass


def find_internal_modules():
    for name in os.listdir("modules"):
        if name[0] == ".":
            continue
        path = os.path.join("modules", name)
        config_file = os.path.join(path, MODULE_CONFIG_FILE)
        if os.path.isdir(path) and os.path.isfile(config_file):
            yield path


def find_external_modules(root):
    for dirpath, dirnames, filenames in os.walk(root):
        # Remove hidden directories
        for dirname in dirnames:
            if dirname[0] == ".":
                dirnames.remove(dirname)
        # See if we are in a module directory
        if MODULE_CONFIG_FILE in filenames:
            yield dirpath
            dirnames[:] = []


def get_mod_name(mod_dir):
    return os.path.basename(mod_dir)


def load_yaml(path):
    with open(path, "r") as handle:
        return yaml.safe_load(handle)


def read_roles(roles, mod_name):
    path = os.path.join("modules", mod_name, MODULE_CONFIG_FILE)
    doc = load_yaml(path)
    doc_roles = doc and doc["roles"]
    if doc_roles:
        if type(doc_roles) is not list:
            raise ConfigError("'roles' must be a list")
        for role in doc_roles:
            if type(role) is not str:
                raise ConfigError("each role must be a string")
            if role in roles:
                roles[role].append(mod_name)
            else:
                roles[role] = [mod_name]


def write_truth(roles):
    with open(ENFORCER_PATH, "w") as out:
        out.write("""
      class truth::enforcer {
        require truth::universal
    """)
        for role in roles:
            out.write("if has_role(\"%s\") {\n" % (role,))
            for mod in roles[role]:
                out.write("include %s\n" % (mod,))
            out.write("notice(\"I am a %s\")\n}\n" % (role,))
        out.write("}\n")


def goto_repo_root():
    os.chdir(subprocess.check_output(
             ["git", "rev-parse", "--show-toplevel"]).rstrip())


def find_config():
    for dirpath, dirnames, filenames in os.walk("."):
        if MAIN_CONFIG_FILE in filenames:
            return os.path.join(dirpath, MAIN_CONFIG_FILE)
    return None


def setup():
    config_file = find_config()
    if config_file:
        config = load_yaml(config_file)
        module_path = config["module_path"]
    else:
        module_path = []
    roles = defaultdict(list)
    for mod_dir in chain(*imap(find_external_modules, module_path)):
        mod_name = get_mod_name(mod_dir)
        os.symlink(os.path.abspath(mod_dir), os.path.join("modules", mod_name))
        read_roles(roles, mod_name)
    for mod_dir in find_internal_modules():
        read_roles(roles, get_mod_name(mod_dir))
    write_truth(roles)


def clean():
    if os.path.isfile(ENFORCER_PATH):
{'fraction_non_alphanumeric': 0.11743656473649967, 'fraction_numerical': 0.0013012361743656475, 'mean_word_length': 2.548182342758223, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2786346', 'n_tokens_mistral': 2048, 'n_tokens_neox': 2008, 'n_words': 478}