<commit_before>"""
Compile our environment from directories and files.
"""
import os

from heapq import heappush

import yaml

from withenv.flatten import flatten

def path_relative_to(root, fname):
    root = os.path.abspath(root)
    if not os.path.isdir(root):
        root = os.path.dirname(root)
    return os.path.normpath(os.path.join(root, fname))


def load_shell_env_file(fname):
    # look for lines with export and parse them
    env = {}
    with open(fname) as fh:
        for line in fh:
            if line.startswith('export'):
                prefix, _, envvar = line.partition(' ')
                k, _, v = envvar.partition('=')
                env[k] = v

    return env

def load_env_file(fname):
    if fname.endswith(['yml', 'yaml']):
        return yaml.safe_load(open(fname))
    return load_shell_env_file(fname)


def find_yml_in_dir(dirname):
    def is_yaml(fn):
        return fn.endswith(('yml', 'yaml'))

    fnames = []  # a heap

    for dirpath, dirnames, filenames in os.walk(dirname):
        for fn in filter(is_yaml, filenames):
            heappush(fnames, os.path.join(dirpath, fn))

    return fnames


def update_env_from_dir(dirname, env):
    for fname in find_yml_in_dir(dirname):
        update_env_from_file(fname, env)


def update_env_from_file(fname, env):
    new_env = yaml.safe_load(open(fname))
    flat_env = {}

    # Order isn't important
    if isinstance(new_env, dict):
        new_env = [new_env]

    if not new_env:
        return

    for item in new_env:
        for k, v in flatten(item):
            env[k] = v

def update_env_from_alias(fname, env):
    action_list = yaml.safe_load(open(fname))
    if not action_list:
        return env

    actions = [
        (k, path_relative_to(fname, v)) for action in action_list
        for k, v in action.items()
    ]
    return compile(actions, env)


def update_env_from_override(override, env):
    k, _, v = override.partition('=')
    env[k] = v


def find_action(name):
    actions = {
        'file': update_env_from_file,<commit_msg>Support executing values as script when backticks are used<commit_after>"""
Compile our environment from directories and files.
"""
import os
import subprocess

from heapq import heappush

import yaml
import json

from withenv.flatten import flatten

def path_relative_to(root, fname):
    root = os.path.abspath(root)
    if not os.path.isdir(root):
        root = os.path.dirname(root)
    return os.path.normpath(os.path.join(root, fname))


def compiled_value(v):
    if v.startswith('`') and v.endswith('`'):
        return subprocess.check_output(v[1:-1], shell=True).strip()
    return v


def load_shell_env_file(fname):
    # look for lines with export and parse them
    env = {}
    with open(fname) as fh:
        for line in fh:
            if line.startswith('export'):
                prefix, _, envvar = line.partition(' ')
                k, _, v = envvar.partition('=')
                env[k] = compiled_value(v)

    return env

def load_env_file(fname):
    if fname.endswith(('yml', 'yaml')):
        return yaml.safe_load(open(fname))
    return load_shell_env_file(fname)


def find_yml_in_dir(dirname):
    def is_yaml(fn):
        return fn.endswith(('yml', 'yaml'))

    fnames = []  # a heap

    for dirpath, dirnames, filenames in os.walk(dirname):
        for fn in filter(is_yaml, filenames):
            heappush(fnames, os.path.join(dirpath, fn))

    return fnames


def update_env_from_dir(dirname, env):
    for fname in find_yml_in_dir(dirname):
        update_env_from_file(fname, env)


def update_env_from_file(fname, env):
    new_env = load_env_file(fname)
    flat_env = {}

    # Order isn't important
    if isinstance(new_env, dict):
        new_env = [new_env]

    if not new_env:
        return

    for item in new_env:
        for k, v in flatten(item):
            env[k] = compiled_value(v)

def update_env_from_alias(fname, env):
    action_list = yaml.safe_load(open(fname))
    if not action_list:
        return env

    actions = []
    for action in action_list:
        for k, v in action.items():
            if not k == 'override':
                v = path_relative_to(fname, v)
            actions.append((k, v))

    return compile(actions, env)


def update_env_from_override(override, env):
    k, _, v = override.partition('=')
    env[k] = compiled_value(v)


def find_action(name):
    actions = {
        'file': update_env_from_file,
{'fraction_non_alphanumeric': 0.11165919282511211, 'fraction_numerical': 0.0004484304932735426, 'mean_word_length': 2.714404662781016, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1250380', 'n_tokens_mistral': 1560, 'n_tokens_neox': 1493, 'n_words': 380}