<commit_before>import os
import stat
import yaml
from pathlib import Path
from tempfile import gettempdir
from functools import wraps
from pyontutils.utils import TermColors as tc, makeSimpleLogger

checkout_ok = 'NIFSTD_CHECKOUT_OK' in os.environ

log = makeSimpleLogger('config')

def get_api_key():
    try: return os.environ['SCICRUNCH_API_KEY']
    except KeyError: return None


class dproperty(property):
    default = None


class dstr(str):
    default = None

    @property
    def isDefault(self):
        return self == self.default


def default(value):
    def decorator(function, default_value=value):
        dv = dstr(default_value)
        dv.default = default_value

        @wraps(function)
        def inner(*args, **kwargs):
            try:
                out = dstr(function(*args, **kwargs))
                out.default = default_value
                return out
            except (TypeError, KeyError, FileNotFoundError) as e:
                return dv

        pinner = dproperty(inner)
        pinner.default = default_value
        return pinner

    return decorator


tempdir = gettempdir()


class Secrets:
    def __init__(self, devconfig):
        self.devconfig = devconfig
        if self.exists:
            fstat = os.stat(self.filename)
            mode = oct(stat.S_IMODE(fstat.st_mode))
            if mode != '0o600' and mode != '0o700':
                raise FileNotFoundError(f'Your secrets file can be read by the whole world! {mode}')

    @property
    def filename(self):
        return self.devconfig.secrets_file

    @property
    def exists(self):
        e = Path(self.filename).exists()
        if not e:
            log.warning(f'secrets file {self.filename} does not exist')

        return e

    @property
    def name_id_map(self):
        # sometimes the easiest solution is just to read from disk every single time
        if self.exists:
            with open(self.filename, 'rt') as f:<commit_msg>config better error messages for secrets<commit_after>import os
import stat
import yaml
from pathlib import Path
from tempfile import gettempdir
from functools import wraps
from pyontutils.utils import TermColors as tc, makeSimpleLogger

checkout_ok = 'NIFSTD_CHECKOUT_OK' in os.environ

log = makeSimpleLogger('config')

def get_api_key():
    try: return os.environ['SCICRUNCH_API_KEY']
    except KeyError: return None


class dproperty(property):
    default = None


class dstr(str):
    default = None

    @property
    def isDefault(self):
        return self == self.default


def default(value):
    def decorator(function, default_value=value):
        dv = dstr(default_value)
        dv.default = default_value

        @wraps(function)
        def inner(*args, **kwargs):
            try:
                out = dstr(function(*args, **kwargs))
                out.default = default_value
                return out
            except (TypeError, KeyError, FileNotFoundError) as e:
                return dv

        pinner = dproperty(inner)
        pinner.default = default_value
        return pinner

    return decorator


tempdir = gettempdir()


class Secrets:
    def __init__(self, devconfig):
        self.devconfig = devconfig
        if self.exists:
            fstat = os.stat(self.filename)
            mode = oct(stat.S_IMODE(fstat.st_mode))
            if mode != '0o600' and mode != '0o700':
                raise FileNotFoundError(f'Your secrets file {self.filename} '
                                        f'can be read by the whole world! {mode}')

    @property
    def filename(self):
        return self.devconfig.secrets_file

    @property
    def exists(self):
        e = Path(self.filename).exists()
        if not e:
            log.warning(f'secrets file {self.filename} does not exist')

        return e

    @property
    def name_id_map(self):
        # sometimes the easiest solution is just to read from disk every single time
        if self.exists:
            with open(self.filename, 'rt') as f:
{'fraction_non_alphanumeric': 0.08302169035153328, 'fraction_numerical': 0.0039890301670406384, 'mean_word_length': 2.3184449958643505, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '471039', 'n_tokens_mistral': 1175, 'n_tokens_neox': 1130, 'n_words': 341}