<commit_before>import os
import sys
import os.path

__DEBUG = False
__RUN_MODE = ''
__env_mode = os.getenv('TSADM_MODE')
if __env_mode == 'test' or __env_mode == 'dev':
    __RUN_MODE = __env_mode
if __env_mode == 'dev':
    __DEBUG = True

__BASE_DIR = '/opt/tsadm'+__RUN_MODE
__CONFIG_PATH = '/etc/opt/tsadm'+__RUN_MODE+'/config.json'

__TSADM = {
    'RUN_MODE': __RUN_MODE,
    'BASE_DIR': __BASE_DIR,
    'CONFIG_PATH': __CONFIG_PATH,
    'DEBUG': __DEBUG,
    'CHARSET': 'utf-8',
    'LANG_CODE': 'en-gb',
    'TIME_ZONE': 'Europe/London',
    'DJANGO_SECRET_KEY': '<KEY>',
    'DJANGO_CACHE_PATH': '/var/tmp/tsadm'+__RUN_MODE+'_cache',
    'DJANGO_CACHE_TIMEOUT': 3600,
    'DJANGO_CACHE_KEY_PREFIX': 'tsadm'+__RUN_MODE+':',
    'OPENSSL': '/usr/bin/openssl',
    'JOBQ_SERVER_PORT': 6100,
    'JOBQ_SERVER_TIMEOUT': 15,
    'JOBQ_SYSLOG_TAG': 'tsadm'+__RUN_MODE+'.jobq',
    'MASTER_SERVER': 'dev.tsadm.local',
    'MASTER_SERVER_PORT': 8000,
    'MASTER_SERVER_SSL': False,
    'SITE_ENV_DOMAIN': 'dev.tsadm.local',
    'SITE_HOME_BASE': '/home/tsadm'+__RUN_MODE+'/sites',
    'LOG_DATE_FMT': '%b%d %H:%M:%S',
    'CUR_TIME_FMT': '%a %b %d %H:%M %Y %Z',
    'JOB_DATE_FMT': '%c %Z',
    'SYSLOG_TAG': 'tsadm'+__RUN_MODE+'.wapp',
    'REGR_TESTS_ENABLE': True,
    'CLEAN_HTML_ENABLE': True,
    'SLAVE_GRAPHS_BASE_URL': 'http://dev.tsadm.local/server-graphs',
    'DB_NAME': 'tsadm'+__RUN_MODE+'db',
    'DB_USER': 'tsadm'+__RUN_MODE,
    'DB_PASS': '<PASSWORD>',
    'CSS_RELPATH': 'static/css/tsadm.css',
    'OFFLINE_FILE': os.path.join(__BASE_DIR, 'OFFLINE'),
}

def get(arg, default=None):
    return __TSADM.get(arg, default)

def update(d):
    return __TSADM.update(d)

def export():
    r = list()
    for sk in sorted(__TSADM.keys()):
        if sk.lower().find("pass") > 0 or sk.lower().find("secret") > 0:
            r.append((sk, '__HIDDEN__'))
        else:
            r.append((sk, __TSADM.get(sk)))
    return r

def setdefault(k, v):
    if k in __TSADM.keys():
        __TSADM[k] = v

if __DEBUG:
    CACHES = {
        'default': {
            'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
        }
    }

def __load_config():
    import json
    cfg = dict()
    try:
        if __DEBUG: print("tsadm config load:", __CONFIG_PATH, file=sys.stderr)
        fh = open(__CONFIG_PATH, 'r')
        cfg = json.load(fh)
        fh.close()
    except Exception as e:<commit_msg>avoid setting RUN_MODE and BASE_DIR from config file<commit_after>import os
import sys
import os.path

__DEBUG = False
__RUN_MODE = ''
__env_mode = os.getenv('TSADM_MODE')
if __env_mode == 'test' or __env_mode == 'dev':
    __RUN_MODE = __env_mode
if __env_mode == 'dev':
    __DEBUG = True

__BASE_DIR = '/opt/tsadm'+__RUN_MODE
__CONFIG_PATH = '/etc/opt/tsadm'+__RUN_MODE+'/config.json'

__TSADM = {
    'RUN_MODE': __RUN_MODE,
    'BASE_DIR': __BASE_DIR,
    'CONFIG_PATH': __CONFIG_PATH,
    'DEBUG': __DEBUG,
    'CHARSET': 'utf-8',
    'LANG_CODE': 'en-gb',
    'TIME_ZONE': 'Europe/London',
    'DJANGO_SECRET_KEY': '<KEY>',
    'DJANGO_CACHE_PATH': '/var/tmp/tsadm'+__RUN_MODE+'_cache',
    'DJANGO_CACHE_TIMEOUT': 3600,
    'DJANGO_CACHE_KEY_PREFIX': 'tsadm'+__RUN_MODE+':',
    'OPENSSL': '/usr/bin/openssl',
    'JOBQ_SERVER_PORT': 6100,
    'JOBQ_SERVER_TIMEOUT': 15,
    'JOBQ_SYSLOG_TAG': 'tsadm'+__RUN_MODE+'.jobq',
    'MASTER_SERVER': 'dev.tsadm.local',
    'MASTER_SERVER_PORT': 8000,
    'MASTER_SERVER_SSL': False,
    'SITE_ENV_DOMAIN': 'dev.tsadm.local',
    'SITE_HOME_BASE': '/home/tsadm'+__RUN_MODE+'/sites',
    'LOG_DATE_FMT': '%b%d %H:%M:%S',
    'CUR_TIME_FMT': '%a %b %d %H:%M %Y %Z',
    'JOB_DATE_FMT': '%c %Z',
    'SYSLOG_TAG': 'tsadm'+__RUN_MODE+'.wapp',
    'REGR_TESTS_ENABLE': True,
    'CLEAN_HTML_ENABLE': True,
    'SLAVE_GRAPHS_BASE_URL': 'http://dev.tsadm.local/server-graphs',
    'DB_NAME': 'tsadm'+__RUN_MODE+'db',
    'DB_USER': 'tsadm'+__RUN_MODE,
    'DB_PASS': '<PASSWORD>',
    'CSS_RELPATH': 'static/css/tsadm.css',
    'OFFLINE_FILE': os.path.join(__BASE_DIR, 'OFFLINE'),
}

def get(arg, default=None):
    return __TSADM.get(arg, default)

def update(d):
    return __TSADM.update(d)

def export():
    r = list()
    for sk in sorted(__TSADM.keys()):
        if sk.lower().find("pass") > 0 or sk.lower().find("secret") > 0:
            r.append((sk, '__HIDDEN__'))
        else:
            r.append((sk, __TSADM.get(sk)))
    return r

def setdefault(k, v):
    if k in __TSADM.keys():
        if k != 'RUN_MODE' and k != 'BASE_DIR':
            __TSADM[k] = v

if __DEBUG:
    CACHES = {
        'default': {
            'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
        }
    }

def __load_config():
    import json
    cfg = dict()
    try:
        if __DEBUG: print("tsadm config load:", __CONFIG_PATH, file=sys.stderr)
        fh = open(__CONFIG_PATH, 'r')
        cfg = json.load(fh)
        fh.close()
    except Exception as e:
{'fraction_non_alphanumeric': 0.1753708595813859, 'fraction_numerical': 0.006909164803901646, 'mean_word_length': 3.3327464788732395, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3898490', 'n_tokens_mistral': 2049, 'n_tokens_neox': 1988, 'n_words': 354}