<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Constant settings for Magpie application.

Constants defined with format ``MAGPIE_[VARIABLE_NAME]`` can be matched with corresponding
settings formatted as ``magpie.[variable_name]`` in the ``magpie.ini`` configuration file.

.. note::
    Since the ``magpie.ini`` file has to be loaded by the application to retrieve various configuration settings,
    constant ``MAGPIE_INI_FILE_PATH`` (or any other `path variable` defined before it - see below) has to be defined
    by environment variable if the default location is not desired (ie: if you want to provide your own configuration).
"""
from magpie.definitions.pyramid_definitions import asbool
from typing import TYPE_CHECKING
import re
import os
import shutil
import dotenv
import logging
import warnings
if TYPE_CHECKING:
    from magpie.definitions.typedefs import Str, Optional, SettingValue, AnySettingsContainer  # noqa: F401

# ===========================
# path variables
# ===========================
MAGPIE_MODULE_DIR = os.path.abspath(os.path.dirname(__file__))
MAGPIE_ROOT = os.path.dirname(MAGPIE_MODULE_DIR)
MAGPIE_CONFIG_DIR = os.getenv(
    "MAGPIE_CONFIG_DIR", os.path.join(MAGPIE_ROOT, "config"))
MAGPIE_PROVIDERS_CONFIG_PATH = os.getenv(
    "MAGPIE_PROVIDERS_CONFIG_PATH", "{}/providers.cfg".format(MAGPIE_CONFIG_DIR))
MAGPIE_PERMISSIONS_CONFIG_PATH = os.getenv(
    "MAGPIE_PERMISSIONS_CONFIG_PATH", "{}/permissions.cfg".format(MAGPIE_CONFIG_DIR))
MAGPIE_INI_FILE_PATH = os.getenv(
    "MAGPIE_INI_FILE_PATH", "{}/magpie.ini".format(MAGPIE_CONFIG_DIR))
MAGPIE_ALEMBIC_INI_FILE_PATH = os.getenv(
    "MAGPIE_ALEMBIC_INI_FILE_PATH", MAGPIE_INI_FILE_PATH)
# allow custom location of env files directory to avoid
# loading from installed magpie in python site-packages
MAGPIE_ENV_DIR = os.getenv("MAGPIE_ENV_DIR", os.path.join(MAGPIE_ROOT, "env"))
MAGPIE_ENV_FILE = os.path.join(MAGPIE_ENV_DIR, "magpie.env")
MAGPIE_POSTGRES_ENV_FILE = os.path.join(MAGPIE_ENV_DIR, "postgres.env")

# create .env from .env.example if not present and load variables into environment
# if files still cannot be found at 'MAGPIE_ENV_DIR' and variables are still not set,
# default values in following sections will be used instead
magpie_env_example = MAGPIE_ENV_FILE + ".example"
postgres_env_example = MAGPIE_ENV_FILE + ".example"
if not os.path.isfile(MAGPIE_ENV_FILE) and os.path.isfile(magpie_env_example):
    shutil.copyfile(magpie_env_example, MAGPIE_ENV_FILE)
if not os.path.isfile(MAGPIE_POSTGRES_ENV_FILE) and os.path.isfile(postgres_env_example):
    shutil.copyfile(postgres_env_example, MAGPIE_POSTGRES_ENV_FILE)
del magpie_env_example
del postgres_env_example
try:
    # if variables already exist, don't override them from defaults in env files
    dotenv.load_dotenv(MAGPIE_ENV_FILE, override=False)
    dotenv.load_dotenv(MAGPIE_POSTGRES_ENV_FILE, override=False)
except IOError:
    warnings.warn("Failed to open environment files [MAGPIE_ENV_DIR={}].".format(MAGPIE_ENV_DIR), RuntimeWarning)
    pass


def _get_default_log_level():
    """Get default configurations from ini file."""
    _default_log_lvl = "INFO"
    # noinspection PyBroadException
    try:
        import magpie.utils
        _settings = magpie.utils.get_settings_from_config_ini(MAGPIE_INI_FILE_PATH,
                                                              ini_main_section_name="logger_magpie")
        _default_log_lvl = _settings.get("level", _default_log_lvl)
    except Exception:
        pass


# ===========================
# variables from magpie.env
# ===========================
MAGPIE_URL = os.getenv("MAGPIE_URL", None)
MAGPIE_SECRET = os.getenv("MAGPIE_SECRET", "seekrit")
MAGPIE_COOKIE_NAME = os.getenv("MAGPIE_COOKIE_NAME", "auth_tkt")
MAGPIE_COOKIE_EXPIRE = os.getenv("MAGPIE_COOKIE_EXPIRE", None)<commit_msg>Add return statement<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Constant settings for Magpie application.

Constants defined with format ``MAGPIE_[VARIABLE_NAME]`` can be matched with corresponding
settings formatted as ``magpie.[variable_name]`` in the ``magpie.ini`` configuration file.

.. note::
    Since the ``magpie.ini`` file has to be loaded by the application to retrieve various configuration settings,
    constant ``MAGPIE_INI_FILE_PATH`` (or any other `path variable` defined before it - see below) has to be defined
    by environment variable if the default location is not desired (ie: if you want to provide your own configuration).
"""
from magpie.definitions.pyramid_definitions import asbool
from typing import TYPE_CHECKING
import re
import os
import shutil
import dotenv
import logging
import warnings
if TYPE_CHECKING:
    from magpie.definitions.typedefs import Str, Optional, SettingValue, AnySettingsContainer  # noqa: F401

# ===========================
# path variables
# ===========================
MAGPIE_MODULE_DIR = os.path.abspath(os.path.dirname(__file__))
MAGPIE_ROOT = os.path.dirname(MAGPIE_MODULE_DIR)
MAGPIE_CONFIG_DIR = os.getenv(
    "MAGPIE_CONFIG_DIR", os.path.join(MAGPIE_ROOT, "config"))
MAGPIE_PROVIDERS_CONFIG_PATH = os.getenv(
    "MAGPIE_PROVIDERS_CONFIG_PATH", "{}/providers.cfg".format(MAGPIE_CONFIG_DIR))
MAGPIE_PERMISSIONS_CONFIG_PATH = os.getenv(
    "MAGPIE_PERMISSIONS_CONFIG_PATH", "{}/permissions.cfg".format(MAGPIE_CONFIG_DIR))
MAGPIE_INI_FILE_PATH = os.getenv(
    "MAGPIE_INI_FILE_PATH", "{}/magpie.ini".format(MAGPIE_CONFIG_DIR))
MAGPIE_ALEMBIC_INI_FILE_PATH = os.getenv(
    "MAGPIE_ALEMBIC_INI_FILE_PATH", MAGPIE_INI_FILE_PATH)
# allow custom location of env files directory to avoid
# loading from installed magpie in python site-packages
MAGPIE_ENV_DIR = os.getenv("MAGPIE_ENV_DIR", os.path.join(MAGPIE_ROOT, "env"))
MAGPIE_ENV_FILE = os.path.join(MAGPIE_ENV_DIR, "magpie.env")
MAGPIE_POSTGRES_ENV_FILE = os.path.join(MAGPIE_ENV_DIR, "postgres.env")

# create .env from .env.example if not present and load variables into environment
# if files still cannot be found at 'MAGPIE_ENV_DIR' and variables are still not set,
# default values in following sections will be used instead
magpie_env_example = MAGPIE_ENV_FILE + ".example"
postgres_env_example = MAGPIE_ENV_FILE + ".example"
if not os.path.isfile(MAGPIE_ENV_FILE) and os.path.isfile(magpie_env_example):
    shutil.copyfile(magpie_env_example, MAGPIE_ENV_FILE)
if not os.path.isfile(MAGPIE_POSTGRES_ENV_FILE) and os.path.isfile(postgres_env_example):
    shutil.copyfile(postgres_env_example, MAGPIE_POSTGRES_ENV_FILE)
del magpie_env_example
del postgres_env_example
try:
    # if variables already exist, don't override them from defaults in env files
    dotenv.load_dotenv(MAGPIE_ENV_FILE, override=False)
    dotenv.load_dotenv(MAGPIE_POSTGRES_ENV_FILE, override=False)
except IOError:
    warnings.warn("Failed to open environment files [MAGPIE_ENV_DIR={}].".format(MAGPIE_ENV_DIR), RuntimeWarning)
    pass


def _get_default_log_level():
    """Get default configurations from ini file."""
    _default_log_lvl = "INFO"
    # noinspection PyBroadException
    try:
        import magpie.utils
        _settings = magpie.utils.get_settings_from_config_ini(MAGPIE_INI_FILE_PATH,
                                                              ini_main_section_name="logger_magpie")
        _default_log_lvl = _settings.get("level", _default_log_lvl)
    except Exception:
        pass
    return _default_log_lvl


# ===========================
# variables from magpie.env
# ===========================
MAGPIE_URL = os.getenv("MAGPIE_URL", None)
MAGPIE_SECRET = os.getenv("MAGPIE_SECRET", "seekrit")
MAGPIE_COOKIE_NAME = os.getenv("MAGPIE_COOKIE_NAME", "auth_tkt")
MAGPIE_COOKIE_EXPIRE = os.getenv("MAGPIE_COOKIE_EXPIRE", None)
{'fraction_non_alphanumeric': 0.11642681187605342, 'fraction_numerical': 0.001037209905354596, 'mean_word_length': 5.338537387017255, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6844657', 'n_tokens_mistral': 2611, 'n_tokens_neox': 2563, 'n_words': 583}