<commit_before>import importlib
import os
import re
import socket
import sys

from django.utils.termcolors import colorize


def log(s):
    sys.stdout.write(colorize(s, fg='cyan') + '\n')


# The normal scenario is that we use the hostname, but let's make it
# overridable, this is useful for dev and debugging.
IDEASCUBE_HOSTNAME = socket.gethostname()  # Store it for later use.
IDEASCUBE_ID = os.environ.get('IDEASCUBE_ID', IDEASCUBE_HOSTNAME)
IDEASCUBE_ID = re.sub('[^\w_]', '', IDEASCUBE_ID)
log('IDEASCUBE_ID={}'.format(IDEASCUBE_ID))

# Every box will have some edge specific needs, such as a specific user model,
# we manage this with per box settings, but we want those specific settings
# to be versionned, for two reasons: easier to debug when there is no hidden
# local config, and easier to manage code upgrade.
try:
    sub = importlib.import_module(".conf." + IDEASCUBE_ID, package="ideascube")

except ImportError:
    # No specific config for this box
    from .conf import base as sub

# Make it available as a settings, to be able to display it in the admin.
SETTINGS_MODULE = sub.__name__
log('Importing settings from %s\n' % SETTINGS_MODULE)
ldict = locals()
for k in sub.__dict__:
    if k.isupper() and not k.startswith('__') or not k.endswith('__'):
        ldict[k] = sub.__dict__[k]
USER_DATA_FIELDS = []
for section, fields in USER_FORM_FIELDS:  # pragma: no flakes
    USER_DATA_FIELDS.extend(fields)

# Allow server settings to only define STORAGE_ROOT without needing to
# redefine all ROOTS like settings.
BACKUPED_ROOT = ldict.get('BACKUPED_ROOT') or os.path.join(STORAGE_ROOT, 'main')  # pragma: no flakes
MEDIA_ROOT = ldict.get('MEDIA_ROOT') or os.path.join(BACKUPED_ROOT, 'media')  # noqa
STATIC_ROOT = ldict.get('STATIC_ROOT') or os.path.join(STORAGE_ROOT, 'static')  # pragma: no flakes
CATALOG_CACHE_ROOT = (
    ldict.get('CATALOG_CACHE_ROOT') or '/var/cache/ideascube/catalog')
CATALOG_STORAGE_ROOT = (
    ldict.get('CATALOG_STORAGE_ROOT')
    or os.path.join(BACKUPED_ROOT, 'catalog'))

if not getattr(ldict, 'DATABASES', None):
    DATABASES = {
        'default': {
            'ENGINE': 'django.db.backends.sqlite3',
            'NAME': os.path.join(BACKUPED_ROOT, 'default.sqlite'),
        },
        'transient': {
            'ENGINE': 'django.db.backends.sqlite3',
            'NAME': os.path.join(STORAGE_ROOT, 'transient.sqlite'),  # pragma: no flakes
        }
    }<commit_msg>Specify the mode of uploaded files<commit_after>import importlib
import os
import re
import socket
import sys

from django.utils.termcolors import colorize


def log(s):
    sys.stdout.write(colorize(s, fg='cyan') + '\n')


# The normal scenario is that we use the hostname, but let's make it
# overridable, this is useful for dev and debugging.
IDEASCUBE_HOSTNAME = socket.gethostname()  # Store it for later use.
IDEASCUBE_ID = os.environ.get('IDEASCUBE_ID', IDEASCUBE_HOSTNAME)
IDEASCUBE_ID = re.sub('[^\w_]', '', IDEASCUBE_ID)
log('IDEASCUBE_ID={}'.format(IDEASCUBE_ID))

# Every box will have some edge specific needs, such as a specific user model,
# we manage this with per box settings, but we want those specific settings
# to be versionned, for two reasons: easier to debug when there is no hidden
# local config, and easier to manage code upgrade.
try:
    sub = importlib.import_module(".conf." + IDEASCUBE_ID, package="ideascube")

except ImportError:
    # No specific config for this box
    from .conf import base as sub

# Make it available as a settings, to be able to display it in the admin.
SETTINGS_MODULE = sub.__name__
log('Importing settings from %s\n' % SETTINGS_MODULE)
ldict = locals()
for k in sub.__dict__:
    if k.isupper() and not k.startswith('__') or not k.endswith('__'):
        ldict[k] = sub.__dict__[k]
USER_DATA_FIELDS = []
for section, fields in USER_FORM_FIELDS:  # pragma: no flakes
    USER_DATA_FIELDS.extend(fields)

# Allow server settings to only define STORAGE_ROOT without needing to
# redefine all ROOTS like settings.
BACKUPED_ROOT = ldict.get('BACKUPED_ROOT') or os.path.join(STORAGE_ROOT, 'main')  # pragma: no flakes
MEDIA_ROOT = ldict.get('MEDIA_ROOT') or os.path.join(BACKUPED_ROOT, 'media')  # noqa
STATIC_ROOT = ldict.get('STATIC_ROOT') or os.path.join(STORAGE_ROOT, 'static')  # pragma: no flakes
CATALOG_CACHE_ROOT = (
    ldict.get('CATALOG_CACHE_ROOT') or '/var/cache/ideascube/catalog')
CATALOG_STORAGE_ROOT = (
    ldict.get('CATALOG_STORAGE_ROOT')
    or os.path.join(BACKUPED_ROOT, 'catalog'))

if not getattr(ldict, 'DATABASES', None):
    DATABASES = {
        'default': {
            'ENGINE': 'django.db.backends.sqlite3',
            'NAME': os.path.join(BACKUPED_ROOT, 'default.sqlite'),
        },
        'transient': {
            'ENGINE': 'django.db.backends.sqlite3',
            'NAME': os.path.join(STORAGE_ROOT, 'transient.sqlite'),  # pragma: no flakes
        }
    }

FILE_UPLOAD_PERMISSIONS = 0o644
{'fraction_non_alphanumeric': 0.11488409922732819, 'fraction_numerical': 0.0016266775111834079, 'mean_word_length': 4.029652351738242, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2058775', 'n_tokens_mistral': 1744, 'n_tokens_neox': 1695, 'n_words': 481}