<commit_before>"""
This module is intended to allow customization of Kolibri settings with the
options.ini file.
The settings can be changed through environment variables or sections and keys
in the options.ini file.
"""
import logging.config
import os
import sys

from configobj import ConfigObj
from configobj import flatten_errors
from configobj import get_extra_values
from django.utils.functional import SimpleLazyObject
from django.utils.six import string_types
from six.moves.urllib.parse import urlparse
from six.moves.urllib.parse import urlunparse
from validate import is_boolean
from validate import Validator
from validate import VdtTypeError
from validate import VdtValueError

try:
    import kolibri.utils.pskolibri as psutil
except NotImplementedError:
    # This module can't work on this OS
    psutil = None


from kolibri.utils.data import bytes_from_humans
from kolibri.utils.i18n import KOLIBRI_LANGUAGE_INFO
from kolibri.utils.i18n import KOLIBRI_SUPPORTED_LANGUAGES
from kolibri.plugins.utils.options import extend_config_spec
from kolibri.deployment.default.sqlite_db_names import (
    ADDITIONAL_SQLITE_DATABASES,
)
from kolibri.utils.system import get_fd_limit


CACHE_SHARDS = 8

# file descriptors per thread
FD_PER_THREAD = sum(
    (
        5,  # minimum allowance
        1 + len(ADDITIONAL_SQLITE_DATABASES),  # DBs assuming SQLite
        CACHE_SHARDS,  # assuming diskcache
    )
)

# Reserve some file descriptors for file operations happening in asynchronous tasks
# when the server is running with threaded task runners.
MIN_RESERVED_FD = 64


def calculate_thread_pool():
    """
    Returns the default value for CherryPY thread_pool:
    - calculated based on the best values obtained in several partners installations
    - servers with more memory can deal with more threads
    - calculations are done for servers with more than 2 Gb of RAM
    - restricts value to avoid exceeding file descriptor limit
    """
    MIN_POOL = 50
    MAX_POOL = 150

    pool_size = MIN_POOL
    if psutil:
        MIN_MEM = 2
        MAX_MEM = 6
        total_memory = psutil.virtual_memory().total / pow(10, 9)  # in GB
        # if it's in the range, scale thread count linearly with available memory
        if MIN_MEM < total_memory < MAX_MEM:
            pool_size = MIN_POOL + int(
                (MAX_POOL - MIN_POOL)
                * float(total_memory - MIN_MEM)
                / (MAX_MEM - MIN_MEM)
            )
        elif total_memory >= MAX_MEM:
            pool_size = MAX_POOL
    elif sys.platform.startswith(
        "darwin"
    ):  # Considering MacOS has at least 4 Gb of RAM
        pool_size = MAX_POOL

    # ensure (number of threads) x (open file descriptors) < (fd limit)
    max_threads = (get_fd_limit() - MIN_RESERVED_FD) // FD_PER_THREAD
    # Ensure that the number of threads never goes below 1
    return max(1, min(pool_size, max_threads))


ALL_LANGUAGES = "kolibri-all"
SUPPORTED_LANGUAGES = "kolibri-supported"


def _process_language_string(value):
    """
    Used to validate string values.
    The only valid argument in this case is that it is a string
    so we first try to coerce it to a string, then do some checks
    to see if it is any of our special values. Then if it is an
    appropriate language code value.
    If no value is appropriate, raise a ValueError.
    """
    value = str(value)
    if value == ALL_LANGUAGES:
        return list(KOLIBRI_LANGUAGE_INFO.keys())
    if value == SUPPORTED_LANGUAGES:
        return list(KOLIBRI_SUPPORTED_LANGUAGES)
    if value in KOLIBRI_LANGUAGE_INFO:
        return [value]
    raise ValueError


def language_list(value):
    """
    Check that the supplied value is a list of languages,
    or a single language, or a special shortcut parameter.
    In the case that it is a special shortcut name, we return the full list
    of relevant languages for that parameter, or throw a validation error
    if that parameter would return an empty list.
    If a single language code is the parameter, this function will return a list
    with that language code as the only member.

    :param Union[str, list[str]] value: Either a string or a list of strings
    String can be any value that is a key of KOLIBRI_LANGUAGE_INFO
    or one of the special strings represented by ALL_LANGUAGES or SUPPORTED_LANGUAGES
    A list must be a list of these strings.
    """
    # Check the supplied value is a list
    if not isinstance(value, list):
        value = [value]

    out = set()
    errors = []
    for entry in value:
        try:
            entry_list = _process_language_string(entry)
            out.update(entry_list)
        except ValueError:
            errors.append(entry)
    if errors:
        raise VdtValueError(errors)

    if not out:
        raise VdtValueError(value)

    return sorted(list(out))


def path(value):
    from kolibri.utils.conf import KOLIBRI_HOME

    if not isinstance(value, string_types):
        raise VdtValueError(repr(value))
    # Allow for blank paths
    if value:
        # ensure all path arguments, e.g. under section "Paths", are fully resolved and expanded, relative to KOLIBRI_HOME
        return os.path.join(KOLIBRI_HOME, os.path.expanduser(value))
    return value


def path_list(value):<commit_msg>Make list processing more flexible.<commit_after>"""
This module is intended to allow customization of Kolibri settings with the
options.ini file.
The settings can be changed through environment variables or sections and keys
in the options.ini file.
"""
import logging.config
import os
import sys

from configobj import ConfigObj
from configobj import flatten_errors
from configobj import get_extra_values
from django.utils.functional import SimpleLazyObject
from django.utils.six import string_types
from six.moves.urllib.parse import urlparse
from six.moves.urllib.parse import urlunparse
from validate import is_boolean
from validate import Validator
from validate import VdtTypeError
from validate import VdtValueError

try:
    import kolibri.utils.pskolibri as psutil
except NotImplementedError:
    # This module can't work on this OS
    psutil = None


from kolibri.utils.data import bytes_from_humans
from kolibri.utils.i18n import KOLIBRI_LANGUAGE_INFO
from kolibri.utils.i18n import KOLIBRI_SUPPORTED_LANGUAGES
from kolibri.plugins.utils.options import extend_config_spec
from kolibri.deployment.default.sqlite_db_names import (
    ADDITIONAL_SQLITE_DATABASES,
)
from kolibri.utils.system import get_fd_limit


CACHE_SHARDS = 8

# file descriptors per thread
FD_PER_THREAD = sum(
    (
        5,  # minimum allowance
        1 + len(ADDITIONAL_SQLITE_DATABASES),  # DBs assuming SQLite
        CACHE_SHARDS,  # assuming diskcache
    )
)

# Reserve some file descriptors for file operations happening in asynchronous tasks
# when the server is running with threaded task runners.
MIN_RESERVED_FD = 64


def calculate_thread_pool():
    """
    Returns the default value for CherryPY thread_pool:
    - calculated based on the best values obtained in several partners installations
    - servers with more memory can deal with more threads
    - calculations are done for servers with more than 2 Gb of RAM
    - restricts value to avoid exceeding file descriptor limit
    """
    MIN_POOL = 50
    MAX_POOL = 150

    pool_size = MIN_POOL
    if psutil:
        MIN_MEM = 2
        MAX_MEM = 6
        total_memory = psutil.virtual_memory().total / pow(10, 9)  # in GB
        # if it's in the range, scale thread count linearly with available memory
        if MIN_MEM < total_memory < MAX_MEM:
            pool_size = MIN_POOL + int(
                (MAX_POOL - MIN_POOL)
                * float(total_memory - MIN_MEM)
                / (MAX_MEM - MIN_MEM)
            )
        elif total_memory >= MAX_MEM:
            pool_size = MAX_POOL
    elif sys.platform.startswith(
        "darwin"
    ):  # Considering MacOS has at least 4 Gb of RAM
        pool_size = MAX_POOL

    # ensure (number of threads) x (open file descriptors) < (fd limit)
    max_threads = (get_fd_limit() - MIN_RESERVED_FD) // FD_PER_THREAD
    # Ensure that the number of threads never goes below 1
    return max(1, min(pool_size, max_threads))


ALL_LANGUAGES = "kolibri-all"
SUPPORTED_LANGUAGES = "kolibri-supported"


def _process_language_string(value):
    """
    Used to validate string values.
    The only valid argument in this case is that it is a string
    so we first try to coerce it to a string, then do some checks
    to see if it is any of our special values. Then if it is an
    appropriate language code value.
    If no value is appropriate, raise a ValueError.
    """
    value = str(value)
    if value == ALL_LANGUAGES:
        return list(KOLIBRI_LANGUAGE_INFO.keys())
    if value == SUPPORTED_LANGUAGES:
        return list(KOLIBRI_SUPPORTED_LANGUAGES)
    if value in KOLIBRI_LANGUAGE_INFO:
        return [value]
    raise ValueError


def _process_list(value, separator=","):
    """
    Used to validate list values.
    The only valid argument in this case is that it is a list
    so we first try to coerce it to a list, then do some checks
    to see if it is any of our special values. Then if it is an
    appropriate list value.
    If no value is appropriate, raise a ValueError.
    """

    # Check the supplied value is a list
    if not isinstance(value, list):
        if not value:
            value = []
        elif isinstance(value, string_types):
            value = value.split(separator)
        else:
            value = [value]
    return value


def language_list(value):
    """
    Check that the supplied value is a list of languages,
    or a single language, or a special shortcut parameter.
    In the case that it is a special shortcut name, we return the full list
    of relevant languages for that parameter, or throw a validation error
    if that parameter would return an empty list.
    If a single language code is the parameter, this function will return a list
    with that language code as the only member.

    :param Union[str, list[str]] value: Either a string or a list of strings
    String can be any value that is a key of KOLIBRI_LANGUAGE_INFO
    or one of the special strings represented by ALL_LANGUAGES or SUPPORTED_LANGUAGES
    A list must be a list of these strings.
    """
    value = _process_list(value)

    out = set()
    errors = []
    for entry in value:
        try:
            entry_list = _process_language_string(entry)
            out.update(entry_list)
        except ValueError:
            errors.append(entry)
    if errors:
        raise VdtValueError(errors)

    if not out:
        raise VdtValueError(value)

    return sorted(list(out))


def path(value):
    from kolibri.utils.conf import KOLIBRI_HOME

    if not isinstance(value, string_types):
        raise VdtValueError(repr(value))
    # Allow for blank paths
    if value:
        # ensure all path arguments, e.g. under section "Paths", are fully resolved and expanded, relative to KOLIBRI_HOME
        return os.path.join(KOLIBRI_HOME, os.path.expanduser(value))
    return value


def path_list(value):
{'fraction_non_alphanumeric': 0.05578438395415473, 'fraction_numerical': 0.0041189111747851, 'mean_word_length': 3.007534983853606, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6505451', 'n_tokens_mistral': 3312, 'n_tokens_neox': 3187, 'n_words': 1248}