<commit_before># -*- coding: utf-8 -*-

"""Settings."""

#------------------------------------------------------------------------------
# Imports
#------------------------------------------------------------------------------

import os
import os.path as op
import re
import pickle

from ..ext.six.moves.cPickle import load, dump
from ._misc import Bunch
from .logging import debug, warn


#------------------------------------------------------------------------------
# Utility functions
#------------------------------------------------------------------------------

def _split_namespace(name, namespace=None):
    if '.' in name:
        namespace, name = name.split('.')
    if namespace is None:
        raise ValueError("The namespace must be specified.")
    return namespace, name


#------------------------------------------------------------------------------
# Base Settings
#------------------------------------------------------------------------------

class BaseSettings(object):
    """Store key-value pairs."""
    def __init__(self):
        self._store = {'global': {}}

    @property
    def _namespaces(self):
        return sorted(self._store['global'])

    def get(self, name, namespace=None, scope='global'):
        """Get a settings value.

        Parameters
        ----------
        name : str
            The settings name
        scope : str (default is 'global')
            The scope for that setting. Can be 'global' or a dataset name.

        """
        namespace, name = _split_namespace(name, namespace=namespace)
        if scope not in self._store:
            scope = 'global'
        return self._store[scope].get(namespace, {}).get(name, None)

    def set(self, key_values=None, namespace=None, scope='global'):
        """Set some settings

        Parameters
        ----------
        key_values : dict
            A {str: object} dictionary with key-value pair settings.
        namespace : str
            The namespace if it is not specified in the keys.
        scope : str (default is 'global')
            The scope for that setting. Can be 'global' or a dataset name.

        """
        assert isinstance(key_values, dict)
        if scope not in self._store:
            self._store[scope] = Bunch({})
        for key, value in key_values.items():
            namespace, name = _split_namespace(key, namespace=namespace)

            # Create dictionaries if they do not exist.
            if namespace not in self._store[scope]:
                self._store[scope][namespace] = Bunch({})

            # Update the settings.
            self._store[scope][namespace][name] = value


#------------------------------------------------------------------------------
# User Settings
#------------------------------------------------------------------------------

class UserSettings(BaseSettings):
    """Support Python settings files."""

    def read_settings_file(self, path, file_namespace=None):
        """Return a dictionary {namespace: {key: value}} dictionary."""
        with open(path, 'r') as f:
            contents = f.read()
        if file_namespace is None:
            file_namespace = {}
        # Executing the code directly updates the internal store.
        # The current store is passed as a global namespace.
        try:
            exec(contents, Bunch(self._store['global']), file_namespace)
        except NameError as e:
            r = re.search("'([^']+)'", e.args[0])
            if r:
                name = r.group(1)
            namespaces = ', '.join(self._namespaces)
            raise NameError("Unknown namespace '{0:s}'. ".format(name) +
                            "Known namespaces are: {0:s}.".format(namespaces))

    def set(self, key_values=None, namespace=None, scope='global',
            path=None, file_namespace=None):
        """Set some settings

        Parameters
        ----------
        key_values : dict
            A {str: object} dictionary with key-value pair settings.
        namespace : str
            The namespace if it is not specified in the keys.
        scope : str (default is 'global')
            The scope for that setting. Can be 'global' or a dataset name.
        path : str
            A path to a Python settings file.
        file_namespace : dict
            A namespace to pass to the Python settings file.

        """
        if path is not None:
            path = op.expanduser(path)
            path = op.realpath(path)
            assert op.exists(path)
            return self.read_settings_file(path,
                                           file_namespace=file_namespace)
        super(UserSettings, self).set(key_values=key_values,
                                      namespace=namespace,
                                      scope=scope,
                                      )


_USER_SETTINGS = UserSettings()


def get_user(*args, **kwargs):
    return _USER_SETTINGS.get(*args, **kwargs)


def set_user(*args, **kwargs):
    return _USER_SETTINGS.set(*args, **kwargs)


#------------------------------------------------------------------------------
# Internal Settings
#------------------------------------------------------------------------------

class InternalSettings(BaseSettings):
    """Settings to be modified by the program, not the user."""
    def load(self, path):
        path = op.realpath(op.expanduser(path))
        if not op.exists(path):
            raise ValueError("The file '{0}' doesn't exist.".format(path))
        try:
            with open(path, 'rb') as f:
                store = load(f)
        except Exception as e:
            warn("Unable to read the internal settings. "
                 "You may want to delete '{0}'.\n{1}".format(path, e.message))
        assert isinstance(store, dict)
        debug("Loaded internal settings from '{0}'.".format(path))
        self._store = store

    def save(self, path):
        path = op.realpath(op.expanduser(path))
        with open(path, 'wb') as f:
            dump(self._store, f)
        debug("Saved internal settings to '{0}'.".format(path))


_INTERNAL_SETTINGS = InternalSettings()


def get_internal(*args, **kwargs):
    return _INTERNAL_SETTINGS.get(*args, **kwargs)


def set_internal(*args, **kwargs):
    return _INTERNAL_SETTINGS.set(*args, **kwargs)<commit_msg>Added save/load_internal global functions in settings.<commit_after># -*- coding: utf-8 -*-

"""Settings."""

#------------------------------------------------------------------------------
# Imports
#------------------------------------------------------------------------------

import os
import os.path as op
import re
import pickle

from ..ext.six.moves.cPickle import load, dump
from ._misc import Bunch
from .logging import debug, warn


#------------------------------------------------------------------------------
# Utility functions
#------------------------------------------------------------------------------

def _split_namespace(name, namespace=None):
    if '.' in name:
        namespace, name = name.split('.')
    if namespace is None:
        raise ValueError("The namespace must be specified.")
    return namespace, name


#------------------------------------------------------------------------------
# Base Settings
#------------------------------------------------------------------------------

class BaseSettings(object):
    """Store key-value pairs."""
    def __init__(self):
        self._store = {'global': {}}

    @property
    def _namespaces(self):
        return sorted(self._store['global'])

    def get(self, name, namespace=None, scope='global'):
        """Get a settings value.

        Parameters
        ----------
        name : str
            The settings name
        scope : str (default is 'global')
            The scope for that setting. Can be 'global' or a dataset name.

        """
        namespace, name = _split_namespace(name, namespace=namespace)
        if scope not in self._store:
            scope = 'global'
        return self._store[scope].get(namespace, {}).get(name, None)

    def set(self, key_values=None, namespace=None, scope='global'):
        """Set some settings

        Parameters
        ----------
        key_values : dict
            A {str: object} dictionary with key-value pair settings.
        namespace : str
            The namespace if it is not specified in the keys.
        scope : str (default is 'global')
            The scope for that setting. Can be 'global' or a dataset name.

        """
        assert isinstance(key_values, dict)
        if scope not in self._store:
            self._store[scope] = Bunch({})
        for key, value in key_values.items():
            namespace, name = _split_namespace(key, namespace=namespace)

            # Create dictionaries if they do not exist.
            if namespace not in self._store[scope]:
                self._store[scope][namespace] = Bunch({})

            # Update the settings.
            self._store[scope][namespace][name] = value


#------------------------------------------------------------------------------
# User Settings
#------------------------------------------------------------------------------

class UserSettings(BaseSettings):
    """Support Python settings files."""

    def read_settings_file(self, path, file_namespace=None):
        """Return a dictionary {namespace: {key: value}} dictionary."""
        with open(path, 'r') as f:
            contents = f.read()
        if file_namespace is None:
            file_namespace = {}
        # Executing the code directly updates the internal store.
        # The current store is passed as a global namespace.
        try:
            exec(contents, Bunch(self._store['global']), file_namespace)
        except NameError as e:
            r = re.search("'([^']+)'", e.args[0])
            if r:
                name = r.group(1)
            namespaces = ', '.join(self._namespaces)
            raise NameError("Unknown namespace '{0:s}'. ".format(name) +
                            "Known namespaces are: {0:s}.".format(namespaces))

    def set(self, key_values=None, namespace=None, scope='global',
            path=None, file_namespace=None):
        """Set some settings

        Parameters
        ----------
        key_values : dict
            A {str: object} dictionary with key-value pair settings.
        namespace : str
            The namespace if it is not specified in the keys.
        scope : str (default is 'global')
            The scope for that setting. Can be 'global' or a dataset name.
        path : str
            A path to a Python settings file.
        file_namespace : dict
            A namespace to pass to the Python settings file.

        """
        if path is not None:
            path = op.expanduser(path)
            path = op.realpath(path)
            assert op.exists(path)
            return self.read_settings_file(path,
                                           file_namespace=file_namespace)
        super(UserSettings, self).set(key_values=key_values,
                                      namespace=namespace,
                                      scope=scope,
                                      )


_USER_SETTINGS = UserSettings()


def get_user(*args, **kwargs):
    return _USER_SETTINGS.get(*args, **kwargs)


def set_user(*args, **kwargs):
    return _USER_SETTINGS.set(*args, **kwargs)


#------------------------------------------------------------------------------
# Internal Settings
#------------------------------------------------------------------------------

class InternalSettings(BaseSettings):
    """Settings to be modified by the program, not the user."""
    def load(self, path):
        path = op.realpath(op.expanduser(path))
        if not op.exists(path):
            raise ValueError("The file '{0}' doesn't exist.".format(path))
        try:
            with open(path, 'rb') as f:
                store = load(f)
        except Exception as e:
            warn("Unable to read the internal settings. "
                 "You may want to delete '{0}'.\n{1}".format(path, e.message))
        assert isinstance(store, dict)
        debug("Loaded internal settings from '{0}'.".format(path))
        self._store = store

    def save(self, path):
        path = op.realpath(op.expanduser(path))
        with open(path, 'wb') as f:
            dump(self._store, f)
        debug("Saved internal settings to '{0}'.".format(path))


_INTERNAL_SETTINGS = InternalSettings()


def get_internal(*args, **kwargs):
    return _INTERNAL_SETTINGS.get(*args, **kwargs)


def set_internal(*args, **kwargs):
    return _INTERNAL_SETTINGS.set(*args, **kwargs)


def save_internal(*args, **kwargs):
    return _INTERNAL_SETTINGS.save(*args, **kwargs)


def load_internal(*args, **kwargs):
    return _INTERNAL_SETTINGS.load(*args, **kwargs)
{'fraction_non_alphanumeric': 0.22242947073910002, 'fraction_numerical': 0.0015543638765835083, 'mean_word_length': 1.4491815759421394, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2719548', 'n_tokens_mistral': 3092, 'n_tokens_neox': 3032, 'n_words': 960}