<commit_before>####################################################################################################
# neuropythy/datasets/core.py
# Implementation of builtin datasets for Neuropythy
# by <NAME>

import os, six, shutil, tempfile, atexit, pimms
import numpy as np

from ..util import (config, to_credentials, ObjectWithMetaData)
from ..freesurfer import subject as freesurfer_subject

# We declare a configuration variable, data_cache_root -- where to put the data that is downloaded.
# If this is None / unset, then we'll use a temporary directory and auto-delete it on exit.
config.declare_dir('data_cache_root')

@pimms.immutable
class Dataset(ObjectWithMetaData):
    '''
    The Dataset class is a simple immutable class that should be implemented by all neuropythy
    datasets. The design is such that neuropythy.data[name] should always (lazily) yield a Dataset
    object specific to the dataset given by name, if it exists and can be loaded.
    
    One reason to require (by convention) that all datasets are distinct classes is that it should
    thus be easy to evaluate help(ny.data[name]) to see help on the given dataset. If you overload
    this class, be sure to overload the documentation.
    '''
    def __init__(self, name, meta_data=None, custom_directory=None,
                 create_directories=True, create_mode=0o755):
        ObjectWithMetaData.__init__(self, meta_data)
        self.custom_directory = custom_directory
        self.name = name
        self.create_directories = create_directories
        self.create_mode = create_mode
    def __repr__(self): return self.repr
    @pimms.value
    def repr(name):
        '''
        dataset.repr is the representation string used for the given dataset.
        '''
        return ("Dataset('%s')" % name) if pimms.is_str(name) else ("Dataset%s" % (name,))
    @staticmethod
    def to_name(nm):
        '''
        Dataset.to_name(name) yields a valid dataset name equivalent to the given name or raises an
          error if name is not valid. In order to be valid, a name must be either strings or a tuple
          of number and strings that start with a string.
        '''
        if pimms.is_str(nm): return nm
        if not pimms.is_vector(nm): raise ValueError('name must be a string or tuple')
        if len(nm) < 1: raise ValueError('names that are tuples must have at least one element')
        if not pimms.is_str(nm): raise ValueError('names that are tuples must begin with a string')
        if not all(pimms.is_str(x) or pimms.is_number(x) for x in nm):
            raise ValueError('dataset names that are tuples must contain only strings and numbers')
        return tuple(nm)
    @pimms.param
    def custom_directory(d):
        '''
        dataset.custom_directory is None if no custom directory was provided for the given dataset;
          otherwise it is the provided custom directory.
        '''
        if d is None: return None
        if not pimms.is_str(d): raise ValueError('custom_directory must be a string')
        else: return d
    @pimms.param
    def create_directories(c):
        '''
        dataset.create_directories is True if the dataset was instructed to create its cache
        directory, should it be found to not exist, and is otherwise False.
        '''
        return bool(c)
    @pimms.param
    def create_mode(c):
        '''
        dataset.create_mode is the octal permision mode used to create the cache directory for the
        given dataset, if the dataset had to create its directory at all.
        '''
        return c
    @pimms.param
    def name(nm):
        '''
        dataset.name is either a string or a tuple of strings and numbers that identifies the given
        dataset. If dataset.name is a tuple, then the first element must be a string.
        '''
        return Dataset.to_name(nm)
    @pimms.value
    def cache_root(custom_directory):
        '''
        dataset.cache_root is the root directory in which the given dataset has been cached.
        '''
        if custom_directory is not None: return None
        elif config['data_cache_root'] is None:
            # we create a data-cache in a temporary directory
            path = tempfile.mkdtemp(prefix='npythy_data_cache_')
            if not os.path.isdir(path): raise ValueError('Could not find or create cache directory')
            config['data_cache_root'] = path
            atexit.register(shutil.rmtree, path)
        return config['data_cache_root']
    @pimms.value
    def cache_directory(cache_root, name, custom_directory):
        '''
        dataset.cache_directory is the directory in which the given dataset is cached.
        '''
        if custom_directory is not None: return custom_directory
        return os.path.join(cache_root, (name    if pimms.is_str(name) else
                                         name[0] if len(name) == 1     else
                                         '%s_%x' % (name[0], hash(name[1:]))))
    @pimms.require
    def ensure_cache_directory(cache_directory, create_directories, create_mode):
        '''
        ensure_cache_directory requires that a dataset's cache directory exists and raises an error
        if it cannot be found.
        '''
        if os.path.isdir(cache_directory): return True
        if not create_directories: raise ValueError('dataset cache directory not found: %s')
        os.makedirs(os.path.abspath(cache_directory), create_mode)
        return True
# We create the dataset repository: this is a lazy map; to add a dataset to it, use the function
# add_dataset(), immediately below
data = pimms.lazy_map({})
def add_dataset(dset, fn=None):
    '''
    add_dataset(dset) adds the given dataset to the neuropythy.data map.
    add_dataset(name, fn) adds a dataset with the given name; fn must be a function of zero
      arguments that yields the dataset.

    add_dataset always yeilds None or raises an error.
    '''
    global data
    if fn is None:
        if not isinstance(dset, Dataset):
            raise ValueError('Cannot add non-Dataset object to neuropythy datasets')
        nm = dset.name
        data = data.set(nm, dset)
    else:
        nm = Dataset.to_name(dset)
        def _load_dset():
            x = fn()
            if not isinstance(x, Dataset):
                raise ValueError('Loader for dataset %s failed to return a dataset' % nm)
            return x
        data = data.set(nm, _load_dset)
    return None

    
            
        <commit_msg>minor improvement to add_dataset<commit_after>####################################################################################################
# neuropythy/datasets/core.py
# Implementation of builtin datasets for Neuropythy
# by <NAME>

import os, six, shutil, tempfile, atexit, pimms
import numpy as np

from ..util import (config, to_credentials, ObjectWithMetaData)
from ..freesurfer import subject as freesurfer_subject

# We declare a configuration variable, data_cache_root -- where to put the data that is downloaded.
# If this is None / unset, then we'll use a temporary directory and auto-delete it on exit.
config.declare_dir('data_cache_root')

@pimms.immutable
class Dataset(ObjectWithMetaData):
    '''
    The Dataset class is a simple immutable class that should be implemented by all neuropythy
    datasets. The design is such that neuropythy.data[name] should always (lazily) yield a Dataset
    object specific to the dataset given by name, if it exists and can be loaded.
    
    One reason to require (by convention) that all datasets are distinct classes is that it should
    thus be easy to evaluate help(ny.data[name]) to see help on the given dataset. If you overload
    this class, be sure to overload the documentation.
    '''
    def __init__(self, name, meta_data=None, custom_directory=None,
                 create_directories=True, create_mode=0o755):
        ObjectWithMetaData.__init__(self, meta_data)
        self.custom_directory = custom_directory
        self.name = name
        self.create_directories = create_directories
        self.create_mode = create_mode
    def __repr__(self): return self.repr
    @pimms.value
    def repr(name):
        '''
        dataset.repr is the representation string used for the given dataset.
        '''
        return ("Dataset('%s')" % name) if pimms.is_str(name) else ("Dataset%s" % (name,))
    @staticmethod
    def to_name(nm):
        '''
        Dataset.to_name(name) yields a valid dataset name equivalent to the given name or raises an
          error if name is not valid. In order to be valid, a name must be either strings or a tuple
          of number and strings that start with a string.
        '''
        if pimms.is_str(nm): return nm
        if not pimms.is_vector(nm): raise ValueError('name must be a string or tuple')
        if len(nm) < 1: raise ValueError('names that are tuples must have at least one element')
        if not pimms.is_str(nm): raise ValueError('names that are tuples must begin with a string')
        if not all(pimms.is_str(x) or pimms.is_number(x) for x in nm):
            raise ValueError('dataset names that are tuples must contain only strings and numbers')
        return tuple(nm)
    @pimms.param
    def custom_directory(d):
        '''
        dataset.custom_directory is None if no custom directory was provided for the given dataset;
          otherwise it is the provided custom directory.
        '''
        if d is None: return None
        if not pimms.is_str(d): raise ValueError('custom_directory must be a string')
        else: return d
    @pimms.param
    def create_directories(c):
        '''
        dataset.create_directories is True if the dataset was instructed to create its cache
        directory, should it be found to not exist, and is otherwise False.
        '''
        return bool(c)
    @pimms.param
    def create_mode(c):
        '''
        dataset.create_mode is the octal permision mode used to create the cache directory for the
        given dataset, if the dataset had to create its directory at all.
        '''
        return c
    @pimms.param
    def name(nm):
        '''
        dataset.name is either a string or a tuple of strings and numbers that identifies the given
        dataset. If dataset.name is a tuple, then the first element must be a string.
        '''
        return Dataset.to_name(nm)
    @pimms.value
    def cache_root(custom_directory):
        '''
        dataset.cache_root is the root directory in which the given dataset has been cached.
        '''
        if custom_directory is not None: return None
        elif config['data_cache_root'] is None:
            # we create a data-cache in a temporary directory
            path = tempfile.mkdtemp(prefix='npythy_data_cache_')
            if not os.path.isdir(path): raise ValueError('Could not find or create cache directory')
            config['data_cache_root'] = path
            atexit.register(shutil.rmtree, path)
        return config['data_cache_root']
    @pimms.value
    def cache_directory(cache_root, name, custom_directory):
        '''
        dataset.cache_directory is the directory in which the given dataset is cached.
        '''
        if custom_directory is not None: return custom_directory
        return os.path.join(cache_root, (name    if pimms.is_str(name) else
                                         name[0] if len(name) == 1     else
                                         '%s_%x' % (name[0], hash(name[1:]))))
    @pimms.require
    def ensure_cache_directory(cache_directory, create_directories, create_mode):
        '''
        ensure_cache_directory requires that a dataset's cache directory exists and raises an error
        if it cannot be found.
        '''
        if os.path.isdir(cache_directory): return True
        if not create_directories: raise ValueError('dataset cache directory not found: %s')
        os.makedirs(os.path.abspath(cache_directory), create_mode)
        return True
# We create the dataset repository: this is just a lazy map; to add a dataset to iit, use the
# function add_dataset(), immediately below
data = pimms.lazy_map({})
def add_dataset(dset, fn=None):
    '''
    add_dataset(dset) adds the given dataset to the neuropythy.data map.
    add_dataset(name, fn) adds a dataset with the given name; fn must be a function of zero
      arguments that yields the dataset.

    add_dataset always yeilds None or raises an error.
    '''
    global data
    if fn is None:
        if not isinstance(dset, Dataset):
            raise ValueError('Cannot add non-Dataset object to neuropythy datasets')
        nm = dset.name
        data = data.set(nm, dset)
    else:
        nm = Dataset.to_name(dset)
        def _load_dset():
            x = fn()
            if not isinstance(x, Dataset):
                raise ValueError('Loader for dataset %s failed to return a dataset' % nm)
            return x
        data = data.set(nm, _load_dset)
    # we want to update neuropythy.data also; this is a bit of a hack, but should work fine
    import neuropythy
    neuropythy.data = data
    return None


            
        
{'fraction_non_alphanumeric': 0.0879538204466049, 'fraction_numerical': 0.0013671578307762418, 'mean_word_length': 2.621287128712871, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1332609', 'n_tokens_mistral': 3582, 'n_tokens_neox': 3547, 'n_words': 1401}