<commit_before>####################################################################################################
# neuropythy/io/core.py
# This file implements the load and save functions that can be used to read and write various kinds
# of neuroscience data

import numpy                        as np
import pyrsistent                   as pyr
import nibabel                      as nib
import os, six, pimms

# The list of import-types we understand
importers = pyr.m()
'''
neuropythy.io.core.importers is the persistent map of file-types that can be imported by neuropythy.
See also neuropythy.io.load.
'''

def guess_import_format(filename, **kwargs):
    '''
    guess_import_format(filename) attempts to guess the file format for the given filename; it does
      this guessing by looking at the file extension and using registered sniff-tests from
      importers. It will not attempt to load the file, so if the extension of the filename is
      missing, it is unlikely that this function will deduce the file-type (though load will often
      succeeed at extracting the data by trying all types exhaustively). If guess_import_format
      cannot deduce the format, it yields None.

    Note that if the filename has an extension that is recognized by neuropythy but the file itself
    is of another format, this function will never look beyond the extention in the filename;
    neither this function nor load perform that level of deduction.
    
    Keyword arguments that are passed to load should also be passed to guess_import_format.
    '''
    # first try file extension
    (_,filename) = os.path.split(filename)
    if '.' in filename:
        fnm = filename.lower()
        fmt = next((k for (k,(_,es,_)) in six.iteritems(importers)
                    if any(fnm.endswith('.' + e) for e in es)),
                   None)
        if fmt: return fmt
    # that didn't work; let's check the sniffers
    for (k,(_,_,sniff)) in six.iteritems(importers):
        try:
            if sniff(filename, **kwargs): return k
        except: pass
    return None
def load(filename, format=None, **kwargs):
    '''
    load(filename) yields the data contained in the file referenced by the given filename in a
      neuropythy or neuropythy-friendly format.
    load(filename, format) specifies that the given format should be used; this should be the name
      of the importer (though a file extension that is recognized also will work).

    Additionally, functions located in load.<format> may be used; so, for example, the following
    are equivalent calls:
      load(filename, 'nifti')
      load.nifti(filename)
    In fact, the load.nifti function is just the nifti importer, so help(load.nifti) will also
    yield documentation for the nifti importer.

    Keyword options may be passed to load; these must match those accepted by the given import
    function.
    '''
    filename = os.path.expanduser(filename)
    if format is None:
        format = guess_import_format(filename, **kwargs)
        if format is None:
            # try formats and see if one works!
            for (k,(f,_,_)) in six.iteritems(importers):
                try:    return f(filename, **kwargs)
                except: pass
            raise ValueError('Could not deduce format of file %s' % filename)
    format = format.lower()
    if format not in importers:
        raise ValueError('Format \'%s\' not recognized by neuropythy' % format)
    (f,_,_) = importers[format]
    return f(filename, **kwargs)
def importer(name, extensions=None, sniff=None):
    '''
    @importer(name) is a decorator that declares that the following function is an file loading
      function that should be registered with the neuropythy load function. See also the
      forget_importer function.

    Any importer function must take, as its first argument, a filename; after that it may take any
    number of keyword arguments, but no other non-keyword arguments. These keyword arguments can be
    passed to the neuropythy load function.
    
    The following options are accepted:
      * extensions (default: None) may be a string or a collection of strings that indicate possible
        file extensions for files of this type.
      * sniff (default: None) may optionally be a function f(s) that yields True when the given
        string s is a filename for a file of this type. If no sniff is given, this type can still
        be detected by running the importer and catching any raised exception.
    '''
    name = name.lower()
    if name in importers:
        raise ValueError('An importer for type %s already exists; see forget_importer' % name)
    if extensions is None:         extensions = ()
    elif pimms.is_str(extensions): (extensions,)
    else:                          extensions = tuple(extensions)<commit_msg>some importers will now auto-document where their returned objects were loaded via meta-data<commit_after>####################################################################################################
# neuropythy/io/core.py
# This file implements the load and save functions that can be used to read and write various kinds
# of neuroscience data

import numpy                        as np
import pyrsistent                   as pyr
import nibabel                      as nib
import os, six, pimms

from ..util import ObjectWithMetaData

# The list of import-types we understand
importers = pyr.m()
'''
neuropythy.io.core.importers is the persistent map of file-types that can be imported by neuropythy.
See also neuropythy.io.load.
'''

def guess_import_format(filename, **kwargs):
    '''
    guess_import_format(filename) attempts to guess the file format for the given filename; it does
      this guessing by looking at the file extension and using registered sniff-tests from
      importers. It will not attempt to load the file, so if the extension of the filename is
      missing, it is unlikely that this function will deduce the file-type (though load will often
      succeeed at extracting the data by trying all types exhaustively). If guess_import_format
      cannot deduce the format, it yields None.

    Note that if the filename has an extension that is recognized by neuropythy but the file itself
    is of another format, this function will never look beyond the extention in the filename;
    neither this function nor load perform that level of deduction.
    
    Keyword arguments that are passed to load should also be passed to guess_import_format.
    '''
    # first try file extension
    (_,filename) = os.path.split(filename)
    if '.' in filename:
        fnm = filename.lower()
        fmt = next((k for (k,(_,es,_)) in six.iteritems(importers)
                    if any(fnm.endswith('.' + e) for e in es)),
                   None)
        if fmt: return fmt
    # that didn't work; let's check the sniffers
    for (k,(_,_,sniff)) in six.iteritems(importers):
        try:
            if sniff(filename, **kwargs): return k
        except: pass
    return None
def load(filename, format=None, **kwargs):
    '''
    load(filename) yields the data contained in the file referenced by the given filename in a
      neuropythy or neuropythy-friendly format.
    load(filename, format) specifies that the given format should be used; this should be the name
      of the importer (though a file extension that is recognized also will work).

    Additionally, functions located in load.<format> may be used; so, for example, the following
    are equivalent calls:
      load(filename, 'nifti')
      load.nifti(filename)
    In fact, the load.nifti function is just the nifti importer, so help(load.nifti) will also
    yield documentation for the nifti importer.

    Keyword options may be passed to load; these must match those accepted by the given import
    function.
    '''
    filename = os.path.expanduser(filename)
    if format is None:
        format = guess_import_format(filename, **kwargs)
        if format is None:
            # try formats and see if one works!
            for (k,(f,_,_)) in six.iteritems(importers):
                try:    return f(filename, **kwargs)
                except: pass
            raise ValueError('Could not deduce format of file %s' % filename)
    format = format.lower()
    if format not in importers:
        raise ValueError('Format \'%s\' not recognized by neuropythy' % format)
    (f,_,_) = importers[format]
    obj = f(filename, **kwargs)
    if isinstance(obj, ObjectWithMetaData): return obj.with_meta(source_filename=filename)
    else: return obj
def importer(name, extensions=None, sniff=None):
    '''
    @importer(name) is a decorator that declares that the following function is an file loading
      function that should be registered with the neuropythy load function. See also the
      forget_importer function.

    Any importer function must take, as its first argument, a filename; after that it may take any
    number of keyword arguments, but no other non-keyword arguments. These keyword arguments can be
    passed to the neuropythy load function.
    
    The following options are accepted:
      * extensions (default: None) may be a string or a collection of strings that indicate possible
        file extensions for files of this type.
      * sniff (default: None) may optionally be a function f(s) that yields True when the given
        string s is a filename for a file of this type. If no sniff is given, this type can still
        be detected by running the importer and catching any raised exception.
    '''
    name = name.lower()
    if name in importers:
        raise ValueError('An importer for type %s already exists; see forget_importer' % name)
    if extensions is None:         extensions = ()
    elif pimms.is_str(extensions): (extensions,)
    else:                          extensions = tuple(extensions)
{'fraction_non_alphanumeric': 0.08408287013643254, 'fraction_numerical': 0.0, 'mean_word_length': 2.9176563737133807, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1837863', 'n_tokens_mistral': 2549, 'n_tokens_neox': 2456, 'n_words': 1195}