<commit_before>"""
Package contains generic functions for data input/output. This includes
methods for accessing file systems and network resources.
"""

import os, gzip, bz2
from urllib2 import urlopen
from urlparse import urlparse

from neuroimaging.utils.path import path

zipexts = (".gz",".bz2")
file_openers = {".gz":gzip.open, ".bz2":bz2.BZ2File, None:file}


def iszip(filename):
    """ Is this filename a zip file. """
    _, ext = path(filename).splitext()
    return ext in zipexts


def splitzipext(filename):
    """
    return (base, zip_extension) from filename.
    If filename does not have a zip extention then
    base = filename and zip_extension = None
    """
    if iszip(filename):
        return path(filename).splitext()
    else:
        return filename, None


def unzip(filename):
    """ Unzip the given file into another file.  Return the new file's name."""
    if not iszip(filename):
        raise ValueError("file %s is not zipped"%filename)
    unzip_name, zipext = splitzipext(filename)
    opener = file_openers[zipext]
    outfile = file(unzip_name,'w')
    outfile.write(opener(filename).read())
    outfile.close()
    return unzip_name


def urlexists(url):
    """
    Test if a url exists by attempting to open it.
    """
    try:
        urlopen(url)
    except Exception:
        return False
    return True


def isurl(pathstr):
    scheme, netloc, _, _, _, _ = urlparse(pathstr)
    return bool(scheme and netloc)


def iswritemode(mode):
    """ Test is the given mode will open a file for writing. """
    return mode.find("w")>-1 or mode.find("+")>-1


def ensuredirs(directory):
    """
    Ensure that the given directory path actually exists.
    If it doesn't, create it.
    """
    if not isinstance(directory, path):
        directory = path(directory)
    if not directory.exists():
        directory.makedirs()


class Cache (object):
    """
    A file cache. The path of the cache can be specified
    or else use ~/.nipy/cache by default.
    """
    
    def __init__(self, cachepath=None):
        if cachepath is not None: 
            self.path = path(cachepath)
        elif os.name == 'posix':
            self.path = path(os.environ["HOME"]).joinpath(".nipy","cache")
        elif os.name == 'nt':
            self.path = path(os.environ["HOMEPATH"]).joinpath(".nipy","cache")
        if not self.path.exists():
            ensuredirs(self.path)


    def filepath(self, uri):
        """
        Return the complete path + filename within the cache.
        """
        (_, netloc, upath, _, _, _) = urlparse(uri)
        return self.path.joinpath(netloc, upath[1:])

    def filename(self, uri): 
        """
        Return the complete path + filename within the cache.
        """
        return str(self.filepath(uri))
    
    def cache(self, uri):
        """
        Copy a file into the cache.
        """
        if self.iscached(uri):
            return
        upath = self.filepath(uri)
        ensuredirs(upath.dirname())
        try:
            openedurl = urlopen(uri)
        except:
            raise IOError("url not found: "+str(uri))
        file(upath, 'w').write(openedurl.read())
        
    def clear(self):
        """ Delete all files in the cache. """
        for f in self.path.files():
            f.rm()
        
    def iscached(self, uri):
        """ Check if a file exists in the cache. """
        return self.filepath(uri).exists()
        
    def retrieve(self, uri):
        """
        Retrieve a file from the cache.
        If not already there, create the file and
        add it to the cache.
        """
        self.cache(uri)
        return file(self.filename(uri))



class DataSource (object):

    def __init__(self, cachepath=os.curdir):
        if cachepath is not None:
            self._cache = Cache(cachepath)
        else:
            self._cache = Cache()

    def _possible_names(self, filename):
        names = (filename,)
        if not iszip(filename):
            for zipext in zipexts:
                names += (filename+zipext,)
        return names

    def cache(self, pathstr):
        if isurl(pathstr):
            self._cache.cache(pathstr)

    def filename(self, pathstr):
        found = None
        for name in self._possible_names(pathstr):
            if isurl(name) and urlexists(name):
                self.cache(name)
                found = self._cache.filename(name)
            elif path(name).exists(): found = name
            if found:
                break
        if found is None:
            raise IOError("%s not found"%pathstr)
        return found

    def exists(self, pathstr):
        try:
            _ = self.filename(pathstr)
            return True
        except IOError:
            return False

    def open(self, pathstr, mode='r'):
        if isurl(pathstr) and iswritemode(mode):
            raise ValueError("URLs are not writeable")
        found = self.filename(pathstr)<commit_msg>remove urlexists from data_io. It was being used to check if a file existed before downloading it, but this was done by trying to download the file. This results in downloading it twice if the file existed. Better just to try to download it and catch when it fails<commit_after>"""
Package contains generic functions for data input/output. This includes
methods for accessing file systems and network resources.
"""

import os, gzip, bz2
from urllib2 import urlopen
from urlparse import urlparse

from neuroimaging.utils.path import path

zipexts = (".gz",".bz2")
file_openers = {".gz":gzip.open, ".bz2":bz2.BZ2File, None:file}


def iszip(filename):
    """ Is this filename a zip file. """
    _, ext = path(filename).splitext()
    return ext in zipexts


def splitzipext(filename):
    """
    return (base, zip_extension) from filename.
    If filename does not have a zip extention then
    base = filename and zip_extension = None
    """
    if iszip(filename):
        return path(filename).splitext()
    else:
        return filename, None


def unzip(filename):
    """ Unzip the given file into another file.  Return the new file's name."""
    if not iszip(filename):
        raise ValueError("file %s is not zipped"%filename)
    unzip_name, zipext = splitzipext(filename)
    opener = file_openers[zipext]
    outfile = file(unzip_name,'w')
    outfile.write(opener(filename).read())
    outfile.close()
    return unzip_name


def isurl(pathstr):
    scheme, netloc, _, _, _, _ = urlparse(pathstr)
    return bool(scheme and netloc)


def iswritemode(mode):
    """ Test is the given mode will open a file for writing. """
    return mode.find("w")>-1 or mode.find("+")>-1


def ensuredirs(directory):
    """
    Ensure that the given directory path actually exists.
    If it doesn't, create it.
    """
    if not isinstance(directory, path):
        directory = path(directory)
    if not directory.exists():
        directory.makedirs()


class Cache (object):
    """
    A file cache. The path of the cache can be specified
    or else use ~/.nipy/cache by default.
    """
    
    def __init__(self, cachepath=None):
        if cachepath is not None: 
            self.path = path(cachepath)
        elif os.name == 'posix':
            self.path = path(os.environ["HOME"]).joinpath(".nipy","cache")
        elif os.name == 'nt':
            self.path = path(os.environ["HOMEPATH"]).joinpath(".nipy","cache")
        if not self.path.exists():
            ensuredirs(self.path)


    def filepath(self, uri):
        """
        Return the complete path + filename within the cache.
        """
        (_, netloc, upath, _, _, _) = urlparse(uri)
        return self.path.joinpath(netloc, upath[1:])

    def filename(self, uri): 
        """
        Return the complete path + filename within the cache.
        """
        return str(self.filepath(uri))
    
    def cache(self, uri):
        """
        Copy a file into the cache.
        """
        if self.iscached(uri):
            return
        upath = self.filepath(uri)
        ensuredirs(upath.dirname())
        try:
            openedurl = urlopen(uri)
        except:
            raise IOError("url not found: "+str(uri))
        file(upath, 'w').write(openedurl.read())
        
    def clear(self):
        """ Delete all files in the cache. """
        for f in self.path.files():
            f.rm()
        
    def iscached(self, uri):
        """ Check if a file exists in the cache. """
        return self.filepath(uri).exists()
        
    def retrieve(self, uri):
        """
        Retrieve a file from the cache.
        If not already there, create the file and
        add it to the cache.
        """
        self.cache(uri)
        return file(self.filename(uri))



class DataSource (object):

    def __init__(self, cachepath=os.curdir):
        if cachepath is not None:
            self._cache = Cache(cachepath)
        else:
            self._cache = Cache()

    def _possible_names(self, filename):
        names = (filename,)
        if not iszip(filename):
            for zipext in zipexts:
                names += (filename+zipext,)
        return names

    def cache(self, pathstr):
        if isurl(pathstr):
            self._cache.cache(pathstr)

    def filename(self, pathstr):
        found = None
        for name in self._possible_names(pathstr):
            try:                
                if isurl(name):
                    self.cache(name)
                    found = self._cache.filename(name)
                else:
                    raise Exception
            except:
                if path(name).exists():
                    found = name
            if found:
                break
        if found is None:
            raise IOError("%s not found"%pathstr)
        return found

    def exists(self, pathstr):
        try:
            _ = self.filename(pathstr)
            return True
        except IOError:
            return False

    def open(self, pathstr, mode='r'):
        if isurl(pathstr) and iswritemode(mode):
            raise ValueError("URLs are not writeable")
        found = self.filename(pathstr)
{'fraction_non_alphanumeric': 0.10879857960149932, 'fraction_numerical': 0.0017754981258630894, 'mean_word_length': 2.0817629179331307, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '957332', 'n_tokens_mistral': 2971, 'n_tokens_neox': 2901, 'n_words': 902}