<commit_before>"""
fs.opener
=========

Open filesystems via a URI.

There are occasions when you want to specify a filesytem from the command line or in a config file.
This module enables that functionality, and can return a FS object given a URI like syntax (http://commons.apache.org/vfs/filesystems.html).

The `OpenerRegistry` class maps the protocol (file, ftp etc.) on to an Opener object, which returns an appropriate filesystem object and path.
You can create a custom opener registry that opens just the filesystems you require, or use the opener registry defined here (also called `opener`) that can open any supported filesystem.
The `parse` method of an `OpenerRegsitry` object returns a tuple of an FS object a path. Here's an example of how to use the default opener registry:: 

    >>> from fs.opener import opener
    >>> opener.parse('ftp://ftp.mozilla.org')
    (<fs.ftpfs.FTPFS object at 0x96e66ec>, u'pub')
    
You can use use the `opendir` method, which just returns an FS object. In the example above, `opendir` will return a FS object for the directory `pub`::

    >>> opener.opendir('ftp://ftp.mozilla.org/pub')
    <SubFS: <FTPFS ftp.mozilla.org>/pub>
    
If you are just interested in a single file, use the `open` method of a registry which returns a file-like object, and has the same signature as FS objects and the `open` builtin::

    >>> opener.open('ftp://ftp.mozilla.org/pub/README')
    <fs.ftpfs._FTPFile object at 0x973764c>

The `opendir` and `open` methods can also be imported from the top-level of this module for sake of convenience.
To avoid shadowing the builtin `open` methd, they are named `fsopendir` and `fsopen`. Here's how you might import them::

    from fs.opener import fsopendir, fsopen
 


"""

__all__ = ['OpenerError',
           'NoOpenerError',
           'OpenerRegistry',
           'opener',
           'fsopen',
           'fsopendir',           
           'OpenerRegistry',
           'Opener',           
           'OSFSOpener',
           'ZipOpener',
           'RPCOpener',
           'FTPOpener',
           'SFTPOpener',
           'MemOpener',
           'DebugOpener',
           'TempOpener',
           'S3Opener',
           'TahoeOpener',
           'DavOpener',
           'HTTPOpener']

import sys
from fs.osfs import OSFS
from fs.path import pathsplit, basename, join, iswildcard, normpath
import os
import os.path
import re
from urlparse import urlparse

class OpenerError(Exception):
    """The base exception thrown by openers"""
    pass

class NoOpenerError(OpenerError):
    """Thrown when there is no opener for the given protocol"""
    pass

def _expand_syspath(path):
    if path is None:
        return path
    path = os.path.expanduser(os.path.expandvars(path))
    path = normpath(path)
    return path
    
def _parse_credentials(url):
    scheme = None
    if '://' in url:
        scheme, url = url.split('://', 1)        
    username = None
    password = <PASSWORD>    
    if '@' in url:
        credentials, url = url.split('@', 1)
        if ':' in credentials:
            username, password = credentials.split(':', 1)
        else:
            username = credentials
    if scheme is not None:
        url = '%s://%s' % (scheme, url)
    return username, password, url

def _parse_name(fs_name):
    if '#' in fs_name:
        fs_name, fs_name_params = fs_name.split('#', 1)
        return fs_name, fs_name_params
    else:
        return fs_name, None
    
def _split_url_path(url):
    if '://' not in url:
        url = 'http://' + url
    scheme, netloc, path, params, query, fragment = urlparse(url)
    url = '%s://%s' % (scheme, netloc)
    return url, path


class OpenerRegistry(object):
     
    """An opener registry that  stores a number of opener objects used to parse FS URIs"""

    re_fs_url = re.compile(r'''
^
(.*?)
:\/\/

(?:
|(.*?)
|(?:(.*?)!)
)

(?:
!(.*?)$
)*$
''', re.VERBOSE)
        
    
     
    def __init__(self, openers=[]):
        self.registry = {}
        self.openers = {}
        self.default_opener = 'osfs'
        for opener in openers:
            self.add(opener)
    
    @classmethod
    def split_segments(self, fs_url):        
        match = self.re_fs_url.match(fs_url)        
        return match        
    
    def get_opener(self, name):
        """Retrieve an opener for the given protocol
        
        :param name: name of the opener to open
        :raises NoOpenerError: if no opener has been registered of that name
        
        """
        if name not in self.registry:
            raise NoOpenerError("No opener for %s" % name)
        index = self.registry[name]
        return self.openers[index]        
    
    def add(self, opener):
        """Adds an opener to the registry
        
        :param opener: a class derived from fs.opener.Opener
        
        """
        
        index = len(self.openers)
        self.openers[index] = opener
        for name in opener.names:
            self.registry[name] = index
    
    def parse(self, fs_url, default_fs_name=None, writeable=False, create_dir=False):
        """Parses a FS url and returns an fs object a path within that FS object
        (if indicated in the path). A tuple of (<FS instance>, <path>) is returned.
        
        :param fs_url: an FS url
        :param default_fs_name: the default FS to use if none is indicated (defaults is OSFS)
        :param writeable: if True, a writeable FS will be returned
        :oaram create_dir: if True, then the directory in the FS will be created
        
        """
                   
        orig_url = fs_url     
        match = self.split_segments(fs_url)
        
        if match:
            fs_name, fs_url, _, path = match.groups()
            path = path or ''
            fs_url = fs_url or ''
            if ':' in fs_name:
                fs_name, sub_protocol = fs_name.split(':', 1)
                fs_url = '%s://%s' % (sub_protocol, fs_url)
            if '!' in path:
                paths = path.split('!')
                path = paths.pop()
                fs_url = '%s!%s' % (fs_url, '!'.join(paths))
            
            fs_name = fs_name or self.default_opener                                                                    
                
        else:
            fs_name = default_fs_name or self.default_opener
            fs_url = _expand_syspath(fs_url) 
            path = ''           
    
        fs_name,  fs_name_params = _parse_name(fs_name)        
        opener = self.get_opener(fs_name)
        
        if fs_url is None:
            raise OpenerError("Unable to parse '%s'" % orig_url)        
        
        fs, fs_path = opener.get_fs(self, fs_name, fs_name_params, fs_url, writeable, create_dir)<commit_msg>Fix for fs url parsing so that credentials can contain an exclamation mark<commit_after>"""
fs.opener
=========

Open filesystems via a URI.

There are occasions when you want to specify a filesytem from the command line or in a config file.
This module enables that functionality, and can return a FS object given a URI like syntax (http://commons.apache.org/vfs/filesystems.html).

The `OpenerRegistry` class maps the protocol (file, ftp etc.) on to an Opener object, which returns an appropriate filesystem object and path.
You can create a custom opener registry that opens just the filesystems you require, or use the opener registry defined here (also called `opener`) that can open any supported filesystem.
The `parse` method of an `OpenerRegsitry` object returns a tuple of an FS object a path. Here's an example of how to use the default opener registry:: 

    >>> from fs.opener import opener
    >>> opener.parse('ftp://ftp.mozilla.org')
    (<fs.ftpfs.FTPFS object at 0x96e66ec>, u'pub')
    
You can use use the `opendir` method, which just returns an FS object. In the example above, `opendir` will return a FS object for the directory `pub`::

    >>> opener.opendir('ftp://ftp.mozilla.org/pub')
    <SubFS: <FTPFS ftp.mozilla.org>/pub>
    
If you are just interested in a single file, use the `open` method of a registry which returns a file-like object, and has the same signature as FS objects and the `open` builtin::

    >>> opener.open('ftp://ftp.mozilla.org/pub/README')
    <fs.ftpfs._FTPFile object at 0x973764c>

The `opendir` and `open` methods can also be imported from the top-level of this module for sake of convenience.
To avoid shadowing the builtin `open` methd, they are named `fsopendir` and `fsopen`. Here's how you might import them::

    from fs.opener import fsopendir, fsopen
 


"""

__all__ = ['OpenerError',
           'NoOpenerError',
           'OpenerRegistry',
           'opener',
           'fsopen',
           'fsopendir',           
           'OpenerRegistry',
           'Opener',           
           'OSFSOpener',
           'ZipOpener',
           'RPCOpener',
           'FTPOpener',
           'SFTPOpener',
           'MemOpener',
           'DebugOpener',
           'TempOpener',
           'S3Opener',
           'TahoeOpener',
           'DavOpener',
           'HTTPOpener']

import sys
from fs.osfs import OSFS
from fs.path import pathsplit, basename, join, iswildcard, normpath
import os
import os.path
import re
from urlparse import urlparse

class OpenerError(Exception):
    """The base exception thrown by openers"""
    pass

class NoOpenerError(OpenerError):
    """Thrown when there is no opener for the given protocol"""
    pass

def _expand_syspath(path):
    if path is None:
        return path
    path = os.path.expanduser(os.path.expandvars(path))
    path = normpath(path)
    return path
    
def _parse_credentials(url):
    scheme = None
    if '://' in url:
        scheme, url = url.split('://', 1)        
    username = None
    password = None    
    if '@' in url:
        credentials, url = url.split('@', 1)
        if ':' in credentials:
            username, password = credentials.split(':', 1)
        else:
            username = credentials
    if scheme is not None:
        url = '%s://%s' % (scheme, url)
    return username, password, url

def _parse_name(fs_name):
    if '#' in fs_name:
        fs_name, fs_name_params = fs_name.split('#', 1)
        return fs_name, fs_name_params
    else:
        return fs_name, None
    
def _split_url_path(url):
    if '://' not in url:
        url = 'http://' + url
    scheme, netloc, path, params, query, fragment = urlparse(url)
    url = '%s://%s' % (scheme, netloc)
    return url, path


class OpenerRegistry(object):
     
    """An opener registry that  stores a number of opener objects used to parse FS URIs"""

    re_fs_url = re.compile(r'''
^
(.*?)
:\/\/

(?:
(?:(.*?)@(.*?))
|(.*?)
)

(?:
!(.*?)$
)*$
''', re.VERBOSE)    
    
     
    def __init__(self, openers=[]):
        self.registry = {}
        self.openers = {}
        self.default_opener = 'osfs'
        for opener in openers:
            self.add(opener)
    
    @classmethod
    def split_segments(self, fs_url):        
        match = self.re_fs_url.match(fs_url)        
        return match        
    
    def get_opener(self, name):
        """Retrieve an opener for the given protocol
        
        :param name: name of the opener to open
        :raises NoOpenerError: if no opener has been registered of that name
        
        """
        if name not in self.registry:
            raise NoOpenerError("No opener for %s" % name)
        index = self.registry[name]
        return self.openers[index]        
    
    def add(self, opener):
        """Adds an opener to the registry
        
        :param opener: a class derived from fs.opener.Opener
        
        """
        
        index = len(self.openers)
        self.openers[index] = opener
        for name in opener.names:
            self.registry[name] = index
    
    def parse(self, fs_url, default_fs_name=None, writeable=False, create_dir=False):
        """Parses a FS url and returns an fs object a path within that FS object
        (if indicated in the path). A tuple of (<FS instance>, <path>) is returned.
        
        :param fs_url: an FS url
        :param default_fs_name: the default FS to use if none is indicated (defaults is OSFS)
        :param writeable: if True, a writeable FS will be returned
        :oaram create_dir: if True, then the directory in the FS will be created
        
        """
                   
        orig_url = fs_url     
        match = self.split_segments(fs_url)
        
        if match:            
            fs_name, credentials, url1, url2, path = match.groups()
            if credentials:
                fs_url = '%s@%s' % (credentials, url1)
            else:
                fs_url = url2
            path = path or ''
            fs_url = fs_url or ''
            if ':' in fs_name:
                fs_name, sub_protocol = fs_name.split(':', 1)
                fs_url = '%s://%s' % (sub_protocol, fs_url)
            if '!' in path:
                paths = path.split('!')
                path = paths.pop()
                fs_url = '%s!%s' % (fs_url, '!'.join(paths))
            
            fs_name = fs_name or self.default_opener                                                                    
                
        else:
            fs_name = default_fs_name or self.default_opener
            fs_url = _expand_syspath(fs_url) 
            path = ''           
    
        fs_name,  fs_name_params = _parse_name(fs_name)        
        opener = self.get_opener(fs_name)
        
        if fs_url is None:
            raise OpenerError("Unable to parse '%s'" % orig_url)        
        
        fs, fs_path = opener.get_fs(self, fs_name, fs_name_params, fs_url, writeable, create_dir)
{'fraction_non_alphanumeric': 0.10236277717193748, 'fraction_numerical': 0.0029080334423845873, 'mean_word_length': 2.0232967032967033, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 40, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '169619', 'n_tokens_mistral': 4232, 'n_tokens_neox': 4048, 'n_words': 1358}