<commit_before>"""
:mod:`disco.ddfs` --- Client interface for Disco Distributed Filesystem
=======================================================================

See also: :ref:`DDFS`.

.. note::

        Parameters below which are indicated as tags can be specified as
        a `tag://` URL, or the name of the tag.
"""
import os, re, random
from cStringIO import StringIO
from urllib import urlencode

from disco.comm import upload, download, json, open_remote
from disco.error import CommError
from disco.settings import DiscoSettings
from disco.util import isiterable, iterify, partition, urlsplit

unsafe_re = re.compile(r'[^A-Za-z0-9_\-@:]')

class InvalidTag(Exception):
    pass

def canonizetag(tag):
    if isiterable(tag):
        for tag in tag:
            return canonizetag(tag)
    elif tag.startswith('tag://'):
        return tag
    elif '://' not in tag:
        return 'tag://%s' % tag
    raise InvalidTag("Invalid tag: %s" % tag)

def canonizetags(tags):
    return [canonizetag(tag) for tag in iterify(tags)]

def istag(tag):
    try:
        return canonizetag(tag)
    except InvalidTag:
        pass

def tagname(tag):
    scheme, netloc, name = urlsplit(canonizetag(tag))
    return name

class DDFS(object):
    """
    Opens and encapsulates a connection to a DDFS master.

    :param master: address of the master,
                   for instance ``disco://localhost``.
    """
    def __init__(self, master=None,
                 proxy=None,
                 settings=DiscoSettings(),
                 read_token=None,
                 write_token=None):
        self.proxy  = proxy or settings['DISCO_PROXY']
        self.master = self.proxy or master or settings['DISCO_MASTER']
        self.read_token = read_token or settings['DDFS_READ_TOKEN']
        self.write_token = write_token or settings['DDFS_WRITE_TOKEN']

    @classmethod
    def safe_name(cls, name):
        return unsafe_re.sub('_', name)

    @classmethod
    def blob_name(cls, url):
        return url.split('/')[-1].split('$')[0]

    def tagmaster(self, tag):
        scheme, (host, port), path = urlsplit(canonizetag(tag))
        if not host:
            return self.master
        if not port:
            return 'disco://%s' % host
        return 'http://%s:%s' % (host, port)

    def attrs(self, tag, token=None):
        """Get a list of the attributes of the tag ``tag`` and their values."""<commit_msg>Remove args to DDFS(); use settings instead.<commit_after>"""
:mod:`disco.ddfs` --- Client interface for Disco Distributed Filesystem
=======================================================================

See also: :ref:`DDFS`.

.. note::

        Parameters below which are indicated as tags can be specified as
        a `tag://` URL, or the name of the tag.
"""
import os, re, random
from cStringIO import StringIO
from urllib import urlencode

from disco.comm import upload, download, json, open_remote
from disco.error import CommError
from disco.settings import DiscoSettings
from disco.util import isiterable, iterify, partition, urlsplit

unsafe_re = re.compile(r'[^A-Za-z0-9_\-@:]')

class InvalidTag(Exception):
    pass

def canonizetag(tag):
    if isiterable(tag):
        for tag in tag:
            return canonizetag(tag)
    elif tag.startswith('tag://'):
        return tag
    elif '://' not in tag:
        return 'tag://%s' % tag
    raise InvalidTag("Invalid tag: %s" % tag)

def canonizetags(tags):
    return [canonizetag(tag) for tag in iterify(tags)]

def istag(tag):
    try:
        return canonizetag(tag)
    except InvalidTag:
        pass

def tagname(tag):
    scheme, netloc, name = urlsplit(canonizetag(tag))
    return name

class DDFS(object):
    """
    Opens and encapsulates a connection to a DDFS master.

    :param master: address of the master,
                   for instance ``disco://localhost``.
    """
    def __init__(self, master=None,
                 proxy=None,
                 settings=DiscoSettings()):
        self.proxy  = proxy or settings['DISCO_PROXY']
        self.master = self.proxy or master or settings['DISCO_MASTER']
        self.read_token = settings['DDFS_READ_TOKEN']
        self.write_token = settings['DDFS_WRITE_TOKEN']

    @classmethod
    def safe_name(cls, name):
        return unsafe_re.sub('_', name)

    @classmethod
    def blob_name(cls, url):
        return url.split('/')[-1].split('$')[0]

    def tagmaster(self, tag):
        scheme, (host, port), path = urlsplit(canonizetag(tag))
        if not host:
            return self.master
        if not port:
            return 'disco://%s' % host
        return 'http://%s:%s' % (host, port)

    def attrs(self, tag, token=None):
        """Get a list of the attributes of the tag ``tag`` and their values."""
{'fraction_non_alphanumeric': 0.14540121516865703, 'fraction_numerical': 0.0016760946993505132, 'mean_word_length': 2.7590551181102363, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1308247', 'n_tokens_mistral': 1461, 'n_tokens_neox': 1439, 'n_words': 437}