<commit_before># connector.py implementation of connector.py, based on
# gocept's functional sketch (connector.py)
# 2007-03-16 tomas
"""Connect to the CMS backend."""
#      RESOLVED: the id gets the standard prefix "http://xml.zeit.de/"

# NOTES, QUESTIONS
# [1]  Connector's __init__ has a named parameter <root> pointing to
#      the repository root URL (class-wide default provided for
#      compatibility to the stub, but I think parameter should be
#      mandatory).
# [2]  Exception for DAVNoCollectionError? Other exceptions?
# [3]  What is _make_id() supposed to do? Can we use url as id?
#      Is (given a connector's context) path and url interchangeable?
#      [RESOLVED] See [8]
# [3a] Even more so here. If we take the URL as id as seems to be the
#      current consensus (ugh!), what would mean to put a resource with
#      an id of "http://north.zeit.de/cms/foo/bar" into a repos rooted
#      at "http://south.zeit.de/cms2/"?
# [4]  See "Performance notes"
# [5]  Having a set of assorted strings as possible resource types
#      ('collection', 'article', 'centerpage', 'feed'...) looks horribly
#      naive.
# [6]  Check whether we need this. I don't quite understand
# [7]  Hrrgrr. davresource.py has create_file and create_collection methods.
#      We'll have to extend them to take attributes (transaction safety!)
# [7a] Where are the props on IResource?
# [8]  The id of a resource is its path counted from the repository's root.
#      For now, we assume it to be the same as the connector's root, which
#      is WRONG and should be fixed at some point.
#      RESOLVED: the id gets the standard prefix "http://xml.zeit.de/"
#      which may be overridden at Connector creation time.
# [9]  WebDAV can lock "non-existing" resources. To be more precise, it
#      locks URLs. This does make sense when trying to avoid race conditions
#      at resource creation.
#      But then we'll have to enhance davresource.py
#      [RESOLVED] Well. Partially. To wit:
#                 * "file" resources support, as of current mod_dav lock-null.
#                    We do use that in the case of file resources.
#                 * mod_dav behaves bizarrely with lock-null then mkcol.
#                    We do live with a "locking gap" when creating collections.
#                 Note that lock-null seems to be on its way to deprecation
#                 anyway. For me that means: avoid WebDAV for future projects.
# [10] we'll have to enhance it anyway because it doesn't support setting
#      lock expiry.
#      [RESOLVED]
# [11] Interface note: This implementation returns a lock token. The davclient
#      implementation keeps a cache of "the" lock token for "this" DAV resource,
#      but I fear our interface won't allow us to make use of that.
#      Maybe if we stick a davresource to each resource object... but then many
#      of the Connector methods would have to migrate to Resource anyway.
# [13] LockingError: we have many variants of that. List them, refine?
# [14] Here we try to decide whether to do a DAVCollection() or a DAVFile().
#      We should rather fix pydavclient's DAVResource() to return the
#      appropriate type (i.e. be an "abstract base class").
#      Or something equivalent, like doing away with the Collection/File difference
#      (which in my view would be more along WebDAV semantics)
# [16] which content-type?
# [17] Try to pass resource content around as IO object

import cStringIO
import datetime
import gocept.lxml.objectify
import httplib
import logging
import os
import pytz
import random
import re
import sys
import threading
import time
import urllib
import urlparse
import zeit.connector.cache
import zeit.connector.dav.davconnection
import zeit.connector.dav.davresource
import zeit.connector.interfaces
import zeit.connector.lockinfo
import zeit.connector.resource
import zeit.connector.search
import zope.cachedescriptors.property
import zope.interface


logger = logging.getLogger(__name__)

# The property holding the "resource type".
RESOURCE_TYPE_PROPERTY = zeit.connector.interfaces.RESOURCE_TYPE_PROPERTY

# Highest possible datetime value. We use datetime-with-timezone everywhere.
# The MAXYEAR-1 is there to protect us from passing this bound when
# transforming into some local time
TIME_ETERNITY = datetime.datetime(
    datetime.MAXYEAR - 1, 12, 31, 23, 59, 59, 999999, tzinfo=pytz.UTC)


class DAVUnexpectedResultError(zeit.connector.dav.interfaces.DAVError):
    """Exception raised on unexpected HTTP return code.
    """
    pass

# id utilities
# IMPLEMENTATION NOTE
# Due to the way we implement IDs, we can "deduct" the ID of a
# resource's parent given the resource's ID (just chop off the
# last path's element). Some notes:
# - Resource ids all have a common prefix (default: http://xml.zeit.de/
#   Given the "correct" environment they might be interpreted as URL
# - Double slashes whithin the path part are treated as single ones (analog to POSIX).
# - Collection resources SHOULD end in slash, non-collections SHOULD NOT
#   (not sure whether we should enforce it, but we comply with it).

_max_timeout_days = ((sys.maxint-1) / 86400) - 1

def _abs2timeout(time):
    # Convert timedelta to int (seconds). Return None when (near) overflow
    # which means infinte.
    # Ain't there anything similar in Python? Grr.
    if time is None:
        return None
    d = time - datetime.datetime.now(pytz.UTC)
    if abs(d.days) > _max_timeout_days:
        return None
    # No negative or zero timeouts:
    return max(d.days * 86400 + d.seconds + int(d.microseconds/1000000.0), 1)


class CannonicalId(unicode):
    """A canonical id."""

    def __repr__(self):
        return '<CannonicalId %s>' % super(CannonicalId, self).__repr__()


class Connector(object):
    """Connect to the CMS backend.
       WebDAV implementation based on pydavclient
    """

    zope.interface.implements(zeit.connector.interfaces.ICachingConnector)

    def __init__(self, roots={}, prefix=u'http://xml.zeit.de/'):
        # NOTE: roots['default'] should be defined
        self._roots = roots # "extra" roots, a dict. ATM only xroots['search']
        self._prefix = prefix
        self.connections = threading.local()

    def get_connection(self, root='default'):
        """Try to get a cached connection suitable for url"""
        try:
            connection = getattr(self.connections, root)
        except AttributeError:
            connection = self.create_connection(root)
            setattr(self.connections, root, connection)
        return connection

    def create_connection(self, root):
        """Create a new connection."""
        logger.debug('New connection')
        url = self._roots[root]
        (scheme, netloc) = urlparse.urlsplit(url)[0:2]
        snetloc = "%s://%s" % (scheme, netloc)
        try: # grmblmmblpython
            host, port = netloc.split(':', 1)
            port = int(port)
        except ValueError:
            host, port = netloc, None
        # FIXME: Argh. DAVConnection should take schema as well!!!

        return zeit.connector.dav.davconnection.DAVConnection(host, port)

    def disconnect(self):
        connections = self.connections
        try:
            del connections.default
        except AttributeError:
            pass

    def listCollection(self, id):
        """List the filenames of a collection identified by <id> (see[8]). """
        # FIXME
        # Performance notes [4]:
        # This one slurps a list of names, keeps a copy and deals name
        # by name via yield(). Our provider, DavCollection, also has a
        # list of (whatever). It might make sense pushing the iterator
        # one layer deeper. But there are other worries (do we get one
        # DAV access per child if our clients request details about our
        # children? Probably yes!
        __traceback_info__ = (id, )
        id = self._get_cannonical_id(id)
        for child_id in self._get_resource_child_ids(id):
            yield (self._id_splitlast(child_id)[1].rstrip('/'), child_id)
<commit_msg>remove old comment which doesn't reflect the current situation<commit_after># connector.py implementation of connector.py, based on
# gocept's functional sketch (connector.py)
# 2007-03-16 tomas
"""Connect to the CMS backend."""
#      RESOLVED: the id gets the standard prefix "http://xml.zeit.de/"

# NOTES, QUESTIONS
# [1]  Connector's __init__ has a named parameter <root> pointing to
#      the repository root URL (class-wide default provided for
#      compatibility to the stub, but I think parameter should be
#      mandatory).
# [2]  Exception for DAVNoCollectionError? Other exceptions?
# [3]  What is _make_id() supposed to do? Can we use url as id?
#      Is (given a connector's context) path and url interchangeable?
#      [RESOLVED] See [8]
# [3a] Even more so here. If we take the URL as id as seems to be the
#      current consensus (ugh!), what would mean to put a resource with
#      an id of "http://north.zeit.de/cms/foo/bar" into a repos rooted
#      at "http://south.zeit.de/cms2/"?
# [4]  See "Performance notes"
# [5]  Having a set of assorted strings as possible resource types
#      ('collection', 'article', 'centerpage', 'feed'...) looks horribly
#      naive.
# [6]  Check whether we need this. I don't quite understand
# [7]  Hrrgrr. davresource.py has create_file and create_collection methods.
#      We'll have to extend them to take attributes (transaction safety!)
# [7a] Where are the props on IResource?
# [8]  The id of a resource is its path counted from the repository's root.
#      For now, we assume it to be the same as the connector's root, which
#      is WRONG and should be fixed at some point.
#      RESOLVED: the id gets the standard prefix "http://xml.zeit.de/"
#      which may be overridden at Connector creation time.
# [9]  WebDAV can lock "non-existing" resources. To be more precise, it
#      locks URLs. This does make sense when trying to avoid race conditions
#      at resource creation.
#      But then we'll have to enhance davresource.py
#      [RESOLVED] Well. Partially. To wit:
#                 * "file" resources support, as of current mod_dav lock-null.
#                    We do use that in the case of file resources.
#                 * mod_dav behaves bizarrely with lock-null then mkcol.
#                    We do live with a "locking gap" when creating collections.
#                 Note that lock-null seems to be on its way to deprecation
#                 anyway. For me that means: avoid WebDAV for future projects.
# [10] we'll have to enhance it anyway because it doesn't support setting
#      lock expiry.
#      [RESOLVED]
# [11] Interface note: This implementation returns a lock token. The davclient
#      implementation keeps a cache of "the" lock token for "this" DAV resource,
#      but I fear our interface won't allow us to make use of that.
#      Maybe if we stick a davresource to each resource object... but then many
#      of the Connector methods would have to migrate to Resource anyway.
# [13] LockingError: we have many variants of that. List them, refine?
# [14] Here we try to decide whether to do a DAVCollection() or a DAVFile().
#      We should rather fix pydavclient's DAVResource() to return the
#      appropriate type (i.e. be an "abstract base class").
#      Or something equivalent, like doing away with the Collection/File difference
#      (which in my view would be more along WebDAV semantics)
# [16] which content-type?
# [17] Try to pass resource content around as IO object

import cStringIO
import datetime
import gocept.lxml.objectify
import httplib
import logging
import os
import pytz
import random
import re
import sys
import threading
import time
import urllib
import urlparse
import zeit.connector.cache
import zeit.connector.dav.davconnection
import zeit.connector.dav.davresource
import zeit.connector.interfaces
import zeit.connector.lockinfo
import zeit.connector.resource
import zeit.connector.search
import zope.cachedescriptors.property
import zope.interface


logger = logging.getLogger(__name__)

# The property holding the "resource type".
RESOURCE_TYPE_PROPERTY = zeit.connector.interfaces.RESOURCE_TYPE_PROPERTY

# Highest possible datetime value. We use datetime-with-timezone everywhere.
# The MAXYEAR-1 is there to protect us from passing this bound when
# transforming into some local time
TIME_ETERNITY = datetime.datetime(
    datetime.MAXYEAR - 1, 12, 31, 23, 59, 59, 999999, tzinfo=pytz.UTC)


class DAVUnexpectedResultError(zeit.connector.dav.interfaces.DAVError):
    """Exception raised on unexpected HTTP return code.
    """
    pass

# id utilities
# IMPLEMENTATION NOTE
# Due to the way we implement IDs, we can "deduct" the ID of a
# resource's parent given the resource's ID (just chop off the
# last path's element). Some notes:
# - Resource ids all have a common prefix (default: http://xml.zeit.de/
#   Given the "correct" environment they might be interpreted as URL
# - Double slashes whithin the path part are treated as single ones (analog to POSIX).
# - Collection resources SHOULD end in slash, non-collections SHOULD NOT
#   (not sure whether we should enforce it, but we comply with it).

_max_timeout_days = ((sys.maxint-1) / 86400) - 1

def _abs2timeout(time):
    # Convert timedelta to int (seconds). Return None when (near) overflow
    # which means infinte.
    # Ain't there anything similar in Python? Grr.
    if time is None:
        return None
    d = time - datetime.datetime.now(pytz.UTC)
    if abs(d.days) > _max_timeout_days:
        return None
    # No negative or zero timeouts:
    return max(d.days * 86400 + d.seconds + int(d.microseconds/1000000.0), 1)


class CannonicalId(unicode):
    """A canonical id."""

    def __repr__(self):
        return '<CannonicalId %s>' % super(CannonicalId, self).__repr__()


class Connector(object):
    """Connect to the CMS backend.
       WebDAV implementation based on pydavclient
    """

    zope.interface.implements(zeit.connector.interfaces.ICachingConnector)

    def __init__(self, roots={}, prefix=u'http://xml.zeit.de/'):
        # NOTE: roots['default'] should be defined
        self._roots = roots # "extra" roots, a dict. ATM only xroots['search']
        self._prefix = prefix
        self.connections = threading.local()

    def get_connection(self, root='default'):
        """Try to get a cached connection suitable for url"""
        try:
            connection = getattr(self.connections, root)
        except AttributeError:
            connection = self.create_connection(root)
            setattr(self.connections, root, connection)
        return connection

    def create_connection(self, root):
        """Create a new connection."""
        logger.debug('New connection')
        url = self._roots[root]
        (scheme, netloc) = urlparse.urlsplit(url)[0:2]
        snetloc = "%s://%s" % (scheme, netloc)
        try: # grmblmmblpython
            host, port = netloc.split(':', 1)
            port = int(port)
        except ValueError:
            host, port = netloc, None
        # FIXME: Argh. DAVConnection should take schema as well!!!

        return zeit.connector.dav.davconnection.DAVConnection(host, port)

    def disconnect(self):
        connections = self.connections
        try:
            del connections.default
        except AttributeError:
            pass

    def listCollection(self, id):
        """List the filenames of a collection identified by <id> (see[8]). """
        __traceback_info__ = (id, )
        id = self._get_cannonical_id(id)
        for child_id in self._get_resource_child_ids(id):
            yield (self._id_splitlast(child_id)[1].rstrip('/'), child_id)

{'fraction_non_alphanumeric': 0.08867589183699377, 'fraction_numerical': 0.009965722990986416, 'mean_word_length': 3.0428534770336158, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4368622', 'n_tokens_mistral': 4653, 'n_tokens_neox': 4400, 'n_words': 1809}