<commit_before>#
# Copyright (c) 2010 rPath, Inc.
#
# This program is distributed under the terms of the MIT License as found 
# in a file called LICENSE. If it is not present, the license
# is always available at http://www.opensource.org/licenses/mit-license.php.
#
# This program is distributed in the hope that it will be useful, but
# without any waranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the MIT License for full details.
#

"""
Module for binding the HTTP client layer to xobj.
"""

from threading import RLock

from xobj import xobj

from robj.proxy import rObjProxy
from robj.errors import HTTPDeleteError
from robj.errors import ExternalUriError
from robj.errors import SerializationError
from robj.http import HTTPClient as _HTTPClient

__ALL__ = ('HTTPClient', )

class HTTPClient(object):
    """
    Wrapper around the basic http client that integrates xobj.
    @param baseUri: URI for connectiong to the root of the desired web service.
                    This may contain user information and must be http or https.
    @type baseUri: str
    @param headers: Any heads that should be included in all requets.
    @type headers: dict
    @param maxClients: The maximum number of workers that will be created to
                       handle requets. Works are created as needed, rather than
                       being preallocated. (default: 10)
    @type maxClients: int
    @param maxConnections: The maximum number of connections each client thread
                           should cache. Client threads only cache one
                           connection per host. This should only matter if you
                           are talking to multiple hosts. (default: 2)
    @type maxConnections: int
    """

    def __init__(self, baseUri, headers=None, maxClients=None,
        maxConnections=None):

        self._client = _HTTPClient(baseUri, headers=headers,
            maxClients=maxClients, maxConnections=maxConnections)

        self._instCache = InstanceCache()

    def _normalize_uri(self, uri):
        """
        Make sure uri is based at the baseUri and shorten to be appened on the
        baseUri.
        """

        # Strip off trailing slashes
        uri = uri.rstrip('/')

        # Trim off the base path.
        if uri.startswith(self._client.path):
            return uri[len(self._client.path):].rstrip('/')

        # Assume the URI is relative if it doesn't start with an
        # expected scheme.
        elif not uri.startswith('http'):
            return uri

        # Strip off the leading base URI.
        elif uri.startswith(self._client.baseURI):
            return uri[len(self._client.baseURI):]

        # Otherwise, raise an exception.
        else:
            raise ExternalUriError(uri=uri, base=self._client.baseURI)

    def _serialize_document(self, doc):
        """
        Serialize a object tree into xml.
        """

        # Serialize document to xml.
        if isinstance(doc, xobj.Document):
            xml = doc.toxml()
        else:
            meta = getattr(doc, '_xobj', None)

            # Raise an exception if we can't figure out the tag.
            if meta is None or not meta.tag:
                raise SerializationError(instance=doc,
                    msg=('Can not determine tag'))

            xml = xobj.toxml(doc, meta.tag)

        return xml

    def _handle_error(self, response):
        """
        Handle all error conditions by raising a reasonable exception.
        """

        raise NotImplementedError

    def _handle_redirect(self, response):
        """
        Handle all redirect conditions. This may include long running jobs
        implemented through a see other (303).
        """

        raise NotImplementedError

    def _handle_request(self, method, uri, xdoc=None, parent=None):
        """
        Process all types of requests.
        """

        # Normalize the URI.
        uri = self._normalize_uri(uri)

        # Check the cache before moving on if this is a GET.
        if method == 'GET' and uri in self._instCache:
            return self._instCache[uri]

        if method in ('POST', 'PUT', ):
            # Make sure there is a document for PUT and POST requests.
            if xdoc is None:
                raise TypeError, 'method requires document instance'

            xml = self._serialize_document(xdoc)
            args = (uri, xml)
        else:
            args = (uri, )

        # Call client method
        func = getattr(self._client, 'do_%s' % method)
        request = func(*args)

        # Wait for request to complete.
        request.wait()

        # Get the response
        response = request.response

        # Special case DELETE method.
        if method == 'DELETE':
            # Raise an exception if the resource could not be deleted.
            if response.status not in (404, 200):
                raise HTTPDeleteError(uri=self._uri, status=response.status,
                    reason=response.reason, response=response)

            self._instCache.clear(uri)

            return response

        # Handle other error codes.
        if response.status >= 400:
            return self._handle_error(response)

        # Handle redirects.
        elif response.status >= 300:
            return self._handle_redirect(response)

        # Parse XML document.
        doc = xobj.parse(response.read())

        # Pull the top level object out of the document.
        assert isinstance(doc, xobj.Document)
        assert len(doc._xobj.elements) == 1
        root = getattr(doc, doc._xobj.elements[0])

        # Cache response and return rObjProxy instance.
        return self._instCache.cache(self, uri, root, parent=parent)

    def do_GET(self, *args, **kwargs):
        """
        Process GET requests.
        @param uri: Full or partial URI (relative to the base URI).
        @type uri: str
        @return rObj representing response.
        @rtype robj.obj.rObjProxy
        """

        return self._handle_request('GET', *args, **kwargs)

    def do_POST(self, *args, **kwargs):
        """
        Process POST requests.
        @param uri: Full or partial URI (relative to the base URI).
        @type uri: str
        @param xdoc: A serializable instance that may include a xobj document
                     wrapper.
        @type xdoc: instance
        @return rObj representing response.
        @rtype robj.obj.rObjProxy
        """

        return self._handle_request('POST', *args, **kwargs)

    def do_PUT(self, *args, **kwargs):
        """
        Process PUT requests.
        @param uri: Full or partial URI (relative to the base URI).
        @type uri: str
        @param xdoc: A serializable instance that may include a xobj document
                     wrapper.
        @type xdoc: instance
        @return rObj representing response.
        @rtype robj.obj.rObjProxy
        """

        return self._handle_request('PUT', *args, **kwargs)

    def do_DELETE(self, *args, **kwargs):
        """
        Process DELETE requests.
        @param uri: Full or partial URI (relative to the base URI).
        @type uri: str
        @return response object
        @rtype robj.http.request.Response
        """

        return self._handle_request('DELETE', *args, **kwargs)


class InstanceCache(dict):
    """
    Cache of all URIs and associated objects.
    """

    def __init__(self):
        dict.__init__(self)
        self._write_lock = RLock()

    def clear(self, uri=None):
        """
        Clear the cache of rObj instances. This will result in orphaned
        instances in memory.
        """

        self._write_lock.acquire()
        if uri:
            self.pop(uri, None)
        else:
            for element in self:
                self.pop(element)
        self._write_lock.release()

    def cache(self, client, uri, root, parent=None):
        """
        Cache the given robj if needed.
        """

        self._write_lock.acquire()

        if uri in self:
            robj = self[uri]
            if not robj._dirty:
                robj._root = root
        else:
            robj = rObjProxy(uri, client, root, parent=parent)
            self[uri] = robj

        self._write_lock.release()
        return robj<commit_msg>rename instance cache so that it is more accessable by the robjproxy class<commit_after>#
# Copyright (c) 2010 rPath, Inc.
#
# This program is distributed under the terms of the MIT License as found 
# in a file called LICENSE. If it is not present, the license
# is always available at http://www.opensource.org/licenses/mit-license.php.
#
# This program is distributed in the hope that it will be useful, but
# without any waranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the MIT License for full details.
#

"""
Module for binding the HTTP client layer to xobj.
"""

from threading import RLock

from xobj import xobj

from robj.proxy import rObjProxy
from robj.errors import HTTPDeleteError
from robj.errors import ExternalUriError
from robj.errors import SerializationError
from robj.http import HTTPClient as _HTTPClient

__ALL__ = ('HTTPClient', )

class HTTPClient(object):
    """
    Wrapper around the basic http client that integrates xobj.
    @param baseUri: URI for connectiong to the root of the desired web service.
                    This may contain user information and must be http or https.
    @type baseUri: str
    @param headers: Any heads that should be included in all requets.
    @type headers: dict
    @param maxClients: The maximum number of workers that will be created to
                       handle requets. Works are created as needed, rather than
                       being preallocated. (default: 10)
    @type maxClients: int
    @param maxConnections: The maximum number of connections each client thread
                           should cache. Client threads only cache one
                           connection per host. This should only matter if you
                           are talking to multiple hosts. (default: 2)
    @type maxConnections: int
    """

    def __init__(self, baseUri, headers=None, maxClients=None,
        maxConnections=None):

        self._client = _HTTPClient(baseUri, headers=headers,
            maxClients=maxClients, maxConnections=maxConnections)

        self.cache = InstanceCache()

    def _normalize_uri(self, uri):
        """
        Make sure uri is based at the baseUri and shorten to be appened on the
        baseUri.
        """

        # Strip off trailing slashes
        uri = uri.rstrip('/')

        # Trim off the base path.
        if uri.startswith(self._client.path):
            return uri[len(self._client.path):].rstrip('/')

        # Assume the URI is relative if it doesn't start with an
        # expected scheme.
        elif not uri.startswith('http'):
            return uri

        # Strip off the leading base URI.
        elif uri.startswith(self._client.baseURI):
            return uri[len(self._client.baseURI):]

        # Otherwise, raise an exception.
        else:
            raise ExternalUriError(uri=uri, base=self._client.baseURI)

    def _serialize_document(self, doc):
        """
        Serialize a object tree into xml.
        """

        # Serialize document to xml.
        if isinstance(doc, xobj.Document):
            xml = doc.toxml()
        else:
            meta = getattr(doc, '_xobj', None)

            # Raise an exception if we can't figure out the tag.
            if meta is None or not meta.tag:
                raise SerializationError(instance=doc,
                    msg=('Can not determine tag'))

            xml = xobj.toxml(doc, meta.tag)

        return xml

    def _handle_error(self, response):
        """
        Handle all error conditions by raising a reasonable exception.
        """

        raise NotImplementedError

    def _handle_redirect(self, response):
        """
        Handle all redirect conditions. This may include long running jobs
        implemented through a see other (303).
        """

        raise NotImplementedError

    def _handle_request(self, method, uri, xdoc=None, parent=None, cache=True):
        """
        Process all types of requests.
        """

        # Normalize the URI.
        uri = self._normalize_uri(uri)

        # Check the cache before moving on if this is a GET.
        if method == 'GET' and uri in self.cache and cache:
            return self.cache[uri]

        if method in ('POST', 'PUT', ):
            # Make sure there is a document for PUT and POST requests.
            if xdoc is None:
                raise TypeError, 'method requires document instance'

            xml = self._serialize_document(xdoc)
            args = (uri, xml)
        else:
            args = (uri, )

        # Call client method
        func = getattr(self._client, 'do_%s' % method)
        request = func(*args)

        # Wait for request to complete.
        request.wait()

        # Get the response
        response = request.response

        # Special case DELETE method.
        if method == 'DELETE':
            # Raise an exception if the resource could not be deleted.
            if response.status not in (404, 200):
                raise HTTPDeleteError(uri=self._uri, status=response.status,
                    reason=response.reason, response=response)

            self.cache.clear(uri)

            return response

        # Handle other error codes.
        if response.status >= 400:
            return self._handle_error(response)

        # Handle redirects.
        elif response.status >= 300:
            return self._handle_redirect(response)

        # Parse XML document.
        doc = xobj.parse(response.read())

        # Pull the top level object out of the document.
        assert isinstance(doc, xobj.Document)
        assert len(doc._xobj.elements) == 1
        root = getattr(doc, doc._xobj.elements[0])

        # If the top level object has an 'id' attribute, use that as its URI.
        # This is here to handle appending to collections, where the resource
        # you get back is the new instance, not the collection itself.
        if 'id' in root._xobj.attributes:
            uri = root.id

        # Cache response and return rObjProxy instance.
        return self.cache.cache(self, uri, root, parent=parent)

    def do_GET(self, *args, **kwargs):
        """
        Process GET requests.
        @param uri: Full or partial URI (relative to the base URI).
        @type uri: str
        @return rObj representing response.
        @rtype robj.obj.rObjProxy
        """

        return self._handle_request('GET', *args, **kwargs)

    def do_POST(self, *args, **kwargs):
        """
        Process POST requests.
        @param uri: Full or partial URI (relative to the base URI).
        @type uri: str
        @param xdoc: A serializable instance that may include a xobj document
                     wrapper.
        @type xdoc: instance
        @return rObj representing response.
        @rtype robj.obj.rObjProxy
        """

        return self._handle_request('POST', *args, **kwargs)

    def do_PUT(self, *args, **kwargs):
        """
        Process PUT requests.
        @param uri: Full or partial URI (relative to the base URI).
        @type uri: str
        @param xdoc: A serializable instance that may include a xobj document
                     wrapper.
        @type xdoc: instance
        @return rObj representing response.
        @rtype robj.obj.rObjProxy
        """

        return self._handle_request('PUT', *args, **kwargs)

    def do_DELETE(self, *args, **kwargs):
        """
        Process DELETE requests.
        @param uri: Full or partial URI (relative to the base URI).
        @type uri: str
        @return response object
        @rtype robj.http.request.Response
        """

        return self._handle_request('DELETE', *args, **kwargs)


class InstanceCache(dict):
    """
    Cache of all URIs and associated objects.
    """

    def __init__(self):
        dict.__init__(self)
        self._write_lock = RLock()

    def clear(self, uri=None):
        """
        Clear the cache of rObj instances. This will result in orphaned
        instances in memory.
        """

        self._write_lock.acquire()
        if uri:
            self.pop(uri, None)
        else:
            for element in self:
                self.pop(element)
        self._write_lock.release()

    def cache(self, client, uri, root, parent=None):
        """
        Cache the given robj if needed.
        """

        self._write_lock.acquire()

        if uri in self:
            robj = self[uri]
            if not robj._dirty:
                robj._root = root
                robj._local_cache = {}
        else:
            robj = rObjProxy(uri, client, root, parent=parent)
            self[uri] = robj

        self._write_lock.release()
        return robj
{'fraction_non_alphanumeric': 0.08210748402977203, 'fraction_numerical': 0.002813104377893688, 'mean_word_length': 1.989488437281009, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 18}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4372968', 'n_tokens_mistral': 4585, 'n_tokens_neox': 4399, 'n_words': 1657}