<commit_before>"""
kombu.connection
================

Broker connection and pools.

:copyright: (c) 2009 - 2011 by <NAME>.
:license: BSD, see LICENSE for more details.

"""
import os
import sys
import socket

from copy import copy
from itertools import count
from Queue import Empty
from urlparse import urlparse
try:
    from urlparse import parse_qsl
except ImportError:
    from cgi import parse_qsl  # noqa

from kombu import exceptions
from kombu.transport import get_transport_cls
from kombu.utils import kwdict, partition, retry_over_time
from kombu.utils.compat import OrderedDict, LifoQueue as _LifoQueue
from kombu.utils.functional import wraps

_LOG_CONNECTION = os.environ.get("KOMBU_LOG_CONNECTION", False)
_LOG_CHANNEL = os.environ.get("KOMBU_LOG_CHANNEL", False)


#: Connection info -> URI
URI_FORMAT = """\
%(transport)s://%(userid)s@%(hostname)s%(port)s/%(virtual_host)s\
"""


def parse_url(url):
    auth = userid = password = None
    scheme = urlparse(url).scheme
    parts = urlparse(url.replace("%s://" % (scheme, ), "http://"))
    netloc = parts.netloc
    if '@' in netloc:
        auth, _, netloc = partition(parts.netloc, '@')
        userid, _, password = partition(auth, ':')
    hostname, _, port = partition(netloc, ':')
    path = parts.path or ""
    if path and path[0] == '/':
        path = path[1:]
    return dict({"hostname": hostname,
                 "port": port and int(port) or None,
                 "userid": userid or None,
                 "password": password or None,
                 "transport": scheme,
                 "virtual_host": path or None},
                **kwdict(dict(parse_qsl(parts.query))))


class BrokerConnection(object):
    """A connection to the broker.

    :param URL:  Connection URL.

    :keyword hostname: Default host name/address if not provided in the URL.
    :keyword userid: Default user name if not provided in the URL.
    :keyword password: Default password if not provided in the URL.
    :keyword virtual_host: Default virtual host if not provided in the URL.
    :keyword port: Default port if not provided in the URL.
    :keyword ssl: Use SSL to connect to the server. Default is ``False``.
      May not be supported by the specified transport.
    :keyword transport: Default transport if not specified in the URL.
    :keyword connect_timeout: Timeout in seconds for connecting to the
      server. May not be supported by the specified transport.
    :keyword transport_options: A dict of additional connection arguments to
      pass to alternate kombu channel implementations.  Consult the transport
      documentation for available options.
    :keyword insist: *Deprecated*

    .. note::

        The connection is established lazily when needed. If you need the
        connection to be established, then force it to do so using
        :meth:`connect`::

            >>> conn.connect()

        Remember to always close the connection::

            >>> conn.release()

    """
    URI_FORMAT = URI_FORMAT

    port = None
    virtual_host = "/"
    connect_timeout = 5

    _closed = None
    _connection = None
    _default_channel = None
    _transport = None
    _logger = None
    skip_uri_transports = set(["sqlalchemy", "sqlakombu.transport.Transport"])

    def __init__(self, hostname="localhost", userid=None,
            password=<PASSWORD>, virtual_host=None, port=None, insist=False,
            ssl=False, transport=None, connect_timeout=5,
            transport_options=None, login_method=None, **kwargs):
        # have to spell the args out, just to get nice docstrings :(
        params = {"hostname": hostname, "userid": userid,
                  "password": password, "virtual_host": virtual_host,
                  "port": port, "insist": insist, "ssl": ssl,
                  "transport": transport, "connect_timeout": connect_timeout,
                  "login_method": login_method}
        if hostname and "://" in hostname \
                and transport not in self.skip_uri_transports:
            params.update(parse_url(hostname))
        self._init_params(**params)

        # backend_cls argument will be removed shortly.
        self.transport_cls = self.transport_cls or kwargs.get("backend_cls")

        if transport_options is None:
            transport_options = {}
        self.transport_options = transport_options

        if _LOG_CONNECTION:
            from kombu.utils.log import get_logger
            self._logger = get_logger("kombu.connection")

    def _init_params(self, hostname, userid, password, virtual_host, port,
            insist, ssl, transport, connect_timeout, login_method):
        self.hostname = hostname
        self.userid = userid
        self.password = password
        self.login_method = login_method
        self.virtual_host = virtual_host or self.virtual_host
        self.port = port or self.port
        self.insist = insist
        self.connect_timeout = connect_timeout
        self.ssl = ssl
        self.transport_cls = transport

    def _debug(self, msg, ident="[Kombu connection:0x%(id)x] ", **kwargs):
        if self._logger:
            self._logger.debug((ident + unicode(msg)) % {"id": id(self)},
                               **kwargs)

    def connect(self):
        """Establish connection to server immediately."""
        self._closed = False
        return self.connection

    def channel(self):
        """Request a new channel."""
        self._debug("create channel")
        chan = self.transport.create_channel(self.connection)
        if _LOG_CHANNEL:
            from kombu.utils.debug import Logwrapped
            return Logwrapped(chan, "kombu.channel",
                    "[Kombu channel:%(channel_id)s] ")
        return chan

    def drain_events(self, **kwargs):
        """Wait for a single event from the server.

        :keyword timeout: Timeout in seconds before we give up.
            Raises :exc:`socket.timeout` if the timeout is exceeded.

        Usually used from an event loop.

        """
        return self.transport.drain_events(self.connection, **kwargs)

    def maybe_close_channel(self, channel):
        try:
            channel.close()
        except (self.connection_errors + self.channel_errors):
            pass

    def _close(self):
        if self._default_channel:
            self.maybe_close_channel(self._default_channel)
        if self._connection:
            try:
                self.transport.close_connection(self._connection)
            except self.connection_errors + (AttributeError, socket.error):
                pass
            self._connection = None
            self._debug("closed")
        if self._transport:
            self._transport.client = None
            self._transport = None
        self._closed = True

    def release(self):
        """Close the connection (if open)."""
        self._close()
    close = release

    def ensure_connection(self, errback=None, max_retries=None,
            interval_start=2, interval_step=2, interval_max=30):
        """Ensure we have a connection to the server.

        If not retry establishing the connection with the settings
        specified.

        :keyword errback: Optional callback called each time the connection
          can't be established. Arguments provided are the exception
          raised and the interval that will be slept ``(exc, interval)``.

        :keyword max_retries: Maximum number of times to retry.
          If this limit is exceeded the connection error will be re-raised.

        :keyword interval_start: The number of seconds we start sleeping for.
        :keyword interval_step: How many seconds added to the interval
          for each retry.
        :keyword interval_max: Maximum number of seconds to sleep between
          each retry.

        """
        retry_over_time(self.connect, self.connection_errors, (), {},
                        errback, max_retries,
                        interval_start, interval_step, interval_max)
        return self

    def revive(self, new_channel):
        if self._default_channel:
            self.maybe_close_channel(self._default_channel)
            self._default_channel = None

    def ensure(self, obj, fun, errback=None, max_retries=None,
            interval_start=1, interval_step=1, interval_max=1, on_revive=None):
        """Ensure operation completes, regardless of any channel/connection
        errors occurring.

        Will retry by establishing the connection, and reapplying
        the function.

        :param fun: Method to apply.

        :keyword errback: Optional callback called each time the connection
          can't be established. Arguments provided are the exception
          raised and the interval that will be slept ``(exc, interval)``.

        :keyword max_retries: Maximum number of times to retry.
          If this limit is exceeded the connection error will be re-raised.

        :keyword interval_start: The number of seconds we start sleeping for.
        :keyword interval_step: How many seconds added to the interval
          for each retry.
        :keyword interval_max: Maximum number of seconds to sleep between
          each retry.

        **Example**

        This is an example ensuring a publish operation::

            >>> def errback(exc, interval):
            ...     print("Couldn't publish message: %r. Retry in %ds" % (
            ...             exc, interval))
            >>> publish = conn.ensure(producer, producer.publish,
            ...                       errback=errback, max_retries=3)
            >>> publish(message, routing_key)

        """

        @wraps(fun)
        def _ensured(*args, **kwargs):
            got_connection = 0
            for retries in count(0):
                try:
                    return fun(*args, **kwargs)
                except self.connection_errors + self.channel_errors, exc:
                    self._debug("ensure got exception: %r" % (exc, ),
                                exc_info=sys.exc_info())
                    if got_connection:
                        raise
                    if max_retries is not None and retries > max_retries:
                        raise
                    errback and errback(exc, 0)
                    self._connection = None
                    self.close()
                    remaining_retries = None
                    if max_retries is not None:
                        remaining_retries = max(max_retries - retries, 1)
                    self.ensure_connection(errback,
                                           remaining_retries,
                                           interval_start,
                                           interval_step,
                                           interval_max)
                    new_channel = self.channel()
                    self.revive(new_channel)
                    obj.revive(new_channel)
                    if on_revive:
                        on_revive(new_channel)
                    got_connection += 1

        _ensured.func_name = _ensured.__name__ = "%s(ensured)" % fun.__name__
        return _ensured

    def autoretry(self, fun, channel=None, **ensure_options):
        """Decorator for functions supporting a ``channel`` keyword argument.

        The resulting callable will retry calling the function if
        it raises connection or channel related errors.
        The return value will be a tuple of ``(retval, last_created_channel)``.

        If a ``channel`` is not provided, then one will be automatically
        acquired (remember to close it afterwards).

        See :meth:`ensure` for the full list of supported keyword arguments.

        Example usage::

            channel = connection.channel()
            try:
                ret, channel = connection.autoretry(publish_messages, channel)
            finally:
                channel.close()
        """
        channels = [channel]
        create_channel = self.channel

        class Revival(object):
            __name__ = fun.__name__
            __module__ = fun.__module__
            __doc__ = fun.__doc__

            def revive(self, channel):
                channels[0] = channel

            def __call__(self, *args, **kwargs):
                if channels[0] is None:
                    self.revive(create_channel())
                kwargs["channel"] = channels[0]
                return fun(*args, **kwargs), channels[0]

        revive = Revival()
        return self.ensure(revive, revive, **ensure_options)

    def create_transport(self):
        return self.get_transport_cls()(client=self)
    create_backend = create_transport   # FIXME

    def get_transport_cls(self):
        """Get the currently used transport class."""
        transport_cls = self.transport_cls
        if not transport_cls or isinstance(transport_cls, basestring):
            transport_cls = get_transport_cls(transport_cls)
        return transport_cls

    def clone(self, **kwargs):
        """Create a copy of the connection with the same connection
        settings."""
        return self.__class__(**dict(self.info(), **kwargs))

    def info(self):
        """Get connection info."""
        transport_cls = self.transport_cls or "amqp"
        transport_cls = {"amqplib": "amqp"}.get(transport_cls, transport_cls)
        defaults = self.transport.default_connection_params
        info = OrderedDict((("hostname", self.hostname),
                            ("userid", self.userid),
                            ("password", <PASSWORD>),
                            ("virtual_host", self.virtual_host),
                            ("port", self.port),
                            ("insist", self.insist),
                            ("ssl", self.ssl),
                            ("transport", transport_cls),
                            ("connect_timeout", self.connect_timeout),
                            ("transport_options", self.transport_options),
                            ("login_method", self.login_method)))
        for key, value in defaults.iteritems():
            if info[key] is None:
                info[key] = value
        return info

    def __hash__(self):
        return hash("|".join(map(str, self.info().itervalues())))

    def as_uri(self, include_password=False):
        fields = self.info()
        port = fields["port"]
        userid = fields["userid"]
        password = fields["password"]
        url = "%s://" % fields["transport"]
        if userid:
            url += userid
            if include_password and password:
                url += ':' + password
            url += '@'
        url += fields["hostname"]
        if port:
            url += ':' + str(port)<commit_msg>First attempt to enable Mongo Replicasets<commit_after>"""
kombu.connection
================

Broker connection and pools.

:copyright: (c) 2009 - 2011 by <NAME>.
:license: BSD, see LICENSE for more details.

"""
import os
import sys
import socket

from copy import copy
from itertools import count
from Queue import Empty
from urlparse import urlparse
try:
    from urlparse import parse_qsl
except ImportError:
    from cgi import parse_qsl  # noqa

from kombu import exceptions
from kombu.transport import get_transport_cls
from kombu.utils import kwdict, partition, retry_over_time
from kombu.utils.compat import OrderedDict, LifoQueue as _LifoQueue
from kombu.utils.functional import wraps

_LOG_CONNECTION = os.environ.get("KOMBU_LOG_CONNECTION", False)
_LOG_CHANNEL = os.environ.get("KOMBU_LOG_CHANNEL", False)


#: Connection info -> URI
URI_FORMAT = """\
%(transport)s://%(userid)s@%(hostname)s%(port)s/%(virtual_host)s\
"""


def parse_url(url):
    port = path = None
    auth = userid = password = None
    scheme = urlparse(url).scheme
    parts = urlparse(url.replace("%s://" % (scheme, ), "http://"))
    if scheme != 'mongodb':
        netloc = parts.netloc
        if '@' in netloc:
            auth, _, netloc = partition(parts.netloc, '@')
            userid, _, password = partition(auth, ':')
        hostname, _, port = partition(netloc, ':')
        path = parts.path or ""
        if path and path[0] == '/':
            path = path[1:]
        port = int(port)
    else:
        hostname = url[len('mongodb://'):]
    return dict({"hostname": hostname,
                 "port": port or None,
                 "userid": userid or None,
                 "password": password or None,
                 "transport": scheme,
                 "virtual_host": path or None},
                **kwdict(dict(parse_qsl(parts.query))))


class BrokerConnection(object):
    """A connection to the broker.

    :param URL:  Connection URL.

    :keyword hostname: Default host name/address if not provided in the URL.
    :keyword userid: Default user name if not provided in the URL.
    :keyword password: Default password if not provided in the URL.
    :keyword virtual_host: Default virtual host if not provided in the URL.
    :keyword port: Default port if not provided in the URL.
    :keyword ssl: Use SSL to connect to the server. Default is ``False``.
      May not be supported by the specified transport.
    :keyword transport: Default transport if not specified in the URL.
    :keyword connect_timeout: Timeout in seconds for connecting to the
      server. May not be supported by the specified transport.
    :keyword transport_options: A dict of additional connection arguments to
      pass to alternate kombu channel implementations.  Consult the transport
      documentation for available options.
    :keyword insist: *Deprecated*

    .. note::

        The connection is established lazily when needed. If you need the
        connection to be established, then force it to do so using
        :meth:`connect`::

            >>> conn.connect()

        Remember to always close the connection::

            >>> conn.release()

    """
    URI_FORMAT = URI_FORMAT

    port = None
    virtual_host = "/"
    connect_timeout = 5

    _closed = None
    _connection = None
    _default_channel = None
    _transport = None
    _logger = None
    skip_uri_transports = set(["sqlalchemy", "sqlakombu.transport.Transport"])

    def __init__(self, hostname="localhost", userid=None,
            password=<PASSWORD>, virtual_host=None, port=None, insist=False,
            ssl=False, transport=None, connect_timeout=5,
            transport_options=None, login_method=None, **kwargs):
        # have to spell the args out, just to get nice docstrings :(
        params = {"hostname": hostname, "userid": userid,
                  "password": password, "virtual_host": virtual_host,
                  "port": port, "insist": insist, "ssl": ssl,
                  "transport": transport, "connect_timeout": connect_timeout,
                  "login_method": login_method}
        if hostname and "://" in hostname \
                and transport not in self.skip_uri_transports:
            params.update(parse_url(hostname))
        self._init_params(**params)

        # backend_cls argument will be removed shortly.
        self.transport_cls = self.transport_cls or kwargs.get("backend_cls")

        if transport_options is None:
            transport_options = {}
        self.transport_options = transport_options

        if _LOG_CONNECTION:
            from kombu.utils.log import get_logger
            self._logger = get_logger("kombu.connection")

    def _init_params(self, hostname, userid, password, virtual_host, port,
            insist, ssl, transport, connect_timeout, login_method):
        self.hostname = hostname
        self.userid = userid
        self.password = password
        self.login_method = login_method
        self.virtual_host = virtual_host or self.virtual_host
        self.port = port or self.port
        self.insist = insist
        self.connect_timeout = connect_timeout
        self.ssl = ssl
        self.transport_cls = transport

    def _debug(self, msg, ident="[Kombu connection:0x%(id)x] ", **kwargs):
        if self._logger:
            self._logger.debug((ident + unicode(msg)) % {"id": id(self)},
                               **kwargs)

    def connect(self):
        """Establish connection to server immediately."""
        self._closed = False
        return self.connection

    def channel(self):
        """Request a new channel."""
        self._debug("create channel")
        chan = self.transport.create_channel(self.connection)
        if _LOG_CHANNEL:
            from kombu.utils.debug import Logwrapped
            return Logwrapped(chan, "kombu.channel",
                    "[Kombu channel:%(channel_id)s] ")
        return chan

    def drain_events(self, **kwargs):
        """Wait for a single event from the server.

        :keyword timeout: Timeout in seconds before we give up.
            Raises :exc:`socket.timeout` if the timeout is exceeded.

        Usually used from an event loop.

        """
        return self.transport.drain_events(self.connection, **kwargs)

    def maybe_close_channel(self, channel):
        try:
            channel.close()
        except (self.connection_errors + self.channel_errors):
            pass

    def _close(self):
        if self._default_channel:
            self.maybe_close_channel(self._default_channel)
        if self._connection:
            try:
                self.transport.close_connection(self._connection)
            except self.connection_errors + (AttributeError, socket.error):
                pass
            self._connection = None
            self._debug("closed")
        if self._transport:
            self._transport.client = None
            self._transport = None
        self._closed = True

    def release(self):
        """Close the connection (if open)."""
        self._close()
    close = release

    def ensure_connection(self, errback=None, max_retries=None,
            interval_start=2, interval_step=2, interval_max=30):
        """Ensure we have a connection to the server.

        If not retry establishing the connection with the settings
        specified.

        :keyword errback: Optional callback called each time the connection
          can't be established. Arguments provided are the exception
          raised and the interval that will be slept ``(exc, interval)``.

        :keyword max_retries: Maximum number of times to retry.
          If this limit is exceeded the connection error will be re-raised.

        :keyword interval_start: The number of seconds we start sleeping for.
        :keyword interval_step: How many seconds added to the interval
          for each retry.
        :keyword interval_max: Maximum number of seconds to sleep between
          each retry.

        """
        retry_over_time(self.connect, self.connection_errors, (), {},
                        errback, max_retries,
                        interval_start, interval_step, interval_max)
        return self

    def revive(self, new_channel):
        if self._default_channel:
            self.maybe_close_channel(self._default_channel)
            self._default_channel = None

    def ensure(self, obj, fun, errback=None, max_retries=None,
            interval_start=1, interval_step=1, interval_max=1, on_revive=None):
        """Ensure operation completes, regardless of any channel/connection
        errors occurring.

        Will retry by establishing the connection, and reapplying
        the function.

        :param fun: Method to apply.

        :keyword errback: Optional callback called each time the connection
          can't be established. Arguments provided are the exception
          raised and the interval that will be slept ``(exc, interval)``.

        :keyword max_retries: Maximum number of times to retry.
          If this limit is exceeded the connection error will be re-raised.

        :keyword interval_start: The number of seconds we start sleeping for.
        :keyword interval_step: How many seconds added to the interval
          for each retry.
        :keyword interval_max: Maximum number of seconds to sleep between
          each retry.

        **Example**

        This is an example ensuring a publish operation::

            >>> def errback(exc, interval):
            ...     print("Couldn't publish message: %r. Retry in %ds" % (
            ...             exc, interval))
            >>> publish = conn.ensure(producer, producer.publish,
            ...                       errback=errback, max_retries=3)
            >>> publish(message, routing_key)

        """

        @wraps(fun)
        def _ensured(*args, **kwargs):
            got_connection = 0
            for retries in count(0):
                try:
                    return fun(*args, **kwargs)
                except self.connection_errors + self.channel_errors, exc:
                    self._debug("ensure got exception: %r" % (exc, ),
                                exc_info=sys.exc_info())
                    if got_connection:
                        raise
                    if max_retries is not None and retries > max_retries:
                        raise
                    errback and errback(exc, 0)
                    self._connection = None
                    self.close()
                    remaining_retries = None
                    if max_retries is not None:
                        remaining_retries = max(max_retries - retries, 1)
                    self.ensure_connection(errback,
                                           remaining_retries,
                                           interval_start,
                                           interval_step,
                                           interval_max)
                    new_channel = self.channel()
                    self.revive(new_channel)
                    obj.revive(new_channel)
                    if on_revive:
                        on_revive(new_channel)
                    got_connection += 1

        _ensured.func_name = _ensured.__name__ = "%s(ensured)" % fun.__name__
        return _ensured

    def autoretry(self, fun, channel=None, **ensure_options):
        """Decorator for functions supporting a ``channel`` keyword argument.

        The resulting callable will retry calling the function if
        it raises connection or channel related errors.
        The return value will be a tuple of ``(retval, last_created_channel)``.

        If a ``channel`` is not provided, then one will be automatically
        acquired (remember to close it afterwards).

        See :meth:`ensure` for the full list of supported keyword arguments.

        Example usage::

            channel = connection.channel()
            try:
                ret, channel = connection.autoretry(publish_messages, channel)
            finally:
                channel.close()
        """
        channels = [channel]
        create_channel = self.channel

        class Revival(object):
            __name__ = fun.__name__
            __module__ = fun.__module__
            __doc__ = fun.__doc__

            def revive(self, channel):
                channels[0] = channel

            def __call__(self, *args, **kwargs):
                if channels[0] is None:
                    self.revive(create_channel())
                kwargs["channel"] = channels[0]
                return fun(*args, **kwargs), channels[0]

        revive = Revival()
        return self.ensure(revive, revive, **ensure_options)

    def create_transport(self):
        return self.get_transport_cls()(client=self)
    create_backend = create_transport   # FIXME

    def get_transport_cls(self):
        """Get the currently used transport class."""
        transport_cls = self.transport_cls
        if not transport_cls or isinstance(transport_cls, basestring):
            transport_cls = get_transport_cls(transport_cls)
        return transport_cls

    def clone(self, **kwargs):
        """Create a copy of the connection with the same connection
        settings."""
        return self.__class__(**dict(self.info(), **kwargs))

    def info(self):
        """Get connection info."""
        transport_cls = self.transport_cls or "amqp"
        transport_cls = {"amqplib": "amqp"}.get(transport_cls, transport_cls)
        defaults = self.transport.default_connection_params
        info = OrderedDict((("hostname", self.hostname),
                            ("userid", self.userid),
                            ("password", <PASSWORD>),
                            ("virtual_host", self.virtual_host),
                            ("port", self.port),
                            ("insist", self.insist),
                            ("ssl", self.ssl),
                            ("transport", transport_cls),
                            ("connect_timeout", self.connect_timeout),
                            ("transport_options", self.transport_options),
                            ("login_method", self.login_method)))
        for key, value in defaults.iteritems():
            if info[key] is None:
                info[key] = value
        return info

    def __hash__(self):
        return hash("|".join(map(str, self.info().itervalues())))

    def as_uri(self, include_password=False):
        fields = self.info()
        port = fields["port"]
        userid = fields["userid"]
        password = fields["password"]
        transport = fields["transport"]
        url = "%s://" % transport
        if userid:
            url += userid
            if include_password and password:
                url += ':' + password
            url += '@'
        url += fields["hostname"]
        if port and transport != "mongodb":
            url += ':' + str(port)
{'fraction_non_alphanumeric': 0.09075929786804762, 'fraction_numerical': 0.0020176205528280315, 'mean_word_length': 2.0145970603142422, 'pattern_counts': {'":': 39, '<': 9, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5679687', 'n_tokens_mistral': 7704, 'n_tokens_neox': 7417, 'n_words': 2348}