<commit_before>"""cl.pools"""

from __future__ import absolute_import

from itertools import chain

from kombu.pools import ProducerPool

__all__ = ["connections", "producers", "set_limit", "reset"]
_limit = [100]


class _Connections(dict):

    def __missing__(self, connection):
        k = self[connection] = connection.Pool(_limit[0])
        return k
connections = _Connections()


class _Producers(dict):

    def __missing__(self, conn):
        k = self[conn] = ProducerPool(connections[conn], limit=_limit[0])
        return k
producers = _Producers()


def _all_pools():
    return chain(connections.itervalues() if connections else iter([]),
                 producers.itervalues() if producers else iter([]))
<commit_msg>Fixes huge connection leak in cl.pools<commit_after>"""cl.pools"""

from __future__ import absolute_import

from itertools import chain

from kombu.pools import ProducerPool

__all__ = ["connections", "producers", "set_limit", "reset"]
_limit = [200]


class HashingDict(dict):

    def __getitem__(self, key):
        h = hash(key)
        if h not in self:
            return self.__missing__(key)
        return dict.__getitem__(self, h)

    def __setitem__(self, key, value):
        return dict.__setitem__(self, hash(key), value)

    def __delitem__(self, key):
        return dict.__delitem__(self, hash(key))


class _Connections(HashingDict):

    def __missing__(self, connection):
        k = self[connection] = connection.Pool(limit=_limit[0])
        return k
connections = _Connections()


class _Producers(HashingDict):

    def __missing__(self, conn):
        k = self[conn] = ProducerPool(connections[conn], limit=_limit[0])
        return k
producers = _Producers()


def _all_pools():
    return chain(connections.itervalues() if connections else iter([]),
                 producers.itervalues() if producers else iter([]))

{'fraction_non_alphanumeric': 0.11695906432748537, 'fraction_numerical': 0.00531632110579479, 'mean_word_length': 3.705, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '166319', 'n_tokens_mistral': 593, 'n_tokens_neox': 565, 'n_words': 152}