<commit_before>"""
Keyed collections of token buckets.
"""


import time
import weakref


__author__ = '<NAME>'
__email__ = '<EMAIL>'
__version__ = '0.1.0'

__all__ = ('TokenBucket', 'Collection')


class TokenBucket(object):
    """
    A token bucket.
    """

    __slots__ = ('clock', 'ts', 'rate', 'limit', '_available')

    def __init__(self, rate, limit, clock=time.time):
        self.clock = clock
        self.ts = self.clock()
        self.rate = rate
        self.limit = limit
        self._available = limit

    def consume(self, tokens):
        """
        Attempt to remove the given number of tokens from this bucket. If
        there are enough tokens in the bucket to fulfil the request, we
        return `True`, otherwise `False`.
        """
        if 0 < tokens <= self.tokens:
            self._available -= tokens
            return True
        return False

    @property
    def tokens(self):
        """
        The number of tokens available in this bucket.
        """
        ts = self.clock()
        if self._available < self.limit:
            self._available += min(
                (ts - self.ts) * self.rate,
                self.limit - self._available)
        self.ts = ts
        return self._available

    def __cmp__(self, other):
        return cmp(self.ts, other.ts)


class Node(object):
    """Linked list node."""

    __slots__ = ('prev_node', 'obj', 'next_node')

    def __init__(self, obj):
        self.obj = obj
        self.prev_node = None
        self.next_node = None

    def detach(self):
        """
        Remove from the linked list and join the adjacent elements to one
        another.
        """
        if self.prev_node is not None:
            self.prev_node.next_node = self.next_node
        if self.next_node is not None:
            self.next_node.prev_node = self.prev_node
        self.prev_node = None
        self.next_node = None

    def insert_after(self, other):
        """
        Insert this node after `other`.
        """
        self.next_node = other
        self.prev_node = other.prev_node
        if self.prev_node is not None:
            self.prev_node.next_node = self
        other.prev_node = self


class Collection(object):
    """
    A keyed collection of token buckets.
    """

    __slots__ = (
        'head_node', 'tail_node', 'node_map',
        'rate', 'limit',
        'timeout',
        'clock')

    def __init__(self, rate, limit, timeout, clock=time.time):
        self.head_node = Node(None)
        self.tail_node = Node(None)
        self.head_node.prev_node = self.tail_node
        self.tail_node.next_node = self.head_node
        self.node_map = weakref.WeakValueDictionary()
        self.rate = rate
        self.limit = limit
        self.timeout = timeout
        self.clock = clock

    def __del__(self):
        node = self.tail_node.next_node
        while node is not self.head_node:
            next_node = node.next_node
            node.detach()
            del node
            node = next_node
        self.head_node.prev_node = None
        self.tail_node.next_node = None

    def __contains__(self, key):<commit_msg>Node already knows how to add nodes to chains.<commit_after>"""
Keyed collections of token buckets.
"""


import time
import weakref


__author__ = '<NAME>'
__email__ = '<EMAIL>'
__version__ = '0.1.0'

__all__ = ('TokenBucket', 'Collection')


class TokenBucket(object):
    """
    A token bucket.
    """

    __slots__ = ('clock', 'ts', 'rate', 'limit', '_available')

    def __init__(self, rate, limit, clock=time.time):
        self.clock = clock
        self.ts = self.clock()
        self.rate = rate
        self.limit = limit
        self._available = limit

    def consume(self, tokens):
        """
        Attempt to remove the given number of tokens from this bucket. If
        there are enough tokens in the bucket to fulfil the request, we
        return `True`, otherwise `False`.
        """
        if 0 < tokens <= self.tokens:
            self._available -= tokens
            return True
        return False

    @property
    def tokens(self):
        """
        The number of tokens available in this bucket.
        """
        ts = self.clock()
        if self._available < self.limit:
            self._available += min(
                (ts - self.ts) * self.rate,
                self.limit - self._available)
        self.ts = ts
        return self._available

    def __cmp__(self, other):
        return cmp(self.ts, other.ts)


class Node(object):
    """Linked list node."""

    __slots__ = ('prev_node', 'obj', 'next_node')

    def __init__(self, obj):
        self.obj = obj
        self.prev_node = None
        self.next_node = None

    def detach(self):
        """
        Remove from the linked list and join the adjacent elements to one
        another.
        """
        if self.prev_node is not None:
            self.prev_node.next_node = self.next_node
        if self.next_node is not None:
            self.next_node.prev_node = self.prev_node
        self.prev_node = None
        self.next_node = None

    def insert_after(self, other):
        """
        Insert this node after `other`.
        """
        self.next_node = other
        self.prev_node = other.prev_node
        if self.prev_node is not None:
            self.prev_node.next_node = self
        other.prev_node = self


class Collection(object):
    """
    A keyed collection of token buckets.
    """

    __slots__ = (
        'head_node', 'tail_node', 'node_map',
        'rate', 'limit',
        'timeout',
        'clock')

    def __init__(self, rate, limit, timeout, clock=time.time):
        self.head_node = Node(None)
        self.tail_node = Node(None)
        self.tail_node.insert_after(self.head_node)
        self.node_map = weakref.WeakValueDictionary()
        self.rate = rate
        self.limit = limit
        self.timeout = timeout
        self.clock = clock

    def __del__(self):
        node = self.tail_node.next_node
        while node is not self.head_node:
            next_node = node.next_node
            node.detach()
            del node
            node = next_node
        self.head_node.prev_node = None
        self.tail_node.next_node = None

    def __contains__(self, key):
{'fraction_non_alphanumeric': 0.10439473266698397, 'fraction_numerical': 0.0012692368713311123, 'mean_word_length': 1.891743119266055, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1934655', 'n_tokens_mistral': 1863, 'n_tokens_neox': 1810, 'n_words': 512}