<commit_before>import uidlib, math, gevent

class FingerTable():
    def __init__(self, self_node, min_count=1):
        self.known = set([])
        self.self = self_node
        self.table = []
        self.min_count = min_count
        self.max_level = int(self._uid_2_level(32*'F', 32*'0'))+1
        for i in range(self.max_level):
            self.table.append(set([]))
        
    def _above_zero(self, item):
        if item < 0: 
            return 0
        return item
        
    def _uid_2_level(self, uid, other_uid=None):
        """
        Convert a uid to a log level in the finger table
        """
        if other_uid == None:
            other_uid = self.self.uid
        level = int(self._above_zero(
                        math.log(
                            uidlib.distance(uid, other_uid)+0.1)
                        )
                    )
        return level
        
    def _level_check(self, level):
        """
        Make sure the level is inside the table range.
        """
        if level > len(self.table) - 1:
            for i in range(len(self.table)-1, level):
                self.table.append(set([]))
                self.max_level += 1
                
    def add(self, Node):
        """
        Add a node to our finger table
        """
        self.known.add(Node)
        level = self._uid_2_level(Node.uid)
        self._level_check(level)
        self.table[level].add(Node)
        
    def remove(self, Node):
        """
        remove a node from our finger table
        """
        if Node not in self.known:
            return
        self.known.remove(Node)
        level = self._uid_2_level(Node.uid)
        self.table[level].remove(Node)
        
    def get_levels(self):
        """
        return what distances we would like to get uids for
        """
        for i in range(len(self.table)):
            if len(self.table[i]) < self.min_count:
                yield int(i)
                
    def get_node_from_level(self, level, uid):
        """
        Gets the node which is a certain level away from another uid
        """
        for i in xrange(
            int(level-self._uid_2_level(uid)), 
            int(level+self._uid_2_level(uid))
            ): #Triangle Inequality
            if i < len(self.table) and i >=0:
                for node in self.table[i]:<commit_msg>Add function to get all nodes in the finger table. Inspired by http://github.com/hoangelos/gevent-dht/commit/8ce9d7d3ed98b18aa84162439d66208bb6bd9fa4<commit_after>import uidlib, math, gevent

class FingerTable():
    def __init__(self, self_node, min_count=1):
        self.known = set([])
        self.self = self_node
        self.table = []
        self.min_count = min_count
        self.max_level = int(self._uid_2_level(32*'F', 32*'0'))+1
        for i in range(self.max_level):
            self.table.append(set([]))
        
    def _above_zero(self, item):
        if item < 0: 
            return 0
        return item
        
    def _uid_2_level(self, uid, other_uid=None):
        """
        Convert a uid to a log level in the finger table
        """
        if other_uid == None:
            other_uid = self.self.uid
        level = int(self._above_zero(
                        math.log(
                            uidlib.distance(uid, other_uid)+0.1)
                        )
                    )
        return level
        
    def _level_check(self, level):
        """
        Make sure the level is inside the table range.
        """
        if level > len(self.table) - 1:
            for i in range(len(self.table)-1, level):
                self.table.append(set([]))
                self.max_level += 1
                
    def add(self, Node):
        """
        Add a node to our finger table
        """
        self.known.add(Node)
        level = self._uid_2_level(Node.uid)
        self._level_check(level)
        self.table[level].add(Node)
        
    def remove(self, Node):
        """
        remove a node from our finger table
        """
        if Node not in self.known:
            return
        self.known.remove(Node)
        level = self._uid_2_level(Node.uid)
        self.table[level].remove(Node)
        
    def get_levels(self):
        """
        return what distances we would like to get uids for
        """
        for i in range(len(self.table)):
            if len(self.table[i]) < self.min_count:
                yield int(i)
                
    def get_nodes(self):
        """
        return a list of nodes in the finger table so far
        """
        nodes = []
        for i in range(len(self.table)):
            for node in self.table[i]:
                host = """%s:%d""" % (node.host, node.port)
                nodes.append(host)
        return nodes
        
    def get_node_from_level(self, level, uid):
        """
        Gets the node which is a certain level away from another uid
        """
        for i in xrange(
            int(level-self._uid_2_level(uid)), 
            int(level+self._uid_2_level(uid))
            ): #Triangle Inequality
            if i < len(self.table) and i >=0:
                for node in self.table[i]:
{'fraction_non_alphanumeric': 0.10567853705486044, 'fraction_numerical': 0.012704523580365737, 'mean_word_length': 1.4754645069080514, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6677466', 'n_tokens_mistral': 1565, 'n_tokens_neox': 1517, 'n_words': 392}