<commit_before>import logging
from twisted.internet import defer
from distance import Distance
from error import TimeoutError
import constants

log = logging.getLogger(__name__)


def get_contact(contact_list, node_id, address, port):
    for contact in contact_list:
        if contact.id == node_id and contact.address == address and contact.port == port:
            return contact
    raise IndexError(node_id)


class _IterativeFind(object):
    # TODO: use polymorphism to search for a value or node
    #       instead of using a find_value flag
    def __init__(self, node, shortlist, key, rpc):
        self.node = node
        self.finished_deferred = defer.Deferred()
        # all distance operations in this class only care about the distance
        # to self.key, so this makes it easier to calculate those
        self.distance = Distance(key)
        # The closest known and active node yet found
        self.closest_node = None if not shortlist else shortlist[0]
        self.prev_closest_node = None
        # Shortlist of contact objects (the k closest known contacts to the key from the routing table)
        self.shortlist = shortlist
        # The search key
        self.key = str(key)
        # The rpc method name (findValue or findNode)
        self.rpc = rpc
        # List of active queries; len() indicates number of active probes
        self.active_probes = []
        # List of contact (address, port) tuples that have already been queried, includes contacts that didn't reply
        self.already_contacted = []
        # A list of found and known-to-be-active remote nodes (Contact objects)
        self.active_contacts = []
        # Ensure only one searchIteration call is running at a time
        self._search_iteration_semaphore = defer.DeferredSemaphore(1)
        self._iteration_count = 0
        self.find_value_result = {}
        self.pending_iteration_calls = []

    @property
    def is_find_node_request(self):
        return self.rpc == "findNode"

    @property
    def is_find_value_request(self):
        return self.rpc == "findValue"

    def is_closer(self, contact):
        if not self.closest_node:
            return True
        return self.distance.is_closer(contact.id, self.closest_node.id)

    def getContactTriples(self, result):
        if self.is_find_value_request:
            contact_triples = result['contacts']
        else:
            contact_triples = result
        for contact_tup in contact_triples:
            if not isinstance(contact_tup, (list, tuple)) or len(contact_tup) != 3:
                raise ValueError("invalid contact triple")
        return contact_triples

    def sortByDistance(self, contact_list):
        """Sort the list of contacts in order by distance from key"""
        contact_list.sort(key=lambda c: self.distance(c.id))

    @defer.inlineCallbacks
    def extendShortlist(self, contact, result):
        # The "raw response" tuple contains the response message and the originating address info
        originAddress = (contact.address, contact.port)
        if self.finished_deferred.called:
            defer.returnValue(contact.id)
        if self.node.contact_manager.is_ignored(originAddress):
            raise ValueError("contact is ignored")
        if contact.id == self.node.node_id:
            defer.returnValue(contact.id)

        if contact not in self.active_contacts:
            self.active_contacts.append(contact)
        if contact not in self.shortlist:
            self.shortlist.append(contact)

        # Now grow extend the (unverified) shortlist with the returned contacts
        # TODO: some validation on the result (for guarding against attacks)
        # If we are looking for a value, first see if this result is the value
        # we are looking for before treating it as a list of contact triples
        if self.is_find_value_request and self.key in result:
            # We have found the value
            self.find_value_result[self.key] = result[self.key]
            self.finished_deferred.callback(self.find_value_result)
        else:
            if self.is_find_value_request:
                # We are looking for a value, and the remote node didn't have it
                # - mark it as the closest "empty" node, if it is
                # TODO: store to this peer after finding the value as per the kademlia spec
                if 'closestNodeNoValue' in self.find_value_result:
                    if self.is_closer(contact):
                        self.find_value_result['closestNodeNoValue'] = contact
                else:
                    self.find_value_result['closestNodeNoValue'] = contact
            contactTriples = self.getContactTriples(result)
            for contactTriple in contactTriples:
                if (contactTriple[1], contactTriple[2]) in ((c.address, c.port) for c in self.already_contacted):
                    continue
                elif self.node.contact_manager.is_ignored((contactTriple[1], contactTriple[2])):
                    raise ValueError("contact is ignored")
                else:
                    found_contact = self.node.contact_manager.make_contact(contactTriple[0], contactTriple[1],
                                                                           contactTriple[2], self.node._protocol)
                    if found_contact not in self.shortlist:
                        self.shortlist.append(found_contact)
<commit_msg>filter out the bad node in results instead of punishing the node we trusted to contact<commit_after>import logging
from twisted.internet import defer
from distance import Distance
from error import TimeoutError
import constants

log = logging.getLogger(__name__)


def get_contact(contact_list, node_id, address, port):
    for contact in contact_list:
        if contact.id == node_id and contact.address == address and contact.port == port:
            return contact
    raise IndexError(node_id)


class _IterativeFind(object):
    # TODO: use polymorphism to search for a value or node
    #       instead of using a find_value flag
    def __init__(self, node, shortlist, key, rpc):
        self.node = node
        self.finished_deferred = defer.Deferred()
        # all distance operations in this class only care about the distance
        # to self.key, so this makes it easier to calculate those
        self.distance = Distance(key)
        # The closest known and active node yet found
        self.closest_node = None if not shortlist else shortlist[0]
        self.prev_closest_node = None
        # Shortlist of contact objects (the k closest known contacts to the key from the routing table)
        self.shortlist = shortlist
        # The search key
        self.key = str(key)
        # The rpc method name (findValue or findNode)
        self.rpc = rpc
        # List of active queries; len() indicates number of active probes
        self.active_probes = []
        # List of contact (address, port) tuples that have already been queried, includes contacts that didn't reply
        self.already_contacted = []
        # A list of found and known-to-be-active remote nodes (Contact objects)
        self.active_contacts = []
        # Ensure only one searchIteration call is running at a time
        self._search_iteration_semaphore = defer.DeferredSemaphore(1)
        self._iteration_count = 0
        self.find_value_result = {}
        self.pending_iteration_calls = []

    @property
    def is_find_node_request(self):
        return self.rpc == "findNode"

    @property
    def is_find_value_request(self):
        return self.rpc == "findValue"

    def is_closer(self, contact):
        if not self.closest_node:
            return True
        return self.distance.is_closer(contact.id, self.closest_node.id)

    def getContactTriples(self, result):
        if self.is_find_value_request:
            contact_triples = result['contacts']
        else:
            contact_triples = result
        for contact_tup in contact_triples:
            if not isinstance(contact_tup, (list, tuple)) or len(contact_tup) != 3:
                raise ValueError("invalid contact triple")
        return contact_triples

    def sortByDistance(self, contact_list):
        """Sort the list of contacts in order by distance from key"""
        contact_list.sort(key=lambda c: self.distance(c.id))

    @defer.inlineCallbacks
    def extendShortlist(self, contact, result):
        # The "raw response" tuple contains the response message and the originating address info
        originAddress = (contact.address, contact.port)
        if self.finished_deferred.called:
            defer.returnValue(contact.id)
        if self.node.contact_manager.is_ignored(originAddress):
            raise ValueError("contact is ignored")
        if contact.id == self.node.node_id:
            defer.returnValue(contact.id)

        if contact not in self.active_contacts:
            self.active_contacts.append(contact)
        if contact not in self.shortlist:
            self.shortlist.append(contact)

        # Now grow extend the (unverified) shortlist with the returned contacts
        # TODO: some validation on the result (for guarding against attacks)
        # If we are looking for a value, first see if this result is the value
        # we are looking for before treating it as a list of contact triples
        if self.is_find_value_request and self.key in result:
            # We have found the value
            self.find_value_result[self.key] = result[self.key]
            self.finished_deferred.callback(self.find_value_result)
        else:
            if self.is_find_value_request:
                # We are looking for a value, and the remote node didn't have it
                # - mark it as the closest "empty" node, if it is
                # TODO: store to this peer after finding the value as per the kademlia spec
                if 'closestNodeNoValue' in self.find_value_result:
                    if self.is_closer(contact):
                        self.find_value_result['closestNodeNoValue'] = contact
                else:
                    self.find_value_result['closestNodeNoValue'] = contact
            contactTriples = self.getContactTriples(result)
            for contactTriple in contactTriples:
                if (contactTriple[1], contactTriple[2]) in ((c.address, c.port) for c in self.already_contacted):
                    continue
                elif self.node.contact_manager.is_ignored((contactTriple[1], contactTriple[2])):
                    continue
                else:
                    found_contact = self.node.contact_manager.make_contact(contactTriple[0], contactTriple[1],
                                                                           contactTriple[2], self.node._protocol)
                    if found_contact not in self.shortlist:
                        self.shortlist.append(found_contact)

{'fraction_non_alphanumeric': 0.06887731798666301, 'fraction_numerical': 0.0020096830181784964, 'mean_word_length': 2.3045578025958346, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3074560', 'n_tokens_mistral': 2901, 'n_tokens_neox': 2817, 'n_words': 928}