<commit_before>doTimingAttackMitigation = False

import base64
import datetime
import errno
import math
import time
import threading
import shared
import hashlib
import os
import select
import socket
import random
import ssl
from struct import unpack, pack
import sys
import traceback
from binascii import hexlify
#import string
#from subprocess import call  # used when the API must execute an outside program
#from pyelliptic.openssl import OpenSSL

#import highlevelcrypto
from addresses import *
from class_objectHashHolder import objectHashHolder
from helper_generic import addDataPadding, isHostInPrivateIPRange
from helper_sql import sqlQuery
from debug import logger
import tr

# This thread is created either by the synSenderThread(for outgoing
# connections) or the singleListenerThread(for incoming connections).

class receiveDataThread(threading.Thread):

    def __init__(self):
        threading.Thread.__init__(self, name="receiveData")
        self.data = ''
        self.verackSent = False
        self.verackReceived = False

    def setup(
        self,
        sock,
        HOST,
        port,
        streamNumber,
        someObjectsOfWhichThisRemoteNodeIsAlreadyAware,
        selfInitiatedConnections,
        sendDataThreadQueue,
        objectHashHolderInstance):
        
        self.sock = sock
        self.peer = shared.Peer(HOST, port)
        self.name = "receiveData-" + self.peer.host.replace(":", ".") # ":" log parser field separator
        self.streamNumber = streamNumber
        self.objectsThatWeHaveYetToGetFromThisPeer = {}
        self.selfInitiatedConnections = selfInitiatedConnections
        self.sendDataThreadQueue = sendDataThreadQueue # used to send commands and data to the sendDataThread
        self.hostIdent = self.peer.port if ".onion" in shared.config.get('bitmessagesettings', 'onionhostname') and shared.checkSocksIP(self.peer.host) else self.peer.host
        shared.connectedHostsList[
            self.hostIdent] = 0  # The very fact that this receiveData thread exists shows that we are connected to the remote host. Let's add it to this list so that an outgoingSynSender thread doesn't try to connect to it.
        self.connectionIsOrWasFullyEstablished = False  # set to true after the remote node and I accept each other's version messages. This is needed to allow the user interface to accurately reflect the current number of connections.
        self.services = 0
        if self.streamNumber == -1:  # This was an incoming connection. Send out a version message if we accept the other node's version message.
            self.initiatedConnection = False
        else:
            self.initiatedConnection = True
            self.selfInitiatedConnections[streamNumber][self] = 0
        self.someObjectsOfWhichThisRemoteNodeIsAlreadyAware = someObjectsOfWhichThisRemoteNodeIsAlreadyAware
        self.objectHashHolderInstance = objectHashHolderInstance
        self.startTime = time.time()

    def run(self):
        logger.debug('receiveDataThread starting. ID ' + str(id(self)) + '. The size of the shared.connectedHostsList is now ' + str(len(shared.connectedHostsList)))

        while True:
            if shared.config.getint('bitmessagesettings', 'maxdownloadrate') == 0:
                downloadRateLimitBytes = float("inf")
            else:
                downloadRateLimitBytes = shared.config.getint('bitmessagesettings', 'maxdownloadrate') * 1000
            with shared.receiveDataLock:
                while shared.numberOfBytesReceivedLastSecond >= downloadRateLimitBytes:
                    if int(time.time()) == shared.lastTimeWeResetBytesReceived:
                        # If it's still the same second that it was last time then sleep.
                        time.sleep(0.3)
                    else:
                        # It's a new second. Let us clear the shared.numberOfBytesReceivedLastSecond.
                        shared.lastTimeWeResetBytesReceived = int(time.time())
                        shared.numberOfBytesReceivedLastSecond = 0
            dataLen = len(self.data)
            try:
                ssl = False
                if ((self.services & shared.NODE_SSL == shared.NODE_SSL) and
                    self.connectionIsOrWasFullyEstablished and
                    shared.haveSSL(not self.initiatedConnection)):
                    ssl = True
                    dataRecv = self.sslSock.recv(1024)
                else:
                    dataRecv = self.sock.recv(1024)
                self.data += dataRecv
                shared.numberOfBytesReceived += len(dataRecv) # for the 'network status' UI tab. The UI clears this value whenever it updates.
                shared.numberOfBytesReceivedLastSecond += len(dataRecv) # for the download rate limit
            except socket.timeout:
                logger.error ('Timeout occurred waiting for data from ' + str(self.peer) + '. Closing receiveData thread. (ID: ' + str(id(self)) + ')')
                break
            except Exception as err:
                if err.errno == 2 or (sys.platform == 'win32' and err.errno == 10035) or (sys.platform != 'win32' and err.errno == errno.EWOULDBLOCK):
                    if ssl:
                        select.select([self.sslSock], [], [])
                    else:
                        select.select([self.sock], [], [])
                    continue
                logger.error('sock.recv error. Closing receiveData thread (' + str(self.peer) + ', Thread ID: ' + str(id(self)) + ').' + str(err.errno) + "/" + str(err))
                if self.initiatedConnection and not self.connectionIsOrWasFullyEstablished:
                    shared.timeOffsetWrongCount += 1
                break
            # print 'Received', repr(self.data)
            if len(self.data) == dataLen: # If self.sock.recv returned no data:
                logger.debug('Connection to ' + str(self.peer) + ' closed. Closing receiveData thread. (ID: ' + str(id(self)) + ')')
                if self.initiatedConnection and not self.connectionIsOrWasFullyEstablished:
                    shared.timeOffsetWrongCount += 1
                break
            else:
                self.processData()

        try:
            del self.selfInitiatedConnections[self.streamNumber][self]
            logger.debug('removed self (a receiveDataThread) from selfInitiatedConnections')
        except:
            pass
        self.sendDataThreadQueue.put((0, 'shutdown','no data')) # commands the corresponding sendDataThread to shut itself down.
        try:
            del shared.connectedHostsList[self.hostIdent]
        except Exception as err:
            logger.error('Could not delete ' + str(self.hostIdent) + ' from shared.connectedHostsList.' + str(err))

        try:
            del shared.numberOfObjectsThatWeHaveYetToGetPerPeer[
                self.peer]
        except:
            pass
        shared.UISignalQueue.put(('updateNetworkStatusTab', 'no data'))
        self.checkTimeOffsetNotification()
        logger.debug('receiveDataThread ending. ID ' + str(id(self)) + '. The size of the shared.connectedHostsList is now ' + str(len(shared.connectedHostsList)))

    def antiIntersectionDelay(self, initial = False):
        # estimated time for a small object to propagate across the whole network
        delay = math.ceil(math.log(len(shared.knownNodes[self.streamNumber]) + 2, 20)) * (0.2 + objectHashHolder.size/2)
        # +2 is to avoid problems with log(0) and log(1)
        # 20 is avg connected nodes count
        # 0.2 is avg message transmission time
        now = time.time()
        if initial and now - delay < self.startTime:
            logger.debug("Initial sleeping for %.2fs", delay - (now - self.startTime))
            time.sleep(delay - (now - self.startTime))
        elif not initial:
            logger.debug("Sleeping due to missing object for %.2fs", delay)
            time.sleep(delay)

    def checkTimeOffsetNotification(self):
        if shared.timeOffsetWrongCount >= 4 and not self.connectionIsOrWasFullyEstablished:
            shared.UISignalQueue.put(('updateStatusBar', tr._translate("MainWindow", "The time on your computer, %1, may be wrong. Please verify your settings.").arg(datetime.datetime.now().strftime("%H:%M:%S"))))

    def processData(self):
        if len(self.data) < shared.Header.size:  # if so little of the data has arrived that we can't even read the checksum then wait for more data.
            return
        
        magic,command,payloadLength,checksum = shared.Header.unpack(self.data[:shared.Header.size])
        if magic != 0xE9BEB4D9:
            self.data = ""
            return
        if payloadLength > 1600100: # ~1.6 MB which is the maximum possible size of an inv message.
            logger.info('The incoming message, which we have not yet download, is too large. Ignoring it. (unfortunately there is no way to tell the other node to stop sending it except to disconnect.) Message size: %s' % payloadLength)
            self.data = self.data[payloadLength + shared.Header.size:]
            del magic,command,payloadLength,checksum # we don't need these anymore and better to clean them now before the recursive call rather than after
            self.processData()
            return
        if len(self.data) < payloadLength + shared.Header.size:  # check if the whole message has arrived yet.
            return
        payload = self.data[shared.Header.size:payloadLength + shared.Header.size]
        if checksum != hashlib.sha512(payload).digest()[0:4]:  # test the checksum in the message.
            logger.error('Checksum incorrect. Clearing this message.')
            self.data = self.data[payloadLength + shared.Header.size:]
            del magic,command,payloadLength,checksum,payload # better to clean up before the recursive call
            self.processData()
            return

        # The time we've last seen this node is obviously right now since we
        # just received valid data from it. So update the knownNodes list so
        # that other peers can be made aware of its existance.
        if self.initiatedConnection and self.connectionIsOrWasFullyEstablished:  # The remote port is only something we should share with others if it is the remote node's incoming port (rather than some random operating-system-assigned outgoing port).
            with shared.knownNodesLock:
                shared.knownNodes[self.streamNumber][self.peer] = int(time.time())
        
        #Strip the nulls
        command = command.rstrip('\x00')
        logger.debug('remoteCommand ' + repr(command) + ' from ' + str(self.peer))
        
        try:
            #TODO: Use a dispatcher here
            if command == 'error':
                self.recerror(payload)
            elif not self.connectionIsOrWasFullyEstablished:
                if command == 'version':
                    self.recversion(payload)
                elif command == 'verack':
                    self.recverack()
            else:
                if command == 'addr':
                    self.recaddr(payload)
                elif command == 'inv':
                    self.recinv(payload)
                elif command == 'getdata':
                    self.recgetdata(payload)
                elif command == 'object':
                    self.recobject(payload)
                elif command == 'ping':
                    self.sendpong(payload)
                #elif command == 'pong':
                #    pass
        except varintDecodeError as e:
            logger.debug("There was a problem with a varint while processing a message from the wire. Some details: %s" % e)
        except Exception as e:
            logger.critical("Critical error in a receiveDataThread: \n%s" % traceback.format_exc())
        
        del payload
        self.data = self.data[payloadLength + shared.Header.size:] # take this message out and then process the next message

        if self.data == '': # if there are no more messages
            while len(self.objectsThatWeHaveYetToGetFromThisPeer) > 0:
                shared.numberOfInventoryLookupsPerformed += 1
                objectHash, = random.sample(
                    self.objectsThatWeHaveYetToGetFromThisPeer, 1)
                if objectHash in shared.inventory:
                    logger.debug('Inventory already has object listed in inv message.')
                    del self.objectsThatWeHaveYetToGetFromThisPeer[objectHash]
                else:
                    # We don't have the object in our inventory. Let's request it.
                    self.sendgetdata(objectHash)
                    del self.objectsThatWeHaveYetToGetFromThisPeer[
                        objectHash]  # It is possible that the remote node might not respond with the object. In that case, we'll very likely get it from someone else anyway.
                    if len(self.objectsThatWeHaveYetToGetFromThisPeer) == 0:
                        logger.debug('(concerning' + str(self.peer) + ') number of objectsThatWeHaveYetToGetFromThisPeer is now 0')
                        try:
                            del shared.numberOfObjectsThatWeHaveYetToGetPerPeer[
                                self.peer]  # this data structure is maintained so that we can keep track of how many total objects, across all connections, are currently outstanding. If it goes too high it can indicate that we are under attack by multiple nodes working together.
                        except:
                            pass
                    break
                if len(self.objectsThatWeHaveYetToGetFromThisPeer) == 0:
                    # We had objectsThatWeHaveYetToGetFromThisPeer but the loop ran, they were all in our inventory, and now we don't have any to get anymore.
                    logger.debug('(concerning' + str(self.peer) + ') number of objectsThatWeHaveYetToGetFromThisPeer is now 0')
                    try:
                        del shared.numberOfObjectsThatWeHaveYetToGetPerPeer[
                            self.peer]  # this data structure is maintained so that we can keep track of how many total objects, across all connections, are currently outstanding. If it goes too high it can indicate that we are under attack by multiple nodes working together.
                    except:
                        pass
            if len(self.objectsThatWeHaveYetToGetFromThisPeer) > 0:
                logger.debug('(concerning' + str(self.peer) + ') number of objectsThatWeHaveYetToGetFromThisPeer is now ' + str(len(self.objectsThatWeHaveYetToGetFromThisPeer)))

                shared.numberOfObjectsThatWeHaveYetToGetPerPeer[self.peer] = len(
                    self.objectsThatWeHaveYetToGetFromThisPeer)  # this data structure is maintained so that we can keep track of how many total objects, across all connections, are currently outstanding. If it goes too high it can indicate that we are under attack by multiple nodes working together.
        self.processData()


    def sendpong(self):
        logger.debug('Sending pong')
        self.sendDataThreadQueue.put((0, 'sendRawData', shared.CreatePacket('pong')))


    def recverack(self):
        logger.debug('verack received')
        self.verackReceived = True
        if self.verackSent:
            # We have thus both sent and received a verack.
            self.connectionFullyEstablished()

    def connectionFullyEstablished(self):
        if self.connectionIsOrWasFullyEstablished:
            # there is no reason to run this function a second time
            return
        self.connectionIsOrWasFullyEstablished = True
        shared.timeOffsetWrongCount = 0

        self.sslSock = self.sock
        if ((self.services & shared.NODE_SSL == shared.NODE_SSL) and
            shared.haveSSL(not self.initiatedConnection)):
            logger.debug("Initialising TLS")
            self.sslSock = ssl.wrap_socket(self.sock, keyfile = os.path.join(shared.codePath(), 'sslkeys', 'key.pem'), certfile = os.path.join(shared.codePath(), 'sslkeys', 'cert.pem'), server_side = not self.initiatedConnection, ssl_version=ssl.PROTOCOL_TLSv1, do_handshake_on_connect=False, ciphers='AECDH-AES256-SHA')
            if hasattr(self.sslSock, "context"):
                self.sslSock.context.set_ecdh_curve("secp256k1")
            while True:
                try:
                    self.sslSock.do_handshake()
                    break
                except ssl.SSLError as e:
                    if e.errno == 2:
                        select.select([self.sslSock], [self.sslSock], [])
                    else:
                        break
                except:
                    break
        # Command the corresponding sendDataThread to set its own connectionIsOrWasFullyEstablished variable to True also
        self.sendDataThreadQueue.put((0, 'connectionIsOrWasFullyEstablished', (self.services, self.sslSock)))

        if not self.initiatedConnection:
            shared.clientHasReceivedIncomingConnections = True
            shared.UISignalQueue.put(('setStatusIcon', 'green'))
        self.sock.settimeout(
            600)  # We'll send out a pong every 5 minutes to make sure the connection stays alive if there has been no other traffic to send lately.<commit_msg>TLS handshake fix<commit_after>doTimingAttackMitigation = False

import base64
import datetime
import errno
import math
import time
import threading
import shared
import hashlib
import os
import select
import socket
import random
import ssl
from struct import unpack, pack
import sys
import traceback
from binascii import hexlify
#import string
#from subprocess import call  # used when the API must execute an outside program
#from pyelliptic.openssl import OpenSSL

#import highlevelcrypto
from addresses import *
from class_objectHashHolder import objectHashHolder
from helper_generic import addDataPadding, isHostInPrivateIPRange
from helper_sql import sqlQuery
from debug import logger
import tr

# This thread is created either by the synSenderThread(for outgoing
# connections) or the singleListenerThread(for incoming connections).

class receiveDataThread(threading.Thread):

    def __init__(self):
        threading.Thread.__init__(self, name="receiveData")
        self.data = ''
        self.verackSent = False
        self.verackReceived = False

    def setup(
        self,
        sock,
        HOST,
        port,
        streamNumber,
        someObjectsOfWhichThisRemoteNodeIsAlreadyAware,
        selfInitiatedConnections,
        sendDataThreadQueue,
        objectHashHolderInstance):
        
        self.sock = sock
        self.peer = shared.Peer(HOST, port)
        self.name = "receiveData-" + self.peer.host.replace(":", ".") # ":" log parser field separator
        self.streamNumber = streamNumber
        self.objectsThatWeHaveYetToGetFromThisPeer = {}
        self.selfInitiatedConnections = selfInitiatedConnections
        self.sendDataThreadQueue = sendDataThreadQueue # used to send commands and data to the sendDataThread
        self.hostIdent = self.peer.port if ".onion" in shared.config.get('bitmessagesettings', 'onionhostname') and shared.checkSocksIP(self.peer.host) else self.peer.host
        shared.connectedHostsList[
            self.hostIdent] = 0  # The very fact that this receiveData thread exists shows that we are connected to the remote host. Let's add it to this list so that an outgoingSynSender thread doesn't try to connect to it.
        self.connectionIsOrWasFullyEstablished = False  # set to true after the remote node and I accept each other's version messages. This is needed to allow the user interface to accurately reflect the current number of connections.
        self.services = 0
        if self.streamNumber == -1:  # This was an incoming connection. Send out a version message if we accept the other node's version message.
            self.initiatedConnection = False
        else:
            self.initiatedConnection = True
            self.selfInitiatedConnections[streamNumber][self] = 0
        self.someObjectsOfWhichThisRemoteNodeIsAlreadyAware = someObjectsOfWhichThisRemoteNodeIsAlreadyAware
        self.objectHashHolderInstance = objectHashHolderInstance
        self.startTime = time.time()

    def run(self):
        logger.debug('receiveDataThread starting. ID ' + str(id(self)) + '. The size of the shared.connectedHostsList is now ' + str(len(shared.connectedHostsList)))

        while True:
            if shared.config.getint('bitmessagesettings', 'maxdownloadrate') == 0:
                downloadRateLimitBytes = float("inf")
            else:
                downloadRateLimitBytes = shared.config.getint('bitmessagesettings', 'maxdownloadrate') * 1000
            with shared.receiveDataLock:
                while shared.numberOfBytesReceivedLastSecond >= downloadRateLimitBytes:
                    if int(time.time()) == shared.lastTimeWeResetBytesReceived:
                        # If it's still the same second that it was last time then sleep.
                        time.sleep(0.3)
                    else:
                        # It's a new second. Let us clear the shared.numberOfBytesReceivedLastSecond.
                        shared.lastTimeWeResetBytesReceived = int(time.time())
                        shared.numberOfBytesReceivedLastSecond = 0
            dataLen = len(self.data)
            try:
                ssl = False
                if ((self.services & shared.NODE_SSL == shared.NODE_SSL) and
                    self.connectionIsOrWasFullyEstablished and
                    shared.haveSSL(not self.initiatedConnection)):
                    ssl = True
                    dataRecv = self.sslSock.recv(1024)
                else:
                    dataRecv = self.sock.recv(1024)
                self.data += dataRecv
                shared.numberOfBytesReceived += len(dataRecv) # for the 'network status' UI tab. The UI clears this value whenever it updates.
                shared.numberOfBytesReceivedLastSecond += len(dataRecv) # for the download rate limit
            except socket.timeout:
                logger.error ('Timeout occurred waiting for data from ' + str(self.peer) + '. Closing receiveData thread. (ID: ' + str(id(self)) + ')')
                break
            except Exception as err:
                if err.errno == 2 or (sys.platform == 'win32' and err.errno == 10035) or (sys.platform != 'win32' and err.errno == errno.EWOULDBLOCK):
                    if ssl:
                        select.select([self.sslSock], [], [])
                    else:
                        select.select([self.sock], [], [])
                    continue
                logger.error('sock.recv error. Closing receiveData thread (' + str(self.peer) + ', Thread ID: ' + str(id(self)) + ').' + str(err.errno) + "/" + str(err))
                if self.initiatedConnection and not self.connectionIsOrWasFullyEstablished:
                    shared.timeOffsetWrongCount += 1
                break
            # print 'Received', repr(self.data)
            if len(self.data) == dataLen: # If self.sock.recv returned no data:
                logger.debug('Connection to ' + str(self.peer) + ' closed. Closing receiveData thread. (ID: ' + str(id(self)) + ')')
                if self.initiatedConnection and not self.connectionIsOrWasFullyEstablished:
                    shared.timeOffsetWrongCount += 1
                break
            else:
                self.processData()

        try:
            del self.selfInitiatedConnections[self.streamNumber][self]
            logger.debug('removed self (a receiveDataThread) from selfInitiatedConnections')
        except:
            pass
        self.sendDataThreadQueue.put((0, 'shutdown','no data')) # commands the corresponding sendDataThread to shut itself down.
        try:
            del shared.connectedHostsList[self.hostIdent]
        except Exception as err:
            logger.error('Could not delete ' + str(self.hostIdent) + ' from shared.connectedHostsList.' + str(err))

        try:
            del shared.numberOfObjectsThatWeHaveYetToGetPerPeer[
                self.peer]
        except:
            pass
        shared.UISignalQueue.put(('updateNetworkStatusTab', 'no data'))
        self.checkTimeOffsetNotification()
        logger.debug('receiveDataThread ending. ID ' + str(id(self)) + '. The size of the shared.connectedHostsList is now ' + str(len(shared.connectedHostsList)))

    def antiIntersectionDelay(self, initial = False):
        # estimated time for a small object to propagate across the whole network
        delay = math.ceil(math.log(len(shared.knownNodes[self.streamNumber]) + 2, 20)) * (0.2 + objectHashHolder.size/2)
        # +2 is to avoid problems with log(0) and log(1)
        # 20 is avg connected nodes count
        # 0.2 is avg message transmission time
        now = time.time()
        if initial and now - delay < self.startTime:
            logger.debug("Initial sleeping for %.2fs", delay - (now - self.startTime))
            time.sleep(delay - (now - self.startTime))
        elif not initial:
            logger.debug("Sleeping due to missing object for %.2fs", delay)
            time.sleep(delay)

    def checkTimeOffsetNotification(self):
        if shared.timeOffsetWrongCount >= 4 and not self.connectionIsOrWasFullyEstablished:
            shared.UISignalQueue.put(('updateStatusBar', tr._translate("MainWindow", "The time on your computer, %1, may be wrong. Please verify your settings.").arg(datetime.datetime.now().strftime("%H:%M:%S"))))

    def processData(self):
        if len(self.data) < shared.Header.size:  # if so little of the data has arrived that we can't even read the checksum then wait for more data.
            return
        
        magic,command,payloadLength,checksum = shared.Header.unpack(self.data[:shared.Header.size])
        if magic != 0xE9BEB4D9:
            self.data = ""
            return
        if payloadLength > 1600100: # ~1.6 MB which is the maximum possible size of an inv message.
            logger.info('The incoming message, which we have not yet download, is too large. Ignoring it. (unfortunately there is no way to tell the other node to stop sending it except to disconnect.) Message size: %s' % payloadLength)
            self.data = self.data[payloadLength + shared.Header.size:]
            del magic,command,payloadLength,checksum # we don't need these anymore and better to clean them now before the recursive call rather than after
            self.processData()
            return
        if len(self.data) < payloadLength + shared.Header.size:  # check if the whole message has arrived yet.
            return
        payload = self.data[shared.Header.size:payloadLength + shared.Header.size]
        if checksum != hashlib.sha512(payload).digest()[0:4]:  # test the checksum in the message.
            logger.error('Checksum incorrect. Clearing this message.')
            self.data = self.data[payloadLength + shared.Header.size:]
            del magic,command,payloadLength,checksum,payload # better to clean up before the recursive call
            self.processData()
            return

        # The time we've last seen this node is obviously right now since we
        # just received valid data from it. So update the knownNodes list so
        # that other peers can be made aware of its existance.
        if self.initiatedConnection and self.connectionIsOrWasFullyEstablished:  # The remote port is only something we should share with others if it is the remote node's incoming port (rather than some random operating-system-assigned outgoing port).
            with shared.knownNodesLock:
                shared.knownNodes[self.streamNumber][self.peer] = int(time.time())
        
        #Strip the nulls
        command = command.rstrip('\x00')
        logger.debug('remoteCommand ' + repr(command) + ' from ' + str(self.peer))
        
        try:
            #TODO: Use a dispatcher here
            if command == 'error':
                self.recerror(payload)
            elif not self.connectionIsOrWasFullyEstablished:
                if command == 'version':
                    self.recversion(payload)
                elif command == 'verack':
                    self.recverack()
            else:
                if command == 'addr':
                    self.recaddr(payload)
                elif command == 'inv':
                    self.recinv(payload)
                elif command == 'getdata':
                    self.recgetdata(payload)
                elif command == 'object':
                    self.recobject(payload)
                elif command == 'ping':
                    self.sendpong(payload)
                #elif command == 'pong':
                #    pass
        except varintDecodeError as e:
            logger.debug("There was a problem with a varint while processing a message from the wire. Some details: %s" % e)
        except Exception as e:
            logger.critical("Critical error in a receiveDataThread: \n%s" % traceback.format_exc())
        
        del payload
        self.data = self.data[payloadLength + shared.Header.size:] # take this message out and then process the next message

        if self.data == '': # if there are no more messages
            while len(self.objectsThatWeHaveYetToGetFromThisPeer) > 0:
                shared.numberOfInventoryLookupsPerformed += 1
                objectHash, = random.sample(
                    self.objectsThatWeHaveYetToGetFromThisPeer, 1)
                if objectHash in shared.inventory:
                    logger.debug('Inventory already has object listed in inv message.')
                    del self.objectsThatWeHaveYetToGetFromThisPeer[objectHash]
                else:
                    # We don't have the object in our inventory. Let's request it.
                    self.sendgetdata(objectHash)
                    del self.objectsThatWeHaveYetToGetFromThisPeer[
                        objectHash]  # It is possible that the remote node might not respond with the object. In that case, we'll very likely get it from someone else anyway.
                    if len(self.objectsThatWeHaveYetToGetFromThisPeer) == 0:
                        logger.debug('(concerning' + str(self.peer) + ') number of objectsThatWeHaveYetToGetFromThisPeer is now 0')
                        try:
                            del shared.numberOfObjectsThatWeHaveYetToGetPerPeer[
                                self.peer]  # this data structure is maintained so that we can keep track of how many total objects, across all connections, are currently outstanding. If it goes too high it can indicate that we are under attack by multiple nodes working together.
                        except:
                            pass
                    break
                if len(self.objectsThatWeHaveYetToGetFromThisPeer) == 0:
                    # We had objectsThatWeHaveYetToGetFromThisPeer but the loop ran, they were all in our inventory, and now we don't have any to get anymore.
                    logger.debug('(concerning' + str(self.peer) + ') number of objectsThatWeHaveYetToGetFromThisPeer is now 0')
                    try:
                        del shared.numberOfObjectsThatWeHaveYetToGetPerPeer[
                            self.peer]  # this data structure is maintained so that we can keep track of how many total objects, across all connections, are currently outstanding. If it goes too high it can indicate that we are under attack by multiple nodes working together.
                    except:
                        pass
            if len(self.objectsThatWeHaveYetToGetFromThisPeer) > 0:
                logger.debug('(concerning' + str(self.peer) + ') number of objectsThatWeHaveYetToGetFromThisPeer is now ' + str(len(self.objectsThatWeHaveYetToGetFromThisPeer)))

                shared.numberOfObjectsThatWeHaveYetToGetPerPeer[self.peer] = len(
                    self.objectsThatWeHaveYetToGetFromThisPeer)  # this data structure is maintained so that we can keep track of how many total objects, across all connections, are currently outstanding. If it goes too high it can indicate that we are under attack by multiple nodes working together.
        self.processData()


    def sendpong(self):
        logger.debug('Sending pong')
        self.sendDataThreadQueue.put((0, 'sendRawData', shared.CreatePacket('pong')))


    def recverack(self):
        logger.debug('verack received')
        self.verackReceived = True
        if self.verackSent:
            # We have thus both sent and received a verack.
            self.connectionFullyEstablished()

    def connectionFullyEstablished(self):
        if self.connectionIsOrWasFullyEstablished:
            # there is no reason to run this function a second time
            return
        self.connectionIsOrWasFullyEstablished = True
        shared.timeOffsetWrongCount = 0

        self.sslSock = self.sock
        if ((self.services & shared.NODE_SSL == shared.NODE_SSL) and
            shared.haveSSL(not self.initiatedConnection)):
            logger.debug("Initialising TLS")
            self.sslSock = ssl.wrap_socket(self.sock, keyfile = os.path.join(shared.codePath(), 'sslkeys', 'key.pem'), certfile = os.path.join(shared.codePath(), 'sslkeys', 'cert.pem'), server_side = not self.initiatedConnection, ssl_version=ssl.PROTOCOL_TLSv1, do_handshake_on_connect=False, ciphers='AECDH-AES256-SHA')
            if hasattr(self.sslSock, "context"):
                self.sslSock.context.set_ecdh_curve("secp256k1")
            while True:
                try:
                    self.sslSock.do_handshake()
                    logger.debug("TLS handshake success")
                    break
                except ssl.SSLWantReadError:
                    logger.debug("Waiting for SSL socket handhake read")
                    select.select([self.sslSock], [], [], 10)
                except ssl.SSLWantWriteError:
                    logger.debug("Waiting for SSL socket handhake write")
                    select.select([], [self.sslSock], [], 10)
                except:
                    logger.debug("SSL socket handhake failed, shutting down connection")
                    self.sendDataThreadQueue.put((0, 'shutdown','tls handshake fail'))
                    return
        # Command the corresponding sendDataThread to set its own connectionIsOrWasFullyEstablished variable to True also
        self.sendDataThreadQueue.put((0, 'connectionIsOrWasFullyEstablished', (self.services, self.sslSock)))

        if not self.initiatedConnection:
            shared.clientHasReceivedIncomingConnections = True
            shared.UISignalQueue.put(('setStatusIcon', 'green'))
        self.sock.settimeout(
            600)  # We'll send out a pong every 5 minutes to make sure the connection stays alive if there has been no other traffic to send lately.
{'fraction_non_alphanumeric': 0.07896090534979423, 'fraction_numerical': 0.005601280292638317, 'mean_word_length': 2.237394763622907, 'pattern_counts': {'":': 4, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7295886', 'n_tokens_mistral': 9170, 'n_tokens_neox': 8642, 'n_words': 2729}