Refactor packet header extraction and generation

Demote payloadLength from class instance variable to processData local variable as no other function was using it
Improve processData:
 -Utilise shared.Header
 -Use a memoryview to reduce memory overhead
 -Clean up variables before a recursive call
 -Strip null bytes from command
Refactor sendData
Various functions:
 -Use shared.CreatePacket to generate packets
Fix typo in _checkIPv4Address

src/class_receiveDataThread.py

@@ -47,7 +47,6 @@ class receiveDataThread(threading.Thread):
         self.sock = sock
         self.peer = shared.Peer(HOST, port)
         self.streamNumber = streamNumber
-        self.payloadLength = 0  # This is the protocol payload length thus it doesn't include the 24 byte message header
         self.objectsThatWeHaveYetToGetFromThisPeer = {}
         self.selfInitiatedConnections = selfInitiatedConnections
         self.sendDataThreadQueue = sendDataThreadQueue # used to send commands and data to the sendDataThread
@@ -113,28 +112,39 @@ class receiveDataThread(threading.Thread):
             # with shared.printLock:
             #   print 'self.data is currently ', repr(self.data)
             #
-        if len(self.data) < 24:  # if so little of the data has arrived that we can't even read the checksum then wait for more data.
+        if len(self.data) < shared.Header.size:  # if so little of the data has arrived that we can't even read the checksum then wait for more data.
             return
-        if self.data[0:4] != '\xe9\xbe\xb4\xd9':
+        #Use a memoryview so we don't copy data unnecessarily
+        view = memoryview(self.data)
+        magic,command,payloadLength,checksum = shared.Header.unpack(view[:shared.Header.size])
+        view = view[shared.Header.size:]
+        if magic != 0xE9BEB4D9:
             #if shared.verbose >= 1:
             #    with shared.printLock:
             #        print 'The magic bytes were not correct. First 40 bytes of data: ' + repr(self.data[0:40])
 
             self.data = ""
             return
-        self.payloadLength, = unpack('>L', self.data[16:20])
+        if payloadLength > 20000000:
-        if self.payloadLength > 20000000:
+            logger.info('The incoming message, which we have not yet download, is too large. Ignoring it. (unfortunately there is no way to tell the other node to stop sending it except to disconnect.) Message size: %s' % payloadLength)
-            logger.info('The incoming message, which we have not yet download, is too large. Ignoring it. (unfortunately there is no way to tell the other node to stop sending it except to disconnect.) Message size: %s' % self.payloadLength)
+            self.data = view[payloadLength:].tobytes()
-            self.data = self.data[self.payloadLength + 24:]
+            del view,magic,command,payloadLength,checksum #we don't need these anymore and better to clean them now before the recursive call rather than after
             self.processData()
             return
-        if len(self.data) < self.payloadLength + 24:  # check if the whole message has arrived yet.
+        if len(view) < payloadLength:  # check if the whole message has arrived yet.
             return
-        if self.data[20:24] != hashlib.sha512(self.data[24:self.payloadLength + 24]).digest()[0:4]:  # test the checksum in the message. If it is correct...
+        payload = view[:payloadLength]
+        if checksum != hashlib.sha512(payload).digest()[0:4]:  # test the checksum in the message. If it is correct...
             print 'Checksum incorrect. Clearing this message.'
-            self.data = self.data[self.payloadLength + 24:]
+            self.data = view[payloadLength:].tobytes()
+            del view,magic,command,payloadLength,checksum,payload #again better to clean up before the recursive call
             self.processData()
             return
+        
+        #We can now revert back to bytestrings and take this message out
+        payload = payload.tobytes()
+        self.data = view[payloadLength:].tobytes()
+        del view,magic,payloadLength,checksum
         # The time we've last seen this node is obviously right now since we
         # just received valid data from it. So update the knownNodes list so
         # that other peers can be made aware of its existance.
@@ -143,37 +153,39 @@ class receiveDataThread(threading.Thread):
             shared.knownNodes[self.streamNumber][self.peer] = int(time.time())
             shared.knownNodesLock.release()
         
-        remoteCommand = self.data[4:16]
+        #Strip the nulls
+        command = command.rstrip('\x00')
         with shared.printLock:
-            print 'remoteCommand', repr(remoteCommand.replace('\x00', '')), ' from', self.peer
+            print 'remoteCommand', repr(command), ' from', self.peer
+        
+        #TODO: Use a dispatcher here
+        if not self.connectionIsOrWasFullyEstablished:
+            if command == 'version':
+                self.recversion(payload)
+            elif command == 'verack':
+                self.recverack()
+        else:
+            if command == 'addr':
+                self.recaddr(payload)
+            elif command == 'getpubkey':
+                shared.checkAndSharegetpubkeyWithPeers(payload)
+            elif command == 'pubkey':
+                self.recpubkey(payload)
+            elif command == 'inv':
+                self.recinv(payload)
+            elif command == 'getdata':
+                self.recgetdata(payload)
+            elif command == 'msg':
+                self.recmsg(payload)
+            elif command == 'broadcast':
+                self.recbroadcast(payload)
+            elif command == 'ping':
+                self.sendpong(payload)
+            #elif command == 'pong':
+            #    pass
+            #elif command == 'alert':
+            #    pass
 
-        if remoteCommand == 'version\x00\x00\x00\x00\x00' and not self.connectionIsOrWasFullyEstablished:
-            self.recversion(self.data[24:self.payloadLength + 24])
-        elif remoteCommand == 'verack\x00\x00\x00\x00\x00\x00' and not self.connectionIsOrWasFullyEstablished:
-            self.recverack()
-        elif remoteCommand == 'addr\x00\x00\x00\x00\x00\x00\x00\x00' and self.connectionIsOrWasFullyEstablished:
-            self.recaddr(self.data[24:self.payloadLength + 24])
-        elif remoteCommand == 'getpubkey\x00\x00\x00' and self.connectionIsOrWasFullyEstablished:
-            shared.checkAndSharegetpubkeyWithPeers(self.data[24:self.payloadLength + 24])
-        elif remoteCommand == 'pubkey\x00\x00\x00\x00\x00\x00' and self.connectionIsOrWasFullyEstablished:
-            self.recpubkey(self.data[24:self.payloadLength + 24])
-        elif remoteCommand == 'inv\x00\x00\x00\x00\x00\x00\x00\x00\x00' and self.connectionIsOrWasFullyEstablished:
-            self.recinv(self.data[24:self.payloadLength + 24])
-        elif remoteCommand == 'getdata\x00\x00\x00\x00\x00' and self.connectionIsOrWasFullyEstablished:
-            self.recgetdata(self.data[24:self.payloadLength + 24])
-        elif remoteCommand == 'msg\x00\x00\x00\x00\x00\x00\x00\x00\x00' and self.connectionIsOrWasFullyEstablished:
-            self.recmsg(self.data[24:self.payloadLength + 24])
-        elif remoteCommand == 'broadcast\x00\x00\x00' and self.connectionIsOrWasFullyEstablished:
-            self.recbroadcast(self.data[24:self.payloadLength + 24])
-        elif remoteCommand == 'ping\x00\x00\x00\x00\x00\x00\x00\x00' and self.connectionIsOrWasFullyEstablished:
-            self.sendpong()
-        elif remoteCommand == 'pong\x00\x00\x00\x00\x00\x00\x00\x00' and self.connectionIsOrWasFullyEstablished:
-            pass
-        elif remoteCommand == 'alert\x00\x00\x00\x00\x00\x00\x00' and self.connectionIsOrWasFullyEstablished:
-            pass
-
-        self.data = self.data[
-            self.payloadLength + 24:]  # take this message out and then process the next message
         if self.data == '':
             while len(self.objectsThatWeHaveYetToGetFromThisPeer) > 0:
                 shared.numberOfInventoryLookupsPerformed += 1
@@ -226,7 +238,7 @@ class receiveDataThread(threading.Thread):
 
     def sendpong(self):
         print 'Sending pong'
-        self.sendDataThreadQueue.put((0, 'sendRawData', '\xE9\xBE\xB4\xD9\x70\x6F\x6E\x67\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xcf\x83\xe1\x35'))
+        self.sendDataThreadQueue.put((0, 'sendRawData', shared.CreatePacket('pong')))
 
 
     def recverack(self):
@@ -297,7 +309,7 @@ class receiveDataThread(threading.Thread):
         for hash, storedValue in bigInvList.items():
             payload += hash
             numberOfObjectsInInvMessage += 1
-            if numberOfObjectsInInvMessage >= 50000:  # We can only send a max of 50000 items per inv message but we may have more objects to advertise. They must be split up into multiple inv messages.
+            if numberOfObjectsInInvMessage == 50000:  # We can only send a max of 50000 items per inv message but we may have more objects to advertise. They must be split up into multiple inv messages.
                 self.sendinvMessageToJustThisOnePeer(
                     numberOfObjectsInInvMessage, payload)
                 payload = ''
@@ -311,13 +323,9 @@ class receiveDataThread(threading.Thread):
     # function for broadcasting invs to everyone in our stream.
     def sendinvMessageToJustThisOnePeer(self, numberOfObjects, payload):
         payload = encodeVarint(numberOfObjects) + payload
-        headerData = '\xe9\xbe\xb4\xd9'  # magic bits, slighly different from Bitcoin's magic bits.
-        headerData += 'inv\x00\x00\x00\x00\x00\x00\x00\x00\x00'
-        headerData += pack('>L', len(payload))
-        headerData += hashlib.sha512(payload).digest()[:4]
         with shared.printLock:
             print 'Sending huge inv message with', numberOfObjects, 'objects to just this one peer'
-        self.sendDataThreadQueue.put((0, 'sendRawData', headerData + payload))
+        self.sendDataThreadQueue.put((0, 'sendRawData', shared.CreatePacket('inv', payload)))
 
     def _sleepForTimingAttackMitigation(self, sleepTime):
         # We don't need to do the timing attack mitigation if we are
@@ -457,12 +465,7 @@ class receiveDataThread(threading.Thread):
             print 'sending getdata to retrieve object with hash:', hash.encode('hex')
 
         payload = '\x01' + hash
-        headerData = '\xe9\xbe\xb4\xd9'  # magic bits, slighly different from Bitcoin's magic bits.
+        self.sendDataThreadQueue.put((0, 'sendRawData', shared.CreatePacket('getdata', payload)))
-        headerData += 'getdata\x00\x00\x00\x00\x00'
-        headerData += pack('>L', len(
-            payload))  # payload length. Note that we add an extra 8 for the nonce.
-        headerData += hashlib.sha512(payload).digest()[:4]
-        self.sendDataThreadQueue.put((0, 'sendRawData', headerData + payload))
 
 
     # We have received a getdata request from our peer
@@ -499,34 +502,17 @@ class receiveDataThread(threading.Thread):
 
     # Our peer has requested (in a getdata message) that we send an object.
     def sendData(self, objectType, payload):
-        headerData = '\xe9\xbe\xb4\xd9'  # magic bits, slighly different from Bitcoin's magic bits.
+        # pubkeyrequest??
-        if objectType == 'pubkey':
+        if objectType == 'pubkeyrequest':
-            with shared.printLock:
+            objectType = 'getpubkey'
-                print 'sending pubkey'
+        elif objectType != 'pubkey' and objectType != 'getpubkey' and
-
+            objectType != 'msg' and objectType != 'broadcast':
-            headerData += 'pubkey\x00\x00\x00\x00\x00\x00'
-        elif objectType == 'getpubkey' or objectType == 'pubkeyrequest':
-            with shared.printLock:
-                print 'sending getpubkey'
-
-            headerData += 'getpubkey\x00\x00\x00'
-        elif objectType == 'msg':
-            with shared.printLock:
-                print 'sending msg'
-
-            headerData += 'msg\x00\x00\x00\x00\x00\x00\x00\x00\x00'
-        elif objectType == 'broadcast':
-            with shared.printLock:
-                print 'sending broadcast'
-
-            headerData += 'broadcast\x00\x00\x00'
-        else:
             sys.stderr.write(
                 'Error: sendData has been asked to send a strange objectType: %s\n' % str(objectType))
             return
-        headerData += pack('>L', len(payload))  # payload length.
+        with shared.printLock:
-        headerData += hashlib.sha512(payload).digest()[:4]
+            print 'sending', objectType
-        self.sendDataThreadQueue.put((0, 'sendRawData', headerData + payload))
+        self.sendDataThreadQueue.put((0, 'sendRawData', shared.CreatePacket(objectType, payload)))
 
 
     # Advertise this object to all of our peers
@@ -545,7 +531,7 @@ class receiveDataThread(threading.Thread):
         if host[0] == '\x0A':
             print 'Ignoring IP address in private range:', hostFromAddrMessage
             return False
-        if host[0:2] == '\xC0A8':
+        if host[0:2] == '\xC0\xA8':
             print 'Ignoring IP address in private range:', hostFromAddrMessage
             return False
         return True
@@ -745,11 +731,7 @@ class receiveDataThread(threading.Thread):
                 payload += pack('>H', PORT)  # remote port
 
         payload = encodeVarint(numberOfAddressesInAddrMessage) + payload
-        datatosend = '\xE9\xBE\xB4\xD9addr\x00\x00\x00\x00\x00\x00\x00\x00'
+        self.sendDataThreadQueue.put((0, 'sendRawData', shared.CreatePacket('addr', payload)))
-        datatosend = datatosend + pack('>L', len(payload))  # payload length
-        datatosend = datatosend + hashlib.sha512(payload).digest()[0:4]
-        datatosend = datatosend + payload
-        self.sendDataThreadQueue.put((0, 'sendRawData', datatosend))
 
 
     # We have received a version message
@@ -832,7 +814,7 @@ class receiveDataThread(threading.Thread):
     def sendverack(self):
         with shared.printLock:
             print 'Sending verack'
-        self.sendDataThreadQueue.put((0, 'sendRawData', '\xE9\xBE\xB4\xD9\x76\x65\x72\x61\x63\x6B\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xcf\x83\xe1\x35'))
+        self.sendDataThreadQueue.put((0, 'sendRawData', shared.CreatePacket('verack')))
         self.verackSent = True
         if self.verackReceived:
             self.connectionFullyEstablished()