preparation for switch from 32 bit time to 64 bit time in protocol

bitmessagemain.py

@@ -511,7 +511,16 @@ class receiveDataThread(QThread):
         if not self.isProofOfWorkSufficient(data):
             print 'Proof of work in broadcast message insufficient.'
             return
-        embeddedTime, = unpack('>I',data[8:12])
+        readPosition = 8 #bypass the nonce
+        embeddedTime, = unpack('>I',data[readPosition:readPosition+4])
+
+        #This section is used for the transition from 32 bit time to 64 bit time in the protocol.
+        if embeddedTime == 0:
+            embeddedTime, = unpack('>Q',data[readPosition:readPosition+8])
+            readPosition += 8
+        else:
+            readPosition += 4
+
         if embeddedTime > (int(time.time())+10800): #prevent funny business
             print 'The embedded time in this broadcast message is more than three hours in the future. That doesn\'t make sense. Ignoring message.'
             return
@@ -539,7 +548,7 @@ class receiveDataThread(QThread):
         self.emit(SIGNAL("incrementNumberOfBroadcastsProcessed()"))
 
 
-        self.processbroadcast(data)#When this function returns, we will have either successfully processed this broadcast because we are interested in it, ignored it because we aren't interested in it, or found problem with the broadcast that warranted ignoring it.
+        self.processbroadcast(readPosition,data)#When this function returns, we will have either successfully processed this broadcast because we are interested in it, ignored it because we aren't interested in it, or found problem with the broadcast that warranted ignoring it.
 
         # Let us now set lengthOfTimeWeShouldUseToProcessThisMessage. If we haven't used the specified amount of time, we shall sleep. These values are mostly the same values used for msg messages although broadcast messages are processed faster.
         if len(data) > 100000000: #Size is greater than 100 megabytes
@@ -563,8 +572,7 @@ class receiveDataThread(QThread):
         printLock.release()
 
     #A broadcast message has a valid time and POW and requires processing. The recbroadcast function calls this one.
-    def processbroadcast(self,data):
+    def processbroadcast(self,readPosition,data):
-        readPosition = 12
         broadcastVersion, broadcastVersionLength = decodeVarint(data[readPosition:readPosition+9])
         if broadcastVersion <> 1:
             #Cannot decode incoming broadcast versions higher than 1. Assuming the sender isn\' being silly, you should upgrade Bitmessage because this message shall be ignored.
@@ -693,13 +701,20 @@ class receiveDataThread(QThread):
 
         readPosition = 8
         embeddedTime, = unpack('>I',data[readPosition:readPosition+4])
+
+        #This section is used for the transition from 32 bit time to 64 bit time in the protocol.
+        if embeddedTime == 0:
+            embeddedTime, = unpack('>Q',data[readPosition:readPosition+8])
+            readPosition += 8
+        else:
+            readPosition += 4
+
         if embeddedTime > int(time.time())+10800:
             print 'The time in the msg message is too new. Ignoring it. Time:', embeddedTime
             return
         if embeddedTime < int(time.time())-maximumAgeOfAnObjectThatIAmWillingToAccept:
             print 'The time in the msg message is too old. Ignoring it. Time:', embeddedTime
             return
-        readPosition += 4
         streamNumberAsClaimedByMsg, streamNumberAsClaimedByMsgLength = decodeVarint(data[readPosition:readPosition+9])
         if streamNumberAsClaimedByMsg != self.streamNumber:
             print 'The stream number encoded in this msg (' + str(streamNumberAsClaimedByMsg) + ') message does not match the stream number on which it was received. Ignoring it.'
@@ -1024,6 +1039,14 @@ class receiveDataThread(QThread):
 
         readPosition = 8 #for the nonce
         embeddedTime, = unpack('>I',data[readPosition:readPosition+4])
+
+        #This section is used for the transition from 32 bit time to 64 bit time in the protocol.
+        if embeddedTime == 0:
+            embeddedTime, = unpack('>Q',data[readPosition:readPosition+8])
+            readPosition += 8
+        else:
+            readPosition += 4
+
         if embeddedTime < int(time.time())-lengthOfTimeToHoldOnToAllPubkeys:
             printLock.acquire()
             print 'The embedded time in this pubkey message is too old. Ignoring. Embedded time is:', embeddedTime
@@ -1034,7 +1057,6 @@ class receiveDataThread(QThread):
             print 'The embedded time in this pubkey message more than several hours in the future. This is irrational. Ignoring message.'
             printLock.release()
             return
-        readPosition += 4 #for the time
         addressVersion, varintLength = decodeVarint(data[readPosition:readPosition+10])
         readPosition += varintLength
         streamNumber, varintLength = decodeVarint(data[readPosition:readPosition+10])
@@ -1154,19 +1176,29 @@ class receiveDataThread(QThread):
         if len(data) < 34:
             print 'getpubkey message doesn\'t contain enough data. Ignoring.'
             return
-        embeddedTime, = unpack('>I',data[8:12])
+        readPosition = 8 #bypass the nonce
+        embeddedTime, = unpack('>I',data[readPosition:readPosition+4])
+
+        #This section is used for the transition from 32 bit time to 64 bit time in the protocol.
+        if embeddedTime == 0:
+            embeddedTime, = unpack('>Q',data[readPosition:readPosition+8])
+            readPosition += 8
+        else:
+            readPosition += 4
+
         if embeddedTime > int(time.time())+10800:
             print 'The time in this getpubkey message is too new. Ignoring it. Time:', embeddedTime
             return
         if embeddedTime < int(time.time())-maximumAgeOfAnObjectThatIAmWillingToAccept:
             print 'The time in this getpubkey message is too old. Ignoring it. Time:', embeddedTime
             return
-
+        addressVersionNumber, addressVersionLength = decodeVarint(data[readPosition:readPosition+10])
-        addressVersionNumber, addressVersionLength = decodeVarint(data[12:22])
+        readPosition += addressVersionLength
-        streamNumber, streamNumberLength = decodeVarint(data[12+addressVersionLength:22+addressVersionLength])
+        streamNumber, streamNumberLength = decodeVarint(data[readPosition:readPosition+10])
         if streamNumber <> self.streamNumber:
             print 'The streamNumber', streamNumber, 'doesn\'t match our stream number:', self.streamNumber
             return
+        readPosition += streamNumberLength
 
         inventoryHash = calculateInventoryHash(data)
         inventoryLock.acquire()
@@ -1195,7 +1227,7 @@ class receiveDataThread(QThread):
             print 'The addressVersionNumber of the pubkey request is too high. Can\'t understand. Ignoring it.'
             return
 
-        requestedHash = data[12+addressVersionLength+streamNumberLength:32+addressVersionLength+streamNumberLength]
+        requestedHash = data[readPosition:readPosition+20]
         if len(requestedHash) != 20:
             print 'The length of the requested hash is not 20 bytes. Something is wrong. Ignoring.'
             return
@@ -1349,12 +1381,13 @@ class receiveDataThread(QThread):
             printLock.acquire()
             print 'addr message contains', numberOfAddressesIncluded, 'IP addresses.'
             printLock.release()
-            #print 'lengthOfNumberOfAddresses', lengthOfNumberOfAddresses
 
+        if self.remoteProtocolVersion == 1:
+            print 'self.remoteProtocolVersion == 1'
             if numberOfAddressesIncluded > 1000 or numberOfAddressesIncluded == 0:
                 return
-        if len(data) < lengthOfNumberOfAddresses + (34 * numberOfAddressesIncluded):
+            if len(data) != lengthOfNumberOfAddresses + (34 * numberOfAddressesIncluded):
-            print 'addr message does not contain enough data. Ignoring.'
+                print 'addr message does not contain the correct amount of data. Ignoring.'
                 return
 
             needToWriteKnownNodesToDisk = False
@@ -1431,6 +1464,98 @@ class receiveDataThread(QThread):
                         knownNodesLock.release()
                         if PORT != recaddrPort:
                             print 'Strange occurance: The port specified in an addr message', str(recaddrPort),'does not match the port',str(PORT),'that this program (or some other peer) used to connect to it',str(hostFromAddrMessage),'. Perhaps they changed their port or are using a strange NAT configuration.'
+            if needToWriteKnownNodesToDisk: #Runs if any nodes were new to us. Also, share those nodes with our peers.
+                output = open(appdata + 'knownnodes.dat', 'wb')
+                knownNodesLock.acquire()
+                pickle.dump(knownNodes, output)
+                knownNodesLock.release()
+                output.close()
+                self.broadcastaddr(listOfAddressDetailsToBroadcastToPeers) #no longer broadcast
+            printLock.acquire()
+            print 'knownNodes currently has', len(knownNodes[self.streamNumber]), 'nodes for this stream.'
+            printLock.release()
+        elif self.remoteProtocolVersion == 2:
+            print 'self.remoteProtocolVersion == 2'
+            if numberOfAddressesIncluded > 1000 or numberOfAddressesIncluded == 0:
+                return
+            if len(data) != lengthOfNumberOfAddresses + (38 * numberOfAddressesIncluded):
+                print 'addr message does not contain the correct amount of data. Ignoring.'
+                return
+
+            needToWriteKnownNodesToDisk = False
+            for i in range(0,numberOfAddressesIncluded):
+                try:
+                    if data[20+lengthOfNumberOfAddresses+(38*i):32+lengthOfNumberOfAddresses+(38*i)] != '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xFF\xFF':
+                        printLock.acquire()
+                        print 'Skipping IPv6 address.', repr(data[20+lengthOfNumberOfAddresses+(38*i):32+lengthOfNumberOfAddresses+(38*i)])
+                        printLock.release()
+                        continue
+                except Exception, err:
+                    printLock.acquire()
+                    sys.stderr.write('ERROR TRYING TO UNPACK recaddr (to test for an IPv6 address). Message: %s\n' % str(err))
+                    printLock.release()
+                    break #giving up on unpacking any more. We should still be connected however.
+
+                try:
+                    recaddrStream, = unpack('>I',data[8+lengthOfNumberOfAddresses+(38*i):12+lengthOfNumberOfAddresses+(38*i)])
+                except Exception, err:
+                    printLock.acquire()
+                    sys.stderr.write('ERROR TRYING TO UNPACK recaddr (recaddrStream). Message: %s\n' % str(err))
+                    printLock.release()
+                    break #giving up on unpacking any more. We should still be connected however.
+                if recaddrStream == 0:
+                    continue
+                if recaddrStream != self.streamNumber and recaddrStream != (self.streamNumber * 2) and recaddrStream != ((self.streamNumber * 2) + 1): #if the embedded stream number is not in my stream or either of my child streams then ignore it. Someone might be trying funny business.
+                    continue
+                try:
+                    recaddrServices, = unpack('>Q',data[12+lengthOfNumberOfAddresses+(38*i):20+lengthOfNumberOfAddresses+(38*i)])
+                except Exception, err:
+                    printLock.acquire()
+                    sys.stderr.write('ERROR TRYING TO UNPACK recaddr (recaddrServices). Message: %s\n' % str(err))
+                    printLock.release()
+                    break #giving up on unpacking any more. We should still be connected however.
+
+                try:
+                    recaddrPort, = unpack('>H',data[36+lengthOfNumberOfAddresses+(38*i):42+lengthOfNumberOfAddresses+(38*i)])
+                except Exception, err:
+                    printLock.acquire()
+                    sys.stderr.write('ERROR TRYING TO UNPACK recaddr (recaddrPort). Message: %s\n' % str(err))
+                    printLock.release()
+                    break #giving up on unpacking any more. We should still be connected however.
+                #print 'Within recaddr(): IP', recaddrIP, ', Port', recaddrPort, ', i', i
+                hostFromAddrMessage = socket.inet_ntoa(data[32+lengthOfNumberOfAddresses+(38*i):36+lengthOfNumberOfAddresses+(38*i)])
+                #print 'hostFromAddrMessage', hostFromAddrMessage
+                if data[32+lengthOfNumberOfAddresses+(38*i)] == '\x7F':
+                    print 'Ignoring IP address in loopback range:', hostFromAddrMessage
+                    continue
+                if data[32+lengthOfNumberOfAddresses+(38*i)] == '\x0A':
+                    print 'Ignoring IP address in private range:', hostFromAddrMessage
+                    continue
+                if data[32+lengthOfNumberOfAddresses+(38*i):34+lengthOfNumberOfAddresses+(38*i)] == '\xC0A8':
+                    print 'Ignoring IP address in private range:', hostFromAddrMessage
+                    continue
+                timeSomeoneElseReceivedMessageFromThisNode, = unpack('>Q',data[lengthOfNumberOfAddresses+(38*i):8+lengthOfNumberOfAddresses+(38*i)]) #This is the 'time' value in the received addr message. 64-bit.
+                if recaddrStream not in knownNodes: #knownNodes is a dictionary of dictionaries with one outer dictionary for each stream. If the outer stream dictionary doesn't exist yet then we must make it.
+                    knownNodesLock.acquire()
+                    knownNodes[recaddrStream] = {}
+                    knownNodesLock.release()
+                if hostFromAddrMessage not in knownNodes[recaddrStream]:
+                    if len(knownNodes[recaddrStream]) < 20000 and timeSomeoneElseReceivedMessageFromThisNode > (int(time.time())-10800) and timeSomeoneElseReceivedMessageFromThisNode < (int(time.time()) + 10800): #If we have more than 20000 nodes in our list already then just forget about adding more. Also, make sure that the time that someone else received a message from this node is within three hours from now.
+                        knownNodesLock.acquire()
+                        knownNodes[recaddrStream][hostFromAddrMessage] = (recaddrPort, timeSomeoneElseReceivedMessageFromThisNode)
+                        knownNodesLock.release()
+                        print 'added new node', hostFromAddrMessage, 'to knownNodes in stream', recaddrStream
+                        needToWriteKnownNodesToDisk = True
+                        hostDetails = (timeSomeoneElseReceivedMessageFromThisNode, recaddrStream, recaddrServices, hostFromAddrMessage, recaddrPort)
+                        listOfAddressDetailsToBroadcastToPeers.append(hostDetails)
+                else:
+                    PORT, timeLastReceivedMessageFromThisNode = knownNodes[recaddrStream][hostFromAddrMessage]#PORT in this case is either the port we used to connect to the remote node, or the port that was specified by someone else in a past addr message.
+                    if (timeLastReceivedMessageFromThisNode < timeSomeoneElseReceivedMessageFromThisNode) and (timeSomeoneElseReceivedMessageFromThisNode < int(time.time())):
+                        knownNodesLock.acquire()
+                        knownNodes[recaddrStream][hostFromAddrMessage] = (PORT, timeSomeoneElseReceivedMessageFromThisNode)
+                        knownNodesLock.release()
+                        if PORT != recaddrPort:
+                            print 'Strange occurance: The port specified in an addr message', str(recaddrPort),'does not match the port',str(PORT),'that this program (or some other peer) used to connect to it',str(hostFromAddrMessage),'. Perhaps they changed their port or are using a strange NAT configuration.'
             if needToWriteKnownNodesToDisk: #Runs if any nodes were new to us. Also, share those nodes with our peers.
                 output = open(appdata + 'knownnodes.dat', 'wb')
                 knownNodesLock.acquire()
@@ -1442,13 +1567,14 @@ class receiveDataThread(QThread):
             print 'knownNodes currently has', len(knownNodes[self.streamNumber]), 'nodes for this stream.'
             printLock.release()
             
+
     #Function runs when we want to broadcast an addr message to all of our peers. Runs when we learn of nodes that we didn't previously know about and want to share them with our peers.
     def broadcastaddr(self,listOfAddressDetailsToBroadcastToPeers):
         numberOfAddressesInAddrMessage = len(listOfAddressDetailsToBroadcastToPeers)
         payload = ''
         for hostDetails in listOfAddressDetailsToBroadcastToPeers:
             timeLastReceivedMessageFromThisNode, streamNumber, services, host, port = hostDetails
-            payload += pack('>I',timeLastReceivedMessageFromThisNode)
+            payload += pack('>Q',timeLastReceivedMessageFromThisNode) #now uses 64-bit time
             payload += pack('>I',streamNumber)
             payload += pack('>q',services) #service bit flags offered by this node
             payload += '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xFF\xFF' + socket.inet_aton(host)
@@ -1503,7 +1629,10 @@ class receiveDataThread(QThread):
             PORT, timeLastReceivedMessageFromThisNode = value
             if timeLastReceivedMessageFromThisNode > (int(time.time())- maximumAgeOfNodesThatIAdvertiseToOthers): #If it is younger than 3 hours old..
                 numberOfAddressesInAddrMessage += 1
-                payload +=  pack('>I',timeLastReceivedMessageFromThisNode)
+                if self.remoteProtocolVersion == 1:
+                    payload +=  pack('>I',timeLastReceivedMessageFromThisNode) #32-bit time
+                else:
+                    payload +=  pack('>Q',timeLastReceivedMessageFromThisNode) #64-bit time
                 payload += pack('>I',self.streamNumber)
                 payload += pack('>q',1) #service bit flags offered by this node
                 payload += '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xFF\xFF' + socket.inet_aton(HOST)
@@ -1512,7 +1641,10 @@ class receiveDataThread(QThread):
             PORT, timeLastReceivedMessageFromThisNode = value
             if timeLastReceivedMessageFromThisNode > (int(time.time())- maximumAgeOfNodesThatIAdvertiseToOthers): #If it is younger than 3 hours old..
                 numberOfAddressesInAddrMessage += 1
-                payload += pack('>I',timeLastReceivedMessageFromThisNode)
+                if self.remoteProtocolVersion == 1:
+                    payload +=  pack('>I',timeLastReceivedMessageFromThisNode) #32-bit time
+                else:
+                    payload +=  pack('>Q',timeLastReceivedMessageFromThisNode) #64-bit time
                 payload += pack('>I',self.streamNumber*2)
                 payload += pack('>q',1) #service bit flags offered by this node
                 payload += '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xFF\xFF' + socket.inet_aton(HOST)
@@ -1521,7 +1653,10 @@ class receiveDataThread(QThread):
             PORT, timeLastReceivedMessageFromThisNode = value
             if timeLastReceivedMessageFromThisNode > (int(time.time())- maximumAgeOfNodesThatIAdvertiseToOthers): #If it is younger than 3 hours old..
                 numberOfAddressesInAddrMessage += 1
-                payload += pack('>I',timeLastReceivedMessageFromThisNode)
+                if self.remoteProtocolVersion == 1:
+                    payload +=  pack('>I',timeLastReceivedMessageFromThisNode) #32-bit time
+                else:
+                    payload +=  pack('>Q',timeLastReceivedMessageFromThisNode) #64-bit time
                 payload += pack('>I',(self.streamNumber*2)+1)
                 payload += pack('>q',1) #service bit flags offered by this node
                 payload += '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xFF\xFF' + socket.inet_aton(HOST)
@@ -1576,6 +1711,7 @@ class receiveDataThread(QThread):
                 print 'Closing connection to myself: ', self.HOST
                 printLock.release()
                 return
+            broadcastToSendDataQueues((0,'setRemoteProtocolVersion',(self.HOST,self.remoteProtocolVersion)))
 
             knownNodesLock.acquire()
             knownNodes[self.streamNumber][self.HOST] = (self.remoteNodeIncomingPort, int(time.time()))
@@ -1728,7 +1864,19 @@ class sendDataThread(QThread):
                         print 'setting the stream number in the sendData thread (ID:',id(self), ') to', specifiedStreamNumber
                         printLock.release()
                         self.streamNumber = specifiedStreamNumber
+                elif command == 'setRemoteProtocolVersion':
+                    hostInMessage, specifiedRemoteProtocolVersion = data
+                    if hostInMessage == self.HOST:
+                        printLock.acquire()
+                        print 'setting the remote node\'s protocol version in the sendData thread (ID:',id(self), ') to', specifiedRemoteProtocolVersion
+                        printLock.release()
+                        self.remoteProtocolVersion = specifiedRemoteProtocolVersion
                 elif command == 'sendaddr':
+                    if self.remoteProtocolVersion == 1:
+                        printLock.acquire()
+                        print 'a sendData thread is not sending an addr message to this particular peer ('+self.HOST+') because their protocol version is 1.'
+                        printLock.release()     
+                    else:
                         try:
                             #To prevent some network analysis, 'leak' the data out to our peer after waiting a random amount of time unless we have a long list of messages in our queue to send.
                             random.seed()
@@ -3993,18 +4141,7 @@ class MyForm(QtGui.QMainWindow):
                         sqlSubmitQueue.put('commit')
                         sqlLock.release()
 
-
-                        
-
-                        """try:
-                            fromLabel = config.get(fromAddress, 'label')
-                        except:
-                            fromLabel = ''
-                        if fromLabel == '':
-                            fromLabel = fromAddress"""
-
                         toLabel = ''
-
                         t = (toAddress,)
                         sqlLock.acquire()
                         sqlSubmitQueue.put('''select label from addressbook where address=?''')
@@ -4018,30 +4155,6 @@ class MyForm(QtGui.QMainWindow):
                         self.displayNewSentMessage(toAddress,toLabel,fromAddress, subject, message, ackdata)
                         workerQueue.put(('sendmessage',toAddress))
 
-                        """self.ui.tableWidgetSent.insertRow(0)
-                        if toLabel == '':
-                            newItem =  QtGui.QTableWidgetItem(unicode(toAddress,'utf-8'))
-                        else:
-                            newItem =  QtGui.QTableWidgetItem(unicode(toLabel,'utf-8'))
-                        newItem.setData(Qt.UserRole,str(toAddress))
-                        self.ui.tableWidgetSent.setItem(0,0,newItem)
-
-                        if fromLabel == '':
-                            newItem =  QtGui.QTableWidgetItem(unicode(fromAddress,'utf-8'))
-                        else:
-                            newItem =  QtGui.QTableWidgetItem(unicode(fromLabel,'utf-8'))
-                        newItem.setData(Qt.UserRole,str(fromAddress))
-                        self.ui.tableWidgetSent.setItem(0,1,newItem)
-                        newItem =  QtGui.QTableWidgetItem(unicode(subject,'utf-8)'))
-                        newItem.setData(Qt.UserRole,unicode(message,'utf-8)'))
-                        self.ui.tableWidgetSent.setItem(0,2,newItem)
-                        newItem =  myTableWidgetItem('Just pressed ''send'' ' + unicode(strftime(config.get('bitmessagesettings', 'timeformat'),localtime(int(time.time()))),'utf-8'))
-                        newItem.setData(Qt.UserRole,QByteArray(ackdata))
-                        newItem.setData(33,int(time.time()))
-                        self.ui.tableWidgetSent.setItem(0,3,newItem)
-
-                        self.ui.textEditSentMessage.setPlainText(self.ui.tableWidgetSent.item(0,2).data(Qt.UserRole).toPyObject())"""
-
                         self.ui.comboBoxSendFrom.setCurrentIndex(0)
                         self.ui.labelFrom.setText('')
                         self.ui.lineEditTo.setText('')