further v4 address work

src/class_addressGenerator.py

@@ -243,8 +243,10 @@ class addressGenerator(threading.Thread):
                                 address)
                             shared.myECCryptorObjects[ripe.digest()] = highlevelcrypto.makeCryptor(
                                 potentialPrivEncryptionKey.encode('hex'))
-                            shared.myAddressesByHash[
+                            shared.myAddressesByHash[ripe.digest()] = address
-                                ripe.digest()] = address
+                            tag = hashlib.sha512(hashlib.sha512(encodeVarint(
+                                addressVersionNumber) + encodeVarint(streamNumber) + ripe.digest()).digest()).digest()[32:]
+                            shared.myAddressesByTag[tag] = address
                             if addressVersionNumber == 3:
                                 shared.workerQueue.put((
                                     'sendOutOrStoreMyV3Pubkey', ripe.digest())) # If this is a chan address,

src/class_receiveDataThread.py

@@ -432,8 +432,8 @@ class receiveDataThread(threading.Thread):
         broadcastVersion, broadcastVersionLength = decodeVarint(
             data[readPosition:readPosition + 9])
         readPosition += broadcastVersionLength
-        if broadcastVersion < 1 or broadcastVersion > 2:
+        if broadcastVersion < 1 or broadcastVersion > 3:
-            print 'Cannot decode incoming broadcast versions higher than 2. Assuming the sender isn\'t being silly, you should upgrade Bitmessage because this message shall be ignored.'
+            print 'Cannot decode incoming broadcast versions higher than 3. Assuming the sender isn\'t being silly, you should upgrade Bitmessage because this message shall be ignored.'
             return
         if broadcastVersion == 1:
             beginningOfPubkeyPosition = readPosition  # used when we add the pubkey to our pubkey table
@@ -709,6 +709,159 @@ class receiveDataThread(threading.Thread):
             with shared.printLock:
                 print 'Time spent processing this interesting broadcast:', time.time() - self.messageProcessingStartTime
 
+        if broadcastVersion == 3:
+            cleartextStreamNumber, cleartextStreamNumberLength = decodeVarint(
+                data[readPosition:readPosition + 10])
+            readPosition += cleartextStreamNumberLength
+            tag = data[readPostion:readPosition+32]
+            readPosition += 32
+            if tag not in shared.MyECSubscriptionCryptorObjects.items():
+                with shared.printLock:
+                    print 'We\'re not interested in this broadcast.'
+                return
+            initialDecryptionSuccessful = False
+            for key, cryptorObject in shared.MyECSubscriptionCryptorObjects.items():
+                try:
+                    decryptedData = cryptorObject.decrypt(data[readPosition:])
+                    toRipe = key  # This is the RIPE hash of the sender's pubkey. We need this below to compare to the RIPE hash of the sender's address to verify that it was encrypted by with their key rather than some other key.
+                    initialDecryptionSuccessful = True
+                    print 'EC decryption successful using key associated with ripe hash:', key.encode('hex')
+                    break
+                except Exception as err:
+                    pass
+                    # print 'cryptorObject.decrypt Exception:', err
+            if not initialDecryptionSuccessful:
+                # This is not a broadcast I am interested in.
+                with shared.printLock:
+                    print 'Length of time program spent failing to decrypt this v2 broadcast:', time.time() - self.messageProcessingStartTime, 'seconds.'
+
+                return
+            # At this point this is a broadcast I have decrypted and thus am
+            # interested in.
+            signedBroadcastVersion, readPosition = decodeVarint(
+                decryptedData[:10])
+            beginningOfPubkeyPosition = readPosition  # used when we add the pubkey to our pubkey table
+            sendersAddressVersion, sendersAddressVersionLength = decodeVarint(
+                decryptedData[readPosition:readPosition + 9])
+            if sendersAddressVersion < 2 or sendersAddressVersion > 3:
+                print 'Cannot decode senderAddressVersion other than 2 or 3. Assuming the sender isn\'t being silly, you should upgrade Bitmessage because this message shall be ignored.'
+                return
+            readPosition += sendersAddressVersionLength
+            sendersStream, sendersStreamLength = decodeVarint(
+                decryptedData[readPosition:readPosition + 9])
+            if sendersStream != cleartextStreamNumber:
+                print 'The stream number outside of the encryption on which the POW was completed doesn\'t match the stream number inside the encryption. Ignoring broadcast.'
+                return
+            readPosition += sendersStreamLength
+            behaviorBitfield = decryptedData[readPosition:readPosition + 4]
+            readPosition += 4
+            sendersPubSigningKey = '\x04' + \
+                decryptedData[readPosition:readPosition + 64]
+            readPosition += 64
+            sendersPubEncryptionKey = '\x04' + \
+                decryptedData[readPosition:readPosition + 64]
+            readPosition += 64
+            if sendersAddressVersion >= 3:
+                requiredAverageProofOfWorkNonceTrialsPerByte, varintLength = decodeVarint(
+                    decryptedData[readPosition:readPosition + 10])
+                readPosition += varintLength
+                print 'sender\'s requiredAverageProofOfWorkNonceTrialsPerByte is', requiredAverageProofOfWorkNonceTrialsPerByte
+                requiredPayloadLengthExtraBytes, varintLength = decodeVarint(
+                    decryptedData[readPosition:readPosition + 10])
+                readPosition += varintLength
+                print 'sender\'s requiredPayloadLengthExtraBytes is', requiredPayloadLengthExtraBytes
+            endOfPubkeyPosition = readPosition
+
+            sha = hashlib.new('sha512')
+            sha.update(sendersPubSigningKey + sendersPubEncryptionKey)
+            ripe = hashlib.new('ripemd160')
+            ripe.update(sha.digest())
+
+            if toRipe != ripe.digest():
+                print 'The encryption key used to encrypt this message doesn\'t match the keys inbedded in the message itself. Ignoring message.'
+                return
+            messageEncodingType, messageEncodingTypeLength = decodeVarint(
+                decryptedData[readPosition:readPosition + 9])
+            if messageEncodingType == 0:
+                return
+            readPosition += messageEncodingTypeLength
+            messageLength, messageLengthLength = decodeVarint(
+                decryptedData[readPosition:readPosition + 9])
+            readPosition += messageLengthLength
+            message = decryptedData[readPosition:readPosition + messageLength]
+            readPosition += messageLength
+            readPositionAtBottomOfMessage = readPosition
+            signatureLength, signatureLengthLength = decodeVarint(
+                decryptedData[readPosition:readPosition + 9])
+            readPosition += signatureLengthLength
+            signature = decryptedData[
+                readPosition:readPosition + signatureLength]
+            try:
+                if not highlevelcrypto.verify(decryptedData[:readPositionAtBottomOfMessage], signature, sendersPubSigningKey.encode('hex')):
+                    print 'ECDSA verify failed'
+                    return
+                print 'ECDSA verify passed'
+            except Exception as err:
+                print 'ECDSA verify failed', err
+                return
+            # verify passed
+
+            # Let's store the public key in case we want to reply to this
+            # person.
+            sqlExecute('''INSERT INTO pubkeys VALUES (?,?,?,?)''',
+                       ripe.digest(),
+                       '\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF' + '\xFF\xFF\xFF\xFF' + decryptedData[beginningOfPubkeyPosition:endOfPubkeyPosition],
+                       int(time.time()),
+                       'yes')
+            # shared.workerQueue.put(('newpubkey',(sendersAddressVersion,sendersStream,ripe.digest())))
+            # This will check to see whether we happen to be awaiting this
+            # pubkey in order to send a message. If we are, it will do the POW
+            # and send it.
+            self.possibleNewPubkey(ripe.digest())
+
+            fromAddress = encodeAddress(
+                sendersAddressVersion, sendersStream, ripe.digest())
+            with shared.printLock:
+                print 'fromAddress:', fromAddress
+
+            if messageEncodingType == 2:
+                subject, body = self.decodeType2Message(message)
+            elif messageEncodingType == 1:
+                body = message
+                subject = ''
+            elif messageEncodingType == 0:
+                print 'messageEncodingType == 0. Doing nothing with the message.'
+            else:
+                body = 'Unknown encoding type.\n\n' + repr(message)
+                subject = ''
+
+            toAddress = '[Broadcast subscribers]'
+            if messageEncodingType != 0:
+
+                t = (self.inventoryHash, toAddress, fromAddress, subject, int(
+                    time.time()), body, 'inbox', messageEncodingType, 0)
+                helper_inbox.insert(t)
+
+                shared.UISignalQueue.put(('displayNewInboxMessage', (
+                    self.inventoryHash, toAddress, fromAddress, subject, body)))
+
+                # If we are behaving as an API then we might need to run an
+                # outside command to let some program know that a new message
+                # has arrived.
+                if shared.safeConfigGetBoolean('bitmessagesettings', 'apienabled'):
+                    try:
+                        apiNotifyPath = shared.config.get(
+                            'bitmessagesettings', 'apinotifypath')
+                    except:
+                        apiNotifyPath = ''
+                    if apiNotifyPath != '':
+                        call([apiNotifyPath, "newBroadcast"])
+
+            # Display timing data
+            with shared.printLock:
+                print 'Time spent processing this interesting broadcast:', time.time() - self.messageProcessingStartTime
+
+
 
     # We have received a msg message.
     def recmsg(self, data):
@@ -1142,6 +1295,7 @@ class receiveDataThread(threading.Thread):
             return
         if addressVersion >= 4:
             tag = data[readPosition:readPosition + 32]
+            print 'tag in received pubkey is:', tag.encode('hex')
         else:
             tag = ''
 
@@ -1198,7 +1352,6 @@ class receiveDataThread(threading.Thread):
         streamNumber, varintLength = decodeVarint(
             data[readPosition:readPosition + 10])
         readPosition += varintLength
-        signedData = data[8:readPosition] # Used only for v4 or higher pubkeys
         if addressVersion == 0:
             print '(Within processpubkey) addressVersion of 0 doesn\'t make sense.'
             return
@@ -1310,12 +1463,15 @@ class receiveDataThread(threading.Thread):
             if len(data) < 350:  # sanity check.
                 print '(within processpubkey) payloadLength less than 350. Sanity check failed.'
                 return
+            signedData = data[8:readPosition] # Used only for v4 or higher pubkeys
             tag = data[readPosition:readPosition + 32]
             readPosition += 32
             encryptedData = data[readPosition:]
             if tag not in shared.neededPubkeys:
                 with shared.printLock:
                     print 'We don\'t need this v4 pubkey. We didn\'t ask for it.'
+                    print 'tag is', repr(tag)
+                    print 'shared.neededPubkeys is', repr(shared.neededPubkeys)
                 return
 
             with shared.printLock:
@@ -1389,7 +1545,7 @@ class receiveDataThread(threading.Thread):
 
             sqlExecute(
                 '''UPDATE sent SET status='doingmsgpow' WHERE toripe=? AND (status='awaitingpubkey' or status='doingpubkeypow') and folder='sent' ''',
-                toRipe)
+                ripe)
             shared.workerQueue.put(('sendmessage', ''))
 
     # We have received a getpubkey message
@@ -1474,7 +1630,8 @@ class receiveDataThread(threading.Thread):
                 return
             with shared.printLock:
                 print 'the tag requested in this getpubkey request is:', requestedTag.encode('hex')
-            if requestedTag in shared.myAddressesByTag[requestedTag]:
+            if requestedTag in shared.myAddressesByTag:
+                
                 myAddress = shared.myAddressesByTag[requestedTag]
 
         if myAddress == '':

src/class_singleWorker.py

@@ -27,12 +27,17 @@ class singleWorker(threading.Thread):
             '''SELECT toripe, toaddress FROM sent WHERE ((status='awaitingpubkey' OR status='doingpubkeypow') AND folder='sent')''')
         for row in queryreturn:
             toripe, toaddress = row
-            toStatus, toAddressVersionNumber, toStreamNumber, toRipe = decodeAddress(toaddress)[1]
+            toStatus, toAddressVersionNumber, toStreamNumber, toRipe = decodeAddress(toaddress)
             if toAddressVersionNumber <= 3 :
                 shared.neededPubkeys[toripe] = 0
             elif toAddressVersionNumber >= 4:
-                privEncryptionKey = hashlib.sha512(hashlib.sha512(encodeVarint(toAddressVersionNumber)+encodeVarint(toStreamNumber)+toRipe).digest()).digest()[:32] # Note that this is the first half of the sha512 hash.
+                with shared.printLock:
-                shared.neededPubkeys[toripe] = highlevelcrypto.makeCryptor(privEncryptionKey.encode('hex')) # We'll need this for when we receive a pubkey reply: it will be encrypted and we'll need to decrypt it.
+                    print 'Loading our list of needed pubkeys...'
+                doubleHashOfAddressData = hashlib.sha512(hashlib.sha512(encodeVarint(
+                    toAddressVersionNumber) + encodeVarint(toStreamNumber) + toRipe).digest()).digest()
+                privEncryptionKey = doubleHashOfAddressData[:32] # Note that this is the first half of the sha512 hash.
+                tag = doubleHashOfAddressData[32:]
+                shared.neededPubkeys[tag] = highlevelcrypto.makeCryptor(privEncryptionKey.encode('hex')) # We'll need this for when we receive a pubkey reply: it will be encrypted and we'll need to decrypt it.
 
         # Initialize the shared.ackdataForWhichImWatching data structure using data
         # from the sql database.
@@ -252,7 +257,6 @@ class singleWorker(threading.Thread):
         payload += encodeVarint(addressVersionNumber)  # Address version number
         payload += encodeVarint(streamNumber)
 
-
         dataToEncrypt = '\x00\x00\x00\x01'  # bitfield of features supported by me (see the wiki).
 
         try:
@@ -264,7 +268,6 @@ class singleWorker(threading.Thread):
             with shared.printLock:
                 sys.stderr.write(
                     'Error within sendOutOrStoreMyV4Pubkey. Could not read the keys from the keys.dat file for a requested address. %s\n' % err)
-
             return
 
         privSigningKeyHex = shared.decodeWalletImportFormat(
@@ -294,16 +297,13 @@ class singleWorker(threading.Thread):
         # so that nodes know which pubkey object to try to decrypt when they
         # want to send a message.
         doubleHashOfAddressData = hashlib.sha512(hashlib.sha512(encodeVarint(
-                addressVersionNumber) + encodeVarint(streamNumber) + hash).digest()).digest()
+            addressVersionNumber) + encodeVarint(streamNumber) + hash).digest()).digest()
+        payload += doubleHashOfAddressData[32:] # the tag
         privEncryptionKey = doubleHashOfAddressData[:32]
         pubEncryptionKey = pointMult(privEncryptionKey)
-        payload += doubleHashOfAddressData[32:] # the tag
         payload += highlevelcrypto.encrypt(
             dataToEncrypt, pubEncryptionKey.encode('hex'))
 
-        ####################
-
-
         if not shared.safeConfigGetBoolean(myAddress, 'chan'):
             # Do the POW for this pubkey message
             target = 2 ** 64 / ((len(payload) + shared.networkDefaultPayloadLengthExtraBytes +
@@ -378,8 +378,15 @@ class singleWorker(threading.Thread):
                 -300, 300)))  # the current time plus or minus five minutes
             payload += encodeVarint(2)  # broadcast version
             payload += encodeVarint(streamNumber)
+            if addressVersionNumber >= 4:
+                doubleHashOfAddressData = hashlib.sha512(hashlib.sha512(encodeVarint(
+                    addressVersionNumber) + encodeVarint(streamNumber) + ripe).digest()).digest()
+                payload += doubleHashOfAddressData[32:]  # the tag
 
-            dataToEncrypt = encodeVarint(2)  # broadcast version
+            if addressVersionNumber <= 3:
+                dataToEncrypt = encodeVarint(2)  # broadcast version
+            else:
+                dataToEncrypt = encodeVarint(3)  # broadcast version
             dataToEncrypt += encodeVarint(addressVersionNumber)
             dataToEncrypt += encodeVarint(streamNumber)
             dataToEncrypt += '\x00\x00\x00\x01'  # behavior bitfield
@@ -399,8 +406,11 @@ class singleWorker(threading.Thread):
             # Encrypt the broadcast with the information contained in the broadcaster's address. Anyone who knows the address can generate 
             # the private encryption key to decrypt the broadcast. This provides virtually no privacy; its purpose is to keep questionable 
             # and illegal content from flowing through the Internet connections and being stored on the disk of 3rd parties. 
-            privEncryptionKey = hashlib.sha512(encodeVarint(
+            if addressVersionNumber <= 3:
-                addressVersionNumber) + encodeVarint(streamNumber) + ripe).digest()[:32]
+                privEncryptionKey = hashlib.sha512(encodeVarint(
+                    addressVersionNumber) + encodeVarint(streamNumber) + ripe).digest()[:32]
+            else:
+                privEncryptionKey = doubleHashOfAddressData[:32]
             pubEncryptionKey = pointMult(privEncryptionKey)
             payload += highlevelcrypto.encrypt(
                 dataToEncrypt, pubEncryptionKey.encode('hex'))
@@ -445,28 +455,108 @@ class singleWorker(threading.Thread):
             '''SELECT DISTINCT toaddress FROM sent WHERE (status='msgqueued' AND folder='sent')''')
         for row in queryreturn:  # For each address to which we need to send a message, check to see if we have its pubkey already.
             toaddress, = row
-            toripe = decodeAddress(toaddress)[3]
+            status, toAddressVersion, toStreamNumber, toRipe = decodeAddress(toaddress)
             queryreturn = sqlQuery(
-                '''SELECT hash FROM pubkeys WHERE hash=? ''', toripe)
+                '''SELECT hash FROM pubkeys WHERE hash=? ''', toRipe)
-            if queryreturn != []:  # If we have the needed pubkey, set the status to doingmsgpow (we'll do it further down)
+            if queryreturn != []:  # If we have the needed pubkey in the pubkey table already, set the status to doingmsgpow (we'll do it further down)
                 sqlExecute(
                     '''UPDATE sent SET status='doingmsgpow' WHERE toaddress=? AND status='msgqueued' ''',
                     toaddress)
-            else:  # We don't have the needed pubkey. Set the status to 'awaitingpubkey' and request it if we haven't already
+            else:  # We don't have the needed pubkey in the pubkey table already.
-                if toripe in shared.neededPubkeys:
+                if toAddressVersion <= 3:
+                    toTag = ''
+                else:
+                    toTag = hashlib.sha512(hashlib.sha512(encodeVarint(toAddressVersion)+encodeVarint(toStreamNumber)+toRipe).digest()).digest()[32:]
+                if toRipe in shared.neededPubkeys or toTag in shared.neededPubkeys:
                     # We already sent a request for the pubkey
                     sqlExecute(
                         '''UPDATE sent SET status='awaitingpubkey' WHERE toaddress=? AND status='msgqueued' ''', toaddress)
                     shared.UISignalQueue.put(('updateSentItemStatusByHash', (
-                        toripe, tr.translateText("MainWindow",'Encryption key was requested earlier.'))))
+                        toRipe, tr.translateText("MainWindow",'Encryption key was requested earlier.'))))
                 else:
                     # We have not yet sent a request for the pubkey
-                    sqlExecute(
+                    needToRequestPubkey = True
-                        '''UPDATE sent SET status='doingpubkeypow' WHERE toaddress=? AND status='msgqueued' ''',
+                    if toAddressVersion >= 4:
-                        toaddress)
+                        # We might have the pubkey in the inventory and need to decrypt it and put it in the pubkeys table.
-                    shared.UISignalQueue.put(('updateSentItemStatusByHash', (
+                        queryreturn = sqlQuery(
-                        toripe, tr.translateText("MainWindow",'Sending a request for the recipient\'s encryption key.'))))
+                            '''SELECT payload FROM inventory WHERE objecttype='pubkey' and tag=? ''', toTag)
-                    self.requestPubKey(toaddress)
+                        if queryreturn != []: # if there was a pubkey in our inventory with the correct tag, we need to try to decrypt it.
+                            for row in queryreturn:
+                                data, = row
+                                readPosition = 8  # for the nonce
+                                readPosition += 8 # for the time
+                                readPosition += 1 # for the address version number
+                                streamNumber, varintLength = decodeVarint(
+                                    data[readPosition:readPosition + 10])
+                                readPosition += varintLength
+                                signedData = data[8:readPosition] # Some of the signed data is not encrypted so let's keep it for now.
+                                readPosition += 32 #for the tag
+                                encryptedData = data[readPosition:]
+                                # Let us try to decrypt the pubkey
+                                privEncryptionKey = hashlib.sha512(hashlib.sha512(encodeVarint(addressVersionNumber)+encodeVarint(streamNumber)+ripe).digest()).digest()[:32]
+                                cryptorObject = highlevelcrypto.makeCryptor(privEncryptionKey.encode('hex'))
+                                try:
+                                    decryptedData = cryptorObject.decrypt(encryptedData)
+                                except:
+                                    # Someone must have encrypted some data with a different key
+                                    # but tagged it with a tag for which we are watching.
+                                    with shared.printLock:
+                                        print 'Pubkey decryption was UNsuccessful.'
+                                    continue
+                                print 'Pubkey decryption successful'
+                                readPosition = 4 # bypass the behavior bitfield
+                                publicSigningKey = '\x04' + decryptedData[readPosition:readPosition + 64]
+                                # Is it possible for a public key to be invalid such that trying to
+                                # encrypt or sign with it will cause an error? If it is, we should
+                                # probably test these keys here.
+                                readPosition += 64
+                                publicEncryptionKey = '\x04' + decryptedData[readPosition:readPosition + 64]
+                                readPosition += 64
+                                specifiedNonceTrialsPerByte, specifiedNonceTrialsPerByteLength = decodeVarint(
+                                    decryptedData[readPosition:readPosition + 10])
+                                readPosition += specifiedNonceTrialsPerByteLength
+                                specifiedPayloadLengthExtraBytes, specifiedPayloadLengthExtraBytesLength = decodeVarint(
+                                    decryptedData[readPosition:readPosition + 10])
+                                readPosition += specifiedPayloadLengthExtraBytesLength
+                                signedData += decryptedData[:readPosition]
+                                signatureLength, signatureLengthLength = decodeVarint(
+                                    decryptedData[readPosition:readPosition + 10])
+                                readPosition += signatureLengthLength
+                                signature = decryptedData[readPosition:readPosition + signatureLength]
+                                try:
+                                    if not highlevelcrypto.verify(signedData, signature, publicSigningKey.encode('hex')):
+                                        print 'ECDSA verify failed (within processpubkey)'
+                                        continue
+                                    print 'ECDSA verify passed (within processpubkey)'
+                                except Exception as err:
+                                    print 'ECDSA verify failed (within processpubkey)', err
+                                    continue
+
+                                sha = hashlib.new('sha512')
+                                sha.update(publicSigningKey + publicEncryptionKey)
+                                ripeHasher = hashlib.new('ripemd160')
+                                ripeHasher.update(sha.digest())
+                                ripe = ripeHasher.digest()
+
+                                # We need to make sure that the tag on the outside of the encryption
+                                # is the one generated from hashing these particular keys.
+                                if toTag != hashlib.sha512(hashlib.sha512(encodeVarint(addressVersion) + encodeVarint(streamNumber) + ripe).digest()).digest()[32:]:
+                                    with shared.printLock:
+                                        print 'Someone was trying to act malicious: tag doesn\'t match the keys in this pubkey message. Ignoring it.'
+                                    continue
+                                else:
+                                    print 'Tag successfully matches keys in pubkey message' # testing. Will remove soon.
+
+                                t = (ripe, signedData, embeddedTime, 'yes')
+                                sqlExecute('''INSERT INTO pubkeys VALUES (?,?,?,?)''', *t)
+                                needToRequestPubkey == False
+                    if needToRequestPubkey:
+                        sqlExecute(
+                            '''UPDATE sent SET status='doingpubkeypow' WHERE toaddress=? AND status='msgqueued' ''',
+                            toaddress)
+                        shared.UISignalQueue.put(('updateSentItemStatusByHash', (
+                            toRipe, tr.translateText("MainWindow",'Sending a request for the recipient\'s encryption key.'))))
+                        self.requestPubKey(toaddress)
         # Get all messages that are ready to be sent, and also all messages
         # which we have sent in the last 28 days which were previously marked
         # as 'toodifficult'. If the user as raised the maximum acceptable
@@ -530,7 +620,7 @@ class singleWorker(threading.Thread):
             for row in queryreturn:
                 pubkeyPayload, = row
 
-            # The v3 pubkey message is stored the way we originally received it
+            # The pubkey message is stored the way we originally received it
             # which means that we need to read beyond things like the nonce and
             # time to get to the actual public keys.
             if toAddressVersionNumber <= 3:
@@ -649,7 +739,7 @@ class singleWorker(threading.Thread):
                 payload += encodeVarint(len(signature))
                 payload += signature
 
-            if fromAddressVersionNumber == 3:
+            if fromAddressVersionNumber >= 3:
                 payload = '\x01'  # Message version.
                 payload += encodeVarint(fromAddressVersionNumber)
                 payload += encodeVarint(fromStreamNumber)
@@ -781,7 +871,8 @@ class singleWorker(threading.Thread):
             shared.neededPubkeys[ripe] = 0
         elif addressVersionNumber >= 4:
             privEncryptionKey = hashlib.sha512(hashlib.sha512(encodeVarint(addressVersionNumber)+encodeVarint(streamNumber)+ripe).digest()).digest()[:32] # Note that this is the first half of the sha512 hash.
-            shared.neededPubkeys[ripe] = highlevelcrypto.makeCryptor(privEncryptionKey.encode('hex')) # We'll need this for when we receive a pubkey reply: it will be encrypted and we'll need to decrypt it.
+            tag = hashlib.sha512(hashlib.sha512(encodeVarint(addressVersionNumber)+encodeVarint(streamNumber)+ripe).digest()).digest()[32:] # Note that this is the second half of the sha512 hash.
+            shared.neededPubkeys[tag] = highlevelcrypto.makeCryptor(privEncryptionKey.encode('hex')) # We'll need this for when we receive a pubkey reply: it will be encrypted and we'll need to decrypt it.
         payload = pack('>Q', (int(time.time()) + random.randrange(
             -300, 300)))  # the current time plus or minus five minutes.
         payload += encodeVarint(addressVersionNumber)
@@ -791,9 +882,9 @@ class singleWorker(threading.Thread):
             with shared.printLock:
                 print 'making request for pubkey with ripe:', ripe.encode('hex')
         else:
-            payload += hashlib.sha512(hashlib.sha512(encodeVarint(addressVersionNumber)+encodeVarint(streamNumber)+ripe).digest()).digest()[32:] # Note that this is the second half of the sha512 hash.
+            payload += tag
             with shared.printLock:
-                print 'making request for v4 pubkey with ripe:', ripe.encode('hex')
+                print 'making request for v4 pubkey with tag:', tag.encode('hex')
 
         # print 'trial value', trialValue
         statusbar = 'Doing the computations necessary to request the recipient\'s public key.'

src/class_sqlThread.py

@@ -48,7 +48,7 @@ class sqlThread(threading.Thread):
             self.cur.execute(
                 '''CREATE TABLE pubkeys (hash blob, transmitdata blob, time int, usedpersonally text, UNIQUE(hash) ON CONFLICT REPLACE)''' )
             self.cur.execute(
-                '''CREATE TABLE inventory (hash blob, objecttype text, streamnumber int, payload blob, receivedtime integer, first20bytesofencryptedmessage blob, tag blob, UNIQUE(hash) ON CONFLICT REPLACE)''' )
+                '''CREATE TABLE inventory (hash blob, objecttype text, streamnumber int, payload blob, receivedtime integer, tag blob, UNIQUE(hash) ON CONFLICT REPLACE)''' )
             self.cur.execute(
                 '''CREATE TABLE knownnodes (timelastseen int, stream int, services blob, host blob, port blob, UNIQUE(host, stream, port) ON CONFLICT REPLACE)''' )
                              # This table isn't used in the program yet but I
@@ -57,7 +57,7 @@ class sqlThread(threading.Thread):
                 '''INSERT INTO subscriptions VALUES('Bitmessage new releases/announcements','BM-GtovgYdgs7qXPkoYaRgrLFuFKz1SFpsw',1)''')
             self.cur.execute(
                 '''CREATE TABLE settings (key blob, value blob, UNIQUE(key) ON CONFLICT REPLACE)''' )
-            self.cur.execute( '''INSERT INTO settings VALUES('version','3')''')
+            self.cur.execute( '''INSERT INTO settings VALUES('version','4')''')
             self.cur.execute( '''INSERT INTO settings VALUES('lastvacuumtime',?)''', (
                 int(time.time()),))
             self.conn.commit()
@@ -193,7 +193,7 @@ class sqlThread(threading.Thread):
             
         ensureNamecoinOptions()
             
-        # Add a new column to the inventory table to store the first 20 bytes of encrypted messages to support Android app
+        """# Add a new column to the inventory table to store the first 20 bytes of encrypted messages to support Android app
         item = '''SELECT value FROM settings WHERE key='version';'''
         parameters = ''
         self.cur.execute(item, parameters)
@@ -204,19 +204,40 @@ class sqlThread(threading.Thread):
             self.cur.execute(item, parameters)
             item = '''update settings set value=? WHERE key='version';'''
             parameters = (2,)
-            self.cur.execute(item, parameters)
+            self.cur.execute(item, parameters)"""
 
-        # Add a new column to the inventory table to store pubkeys' tags.
+        # Let's get rid of the first20bytesofencryptedmessage field in the inventory table.
         item = '''SELECT value FROM settings WHERE key='version';'''
         parameters = ''
         self.cur.execute(item, parameters)
         if int(self.cur.fetchall()[0][0]) == 2:
-            print 'upgrading database'
+            logger.debug('In messages.dat database, removing an obsolete field from the inventory table.')
+            self.cur.execute(
+                '''CREATE TEMPORARY TABLE inventory_backup(hash blob, objecttype text, streamnumber int, payload blob, receivedtime integer, UNIQUE(hash) ON CONFLICT REPLACE);''')
+            self.cur.execute(
+                '''INSERT INTO inventory_backup SELECT hash, objecttype, streamnumber, payload, receivedtime FROM inventory;''')
+            self.cur.execute( '''DROP TABLE inventory''')
+            self.cur.execute(
+                '''CREATE TABLE inventory (hash blob, objecttype text, streamnumber int, payload blob, receivedtime integer, UNIQUE(hash) ON CONFLICT REPLACE)''' )
+            self.cur.execute(
+                '''INSERT INTO inventory SELECT hash, objecttype, streamnumber, payload, receivedtime FROM inventory_backup;''')
+            self.cur.execute( '''DROP TABLE inventory_backup;''')
+            item = '''update settings set value=? WHERE key='version';'''
+            parameters = (3,)
+            self.cur.execute(item, parameters)
+
+        # Add a new column to the inventory table to store tags.
+        item = '''SELECT value FROM settings WHERE key='version';'''
+        parameters = ''
+        self.cur.execute(item, parameters)
+        currentVersion = int(self.cur.fetchall()[0][0])
+        if currentVersion == 1 or currentVersion == 3:
+            logger.debug('In messages.dat database, adding tag field to the inventory table.')
             item = '''ALTER TABLE inventory ADD tag blob DEFAULT '' '''
             parameters = ''
             self.cur.execute(item, parameters)
             item = '''update settings set value=? WHERE key='version';'''
-            parameters = (3,)
+            parameters = (4,)
             self.cur.execute(item, parameters)
 
         if not shared.config.has_option('bitmessagesettings', 'userlocale'):

src/shared.py

@@ -34,6 +34,7 @@ config = ConfigParser.SafeConfigParser()
 myECCryptorObjects = {}
 MyECSubscriptionCryptorObjects = {}
 myAddressesByHash = {} #The key in this dictionary is the RIPE hash which is encoded in an address and value is the address itself.
+myAddressesByTag = {} # The key in this dictionary is the tag generated from the address.
 broadcastSendersForWhichImWatching = {}
 workerQueue = Queue.Queue()
 UISignalQueue = Queue.Queue()
@@ -222,6 +223,7 @@ def reloadMyAddressHashes():
     logger.debug('reloading keys from keys.dat file')
     myECCryptorObjects.clear()
     myAddressesByHash.clear()
+    myAddressesByTag.clear()
     #myPrivateKeys.clear()
 
     keyfileSecure = checkSensitiveFilePermissions(appdata + 'keys.dat')
@@ -242,6 +244,9 @@ def reloadMyAddressHashes():
                     if len(privEncryptionKey) == 64:#It is 32 bytes encoded as 64 hex characters
                         myECCryptorObjects[hash] = highlevelcrypto.makeCryptor(privEncryptionKey)
                         myAddressesByHash[hash] = addressInKeysFile
+                        tag = hashlib.sha512(hashlib.sha512(encodeVarint(
+                            addressVersionNumber) + encodeVarint(streamNumber) + hash).digest()).digest()[32:]
+                        myAddressesByTag[tag] = addressInKeysFile
 
                 else:
                     logger.error('Error in reloadMyAddressHashes: Can\'t handle address versions other than 2, 3, or 4.\n')
@@ -250,18 +255,26 @@ def reloadMyAddressHashes():
         fixSensitiveFilePermissions(appdata + 'keys.dat', hasEnabledKeys)
 
 def reloadBroadcastSendersForWhichImWatching():
-    logger.debug('reloading subscriptions...')
     broadcastSendersForWhichImWatching.clear()
     MyECSubscriptionCryptorObjects.clear()
     queryreturn = sqlQuery('SELECT address FROM subscriptions where enabled=1')
+    logger.debug('reloading subscriptions...')
     for row in queryreturn:
         address, = row
         status,addressVersionNumber,streamNumber,hash = decodeAddress(address)
         if addressVersionNumber == 2:
             broadcastSendersForWhichImWatching[hash] = 0
         #Now, for all addresses, even version 2 addresses, we should create Cryptor objects in a dictionary which we will use to attempt to decrypt encrypted broadcast messages.
-        privEncryptionKey = hashlib.sha512(encodeVarint(addressVersionNumber)+encodeVarint(streamNumber)+hash).digest()[:32]
+        
-        MyECSubscriptionCryptorObjects[hash] = highlevelcrypto.makeCryptor(privEncryptionKey.encode('hex'))
+        if addressVersionNumber <= 3:
+            privEncryptionKey = hashlib.sha512(encodeVarint(addressVersionNumber)+encodeVarint(streamNumber)+hash).digest()[:32]
+            MyECSubscriptionCryptorObjects[hash] = highlevelcrypto.makeCryptor(privEncryptionKey.encode('hex'))
+        else:
+            doubleHashOfAddressData = hashlib.sha512(hashlib.sha512(encodeVarint(
+                addressVersionNumber) + encodeVarint(streamNumber) + hash).digest()).digest()
+            tag = doubleHashOfAddressData[32:]
+            privEncryptionKey = doubleHashOfAddressData[:32]
+            MyECSubscriptionCryptorObjects[tag] = highlevelcrypto.makeCryptor(privEncryptionKey.encode('hex'))
 
 def doCleanShutdown():
     global shutdown
@@ -312,8 +325,8 @@ def flushInventory():
     with SqlBulkExecute() as sql:
         for hash, storedValue in inventory.items():
             objectType, streamNumber, payload, receivedTime, tag = storedValue
-            sql.execute('''INSERT INTO inventory VALUES (?,?,?,?,?,?,?)''',
+            sql.execute('''INSERT INTO inventory VALUES (?,?,?,?,?,?)''',
-                       hash,objectType,streamNumber,payload,receivedTime,'',tag)
+                       hash,objectType,streamNumber,payload,receivedTime,tag)
             del inventory[hash]
 
 def fixPotentiallyInvalidUTF8Data(text):