Start adding hashes with double SHA512

src/addresses.py

@@ -2,11 +2,16 @@
 Operations with addresses
 """
 # pylint: disable=inconsistent-return-statements
-import hashlib
+
 import logging
 from binascii import hexlify, unhexlify
 from struct import pack, unpack
 
+try:
+    from highlevelcrypto import double_sha512
+except ImportError:
+    from .highlevelcrypto import double_sha512
+
 
 logger = logging.getLogger('default')
 
@@ -134,15 +139,6 @@ def decodeVarint(data):
         return (encodedValue, 9)
 
 
-def calculateInventoryHash(data):
-    """Calculate inventory hash from object data"""
-    sha = hashlib.new('sha512')
-    sha2 = hashlib.new('sha512')
-    sha.update(data)
-    sha2.update(sha.digest())
-    return sha2.digest()[0:32]
-
-
 def encodeAddress(version, stream, ripe):
     """Convert ripe to address"""
     if version >= 2 and version < 4:
@@ -166,12 +162,7 @@ def encodeAddress(version, stream, ripe):
     storedBinaryData = encodeVarint(version) + encodeVarint(stream) + ripe
 
     # Generate the checksum
-    sha = hashlib.new('sha512')
+    checksum = double_sha512(storedBinaryData)[0:4]
-    sha.update(storedBinaryData)
-    currentHash = sha.digest()
-    sha = hashlib.new('sha512')
-    sha.update(currentHash)
-    checksum = sha.digest()[0:4]
 
     # FIXME: encodeBase58 should take binary data, to reduce conversions
     # encodeBase58(storedBinaryData + checksum)
@@ -207,13 +198,7 @@ def decodeAddress(address):
     data = unhexlify(hexdata)
     checksum = data[-4:]
 
-    sha = hashlib.new('sha512')
+    if checksum != double_sha512(data[:-4])[0:4]:
-    sha.update(data[:-4])
-    currentHash = sha.digest()
-    sha = hashlib.new('sha512')
-    sha.update(currentHash)
-
-    if checksum != sha.digest()[0:4]:
         status = 'checksumfailed'
         return status, 0, 0, ''
 

src/api.py

@@ -71,6 +71,8 @@ from binascii import hexlify, unhexlify
 from SimpleXMLRPCServer import SimpleXMLRPCRequestHandler, SimpleXMLRPCServer
 from struct import pack
 
+from six.moves import queue
+
 import defaults
 import helper_inbox
 import helper_sent
@@ -82,17 +84,17 @@ import shutdown
 import state
 from addresses import (
     addBMIfNotPresent,
-    calculateInventoryHash,
     decodeAddress,
     decodeVarint,
     varintDecodeError
 )
 from bmconfigparser import BMConfigParser
 from debug import logger
-from helper_sql import SqlBulkExecute, sqlExecute, sqlQuery, sqlStoredProcedure, sql_ready
+from helper_sql import (
+    SqlBulkExecute, sqlExecute, sqlQuery, sql_ready, sqlStoredProcedure)
+from highlevelcrypto import calculateInventoryHash
 from inventory import Inventory
 from network.threads import StoppableThread
-from six.moves import queue
 from version import softwareVersion
 
 try:  # TODO: write tests for XML vulnerabilities

src/class_addressGenerator.py

@@ -367,10 +367,10 @@ class addressGenerator(StoppableThread):
                                 highlevelcrypto.makeCryptor(
                                     hexlify(potentialPrivEncryptionKey))
                             shared.myAddressesByHash[ripe] = address
-                            tag = hashlib.sha512(hashlib.sha512(
+                            tag = highlevelcrypto.double_sha512(
                                 encodeVarint(addressVersionNumber)
                                 + encodeVarint(streamNumber) + ripe
-                            ).digest()).digest()[32:]
+                            )[32:]
                             shared.myAddressesByTag[tag] = address
                             if addressVersionNumber == 3:
                                 # If this is a chan address,

src/class_objectProcessor.py

@@ -23,7 +23,7 @@ import queues
 import shared
 import state
 from addresses import (
-    calculateInventoryHash, decodeAddress, decodeVarint,
+    decodeAddress, decodeVarint,
     encodeAddress, encodeVarint, varintDecodeError
 )
 from bmconfigparser import BMConfigParser
@@ -450,7 +450,7 @@ class objectProcessor(threading.Thread):
         streamNumberAsClaimedByMsg, streamNumberAsClaimedByMsgLength = \
             decodeVarint(data[readPosition:readPosition + 9])
         readPosition += streamNumberAsClaimedByMsgLength
-        inventoryHash = calculateInventoryHash(data)
+        inventoryHash = highlevelcrypto.calculateInventoryHash(data)
         initialDecryptionSuccessful = False
 
         # This is not an acknowledgement bound for me. See if it is a message
@@ -580,8 +580,7 @@ class objectProcessor(threading.Thread):
                 helper_bitcoin.calculateTestnetAddressFromPubkey(pubSigningKey)
             )
         # Used to detect and ignore duplicate messages in our inbox
-        sigHash = hashlib.sha512(
+        sigHash = highlevelcrypto.double_sha512(signature)[32:]
-            hashlib.sha512(signature).digest()).digest()[32:]
 
         # calculate the fromRipe.
         sha = hashlib.new('sha512')
@@ -751,7 +750,7 @@ class objectProcessor(threading.Thread):
         state.numberOfBroadcastsProcessed += 1
         queues.UISignalQueue.put((
             'updateNumberOfBroadcastsProcessed', 'no data'))
-        inventoryHash = calculateInventoryHash(data)
+        inventoryHash = highlevelcrypto.calculateInventoryHash(data)
         readPosition = 20  # bypass the nonce, time, and object type
         broadcastVersion, broadcastVersionLength = decodeVarint(
             data[readPosition:readPosition + 9])
@@ -885,10 +884,10 @@ class objectProcessor(threading.Thread):
                     ' itself. Ignoring message.'
                 )
         elif broadcastVersion == 5:
-            calculatedTag = hashlib.sha512(hashlib.sha512(
+            calculatedTag = highlevelcrypto.double_sha512(
                 encodeVarint(sendersAddressVersion)
                 + encodeVarint(sendersStream) + calculatedRipe
-            ).digest()).digest()[32:]
+            )[32:]
             if calculatedTag != embeddedTag:
                 return logger.debug(
                     'The tag and encryption key used to encrypt this'
@@ -918,8 +917,7 @@ class objectProcessor(threading.Thread):
             return
         logger.debug('ECDSA verify passed')
         # Used to detect and ignore duplicate messages in our inbox
-        sigHash = hashlib.sha512(
+        sigHash = highlevelcrypto.double_sha512(signature)[32:]
-            hashlib.sha512(signature).digest()).digest()[32:]
 
         fromAddress = encodeAddress(
             sendersAddressVersion, sendersStream, calculatedRipe)
@@ -993,10 +991,10 @@ class objectProcessor(threading.Thread):
         # Let us create the tag from the address and see if we were waiting
         # for it.
         elif addressVersion >= 4:
-            tag = hashlib.sha512(hashlib.sha512(
+            tag = highlevelcrypto.double_sha512(
                 encodeVarint(addressVersion) + encodeVarint(streamNumber)
                 + ripe
-            ).digest()).digest()[32:]
+            )[32:]
             if tag in state.neededPubkeys:
                 del state.neededPubkeys[tag]
                 self.sendMessages(address)

src/class_singleWorker.py

@@ -25,9 +25,7 @@ import queues
 import shared
 import state
 import tr
-from addresses import (
+from addresses import decodeAddress, decodeVarint, encodeVarint
-    calculateInventoryHash, decodeAddress, decodeVarint, encodeVarint
-)
 from bmconfigparser import BMConfigParser
 from helper_sql import sqlExecute, sqlQuery
 from inventory import Inventory
@@ -75,18 +73,16 @@ class singleWorker(StoppableThread):
         queryreturn = sqlQuery(
             '''SELECT DISTINCT toaddress FROM sent'''
             ''' WHERE (status='awaitingpubkey' AND folder='sent')''')
-        for row in queryreturn:
+        for toAddress, in queryreturn:
-            toAddress, = row
+            toAddressVersionNumber, toStreamNumber, toRipe = \
-            # toStatus
+                decodeAddress(toAddress)[1:]
-            _, toAddressVersionNumber, toStreamNumber, toRipe = \
-                decodeAddress(toAddress)
             if toAddressVersionNumber <= 3:
                 state.neededPubkeys[toAddress] = 0
             elif toAddressVersionNumber >= 4:
-                doubleHashOfAddressData = hashlib.sha512(hashlib.sha512(
+                doubleHashOfAddressData = highlevelcrypto.double_sha512(
                     encodeVarint(toAddressVersionNumber)
                     + encodeVarint(toStreamNumber) + toRipe
-                ).digest()).digest()
+                )
                 # Note that this is the first half of the sha512 hash.
                 privEncryptionKey = doubleHashOfAddressData[:32]
                 tag = doubleHashOfAddressData[32:]
@@ -290,7 +286,7 @@ class singleWorker(StoppableThread):
         payload = self._doPOWDefaults(
             payload, TTL, log_prefix='(For pubkey message)')
 
-        inventoryHash = calculateInventoryHash(payload)
+        inventoryHash = highlevelcrypto.calculateInventoryHash(payload)
         objectType = 1
         Inventory()[inventoryHash] = (
             objectType, streamNumber, payload, embeddedTime, '')
@@ -379,7 +375,7 @@ class singleWorker(StoppableThread):
         payload = self._doPOWDefaults(
             payload, TTL, log_prefix='(For pubkey message)')
 
-        inventoryHash = calculateInventoryHash(payload)
+        inventoryHash = highlevelcrypto.calculateInventoryHash(payload)
         objectType = 1
         Inventory()[inventoryHash] = (
             objectType, streamNumber, payload, embeddedTime, '')
@@ -452,10 +448,10 @@ class singleWorker(StoppableThread):
         # unencrypted, the pubkey with part of the hash so that nodes
         # know which pubkey object to try to decrypt
         # when they want to send a message.
-        doubleHashOfAddressData = hashlib.sha512(hashlib.sha512(
+        doubleHashOfAddressData = highlevelcrypto.double_sha512(
             encodeVarint(addressVersionNumber)
             + encodeVarint(streamNumber) + addressHash
-        ).digest()).digest()
+        )
         payload += doubleHashOfAddressData[32:]  # the tag
         signature = highlevelcrypto.sign(
             payload + dataToEncrypt, privSigningKeyHex, self.digestAlg)
@@ -471,7 +467,7 @@ class singleWorker(StoppableThread):
         payload = self._doPOWDefaults(
             payload, TTL, log_prefix='(For pubkey message)')
 
-        inventoryHash = calculateInventoryHash(payload)
+        inventoryHash = highlevelcrypto.calculateInventoryHash(payload)
         objectType = 1
         Inventory()[inventoryHash] = (
             objectType, streamNumber, payload, embeddedTime,
@@ -507,7 +503,7 @@ class singleWorker(StoppableThread):
         objectType = protocol.OBJECT_ONIONPEER
         # FIXME: ideally the objectPayload should be signed
         objectPayload = encodeVarint(peer.port) + protocol.encodeHost(peer.host)
-        tag = calculateInventoryHash(objectPayload)
+        tag = highlevelcrypto.calculateInventoryHash(objectPayload)
 
         if Inventory().by_type_and_tag(objectType, tag):
             return  # not expired
@@ -521,7 +517,7 @@ class singleWorker(StoppableThread):
         payload = self._doPOWDefaults(
             payload, TTL, log_prefix='(For onionpeer object)')
 
-        inventoryHash = calculateInventoryHash(payload)
+        inventoryHash = highlevelcrypto.calculateInventoryHash(payload)
         Inventory()[inventoryHash] = (
             objectType, streamNumber, buffer(payload),
             embeddedTime, buffer(tag)
@@ -615,10 +611,10 @@ class singleWorker(StoppableThread):
 
             payload += encodeVarint(streamNumber)
             if addressVersionNumber >= 4:
-                doubleHashOfAddressData = hashlib.sha512(hashlib.sha512(
+                doubleHashOfAddressData = highlevelcrypto.double_sha512(
                     encodeVarint(addressVersionNumber)
                     + encodeVarint(streamNumber) + ripe
-                ).digest()).digest()
+                )
                 tag = doubleHashOfAddressData[32:]
                 payload += tag
             else:
@@ -688,7 +684,7 @@ class singleWorker(StoppableThread):
                 )
                 continue
 
-            inventoryHash = calculateInventoryHash(payload)
+            inventoryHash = highlevelcrypto.calculateInventoryHash(payload)
             objectType = 3
             Inventory()[inventoryHash] = (
                 objectType, streamNumber, payload, embeddedTime, tag)
@@ -797,10 +793,10 @@ class singleWorker(StoppableThread):
                     if toAddressVersionNumber <= 3:
                         toTag = ''
                     else:
-                        toTag = hashlib.sha512(hashlib.sha512(
+                        toTag = highlevelcrypto.double_sha512(
                             encodeVarint(toAddressVersionNumber)
                             + encodeVarint(toStreamNumber) + toRipe
-                        ).digest()).digest()[32:]
+                        )[32:]
                     if toaddress in state.neededPubkeys or \
                             toTag in state.neededPubkeys:
                         # We already sent a request for the pubkey
@@ -834,11 +830,11 @@ class singleWorker(StoppableThread):
                         # already contains the toAddress and cryptor
                         # object associated with the tag for this toAddress.
                         if toAddressVersionNumber >= 4:
-                            doubleHashOfToAddressData = hashlib.sha512(
+                            doubleHashOfToAddressData = \
-                                hashlib.sha512(
+                                highlevelcrypto.double_sha512(
-                                    encodeVarint(toAddressVersionNumber) + encodeVarint(toStreamNumber) + toRipe
+                                    encodeVarint(toAddressVersionNumber)
-                                ).digest()
+                                    + encodeVarint(toStreamNumber) + toRipe
-                            ).digest()
+                                )
                             # The first half of the sha512 hash.
                             privEncryptionKey = doubleHashOfToAddressData[:32]
                             # The second half of the sha512 hash.
@@ -1304,7 +1300,7 @@ class singleWorker(StoppableThread):
                 )
                 continue
 
-            inventoryHash = calculateInventoryHash(encryptedPayload)
+            inventoryHash = highlevelcrypto.calculateInventoryHash(encryptedPayload)
             objectType = 2
             Inventory()[inventoryHash] = (
                 objectType, toStreamNumber, encryptedPayload, embeddedTime, '')
@@ -1354,8 +1350,7 @@ class singleWorker(StoppableThread):
             # the message in our own inbox.
             if BMConfigParser().has_section(toaddress):
                 # Used to detect and ignore duplicate messages in our inbox
-                sigHash = hashlib.sha512(hashlib.sha512(
+                sigHash = highlevelcrypto.double_sha512(signature)[32:]
-                    signature).digest()).digest()[32:]
                 t = (inventoryHash, toaddress, fromaddress, subject, int(
                     time.time()), message, 'inbox', encoding, 0, sigHash)
                 helper_inbox.insert(t)
@@ -1410,16 +1405,13 @@ class singleWorker(StoppableThread):
             # neededPubkeys dictionary. But if we are recovering
             # from a restart of the client then we have to put it in now.
 
-            # Note that this is the first half of the sha512 hash.
+            doubleHashOfAddressData = highlevelcrypto.double_sha512(
-            privEncryptionKey = hashlib.sha512(hashlib.sha512(
                 encodeVarint(addressVersionNumber)
                 + encodeVarint(streamNumber) + ripe
-            ).digest()).digest()[:32]
+            )
+            privEncryptionKey = doubleHashOfAddressData[:32]
             # Note that this is the second half of the sha512 hash.
-            tag = hashlib.sha512(hashlib.sha512(
+            tag = doubleHashOfAddressData[32:]
-                encodeVarint(addressVersionNumber)
-                + encodeVarint(streamNumber) + ripe
-            ).digest()).digest()[32:]
             if tag not in state.neededPubkeys:
                 # We'll need this for when we receive a pubkey reply:
                 # it will be encrypted and we'll need to decrypt it.
@@ -1462,7 +1454,7 @@ class singleWorker(StoppableThread):
 
         payload = self._doPOWDefaults(payload, TTL)
 
-        inventoryHash = calculateInventoryHash(payload)
+        inventoryHash = highlevelcrypto.calculateInventoryHash(payload)
         objectType = 1
         Inventory()[inventoryHash] = (
             objectType, streamNumber, payload, embeddedTime, '')

src/highlevelcrypto.py

@@ -7,6 +7,7 @@ High level cryptographic functions based on `.pyelliptic` OpenSSL bindings.
   `More discussion. <https://github.com/yann2192/pyelliptic/issues/32>`_
 """
 
+import hashlib
 from binascii import hexlify
 
 import pyelliptic
@@ -17,6 +18,18 @@ from pyelliptic import arithmetic as a
 __all__ = ['encrypt', 'makeCryptor', 'pointMult', 'privToPub', 'sign', 'verify']
 
 
+# Hashes
+
+def double_sha512(data):
+    """Binary double SHA512 digest"""
+    return hashlib.sha512(hashlib.sha512(data).digest()).digest()
+
+
+def calculateInventoryHash(data):
+    """Calculate inventory hash from object data"""
+    return double_sha512(data)[:32]
+
+
 def makeCryptor(privkey):
     """Return a private `.pyelliptic.ECC` instance"""
     private_key = a.changebase(privkey, 16, 256, minlen=32)

src/network/bmobject.py

@@ -6,7 +6,7 @@ import time
 
 import protocol
 import state
-from addresses import calculateInventoryHash
+from highlevelcrypto import calculateInventoryHash
 from inventory import Inventory
 from network.dandelion import Dandelion
 

src/proofofwork.py

@@ -4,7 +4,6 @@ Proof of work calculation
 """
 
 import ctypes
-import hashlib
 import os
 import sys
 import tempfile
@@ -12,6 +11,7 @@ import time
 from struct import pack, unpack
 from subprocess import call
 
+import highlevelcrypto
 import openclpow
 import paths
 import queues
@@ -87,13 +87,20 @@ def _set_idle():
             pass
 
 
+def trial_value(nonce, initialHash):
+    """Calculate PoW trial value"""
+    trialValue, = unpack(
+        '>Q', highlevelcrypto.double_sha512(
+            pack('>Q', nonce) + initialHash)[0:8])
+    return trialValue
+
+
 def _pool_worker(nonce, initialHash, target, pool_size):
     _set_idle()
     trialValue = float('inf')
     while trialValue > target:
         nonce += pool_size
-        trialValue, = unpack('>Q', hashlib.sha512(hashlib.sha512(
+        trialValue = trial_value(nonce, initialHash)
-            pack('>Q', nonce) + initialHash).digest()).digest()[0:8])
     return [trialValue, nonce]
 
 
@@ -103,10 +110,9 @@ def _doSafePoW(target, initialHash):
     trialValue = float('inf')
     while trialValue > target and state.shutdown == 0:
         nonce += 1
-        trialValue, = unpack('>Q', hashlib.sha512(hashlib.sha512(
+        trialValue = trial_value(nonce, initialHash)
-            pack('>Q', nonce) + initialHash).digest()).digest()[0:8])
     if state.shutdown != 0:
-        raise StopIteration("Interrupted")  # pylint: misplaced-bare-raise
+        raise StopIteration("Interrupted")
     logger.debug("Safe PoW done")
     return [trialValue, nonce]
 
@@ -163,7 +169,7 @@ def _doCPoW(target, initialHash):
         logger.debug("C PoW start")
         nonce = bmpow(out_h, out_m)
 
-    trialValue, = unpack('>Q', hashlib.sha512(hashlib.sha512(pack('>Q', nonce) + initialHash).digest()).digest()[0:8])
+    trialValue = trial_value(nonce, initialHash)
     if state.shutdown != 0:
         raise StopIteration("Interrupted")
     logger.debug("C PoW done")
@@ -173,7 +179,7 @@ def _doCPoW(target, initialHash):
 def _doGPUPoW(target, initialHash):
     logger.debug("GPU PoW start")
     nonce = openclpow.do_opencl_pow(initialHash.encode("hex"), target)
-    trialValue, = unpack('>Q', hashlib.sha512(hashlib.sha512(pack('>Q', nonce) + initialHash).digest()).digest()[0:8])
+    trialValue = trial_value(nonce, initialHash)
     if trialValue > target:
         deviceNames = ", ".join(gpu.name for gpu in openclpow.enabledGpus)
         queues.UISignalQueue.put((

src/protocol.py

@@ -269,12 +269,11 @@ def isProofOfWorkSufficient(
     if payloadLengthExtraBytes < defaults.networkDefaultPayloadLengthExtraBytes:
         payloadLengthExtraBytes = defaults.networkDefaultPayloadLengthExtraBytes
     endOfLifeTime, = unpack('>Q', data[8:16])
-    TTL = endOfLifeTime - (int(recvTime) if recvTime else int(time.time()))
+    TTL = endOfLifeTime - int(recvTime if recvTime else time.time())
     if TTL < 300:
         TTL = 300
-    POW, = unpack('>Q', hashlib.sha512(hashlib.sha512(
+    POW, = unpack('>Q', highlevelcrypto.double_sha512(
-        data[:8] + hashlib.sha512(data[8:]).digest()
+        data[:8] + hashlib.sha512(data[8:]).digest())[0:8])
-    ).digest()).digest()[0:8])
     return POW <= 2 ** 64 / (
         nonceTrialsPerByte * (
             len(data) + payloadLengthExtraBytes

src/shared.py

@@ -121,7 +121,8 @@ def reloadMyAddressHashes():
         if isEnabled:
             hasEnabledKeys = True
             # status
-            addressVersionNumber, streamNumber, hashobj = decodeAddress(addressInKeysFile)[1:]
+            addressVersionNumber, streamNumber, hashobj = decodeAddress(
+                addressInKeysFile)[1:]
             if addressVersionNumber in (2, 3, 4):
                 # Returns a simple 32 bytes of information encoded
                 # in 64 Hex characters, or null if there was an error.
@@ -132,9 +133,9 @@ def reloadMyAddressHashes():
                     myECCryptorObjects[hashobj] = \
                         highlevelcrypto.makeCryptor(privEncryptionKey)
                     myAddressesByHash[hashobj] = addressInKeysFile
-                    tag = hashlib.sha512(hashlib.sha512(
+                    tag = highlevelcrypto.double_sha512(
                         encodeVarint(addressVersionNumber)
-                        + encodeVarint(streamNumber) + hashobj).digest()).digest()[32:]
+                        + encodeVarint(streamNumber) + hashobj)[32:]
                     myAddressesByTag[tag] = addressInKeysFile
             else:
                 logger.error(
@@ -174,10 +175,10 @@ def reloadBroadcastSendersForWhichImWatching():
             MyECSubscriptionCryptorObjects[hashobj] = \
                 highlevelcrypto.makeCryptor(hexlify(privEncryptionKey))
         else:
-            doubleHashOfAddressData = hashlib.sha512(hashlib.sha512(
+            doubleHashOfAddressData = highlevelcrypto.double_sha512(
                 encodeVarint(addressVersionNumber)
                 + encodeVarint(streamNumber) + hashobj
-            ).digest()).digest()
+            )
             tag = doubleHashOfAddressData[32:]
             privEncryptionKey = doubleHashOfAddressData[:32]
             MyECSubscriptionCryptorObjects[tag] = \