diff --git a/src/addresses.py b/src/addresses.py index e48873a1..885c1f64 100644 --- a/src/addresses.py +++ b/src/addresses.py @@ -2,11 +2,16 @@ Operations with addresses """ # pylint: disable=inconsistent-return-statements -import hashlib + import logging from binascii import hexlify, unhexlify from struct import pack, unpack +try: + from highlevelcrypto import double_sha512 +except ImportError: + from .highlevelcrypto import double_sha512 + logger = logging.getLogger('default') @@ -134,15 +139,6 @@ def decodeVarint(data): return (encodedValue, 9) -def calculateInventoryHash(data): - """Calculate inventory hash from object data""" - sha = hashlib.new('sha512') - sha2 = hashlib.new('sha512') - sha.update(data) - sha2.update(sha.digest()) - return sha2.digest()[0:32] - - def encodeAddress(version, stream, ripe): """Convert ripe to address""" if version >= 2 and version < 4: @@ -166,12 +162,7 @@ def encodeAddress(version, stream, ripe): storedBinaryData = encodeVarint(version) + encodeVarint(stream) + ripe # Generate the checksum - sha = hashlib.new('sha512') - sha.update(storedBinaryData) - currentHash = sha.digest() - sha = hashlib.new('sha512') - sha.update(currentHash) - checksum = sha.digest()[0:4] + checksum = double_sha512(storedBinaryData)[0:4] # FIXME: encodeBase58 should take binary data, to reduce conversions # encodeBase58(storedBinaryData + checksum) @@ -207,13 +198,7 @@ def decodeAddress(address): data = unhexlify(hexdata) checksum = data[-4:] - sha = hashlib.new('sha512') - sha.update(data[:-4]) - currentHash = sha.digest() - sha = hashlib.new('sha512') - sha.update(currentHash) - - if checksum != sha.digest()[0:4]: + if checksum != double_sha512(data[:-4])[0:4]: status = 'checksumfailed' return status, 0, 0, '' diff --git a/src/api.py b/src/api.py index c27a24e7..4bf316f2 100644 --- a/src/api.py +++ b/src/api.py @@ -83,7 +83,6 @@ import shutdown import state from addresses import ( addBMIfNotPresent, - calculateInventoryHash, decodeAddress, decodeVarint, varintDecodeError @@ -92,6 +91,7 @@ from bmconfigparser import config from debug import logger from helper_sql import ( SqlBulkExecute, sqlExecute, sqlQuery, sqlStoredProcedure, sql_ready) +from highlevelcrypto import calculateInventoryHash from inventory import Inventory try: diff --git a/src/class_addressGenerator.py b/src/class_addressGenerator.py index d42469c3..c32c3994 100644 --- a/src/class_addressGenerator.py +++ b/src/class_addressGenerator.py @@ -349,10 +349,10 @@ class addressGenerator(StoppableThread): highlevelcrypto.makeCryptor( hexlify(potentialPrivEncryptionKey)) shared.myAddressesByHash[ripe] = address - tag = hashlib.sha512(hashlib.sha512( + tag = highlevelcrypto.double_sha512( encodeVarint(addressVersionNumber) + encodeVarint(streamNumber) + ripe - ).digest()).digest()[32:] + )[32:] shared.myAddressesByTag[tag] = address if addressVersionNumber == 3: # If this is a chan address, diff --git a/src/class_objectProcessor.py b/src/class_objectProcessor.py index 8d2e12a8..76d0209b 100644 --- a/src/class_objectProcessor.py +++ b/src/class_objectProcessor.py @@ -24,7 +24,7 @@ import queues import shared import state from addresses import ( - calculateInventoryHash, decodeAddress, decodeVarint, + decodeAddress, decodeVarint, encodeAddress, encodeVarint, varintDecodeError ) from bmconfigparser import config @@ -456,7 +456,7 @@ class objectProcessor(threading.Thread): streamNumberAsClaimedByMsg, streamNumberAsClaimedByMsgLength = \ decodeVarint(data[readPosition:readPosition + 9]) readPosition += streamNumberAsClaimedByMsgLength - inventoryHash = calculateInventoryHash(data) + inventoryHash = highlevelcrypto.calculateInventoryHash(data) initialDecryptionSuccessful = False # This is not an acknowledgement bound for me. See if it is a message @@ -586,8 +586,7 @@ class objectProcessor(threading.Thread): helper_bitcoin.calculateTestnetAddressFromPubkey(pubSigningKey) ) # Used to detect and ignore duplicate messages in our inbox - sigHash = hashlib.sha512( - hashlib.sha512(signature).digest()).digest()[32:] + sigHash = highlevelcrypto.double_sha512(signature)[32:] # calculate the fromRipe. sha = hashlib.new('sha512') @@ -736,7 +735,7 @@ class objectProcessor(threading.Thread): ackPayload = ackData[24:] objectType, toStreamNumber, expiresTime = \ protocol.decodeObjectParameters(ackPayload) - inventoryHash = calculateInventoryHash(ackPayload) + inventoryHash = highlevelcrypto.calculateInventoryHash(ackPayload) Inventory()[inventoryHash] = ( objectType, toStreamNumber, ackPayload, expiresTime, b'') queues.invQueue.put((toStreamNumber, inventoryHash)) @@ -763,7 +762,7 @@ class objectProcessor(threading.Thread): state.numberOfBroadcastsProcessed += 1 queues.UISignalQueue.put(( 'updateNumberOfBroadcastsProcessed', 'no data')) - inventoryHash = calculateInventoryHash(data) + inventoryHash = highlevelcrypto.calculateInventoryHash(data) readPosition = 20 # bypass the nonce, time, and object type broadcastVersion, broadcastVersionLength = decodeVarint( data[readPosition:readPosition + 9]) @@ -897,10 +896,10 @@ class objectProcessor(threading.Thread): ' itself. Ignoring message.' ) elif broadcastVersion == 5: - calculatedTag = hashlib.sha512(hashlib.sha512( + calculatedTag = highlevelcrypto.double_sha512( encodeVarint(sendersAddressVersion) + encodeVarint(sendersStream) + calculatedRipe - ).digest()).digest()[32:] + )[32:] if calculatedTag != embeddedTag: return logger.debug( 'The tag and encryption key used to encrypt this' @@ -930,8 +929,7 @@ class objectProcessor(threading.Thread): return logger.debug('ECDSA verify passed') # Used to detect and ignore duplicate messages in our inbox - sigHash = hashlib.sha512( - hashlib.sha512(signature).digest()).digest()[32:] + sigHash = highlevelcrypto.double_sha512(signature)[32:] fromAddress = encodeAddress( sendersAddressVersion, sendersStream, calculatedRipe) @@ -1005,10 +1003,10 @@ class objectProcessor(threading.Thread): # Let us create the tag from the address and see if we were waiting # for it. elif addressVersion >= 4: - tag = hashlib.sha512(hashlib.sha512( + tag = highlevelcrypto.double_sha512( encodeVarint(addressVersion) + encodeVarint(streamNumber) + ripe - ).digest()).digest()[32:] + )[32:] if tag in state.neededPubkeys: del state.neededPubkeys[tag] self.sendMessages(address) diff --git a/src/class_singleWorker.py b/src/class_singleWorker.py index d9ee69e7..eb5e8406 100644 --- a/src/class_singleWorker.py +++ b/src/class_singleWorker.py @@ -25,9 +25,7 @@ import queues import shared import state import tr -from addresses import ( - calculateInventoryHash, decodeAddress, decodeVarint, encodeVarint -) +from addresses import decodeAddress, decodeVarint, encodeVarint from bmconfigparser import config from helper_sql import sqlExecute, sqlQuery from inventory import Inventory @@ -75,18 +73,16 @@ class singleWorker(StoppableThread): queryreturn = sqlQuery( '''SELECT DISTINCT toaddress FROM sent''' ''' WHERE (status='awaitingpubkey' AND folder='sent')''') - for row in queryreturn: - toAddress, = row - # toStatus - _, toAddressVersionNumber, toStreamNumber, toRipe = \ - decodeAddress(toAddress) + for toAddress, in queryreturn: + toAddressVersionNumber, toStreamNumber, toRipe = \ + decodeAddress(toAddress)[1:] if toAddressVersionNumber <= 3: state.neededPubkeys[toAddress] = 0 elif toAddressVersionNumber >= 4: - doubleHashOfAddressData = hashlib.sha512(hashlib.sha512( + doubleHashOfAddressData = highlevelcrypto.double_sha512( encodeVarint(toAddressVersionNumber) + encodeVarint(toStreamNumber) + toRipe - ).digest()).digest() + ) # Note that this is the first half of the sha512 hash. privEncryptionKey = doubleHashOfAddressData[:32] tag = doubleHashOfAddressData[32:] @@ -289,7 +285,7 @@ class singleWorker(StoppableThread): payload = self._doPOWDefaults( payload, TTL, log_prefix='(For pubkey message)') - inventoryHash = calculateInventoryHash(payload) + inventoryHash = highlevelcrypto.calculateInventoryHash(payload) objectType = 1 Inventory()[inventoryHash] = ( objectType, streamNumber, payload, embeddedTime, '') @@ -377,7 +373,7 @@ class singleWorker(StoppableThread): payload = self._doPOWDefaults( payload, TTL, log_prefix='(For pubkey message)') - inventoryHash = calculateInventoryHash(payload) + inventoryHash = highlevelcrypto.calculateInventoryHash(payload) objectType = 1 Inventory()[inventoryHash] = ( objectType, streamNumber, payload, embeddedTime, '') @@ -449,10 +445,10 @@ class singleWorker(StoppableThread): # unencrypted, the pubkey with part of the hash so that nodes # know which pubkey object to try to decrypt # when they want to send a message. - doubleHashOfAddressData = hashlib.sha512(hashlib.sha512( + doubleHashOfAddressData = highlevelcrypto.double_sha512( encodeVarint(addressVersionNumber) + encodeVarint(streamNumber) + addressHash - ).digest()).digest() + ) payload += doubleHashOfAddressData[32:] # the tag signature = highlevelcrypto.sign( payload + dataToEncrypt, privSigningKeyHex, self.digestAlg) @@ -468,7 +464,7 @@ class singleWorker(StoppableThread): payload = self._doPOWDefaults( payload, TTL, log_prefix='(For pubkey message)') - inventoryHash = calculateInventoryHash(payload) + inventoryHash = highlevelcrypto.calculateInventoryHash(payload) objectType = 1 Inventory()[inventoryHash] = ( objectType, streamNumber, payload, embeddedTime, @@ -504,7 +500,7 @@ class singleWorker(StoppableThread): objectType = protocol.OBJECT_ONIONPEER # FIXME: ideally the objectPayload should be signed objectPayload = encodeVarint(peer.port) + protocol.encodeHost(peer.host) - tag = calculateInventoryHash(objectPayload) + tag = highlevelcrypto.calculateInventoryHash(objectPayload) if Inventory().by_type_and_tag(objectType, tag): return # not expired @@ -518,7 +514,7 @@ class singleWorker(StoppableThread): payload = self._doPOWDefaults( payload, TTL, log_prefix='(For onionpeer object)') - inventoryHash = calculateInventoryHash(payload) + inventoryHash = highlevelcrypto.calculateInventoryHash(payload) Inventory()[inventoryHash] = ( objectType, streamNumber, buffer(payload), # noqa: F821 embeddedTime, buffer(tag) # noqa: F821 @@ -612,10 +608,10 @@ class singleWorker(StoppableThread): payload += encodeVarint(streamNumber) if addressVersionNumber >= 4: - doubleHashOfAddressData = hashlib.sha512(hashlib.sha512( + doubleHashOfAddressData = highlevelcrypto.double_sha512( encodeVarint(addressVersionNumber) + encodeVarint(streamNumber) + ripe - ).digest()).digest() + ) tag = doubleHashOfAddressData[32:] payload += tag else: @@ -685,7 +681,7 @@ class singleWorker(StoppableThread): ) continue - inventoryHash = calculateInventoryHash(payload) + inventoryHash = highlevelcrypto.calculateInventoryHash(payload) objectType = 3 Inventory()[inventoryHash] = ( objectType, streamNumber, payload, embeddedTime, tag) @@ -794,10 +790,10 @@ class singleWorker(StoppableThread): if toAddressVersionNumber <= 3: toTag = '' else: - toTag = hashlib.sha512(hashlib.sha512( + toTag = highlevelcrypto.double_sha512( encodeVarint(toAddressVersionNumber) + encodeVarint(toStreamNumber) + toRipe - ).digest()).digest()[32:] + )[32:] if toaddress in state.neededPubkeys or \ toTag in state.neededPubkeys: # We already sent a request for the pubkey @@ -831,11 +827,11 @@ class singleWorker(StoppableThread): # already contains the toAddress and cryptor # object associated with the tag for this toAddress. if toAddressVersionNumber >= 4: - doubleHashOfToAddressData = hashlib.sha512( - hashlib.sha512( - encodeVarint(toAddressVersionNumber) + encodeVarint(toStreamNumber) + toRipe - ).digest() - ).digest() + doubleHashOfToAddressData = \ + highlevelcrypto.double_sha512( + encodeVarint(toAddressVersionNumber) + + encodeVarint(toStreamNumber) + toRipe + ) # The first half of the sha512 hash. privEncryptionKey = doubleHashOfToAddressData[:32] # The second half of the sha512 hash. @@ -1302,7 +1298,7 @@ class singleWorker(StoppableThread): ) continue - inventoryHash = calculateInventoryHash(encryptedPayload) + inventoryHash = highlevelcrypto.calculateInventoryHash(encryptedPayload) objectType = 2 Inventory()[inventoryHash] = ( objectType, toStreamNumber, encryptedPayload, embeddedTime, '') @@ -1352,8 +1348,7 @@ class singleWorker(StoppableThread): # the message in our own inbox. if config.has_section(toaddress): # Used to detect and ignore duplicate messages in our inbox - sigHash = hashlib.sha512(hashlib.sha512( - signature).digest()).digest()[32:] + sigHash = highlevelcrypto.double_sha512(signature)[32:] t = (inventoryHash, toaddress, fromaddress, subject, int( time.time()), message, 'inbox', encoding, 0, sigHash) helper_inbox.insert(t) @@ -1410,16 +1405,13 @@ class singleWorker(StoppableThread): # neededPubkeys dictionary. But if we are recovering # from a restart of the client then we have to put it in now. - # Note that this is the first half of the sha512 hash. - privEncryptionKey = hashlib.sha512(hashlib.sha512( + doubleHashOfAddressData = highlevelcrypto.double_sha512( encodeVarint(addressVersionNumber) + encodeVarint(streamNumber) + ripe - ).digest()).digest()[:32] + ) + privEncryptionKey = doubleHashOfAddressData[:32] # Note that this is the second half of the sha512 hash. - tag = hashlib.sha512(hashlib.sha512( - encodeVarint(addressVersionNumber) - + encodeVarint(streamNumber) + ripe - ).digest()).digest()[32:] + tag = doubleHashOfAddressData[32:] if tag not in state.neededPubkeys: # We'll need this for when we receive a pubkey reply: # it will be encrypted and we'll need to decrypt it. @@ -1462,7 +1454,7 @@ class singleWorker(StoppableThread): payload = self._doPOWDefaults(payload, TTL) - inventoryHash = calculateInventoryHash(payload) + inventoryHash = highlevelcrypto.calculateInventoryHash(payload) objectType = 1 Inventory()[inventoryHash] = ( objectType, streamNumber, payload, embeddedTime, '') diff --git a/src/highlevelcrypto.py b/src/highlevelcrypto.py index 2604594e..e497c31c 100644 --- a/src/highlevelcrypto.py +++ b/src/highlevelcrypto.py @@ -18,6 +18,7 @@ from pyelliptic import arithmetic as a __all__ = [ 'decodeWalletImportFormat', 'encodeWalletImportFormat', + 'double_sha512', 'calculateInventoryHash', 'encrypt', 'makeCryptor', 'pointMult', 'privToPub', 'randomBytes', 'sign', 'verify'] @@ -61,6 +62,18 @@ def randomBytes(n): return OpenSSL.rand(n) +# Hashes + +def double_sha512(data): + """Binary double SHA512 digest""" + return hashlib.sha512(hashlib.sha512(data).digest()).digest() + + +def calculateInventoryHash(data): + """Calculate inventory hash from object data""" + return double_sha512(data)[:32] + + def makeCryptor(privkey, curve='secp256k1'): """Return a private `.pyelliptic.ECC` instance""" private_key = a.changebase(privkey, 16, 256, minlen=32) diff --git a/src/network/bmobject.py b/src/network/bmobject.py index 5d7fdcbd..c5d4c8fd 100644 --- a/src/network/bmobject.py +++ b/src/network/bmobject.py @@ -6,7 +6,7 @@ import time import protocol import state -from addresses import calculateInventoryHash +from highlevelcrypto import calculateInventoryHash from inventory import Inventory from network.dandelion import Dandelion diff --git a/src/proofofwork.py b/src/proofofwork.py index f77f455a..49e86f59 100644 --- a/src/proofofwork.py +++ b/src/proofofwork.py @@ -4,7 +4,6 @@ Proof of work calculation """ import ctypes -import hashlib import os import sys import tempfile @@ -12,6 +11,7 @@ import time from struct import pack, unpack from subprocess import call # nosec B404 +import highlevelcrypto import openclpow import paths import queues @@ -87,13 +87,20 @@ def _set_idle(): pass +def trial_value(nonce, initialHash): + """Calculate PoW trial value""" + trialValue, = unpack( + '>Q', highlevelcrypto.double_sha512( + pack('>Q', nonce) + initialHash)[0:8]) + return trialValue + + def _pool_worker(nonce, initialHash, target, pool_size): _set_idle() trialValue = float('inf') while trialValue > target: nonce += pool_size - trialValue, = unpack('>Q', hashlib.sha512(hashlib.sha512( - pack('>Q', nonce) + initialHash).digest()).digest()[0:8]) + trialValue = trial_value(nonce, initialHash) return [trialValue, nonce] @@ -103,10 +110,9 @@ def _doSafePoW(target, initialHash): trialValue = float('inf') while trialValue > target and state.shutdown == 0: nonce += 1 - trialValue, = unpack('>Q', hashlib.sha512(hashlib.sha512( - pack('>Q', nonce) + initialHash).digest()).digest()[0:8]) + trialValue = trial_value(nonce, initialHash) if state.shutdown != 0: - raise StopIteration("Interrupted") # pylint: misplaced-bare-raise + raise StopIteration("Interrupted") logger.debug("Safe PoW done") return [trialValue, nonce] @@ -163,7 +169,7 @@ def _doCPoW(target, initialHash): logger.debug("C PoW start") nonce = bmpow(out_h, out_m) - trialValue, = unpack('>Q', hashlib.sha512(hashlib.sha512(pack('>Q', nonce) + initialHash).digest()).digest()[0:8]) + trialValue = trial_value(nonce, initialHash) if state.shutdown != 0: raise StopIteration("Interrupted") logger.debug("C PoW done") @@ -173,7 +179,7 @@ def _doCPoW(target, initialHash): def _doGPUPoW(target, initialHash): logger.debug("GPU PoW start") nonce = openclpow.do_opencl_pow(initialHash.encode("hex"), target) - trialValue, = unpack('>Q', hashlib.sha512(hashlib.sha512(pack('>Q', nonce) + initialHash).digest()).digest()[0:8]) + trialValue = trial_value(nonce, initialHash) if trialValue > target: deviceNames = ", ".join(gpu.name for gpu in openclpow.enabledGpus) queues.UISignalQueue.put(( diff --git a/src/protocol.py b/src/protocol.py index 722ce9c1..d6ea884d 100644 --- a/src/protocol.py +++ b/src/protocol.py @@ -290,12 +290,11 @@ def isProofOfWorkSufficient( if payloadLengthExtraBytes < defaults.networkDefaultPayloadLengthExtraBytes: payloadLengthExtraBytes = defaults.networkDefaultPayloadLengthExtraBytes endOfLifeTime, = unpack('>Q', data[8:16]) - TTL = endOfLifeTime - (int(recvTime) if recvTime else int(time.time())) + TTL = endOfLifeTime - int(recvTime if recvTime else time.time()) if TTL < 300: TTL = 300 - POW, = unpack('>Q', hashlib.sha512(hashlib.sha512( - data[:8] + hashlib.sha512(data[8:]).digest() - ).digest()).digest()[0:8]) + POW, = unpack('>Q', highlevelcrypto.double_sha512( + data[:8] + hashlib.sha512(data[8:]).digest())[0:8]) return POW <= 2 ** 64 / ( nonceTrialsPerByte * ( len(data) + payloadLengthExtraBytes diff --git a/src/shared.py b/src/shared.py index 114bd916..3570864a 100644 --- a/src/shared.py +++ b/src/shared.py @@ -86,8 +86,7 @@ def reloadMyAddressHashes(): state.appdata, 'keys.dat')) hasEnabledKeys = False for addressInKeysFile in config.addresses(): - isEnabled = config.getboolean(addressInKeysFile, 'enabled') - if not isEnabled: + if not config.getboolean(addressInKeysFile, 'enabled'): continue hasEnabledKeys = True @@ -116,9 +115,9 @@ def reloadMyAddressHashes(): myECCryptorObjects[hashobj] = \ highlevelcrypto.makeCryptor(privEncryptionKey) myAddressesByHash[hashobj] = addressInKeysFile - tag = hashlib.sha512(hashlib.sha512( + tag = highlevelcrypto.double_sha512( encodeVarint(addressVersionNumber) - + encodeVarint(streamNumber) + hashobj).digest()).digest()[32:] + + encodeVarint(streamNumber) + hashobj)[32:] myAddressesByTag[tag] = addressInKeysFile if not keyfileSecure: @@ -153,10 +152,10 @@ def reloadBroadcastSendersForWhichImWatching(): MyECSubscriptionCryptorObjects[hashobj] = \ highlevelcrypto.makeCryptor(hexlify(privEncryptionKey)) else: - doubleHashOfAddressData = hashlib.sha512(hashlib.sha512( + doubleHashOfAddressData = highlevelcrypto.double_sha512( encodeVarint(addressVersionNumber) + encodeVarint(streamNumber) + hashobj - ).digest()).digest() + ) tag = doubleHashOfAddressData[32:] privEncryptionKey = doubleHashOfAddressData[:32] MyECSubscriptionCryptorObjects[tag] = \ diff --git a/src/tests/test_inventory.py b/src/tests/test_inventory.py index b6d0cc85..5978f9a5 100644 --- a/src/tests/test_inventory.py +++ b/src/tests/test_inventory.py @@ -7,8 +7,8 @@ import tempfile import time import unittest +from pybitmessage import highlevelcrypto from pybitmessage.storage import storage -from pybitmessage.addresses import calculateInventoryHash from .partial import TestPartialRun @@ -35,7 +35,7 @@ class TestFilesystemInventory(TestPartialRun): TTL = 24 * 60 * 60 embedded_time = int(time.time() + TTL) msg = struct.pack('>Q', embedded_time) + os.urandom(166) - invhash = calculateInventoryHash(msg) + invhash = highlevelcrypto.calculateInventoryHash(msg) self.inventory[invhash] = (2, 1, msg, embedded_time, b'') @classmethod diff --git a/src/tests/test_openclpow.py b/src/tests/test_openclpow.py index 341beec9..4770072e 100644 --- a/src/tests/test_openclpow.py +++ b/src/tests/test_openclpow.py @@ -1,10 +1,10 @@ """ Tests for openclpow module """ -import hashlib + import unittest -from struct import pack, unpack -from pybitmessage import openclpow + +from pybitmessage import openclpow, proofofwork class TestOpenClPow(unittest.TestCase): @@ -25,7 +25,5 @@ class TestOpenClPow(unittest.TestCase): "b93f3ffeba0ef2fd08a8dc2f87b68ae5a0dc819ab57f22ad2c4c9c8618a43b3" ).decode("hex") nonce = openclpow.do_opencl_pow(initialHash.encode("hex"), target_) - trialValue, = unpack( - '>Q', hashlib.sha512(hashlib.sha512( - pack('>Q', nonce) + initialHash).digest()).digest()[0:8]) - self.assertLess((nonce - trialValue), target_) + self.assertLess( + nonce - proofofwork.trial_value(nonce, initialHash), target_)