From ba2d0e26878a3e43d8771cd3a998d940863290fc Mon Sep 17 00:00:00 2001 From: Dmitri Bogomolov <4glitch@gmail.com> Date: Thu, 29 Jul 2021 22:16:37 +0300 Subject: [PATCH] Start adding hashes with double SHA512 --- src/addresses.py | 31 +++++----------- src/api.py | 8 +++-- src/class_addressGenerator.py | 4 +-- src/class_objectProcessor.py | 20 +++++------ src/class_singleWorker.py | 68 ++++++++++++++++------------------- src/highlevelcrypto.py | 13 +++++++ src/network/bmobject.py | 2 +- src/proofofwork.py | 22 +++++++----- src/protocol.py | 7 ++-- src/shared.py | 11 +++--- 10 files changed, 91 insertions(+), 95 deletions(-) diff --git a/src/addresses.py b/src/addresses.py index e48873a1..885c1f64 100644 --- a/src/addresses.py +++ b/src/addresses.py @@ -2,11 +2,16 @@ Operations with addresses """ # pylint: disable=inconsistent-return-statements -import hashlib + import logging from binascii import hexlify, unhexlify from struct import pack, unpack +try: + from highlevelcrypto import double_sha512 +except ImportError: + from .highlevelcrypto import double_sha512 + logger = logging.getLogger('default') @@ -134,15 +139,6 @@ def decodeVarint(data): return (encodedValue, 9) -def calculateInventoryHash(data): - """Calculate inventory hash from object data""" - sha = hashlib.new('sha512') - sha2 = hashlib.new('sha512') - sha.update(data) - sha2.update(sha.digest()) - return sha2.digest()[0:32] - - def encodeAddress(version, stream, ripe): """Convert ripe to address""" if version >= 2 and version < 4: @@ -166,12 +162,7 @@ def encodeAddress(version, stream, ripe): storedBinaryData = encodeVarint(version) + encodeVarint(stream) + ripe # Generate the checksum - sha = hashlib.new('sha512') - sha.update(storedBinaryData) - currentHash = sha.digest() - sha = hashlib.new('sha512') - sha.update(currentHash) - checksum = sha.digest()[0:4] + checksum = double_sha512(storedBinaryData)[0:4] # FIXME: encodeBase58 should take binary data, to reduce conversions # encodeBase58(storedBinaryData + checksum) @@ -207,13 +198,7 @@ def decodeAddress(address): data = unhexlify(hexdata) checksum = data[-4:] - sha = hashlib.new('sha512') - sha.update(data[:-4]) - currentHash = sha.digest() - sha = hashlib.new('sha512') - sha.update(currentHash) - - if checksum != sha.digest()[0:4]: + if checksum != double_sha512(data[:-4])[0:4]: status = 'checksumfailed' return status, 0, 0, '' diff --git a/src/api.py b/src/api.py index de220cc4..536768e8 100644 --- a/src/api.py +++ b/src/api.py @@ -71,6 +71,8 @@ from binascii import hexlify, unhexlify from SimpleXMLRPCServer import SimpleXMLRPCRequestHandler, SimpleXMLRPCServer from struct import pack +from six.moves import queue + import defaults import helper_inbox import helper_sent @@ -82,17 +84,17 @@ import shutdown import state from addresses import ( addBMIfNotPresent, - calculateInventoryHash, decodeAddress, decodeVarint, varintDecodeError ) from bmconfigparser import BMConfigParser from debug import logger -from helper_sql import SqlBulkExecute, sqlExecute, sqlQuery, sqlStoredProcedure, sql_ready +from helper_sql import ( + SqlBulkExecute, sqlExecute, sqlQuery, sql_ready, sqlStoredProcedure) +from highlevelcrypto import calculateInventoryHash from inventory import Inventory from network.threads import StoppableThread -from six.moves import queue from version import softwareVersion try: # TODO: write tests for XML vulnerabilities diff --git a/src/class_addressGenerator.py b/src/class_addressGenerator.py index 25b0c5df..f4a0f008 100644 --- a/src/class_addressGenerator.py +++ b/src/class_addressGenerator.py @@ -367,10 +367,10 @@ class addressGenerator(StoppableThread): highlevelcrypto.makeCryptor( hexlify(potentialPrivEncryptionKey)) shared.myAddressesByHash[ripe] = address - tag = hashlib.sha512(hashlib.sha512( + tag = highlevelcrypto.double_sha512( encodeVarint(addressVersionNumber) + encodeVarint(streamNumber) + ripe - ).digest()).digest()[32:] + )[32:] shared.myAddressesByTag[tag] = address if addressVersionNumber == 3: # If this is a chan address, diff --git a/src/class_objectProcessor.py b/src/class_objectProcessor.py index 1bacf639..86ed2bcc 100644 --- a/src/class_objectProcessor.py +++ b/src/class_objectProcessor.py @@ -23,7 +23,7 @@ import queues import shared import state from addresses import ( - calculateInventoryHash, decodeAddress, decodeVarint, + decodeAddress, decodeVarint, encodeAddress, encodeVarint, varintDecodeError ) from bmconfigparser import BMConfigParser @@ -450,7 +450,7 @@ class objectProcessor(threading.Thread): streamNumberAsClaimedByMsg, streamNumberAsClaimedByMsgLength = \ decodeVarint(data[readPosition:readPosition + 9]) readPosition += streamNumberAsClaimedByMsgLength - inventoryHash = calculateInventoryHash(data) + inventoryHash = highlevelcrypto.calculateInventoryHash(data) initialDecryptionSuccessful = False # This is not an acknowledgement bound for me. See if it is a message @@ -580,8 +580,7 @@ class objectProcessor(threading.Thread): helper_bitcoin.calculateTestnetAddressFromPubkey(pubSigningKey) ) # Used to detect and ignore duplicate messages in our inbox - sigHash = hashlib.sha512( - hashlib.sha512(signature).digest()).digest()[32:] + sigHash = highlevelcrypto.double_sha512(signature)[32:] # calculate the fromRipe. sha = hashlib.new('sha512') @@ -751,7 +750,7 @@ class objectProcessor(threading.Thread): state.numberOfBroadcastsProcessed += 1 queues.UISignalQueue.put(( 'updateNumberOfBroadcastsProcessed', 'no data')) - inventoryHash = calculateInventoryHash(data) + inventoryHash = highlevelcrypto.calculateInventoryHash(data) readPosition = 20 # bypass the nonce, time, and object type broadcastVersion, broadcastVersionLength = decodeVarint( data[readPosition:readPosition + 9]) @@ -885,10 +884,10 @@ class objectProcessor(threading.Thread): ' itself. Ignoring message.' ) elif broadcastVersion == 5: - calculatedTag = hashlib.sha512(hashlib.sha512( + calculatedTag = highlevelcrypto.double_sha512( encodeVarint(sendersAddressVersion) + encodeVarint(sendersStream) + calculatedRipe - ).digest()).digest()[32:] + )[32:] if calculatedTag != embeddedTag: return logger.debug( 'The tag and encryption key used to encrypt this' @@ -918,8 +917,7 @@ class objectProcessor(threading.Thread): return logger.debug('ECDSA verify passed') # Used to detect and ignore duplicate messages in our inbox - sigHash = hashlib.sha512( - hashlib.sha512(signature).digest()).digest()[32:] + sigHash = highlevelcrypto.double_sha512(signature)[32:] fromAddress = encodeAddress( sendersAddressVersion, sendersStream, calculatedRipe) @@ -993,10 +991,10 @@ class objectProcessor(threading.Thread): # Let us create the tag from the address and see if we were waiting # for it. elif addressVersion >= 4: - tag = hashlib.sha512(hashlib.sha512( + tag = highlevelcrypto.double_sha512( encodeVarint(addressVersion) + encodeVarint(streamNumber) + ripe - ).digest()).digest()[32:] + )[32:] if tag in state.neededPubkeys: del state.neededPubkeys[tag] self.sendMessages(address) diff --git a/src/class_singleWorker.py b/src/class_singleWorker.py index 49c41c07..23631d71 100644 --- a/src/class_singleWorker.py +++ b/src/class_singleWorker.py @@ -25,9 +25,7 @@ import queues import shared import state import tr -from addresses import ( - calculateInventoryHash, decodeAddress, decodeVarint, encodeVarint -) +from addresses import decodeAddress, decodeVarint, encodeVarint from bmconfigparser import BMConfigParser from helper_sql import sqlExecute, sqlQuery from inventory import Inventory @@ -75,18 +73,16 @@ class singleWorker(StoppableThread): queryreturn = sqlQuery( '''SELECT DISTINCT toaddress FROM sent''' ''' WHERE (status='awaitingpubkey' AND folder='sent')''') - for row in queryreturn: - toAddress, = row - # toStatus - _, toAddressVersionNumber, toStreamNumber, toRipe = \ - decodeAddress(toAddress) + for toAddress, in queryreturn: + toAddressVersionNumber, toStreamNumber, toRipe = \ + decodeAddress(toAddress)[1:] if toAddressVersionNumber <= 3: state.neededPubkeys[toAddress] = 0 elif toAddressVersionNumber >= 4: - doubleHashOfAddressData = hashlib.sha512(hashlib.sha512( + doubleHashOfAddressData = highlevelcrypto.double_sha512( encodeVarint(toAddressVersionNumber) + encodeVarint(toStreamNumber) + toRipe - ).digest()).digest() + ) # Note that this is the first half of the sha512 hash. privEncryptionKey = doubleHashOfAddressData[:32] tag = doubleHashOfAddressData[32:] @@ -290,7 +286,7 @@ class singleWorker(StoppableThread): payload = self._doPOWDefaults( payload, TTL, log_prefix='(For pubkey message)') - inventoryHash = calculateInventoryHash(payload) + inventoryHash = highlevelcrypto.calculateInventoryHash(payload) objectType = 1 Inventory()[inventoryHash] = ( objectType, streamNumber, payload, embeddedTime, '') @@ -379,7 +375,7 @@ class singleWorker(StoppableThread): payload = self._doPOWDefaults( payload, TTL, log_prefix='(For pubkey message)') - inventoryHash = calculateInventoryHash(payload) + inventoryHash = highlevelcrypto.calculateInventoryHash(payload) objectType = 1 Inventory()[inventoryHash] = ( objectType, streamNumber, payload, embeddedTime, '') @@ -452,10 +448,10 @@ class singleWorker(StoppableThread): # unencrypted, the pubkey with part of the hash so that nodes # know which pubkey object to try to decrypt # when they want to send a message. - doubleHashOfAddressData = hashlib.sha512(hashlib.sha512( + doubleHashOfAddressData = highlevelcrypto.double_sha512( encodeVarint(addressVersionNumber) + encodeVarint(streamNumber) + addressHash - ).digest()).digest() + ) payload += doubleHashOfAddressData[32:] # the tag signature = highlevelcrypto.sign( payload + dataToEncrypt, privSigningKeyHex, self.digestAlg) @@ -471,7 +467,7 @@ class singleWorker(StoppableThread): payload = self._doPOWDefaults( payload, TTL, log_prefix='(For pubkey message)') - inventoryHash = calculateInventoryHash(payload) + inventoryHash = highlevelcrypto.calculateInventoryHash(payload) objectType = 1 Inventory()[inventoryHash] = ( objectType, streamNumber, payload, embeddedTime, @@ -507,7 +503,7 @@ class singleWorker(StoppableThread): objectType = protocol.OBJECT_ONIONPEER # FIXME: ideally the objectPayload should be signed objectPayload = encodeVarint(peer.port) + protocol.encodeHost(peer.host) - tag = calculateInventoryHash(objectPayload) + tag = highlevelcrypto.calculateInventoryHash(objectPayload) if Inventory().by_type_and_tag(objectType, tag): return # not expired @@ -521,7 +517,7 @@ class singleWorker(StoppableThread): payload = self._doPOWDefaults( payload, TTL, log_prefix='(For onionpeer object)') - inventoryHash = calculateInventoryHash(payload) + inventoryHash = highlevelcrypto.calculateInventoryHash(payload) Inventory()[inventoryHash] = ( objectType, streamNumber, buffer(payload), embeddedTime, buffer(tag) @@ -615,10 +611,10 @@ class singleWorker(StoppableThread): payload += encodeVarint(streamNumber) if addressVersionNumber >= 4: - doubleHashOfAddressData = hashlib.sha512(hashlib.sha512( + doubleHashOfAddressData = highlevelcrypto.double_sha512( encodeVarint(addressVersionNumber) + encodeVarint(streamNumber) + ripe - ).digest()).digest() + ) tag = doubleHashOfAddressData[32:] payload += tag else: @@ -688,7 +684,7 @@ class singleWorker(StoppableThread): ) continue - inventoryHash = calculateInventoryHash(payload) + inventoryHash = highlevelcrypto.calculateInventoryHash(payload) objectType = 3 Inventory()[inventoryHash] = ( objectType, streamNumber, payload, embeddedTime, tag) @@ -797,10 +793,10 @@ class singleWorker(StoppableThread): if toAddressVersionNumber <= 3: toTag = '' else: - toTag = hashlib.sha512(hashlib.sha512( + toTag = highlevelcrypto.double_sha512( encodeVarint(toAddressVersionNumber) + encodeVarint(toStreamNumber) + toRipe - ).digest()).digest()[32:] + )[32:] if toaddress in state.neededPubkeys or \ toTag in state.neededPubkeys: # We already sent a request for the pubkey @@ -834,11 +830,11 @@ class singleWorker(StoppableThread): # already contains the toAddress and cryptor # object associated with the tag for this toAddress. if toAddressVersionNumber >= 4: - doubleHashOfToAddressData = hashlib.sha512( - hashlib.sha512( - encodeVarint(toAddressVersionNumber) + encodeVarint(toStreamNumber) + toRipe - ).digest() - ).digest() + doubleHashOfToAddressData = \ + highlevelcrypto.double_sha512( + encodeVarint(toAddressVersionNumber) + + encodeVarint(toStreamNumber) + toRipe + ) # The first half of the sha512 hash. privEncryptionKey = doubleHashOfToAddressData[:32] # The second half of the sha512 hash. @@ -1304,7 +1300,7 @@ class singleWorker(StoppableThread): ) continue - inventoryHash = calculateInventoryHash(encryptedPayload) + inventoryHash = highlevelcrypto.calculateInventoryHash(encryptedPayload) objectType = 2 Inventory()[inventoryHash] = ( objectType, toStreamNumber, encryptedPayload, embeddedTime, '') @@ -1354,8 +1350,7 @@ class singleWorker(StoppableThread): # the message in our own inbox. if BMConfigParser().has_section(toaddress): # Used to detect and ignore duplicate messages in our inbox - sigHash = hashlib.sha512(hashlib.sha512( - signature).digest()).digest()[32:] + sigHash = highlevelcrypto.double_sha512(signature)[32:] t = (inventoryHash, toaddress, fromaddress, subject, int( time.time()), message, 'inbox', encoding, 0, sigHash) helper_inbox.insert(t) @@ -1410,16 +1405,13 @@ class singleWorker(StoppableThread): # neededPubkeys dictionary. But if we are recovering # from a restart of the client then we have to put it in now. - # Note that this is the first half of the sha512 hash. - privEncryptionKey = hashlib.sha512(hashlib.sha512( + doubleHashOfAddressData = highlevelcrypto.double_sha512( encodeVarint(addressVersionNumber) + encodeVarint(streamNumber) + ripe - ).digest()).digest()[:32] + ) + privEncryptionKey = doubleHashOfAddressData[:32] # Note that this is the second half of the sha512 hash. - tag = hashlib.sha512(hashlib.sha512( - encodeVarint(addressVersionNumber) - + encodeVarint(streamNumber) + ripe - ).digest()).digest()[32:] + tag = doubleHashOfAddressData[32:] if tag not in state.neededPubkeys: # We'll need this for when we receive a pubkey reply: # it will be encrypted and we'll need to decrypt it. @@ -1462,7 +1454,7 @@ class singleWorker(StoppableThread): payload = self._doPOWDefaults(payload, TTL) - inventoryHash = calculateInventoryHash(payload) + inventoryHash = highlevelcrypto.calculateInventoryHash(payload) objectType = 1 Inventory()[inventoryHash] = ( objectType, streamNumber, payload, embeddedTime, '') diff --git a/src/highlevelcrypto.py b/src/highlevelcrypto.py index 9a31ad97..3c084b12 100644 --- a/src/highlevelcrypto.py +++ b/src/highlevelcrypto.py @@ -7,6 +7,7 @@ High level cryptographic functions based on `.pyelliptic` OpenSSL bindings. `More discussion. `_ """ +import hashlib from binascii import hexlify import pyelliptic @@ -17,6 +18,18 @@ from pyelliptic import arithmetic as a __all__ = ['encrypt', 'makeCryptor', 'pointMult', 'privToPub', 'sign', 'verify'] +# Hashes + +def double_sha512(data): + """Binary double SHA512 digest""" + return hashlib.sha512(hashlib.sha512(data).digest()).digest() + + +def calculateInventoryHash(data): + """Calculate inventory hash from object data""" + return double_sha512(data)[:32] + + def makeCryptor(privkey): """Return a private `.pyelliptic.ECC` instance""" private_key = a.changebase(privkey, 16, 256, minlen=32) diff --git a/src/network/bmobject.py b/src/network/bmobject.py index 12b997d7..49e3c2de 100644 --- a/src/network/bmobject.py +++ b/src/network/bmobject.py @@ -6,7 +6,7 @@ import time import protocol import state -from addresses import calculateInventoryHash +from highlevelcrypto import calculateInventoryHash from inventory import Inventory from network.dandelion import Dandelion diff --git a/src/proofofwork.py b/src/proofofwork.py index 148d6734..69b04e2d 100644 --- a/src/proofofwork.py +++ b/src/proofofwork.py @@ -4,7 +4,6 @@ Proof of work calculation """ import ctypes -import hashlib import os import sys import tempfile @@ -12,6 +11,7 @@ import time from struct import pack, unpack from subprocess import call +import highlevelcrypto import openclpow import paths import queues @@ -87,13 +87,20 @@ def _set_idle(): pass +def trial_value(nonce, initialHash): + """Calculate PoW trial value""" + trialValue, = unpack( + '>Q', highlevelcrypto.double_sha512( + pack('>Q', nonce) + initialHash)[0:8]) + return trialValue + + def _pool_worker(nonce, initialHash, target, pool_size): _set_idle() trialValue = float('inf') while trialValue > target: nonce += pool_size - trialValue, = unpack('>Q', hashlib.sha512(hashlib.sha512( - pack('>Q', nonce) + initialHash).digest()).digest()[0:8]) + trialValue = trial_value(nonce, initialHash) return [trialValue, nonce] @@ -103,10 +110,9 @@ def _doSafePoW(target, initialHash): trialValue = float('inf') while trialValue > target and state.shutdown == 0: nonce += 1 - trialValue, = unpack('>Q', hashlib.sha512(hashlib.sha512( - pack('>Q', nonce) + initialHash).digest()).digest()[0:8]) + trialValue = trial_value(nonce, initialHash) if state.shutdown != 0: - raise StopIteration("Interrupted") # pylint: misplaced-bare-raise + raise StopIteration("Interrupted") logger.debug("Safe PoW done") return [trialValue, nonce] @@ -163,7 +169,7 @@ def _doCPoW(target, initialHash): logger.debug("C PoW start") nonce = bmpow(out_h, out_m) - trialValue, = unpack('>Q', hashlib.sha512(hashlib.sha512(pack('>Q', nonce) + initialHash).digest()).digest()[0:8]) + trialValue = trial_value(nonce, initialHash) if state.shutdown != 0: raise StopIteration("Interrupted") logger.debug("C PoW done") @@ -173,7 +179,7 @@ def _doCPoW(target, initialHash): def _doGPUPoW(target, initialHash): logger.debug("GPU PoW start") nonce = openclpow.do_opencl_pow(initialHash.encode("hex"), target) - trialValue, = unpack('>Q', hashlib.sha512(hashlib.sha512(pack('>Q', nonce) + initialHash).digest()).digest()[0:8]) + trialValue = trial_value(nonce, initialHash) if trialValue > target: deviceNames = ", ".join(gpu.name for gpu in openclpow.enabledGpus) queues.UISignalQueue.put(( diff --git a/src/protocol.py b/src/protocol.py index 1934d9cc..ac00c1e2 100644 --- a/src/protocol.py +++ b/src/protocol.py @@ -269,12 +269,11 @@ def isProofOfWorkSufficient( if payloadLengthExtraBytes < defaults.networkDefaultPayloadLengthExtraBytes: payloadLengthExtraBytes = defaults.networkDefaultPayloadLengthExtraBytes endOfLifeTime, = unpack('>Q', data[8:16]) - TTL = endOfLifeTime - (int(recvTime) if recvTime else int(time.time())) + TTL = endOfLifeTime - int(recvTime if recvTime else time.time()) if TTL < 300: TTL = 300 - POW, = unpack('>Q', hashlib.sha512(hashlib.sha512( - data[:8] + hashlib.sha512(data[8:]).digest() - ).digest()).digest()[0:8]) + POW, = unpack('>Q', highlevelcrypto.double_sha512( + data[:8] + hashlib.sha512(data[8:]).digest())[0:8]) return POW <= 2 ** 64 / ( nonceTrialsPerByte * ( len(data) + payloadLengthExtraBytes diff --git a/src/shared.py b/src/shared.py index 4a654932..8adb73b2 100644 --- a/src/shared.py +++ b/src/shared.py @@ -121,7 +121,8 @@ def reloadMyAddressHashes(): if isEnabled: hasEnabledKeys = True # status - addressVersionNumber, streamNumber, hashobj = decodeAddress(addressInKeysFile)[1:] + addressVersionNumber, streamNumber, hashobj = decodeAddress( + addressInKeysFile)[1:] if addressVersionNumber in (2, 3, 4): # Returns a simple 32 bytes of information encoded # in 64 Hex characters, or null if there was an error. @@ -132,9 +133,9 @@ def reloadMyAddressHashes(): myECCryptorObjects[hashobj] = \ highlevelcrypto.makeCryptor(privEncryptionKey) myAddressesByHash[hashobj] = addressInKeysFile - tag = hashlib.sha512(hashlib.sha512( + tag = highlevelcrypto.double_sha512( encodeVarint(addressVersionNumber) - + encodeVarint(streamNumber) + hashobj).digest()).digest()[32:] + + encodeVarint(streamNumber) + hashobj)[32:] myAddressesByTag[tag] = addressInKeysFile else: logger.error( @@ -174,10 +175,10 @@ def reloadBroadcastSendersForWhichImWatching(): MyECSubscriptionCryptorObjects[hashobj] = \ highlevelcrypto.makeCryptor(hexlify(privEncryptionKey)) else: - doubleHashOfAddressData = hashlib.sha512(hashlib.sha512( + doubleHashOfAddressData = highlevelcrypto.double_sha512( encodeVarint(addressVersionNumber) + encodeVarint(streamNumber) + hashobj - ).digest()).digest() + ) tag = doubleHashOfAddressData[32:] privEncryptionKey = doubleHashOfAddressData[:32] MyECSubscriptionCryptorObjects[tag] = \