Bitmessage/PyBitmessage-2024-08-21
Archived
2
0
Fork 0
Code Issues 168 Pull Requests 53 Actions Packages Projects Releases 5 Wiki Activity

Start adding hashes with double SHA512

Browse Source
This commit is contained in:
Dmitri Bogomolov 2021-07-29 22:16:37 +03:00 committed by Lee Miller
parent 1b9773f2cf
commit 3ed84a5863
Signed by untrusted user: lee.miller
GPG Key ID: 4F97A5EA88F4AB63
12 changed files with 94 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
src/addresses.py
View File

@ -2,11 +2,16 @@
Operations with addresses
"""
# pylint: disable=inconsistent-return-statements
import hashlib
import logging
from binascii import hexlify, unhexlify
from struct import pack, unpack
try:
from highlevelcrypto import double_sha512
except ImportError:
from .highlevelcrypto import double_sha512
logger = logging.getLogger('default')
@ -134,15 +139,6 @@ def decodeVarint(data):
return (encodedValue, 9)
def calculateInventoryHash(data):
"""Calculate inventory hash from object data"""
sha = hashlib.new('sha512')
sha2 = hashlib.new('sha512')
sha.update(data)
sha2.update(sha.digest())
return sha2.digest()[0:32]
def encodeAddress(version, stream, ripe):
"""Convert ripe to address"""
if version >= 2 and version < 4:
@ -166,12 +162,7 @@ def encodeAddress(version, stream, ripe):
storedBinaryData = encodeVarint(version) + encodeVarint(stream) + ripe
# Generate the checksum
sha = hashlib.new('sha512')
sha.update(storedBinaryData)
currentHash = sha.digest()
sha = hashlib.new('sha512')
sha.update(currentHash)
checksum = sha.digest()[0:4]
checksum = double_sha512(storedBinaryData)[0:4]
# FIXME: encodeBase58 should take binary data, to reduce conversions
# encodeBase58(storedBinaryData + checksum)
@ -207,13 +198,7 @@ def decodeAddress(address):
data = unhexlify(hexdata)
checksum = data[-4:]
sha = hashlib.new('sha512')
sha.update(data[:-4])
currentHash = sha.digest()
sha = hashlib.new('sha512')
sha.update(currentHash)
if checksum != sha.digest()[0:4]:
if checksum != double_sha512(data[:-4])[0:4]:
status = 'checksumfailed'
return status, 0, 0, ''

2
src/api.py
View File

@ -83,7 +83,6 @@ import shutdown
import state
from addresses import (
addBMIfNotPresent,
calculateInventoryHash,
decodeAddress,
decodeVarint,
varintDecodeError
@ -92,6 +91,7 @@ from bmconfigparser import config
from debug import logger
from helper_sql import (
SqlBulkExecute, sqlExecute, sqlQuery, sqlStoredProcedure, sql_ready)
from highlevelcrypto import calculateInventoryHash
from inventory import Inventory
try:

4
src/class_addressGenerator.py
View File

@ -349,10 +349,10 @@ class addressGenerator(StoppableThread):
highlevelcrypto.makeCryptor(
hexlify(potentialPrivEncryptionKey))
shared.myAddressesByHash[ripe] = address
tag = hashlib.sha512(hashlib.sha512(
tag = highlevelcrypto.double_sha512(
encodeVarint(addressVersionNumber)
+ encodeVarint(streamNumber) + ripe
).digest()).digest()[32:]
)[32:]
shared.myAddressesByTag[tag] = address
if addressVersionNumber == 3:
# If this is a chan address,

22
src/class_objectProcessor.py
View File

@ -24,7 +24,7 @@ import queues
import shared
import state
from addresses import (
calculateInventoryHash, decodeAddress, decodeVarint,
decodeAddress, decodeVarint,
encodeAddress, encodeVarint, varintDecodeError
)
from bmconfigparser import config
@ -456,7 +456,7 @@ class objectProcessor(threading.Thread):
streamNumberAsClaimedByMsg, streamNumberAsClaimedByMsgLength = \
decodeVarint(data[readPosition:readPosition + 9])
readPosition += streamNumberAsClaimedByMsgLength
inventoryHash = calculateInventoryHash(data)
inventoryHash = highlevelcrypto.calculateInventoryHash(data)
initialDecryptionSuccessful = False
# This is not an acknowledgement bound for me. See if it is a message
@ -586,8 +586,7 @@ class objectProcessor(threading.Thread):
helper_bitcoin.calculateTestnetAddressFromPubkey(pubSigningKey)
)
# Used to detect and ignore duplicate messages in our inbox
sigHash = hashlib.sha512(
hashlib.sha512(signature).digest()).digest()[32:]
sigHash = highlevelcrypto.double_sha512(signature)[32:]
# calculate the fromRipe.
sha = hashlib.new('sha512')
@ -736,7 +735,7 @@ class objectProcessor(threading.Thread):
ackPayload = ackData[24:]
objectType, toStreamNumber, expiresTime = \
protocol.decodeObjectParameters(ackPayload)
inventoryHash = calculateInventoryHash(ackPayload)
inventoryHash = highlevelcrypto.calculateInventoryHash(ackPayload)
Inventory()[inventoryHash] = (
objectType, toStreamNumber, ackPayload, expiresTime, b'')
queues.invQueue.put((toStreamNumber, inventoryHash))
@ -763,7 +762,7 @@ class objectProcessor(threading.Thread):
state.numberOfBroadcastsProcessed += 1
queues.UISignalQueue.put((
'updateNumberOfBroadcastsProcessed', 'no data'))
inventoryHash = calculateInventoryHash(data)
inventoryHash = highlevelcrypto.calculateInventoryHash(data)
readPosition = 20 # bypass the nonce, time, and object type
broadcastVersion, broadcastVersionLength = decodeVarint(
data[readPosition:readPosition + 9])
@ -897,10 +896,10 @@ class objectProcessor(threading.Thread):
' itself. Ignoring message.'
)
elif broadcastVersion == 5:
calculatedTag = hashlib.sha512(hashlib.sha512(
calculatedTag = highlevelcrypto.double_sha512(
encodeVarint(sendersAddressVersion)
+ encodeVarint(sendersStream) + calculatedRipe
).digest()).digest()[32:]
)[32:]
if calculatedTag != embeddedTag:
return logger.debug(
'The tag and encryption key used to encrypt this'
@ -930,8 +929,7 @@ class objectProcessor(threading.Thread):
return
logger.debug('ECDSA verify passed')
# Used to detect and ignore duplicate messages in our inbox
sigHash = hashlib.sha512(
hashlib.sha512(signature).digest()).digest()[32:]
sigHash = highlevelcrypto.double_sha512(signature)[32:]
fromAddress = encodeAddress(
sendersAddressVersion, sendersStream, calculatedRipe)
@ -1005,10 +1003,10 @@ class objectProcessor(threading.Thread):
# Let us create the tag from the address and see if we were waiting
# for it.
elif addressVersion >= 4:
tag = hashlib.sha512(hashlib.sha512(
tag = highlevelcrypto.double_sha512(
encodeVarint(addressVersion) + encodeVarint(streamNumber)
+ ripe
).digest()).digest()[32:]
)[32:]
if tag in state.neededPubkeys:
del state.neededPubkeys[tag]
self.sendMessages(address)

68
src/class_singleWorker.py
View File

@ -25,9 +25,7 @@ import queues
import shared
import state
import tr
from addresses import (
calculateInventoryHash, decodeAddress, decodeVarint, encodeVarint
)
from addresses import decodeAddress, decodeVarint, encodeVarint
from bmconfigparser import config
from helper_sql import sqlExecute, sqlQuery
from inventory import Inventory
@ -75,18 +73,16 @@ class singleWorker(StoppableThread):
queryreturn = sqlQuery(
'''SELECT DISTINCT toaddress FROM sent'''
''' WHERE (status='awaitingpubkey' AND folder='sent')''')
for row in queryreturn:
toAddress, = row
# toStatus
_, toAddressVersionNumber, toStreamNumber, toRipe = \
decodeAddress(toAddress)
for toAddress, in queryreturn:
toAddressVersionNumber, toStreamNumber, toRipe = \
decodeAddress(toAddress)[1:]
if toAddressVersionNumber <= 3:
state.neededPubkeys[toAddress] = 0
elif toAddressVersionNumber >= 4:
doubleHashOfAddressData = hashlib.sha512(hashlib.sha512(
doubleHashOfAddressData = highlevelcrypto.double_sha512(
encodeVarint(toAddressVersionNumber)
+ encodeVarint(toStreamNumber) + toRipe
).digest()).digest()
)
# Note that this is the first half of the sha512 hash.
privEncryptionKey = doubleHashOfAddressData[:32]
tag = doubleHashOfAddressData[32:]
@ -289,7 +285,7 @@ class singleWorker(StoppableThread):
payload = self._doPOWDefaults(
payload, TTL, log_prefix='(For pubkey message)')
inventoryHash = calculateInventoryHash(payload)
inventoryHash = highlevelcrypto.calculateInventoryHash(payload)
objectType = 1
Inventory()[inventoryHash] = (
objectType, streamNumber, payload, embeddedTime, '')
@ -377,7 +373,7 @@ class singleWorker(StoppableThread):
payload = self._doPOWDefaults(
payload, TTL, log_prefix='(For pubkey message)')
inventoryHash = calculateInventoryHash(payload)
inventoryHash = highlevelcrypto.calculateInventoryHash(payload)
objectType = 1
Inventory()[inventoryHash] = (
objectType, streamNumber, payload, embeddedTime, '')
@ -449,10 +445,10 @@ class singleWorker(StoppableThread):
# unencrypted, the pubkey with part of the hash so that nodes
# know which pubkey object to try to decrypt
# when they want to send a message.
doubleHashOfAddressData = hashlib.sha512(hashlib.sha512(
doubleHashOfAddressData = highlevelcrypto.double_sha512(
encodeVarint(addressVersionNumber)
+ encodeVarint(streamNumber) + addressHash
).digest()).digest()
)
payload += doubleHashOfAddressData[32:] # the tag
signature = highlevelcrypto.sign(
payload + dataToEncrypt, privSigningKeyHex, self.digestAlg)
@ -468,7 +464,7 @@ class singleWorker(StoppableThread):
payload = self._doPOWDefaults(
payload, TTL, log_prefix='(For pubkey message)')
inventoryHash = calculateInventoryHash(payload)
inventoryHash = highlevelcrypto.calculateInventoryHash(payload)
objectType = 1
Inventory()[inventoryHash] = (
objectType, streamNumber, payload, embeddedTime,
@ -504,7 +500,7 @@ class singleWorker(StoppableThread):
objectType = protocol.OBJECT_ONIONPEER
# FIXME: ideally the objectPayload should be signed
objectPayload = encodeVarint(peer.port) + protocol.encodeHost(peer.host)
tag = calculateInventoryHash(objectPayload)
tag = highlevelcrypto.calculateInventoryHash(objectPayload)
if Inventory().by_type_and_tag(objectType, tag):
return # not expired
@ -518,7 +514,7 @@ class singleWorker(StoppableThread):
payload = self._doPOWDefaults(
payload, TTL, log_prefix='(For onionpeer object)')
inventoryHash = calculateInventoryHash(payload)
inventoryHash = highlevelcrypto.calculateInventoryHash(payload)
Inventory()[inventoryHash] = (
objectType, streamNumber, buffer(payload), # noqa: F821
embeddedTime, buffer(tag) # noqa: F821
@ -612,10 +608,10 @@ class singleWorker(StoppableThread):
payload += encodeVarint(streamNumber)
if addressVersionNumber >= 4:
doubleHashOfAddressData = hashlib.sha512(hashlib.sha512(
doubleHashOfAddressData = highlevelcrypto.double_sha512(
encodeVarint(addressVersionNumber)
+ encodeVarint(streamNumber) + ripe
).digest()).digest()
)
tag = doubleHashOfAddressData[32:]
payload += tag
else:
@ -685,7 +681,7 @@ class singleWorker(StoppableThread):
)
continue
inventoryHash = calculateInventoryHash(payload)
inventoryHash = highlevelcrypto.calculateInventoryHash(payload)
objectType = 3
Inventory()[inventoryHash] = (
objectType, streamNumber, payload, embeddedTime, tag)
@ -794,10 +790,10 @@ class singleWorker(StoppableThread):
if toAddressVersionNumber <= 3:
toTag = ''
else:
toTag = hashlib.sha512(hashlib.sha512(
toTag = highlevelcrypto.double_sha512(
encodeVarint(toAddressVersionNumber)
+ encodeVarint(toStreamNumber) + toRipe
).digest()).digest()[32:]
)[32:]
if toaddress in state.neededPubkeys or \
toTag in state.neededPubkeys:
# We already sent a request for the pubkey
@ -831,11 +827,11 @@ class singleWorker(StoppableThread):
# already contains the toAddress and cryptor
# object associated with the tag for this toAddress.
if toAddressVersionNumber >= 4:
doubleHashOfToAddressData = hashlib.sha512(
hashlib.sha512(
encodeVarint(toAddressVersionNumber) + encodeVarint(toStreamNumber) + toRipe
).digest()
).digest()
doubleHashOfToAddressData = \
highlevelcrypto.double_sha512(
encodeVarint(toAddressVersionNumber)
+ encodeVarint(toStreamNumber) + toRipe
)
# The first half of the sha512 hash.
privEncryptionKey = doubleHashOfToAddressData[:32]
# The second half of the sha512 hash.
@ -1302,7 +1298,7 @@ class singleWorker(StoppableThread):
)
continue
inventoryHash = calculateInventoryHash(encryptedPayload)
inventoryHash = highlevelcrypto.calculateInventoryHash(encryptedPayload)
objectType = 2
Inventory()[inventoryHash] = (
objectType, toStreamNumber, encryptedPayload, embeddedTime, '')
@ -1352,8 +1348,7 @@ class singleWorker(StoppableThread):
# the message in our own inbox.
if config.has_section(toaddress):
# Used to detect and ignore duplicate messages in our inbox
sigHash = hashlib.sha512(hashlib.sha512(
signature).digest()).digest()[32:]
sigHash = highlevelcrypto.double_sha512(signature)[32:]
t = (inventoryHash, toaddress, fromaddress, subject, int(
time.time()), message, 'inbox', encoding, 0, sigHash)
helper_inbox.insert(t)
@ -1410,16 +1405,13 @@ class singleWorker(StoppableThread):
# neededPubkeys dictionary. But if we are recovering
# from a restart of the client then we have to put it in now.
# Note that this is the first half of the sha512 hash.
privEncryptionKey = hashlib.sha512(hashlib.sha512(
doubleHashOfAddressData = highlevelcrypto.double_sha512(
encodeVarint(addressVersionNumber)
+ encodeVarint(streamNumber) + ripe
).digest()).digest()[:32]
)
privEncryptionKey = doubleHashOfAddressData[:32]
# Note that this is the second half of the sha512 hash.
tag = hashlib.sha512(hashlib.sha512(
encodeVarint(addressVersionNumber)
+ encodeVarint(streamNumber) + ripe
).digest()).digest()[32:]
tag = doubleHashOfAddressData[32:]
if tag not in state.neededPubkeys:
# We'll need this for when we receive a pubkey reply:
# it will be encrypted and we'll need to decrypt it.
@ -1462,7 +1454,7 @@ class singleWorker(StoppableThread):
payload = self._doPOWDefaults(payload, TTL)
inventoryHash = calculateInventoryHash(payload)
inventoryHash = highlevelcrypto.calculateInventoryHash(payload)
objectType = 1
Inventory()[inventoryHash] = (
objectType, streamNumber, payload, embeddedTime, '')

13
src/highlevelcrypto.py
View File

@ -18,6 +18,7 @@ from pyelliptic import arithmetic as a
__all__ = [
'decodeWalletImportFormat', 'encodeWalletImportFormat',
'double_sha512', 'calculateInventoryHash',
'encrypt', 'makeCryptor', 'pointMult', 'privToPub', 'randomBytes',
'sign', 'verify']
@ -61,6 +62,18 @@ def randomBytes(n):
return OpenSSL.rand(n)
# Hashes
def double_sha512(data):
"""Binary double SHA512 digest"""
return hashlib.sha512(hashlib.sha512(data).digest()).digest()
def calculateInventoryHash(data):
"""Calculate inventory hash from object data"""
return double_sha512(data)[:32]
def makeCryptor(privkey, curve='secp256k1'):
"""Return a private `.pyelliptic.ECC` instance"""
private_key = a.changebase(privkey, 16, 256, minlen=32)

2
src/network/bmobject.py
View File

@ -6,7 +6,7 @@ import time
import protocol
import state
from addresses import calculateInventoryHash
from highlevelcrypto import calculateInventoryHash
from inventory import Inventory
from network.dandelion import Dandelion

22
src/proofofwork.py
View File

@ -4,7 +4,6 @@ Proof of work calculation
"""
import ctypes
import hashlib
import os
import sys
import tempfile
@ -12,6 +11,7 @@ import time
from struct import pack, unpack
from subprocess import call # nosec B404
import highlevelcrypto
import openclpow
import paths
import queues
@ -87,13 +87,20 @@ def _set_idle():
pass
def trial_value(nonce, initialHash):
"""Calculate PoW trial value"""
trialValue, = unpack(
'>Q', highlevelcrypto.double_sha512(
pack('>Q', nonce) + initialHash)[0:8])
return trialValue
def _pool_worker(nonce, initialHash, target, pool_size):
_set_idle()
trialValue = float('inf')
while trialValue > target:
nonce += pool_size
trialValue, = unpack('>Q', hashlib.sha512(hashlib.sha512(
pack('>Q', nonce) + initialHash).digest()).digest()[0:8])
trialValue = trial_value(nonce, initialHash)
return [trialValue, nonce]
@ -103,10 +110,9 @@ def _doSafePoW(target, initialHash):
trialValue = float('inf')
while trialValue > target and state.shutdown == 0:
nonce += 1
trialValue, = unpack('>Q', hashlib.sha512(hashlib.sha512(
pack('>Q', nonce) + initialHash).digest()).digest()[0:8])
trialValue = trial_value(nonce, initialHash)
if state.shutdown != 0:
raise StopIteration("Interrupted") # pylint: misplaced-bare-raise
raise StopIteration("Interrupted")
logger.debug("Safe PoW done")
return [trialValue, nonce]
@ -163,7 +169,7 @@ def _doCPoW(target, initialHash):
logger.debug("C PoW start")
nonce = bmpow(out_h, out_m)
trialValue, = unpack('>Q', hashlib.sha512(hashlib.sha512(pack('>Q', nonce) + initialHash).digest()).digest()[0:8])
trialValue = trial_value(nonce, initialHash)
if state.shutdown != 0:
raise StopIteration("Interrupted")
logger.debug("C PoW done")
@ -173,7 +179,7 @@ def _doCPoW(target, initialHash):
def _doGPUPoW(target, initialHash):
logger.debug("GPU PoW start")
nonce = openclpow.do_opencl_pow(initialHash.encode("hex"), target)
trialValue, = unpack('>Q', hashlib.sha512(hashlib.sha512(pack('>Q', nonce) + initialHash).digest()).digest()[0:8])
trialValue = trial_value(nonce, initialHash)
if trialValue > target:
deviceNames = ", ".join(gpu.name for gpu in openclpow.enabledGpus)
queues.UISignalQueue.put((

7
src/protocol.py
View File

@ -290,12 +290,11 @@ def isProofOfWorkSufficient(
if payloadLengthExtraBytes < defaults.networkDefaultPayloadLengthExtraBytes:
payloadLengthExtraBytes = defaults.networkDefaultPayloadLengthExtraBytes
endOfLifeTime, = unpack('>Q', data[8:16])
TTL = endOfLifeTime - (int(recvTime) if recvTime else int(time.time()))
TTL = endOfLifeTime - int(recvTime if recvTime else time.time())
if TTL < 300:
TTL = 300
POW, = unpack('>Q', hashlib.sha512(hashlib.sha512(
data[:8] + hashlib.sha512(data[8:]).digest()
).digest()).digest()[0:8])
POW, = unpack('>Q', highlevelcrypto.double_sha512(
data[:8] + hashlib.sha512(data[8:]).digest())[0:8])
return POW <= 2 ** 64 / (
nonceTrialsPerByte * (
len(data) + payloadLengthExtraBytes

11
src/shared.py
View File

@ -86,8 +86,7 @@ def reloadMyAddressHashes():
state.appdata, 'keys.dat'))
hasEnabledKeys = False
for addressInKeysFile in config.addresses():
isEnabled = config.getboolean(addressInKeysFile, 'enabled')
if not isEnabled:
if not config.getboolean(addressInKeysFile, 'enabled'):
continue
hasEnabledKeys = True
@ -116,9 +115,9 @@ def reloadMyAddressHashes():
myECCryptorObjects[hashobj] = \
highlevelcrypto.makeCryptor(privEncryptionKey)
myAddressesByHash[hashobj] = addressInKeysFile
tag = hashlib.sha512(hashlib.sha512(
tag = highlevelcrypto.double_sha512(
encodeVarint(addressVersionNumber)
+ encodeVarint(streamNumber) + hashobj).digest()).digest()[32:]
+ encodeVarint(streamNumber) + hashobj)[32:]
myAddressesByTag[tag] = addressInKeysFile
if not keyfileSecure:
@ -153,10 +152,10 @@ def reloadBroadcastSendersForWhichImWatching():
MyECSubscriptionCryptorObjects[hashobj] = \
highlevelcrypto.makeCryptor(hexlify(privEncryptionKey))
else:
doubleHashOfAddressData = hashlib.sha512(hashlib.sha512(
doubleHashOfAddressData = highlevelcrypto.double_sha512(
encodeVarint(addressVersionNumber)
+ encodeVarint(streamNumber) + hashobj
).digest()).digest()
)
tag = doubleHashOfAddressData[32:]
privEncryptionKey = doubleHashOfAddressData[:32]
MyECSubscriptionCryptorObjects[tag] = \

4
src/tests/test_inventory.py
View File

@ -7,8 +7,8 @@ import tempfile
import time
import unittest
from pybitmessage import highlevelcrypto
from pybitmessage.storage import storage
from pybitmessage.addresses import calculateInventoryHash
from .partial import TestPartialRun
@ -35,7 +35,7 @@ class TestFilesystemInventory(TestPartialRun):
TTL = 24 * 60 * 60
embedded_time = int(time.time() + TTL)
msg = struct.pack('>Q', embedded_time) + os.urandom(166)
invhash = calculateInventoryHash(msg)
invhash = highlevelcrypto.calculateInventoryHash(msg)
self.inventory[invhash] = (2, 1, msg, embedded_time, b'')
@classmethod

12
src/tests/test_openclpow.py
View File

@ -1,10 +1,10 @@
"""
Tests for openclpow module
"""
import hashlib
import unittest
from struct import pack, unpack
from pybitmessage import openclpow
from pybitmessage import openclpow, proofofwork
class TestOpenClPow(unittest.TestCase):
@ -25,7 +25,5 @@ class TestOpenClPow(unittest.TestCase):
"b93f3ffeba0ef2fd08a8dc2f87b68ae5a0dc819ab57f22ad2c4c9c8618a43b3"
).decode("hex")
nonce = openclpow.do_opencl_pow(initialHash.encode("hex"), target_)
trialValue, = unpack(
'>Q', hashlib.sha512(hashlib.sha512(
pack('>Q', nonce) + initialHash).digest()).digest()[0:8])
self.assertLess((nonce - trialValue), target_)
self.assertLess(
nonce - proofofwork.trial_value(nonce, initialHash), target_)