From 9747f65884cf8745a9d277a682b75e0be40d747a Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Thu, 12 Oct 2023 03:35:44 +0300 Subject: [PATCH 1/6] Define a protocol function for decoding object parameters outside of network --- src/protocol.py | 11 +++++++++++ src/tests/samples.py | 9 +++++++++ src/tests/test_packets.py | 11 ++++++++++- 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/protocol.py b/src/protocol.py index f5aa2750..722ce9c1 100644 --- a/src/protocol.py +++ b/src/protocol.py @@ -436,6 +436,17 @@ def assembleErrorMessage(fatal=0, banTime=0, inventoryVector='', errorText=''): # Packet decoding +def decodeObjectParameters(data): + """Decode the parameters of a raw object needed to put it in inventory""" + # BMProto.decode_payload_content("QQIvv") + expiresTime = unpack('>Q', data[8:16])[0] + objectType = unpack('>I', data[16:20])[0] + parserPos = 20 + decodeVarint(data[20:30])[1] + toStreamNumber = decodeVarint(data[parserPos:parserPos + 10])[0] + + return objectType, toStreamNumber, expiresTime + + def decryptAndCheckPubkeyPayload(data, address): """ Version 4 pubkeys are encrypted. This function is run when we diff --git a/src/tests/samples.py b/src/tests/samples.py index aa4bac9e..93e5ed61 100644 --- a/src/tests/samples.py +++ b/src/tests/samples.py @@ -54,3 +54,12 @@ sample_subscription_addresses = [ 'BM-2cWQLCBGorT9pUGkYSuGGVr9LzE4mRnQaq', 'BM-GtovgYdgs7qXPkoYaRgrLFuFKz1SFpsw'] sample_subscription_name = 'test sub' + + +sample_object_expires = 1712271487 +# from minode import structure +# obj = structure.Object( +# b'\x00' * 8, sample_object_expires, 42, 1, 2, b'HELLO') +# .. do pow and obj.to_bytes() +sample_object_data = unhexlify( + '00000000001be7fc00000000660f307f0000002a010248454c4c4f') diff --git a/src/tests/test_packets.py b/src/tests/test_packets.py index f030912a..9dfb1d23 100644 --- a/src/tests/test_packets.py +++ b/src/tests/test_packets.py @@ -5,7 +5,8 @@ from struct import pack from pybitmessage import addresses, protocol -from .samples import sample_addr_data +from .samples import ( + sample_addr_data, sample_object_data, sample_object_expires) from .test_protocol import TestSocketInet @@ -49,6 +50,14 @@ class TestSerialize(TestSocketInet): self.assertEqual( protocol.CreatePacket(b'ping')[:len(head)], head) + def test_decode_obj_parameters(self): + """Check parameters decoded from a sample object""" + objectType, toStreamNumber, expiresTime = \ + protocol.decodeObjectParameters(sample_object_data) + self.assertEqual(objectType, 42) + self.assertEqual(toStreamNumber, 2) + self.assertEqual(expiresTime, sample_object_expires) + def test_encodehost(self): """Check the result of protocol.encodeHost()""" self.assertEqual( -- 2.45.1 From 1794384f01e249847ac2f93fb4ddef3bbef9e136 Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Thu, 12 Oct 2023 04:03:44 +0300 Subject: [PATCH 2/6] Use decodeObjectParameters() in disseminatePreEncryptedMsg API command handler --- src/api.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/src/api.py b/src/api.py index a4b0aed5..69e7523a 100644 --- a/src/api.py +++ b/src/api.py @@ -66,7 +66,7 @@ import socket import subprocess # nosec B404 import time from binascii import hexlify, unhexlify -from struct import pack, unpack +from struct import pack import six from six.moves import configparser, http_client, xmlrpc_server @@ -74,6 +74,7 @@ from six.moves import configparser, http_client, xmlrpc_server import defaults import helper_inbox import helper_sent +import protocol import proofofwork import queues import shared @@ -1284,7 +1285,7 @@ class BMRPCDispatcher(object): return {'subscriptions': data} @command('disseminatePreEncryptedMsg') - def HandleDisseminatePreEncryptedMsg( # pylint: disable=too-many-locals + def HandleDisseminatePreEncryptedMsg( self, encryptedPayload, requiredAverageProofOfWorkNonceTrialsPerByte, requiredPayloadLengthExtraBytes): """Handle a request to disseminate an encrypted message""" @@ -1294,9 +1295,12 @@ class BMRPCDispatcher(object): # to be done. PyBitmessage accepts this msg object and sends it out # to the rest of the Bitmessage network as if it had generated # the message itself. Please do not yet add this to the api doc. - encryptedPayload = self._decode(encryptedPayload, "hex") - expiresTime = unpack('>Q', encryptedPayload[0:8])[0] - objectType = unpack('>I', encryptedPayload[8:12])[0] + encryptedPayload = b'\x00' * 8 + self._decode(encryptedPayload, "hex") + # compatibility stub ^, since disseminatePreEncryptedMsg + # still expects the encryptedPayload without a nonce + objectType, toStreamNumber, expiresTime = \ + protocol.decodeObjectParameters(encryptedPayload) + encryptedPayload = encryptedPayload[8:] TTL = expiresTime - time.time() + 300 # a bit of extra padding # Let us do the POW and attach it to the front target = 2**64 / ( @@ -1329,12 +1333,6 @@ class BMRPCDispatcher(object): nonce / (time.time() - powStartTime) ) encryptedPayload = pack('>Q', nonce) + encryptedPayload - parserPos = 20 - _, objectVersionLength = decodeVarint( - encryptedPayload[parserPos:parserPos + 10]) - parserPos += objectVersionLength - toStreamNumber, _ = decodeVarint( - encryptedPayload[parserPos:parserPos + 10]) inventoryHash = calculateInventoryHash(encryptedPayload) Inventory()[inventoryHash] = ( objectType, toStreamNumber, encryptedPayload, -- 2.45.1 From 2030e08db312a62da40551fa4023e815446f2be2 Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Mon, 18 Mar 2024 02:27:53 +0200 Subject: [PATCH 3/6] A test for disseminatePreEncryptedMsg API command --- src/tests/test_api_thread.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/src/tests/test_api_thread.py b/src/tests/test_api_thread.py index e3218b85..5b004066 100644 --- a/src/tests/test_api_thread.py +++ b/src/tests/test_api_thread.py @@ -2,11 +2,18 @@ import sys import time +from binascii import hexlify, unhexlify +from struct import pack from six.moves import queue, xmlrpc_client +from pybitmessage import protocol +from pybitmessage.defaults import ( + networkDefaultProofOfWorkNonceTrialsPerByte, + networkDefaultPayloadLengthExtraBytes) + from .partial import TestPartialRun -from .samples import sample_statusbar_msg # any +from .samples import sample_statusbar_msg, sample_object_data class TestAPIThread(TestPartialRun): @@ -66,3 +73,22 @@ class TestAPIThread(TestPartialRun): if sys.hexversion >= 0x3000000: self.assertEqual(status["networkConnections"], 4) self.assertEqual(status["pendingDownload"], 0) + + def test_disseminate_preencrypted(self): + """Call disseminatePreEncryptedMsg API command and check inventory""" + import proofofwork + from inventory import Inventory + + proofofwork.init() + update_object = pack( + '>Q', int(time.time() + 7200)) + sample_object_data[16:] + invhash = unhexlify(self.api.disseminatePreEncryptedMsg( + hexlify(update_object).decode(), + networkDefaultProofOfWorkNonceTrialsPerByte, + networkDefaultPayloadLengthExtraBytes + )) + obj_type, obj_stream, obj_data = Inventory()[invhash][:3] + self.assertEqual(obj_type, 42) + self.assertEqual(obj_stream, 2) + self.assertEqual(sample_object_data[16:], obj_data[16:]) + self.assertTrue(protocol.isProofOfWorkSufficient(obj_data)) -- 2.45.1 From 8fb6410097344f81a4e35eaa43a6dc3b90be10d8 Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Mon, 18 Mar 2024 18:53:57 +0200 Subject: [PATCH 4/6] Enable inventory item setting for python3 --- src/inventory.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/inventory.py b/src/inventory.py index e6d4a24c..dc8e36bf 100644 --- a/src/inventory.py +++ b/src/inventory.py @@ -45,3 +45,6 @@ class Inventory(): # hint for pylint: this is dictionary like object def __getitem__(self, key): return self._realInventory[key] + + def __setitem__(self, key, value): + self._realInventory[key] = value -- 2.45.1 From 2a93b04332050a47e4c205bee7e09a3b4746f09d Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Tue, 19 Mar 2024 04:12:35 +0200 Subject: [PATCH 5/6] Fix api.HandleDisseminatePreEncryptedMsg() for python3 --- src/api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api.py b/src/api.py index 69e7523a..c27a24e7 100644 --- a/src/api.py +++ b/src/api.py @@ -1336,13 +1336,13 @@ class BMRPCDispatcher(object): inventoryHash = calculateInventoryHash(encryptedPayload) Inventory()[inventoryHash] = ( objectType, toStreamNumber, encryptedPayload, - expiresTime, '' + expiresTime, b'' ) logger.info( 'Broadcasting inv for msg(API disseminatePreEncryptedMsg' ' command): %s', hexlify(inventoryHash)) queues.invQueue.put((toStreamNumber, inventoryHash)) - return hexlify(inventoryHash) + return hexlify(inventoryHash).decode() @command('trashSentMessageByAckData') def HandleTrashSentMessageByAckDAta(self, ackdata): -- 2.45.1 From 13d090e344aea54474739ab1c17353123d87af25 Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Fri, 13 Oct 2023 04:38:19 +0300 Subject: [PATCH 6/6] Use protocol.decodeObjectParameters() for sending ACK - put it in the inventory --- src/class_objectProcessor.py | 12 +++++++++--- src/network/bmproto.py | 30 ------------------------------ 2 files changed, 9 insertions(+), 33 deletions(-) diff --git a/src/class_objectProcessor.py b/src/class_objectProcessor.py index b7d351ab..8d2e12a8 100644 --- a/src/class_objectProcessor.py +++ b/src/class_objectProcessor.py @@ -31,7 +31,8 @@ from bmconfigparser import config from fallback import RIPEMD160Hash from helper_sql import ( sql_ready, sql_timeout, SqlBulkExecute, sqlExecute, sqlQuery) -from network import bmproto, knownnodes +from inventory import Inventory +from network import knownnodes from network.node import Peer from tr import _translate @@ -64,7 +65,6 @@ class objectProcessor(threading.Thread): logger.debug( 'Loaded %s objects from disk into the objectProcessorQueue.', len(queryreturn)) - self._ack_obj = bmproto.BMStringParser() self.successfullyDecryptMessageTimings = [] def run(self): @@ -733,7 +733,13 @@ class objectProcessor(threading.Thread): and not config.safeGetBoolean(toAddress, 'dontsendack') and not config.safeGetBoolean(toAddress, 'chan') ): - self._ack_obj.send_data(ackData[24:]) + ackPayload = ackData[24:] + objectType, toStreamNumber, expiresTime = \ + protocol.decodeObjectParameters(ackPayload) + inventoryHash = calculateInventoryHash(ackPayload) + Inventory()[inventoryHash] = ( + objectType, toStreamNumber, ackPayload, expiresTime, b'') + queues.invQueue.put((toStreamNumber, inventoryHash)) # Display timing data timeRequiredToAttemptToDecryptMessage = time.time( diff --git a/src/network/bmproto.py b/src/network/bmproto.py index c8f298b7..b03626eb 100644 --- a/src/network/bmproto.py +++ b/src/network/bmproto.py @@ -9,7 +9,6 @@ import re import socket import struct import time -from binascii import hexlify # magic imports! import addresses @@ -678,32 +677,3 @@ class BMProto(AdvancedDispatcher, ObjectTracker): except AttributeError: logger.debug('Disconnected socket closing') AdvancedDispatcher.handle_close(self) - - -class BMStringParser(BMProto): - """ - A special case of BMProto used by objectProcessor to send ACK - """ - def __init__(self): - super(BMStringParser, self).__init__() - self.destination = Peer('127.0.0.1', 8444) - self.payload = None - ObjectTracker.__init__(self) - - def send_data(self, data): - """Send object given by the data string""" - # This class is introduced specially for ACK sending, please - # change log strings if you are going to use it for something else - self.bm_proto_reset() - self.payload = data - try: - self.bm_command_object() - except BMObjectAlreadyHaveError: - pass # maybe the same msg received on different nodes - except BMObjectExpiredError: - logger.debug( - 'Sending ACK failure (expired): %s', hexlify(data)) - except Exception as e: - logger.debug( - 'Exception of type %s while sending ACK', - type(e), exc_info=True) -- 2.45.1