From f07af4e80395b33d716d8762680c5a1d1f335339 Mon Sep 17 00:00:00 2001 From: Kashiko Koibumi Date: Fri, 31 May 2024 00:49:37 +0900 Subject: [PATCH 01/30] fix TLS configuration bug --- src/network/tls.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/network/tls.py b/src/network/tls.py index 1ad5d1a4..477ef8c6 100644 --- a/src/network/tls.py +++ b/src/network/tls.py @@ -72,14 +72,15 @@ class TLSDispatcher(AdvancedDispatcher): self.set_state("tls_handshake") return False - self.do_tls_init() + return self.do_tls_init() def do_tls_init(self): # Once the connection has been established, # it's safe to wrap the socket. if sys.version_info >= (2, 7, 9): if ssl.OPENSSL_VERSION_NUMBER >= 0x30000000: - context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER + if self.server_side else ssl.PROTOCOL_TLS_CLIENT) else: context = ssl.create_default_context( purpose=ssl.Purpose.SERVER_AUTH @@ -92,7 +93,7 @@ class TLSDispatcher(AdvancedDispatcher): if ssl.OPENSSL_VERSION_NUMBER >= 0x30000000: context.options = ssl.OP_ALL | ssl.OP_NO_SSLv2 |\ ssl.OP_NO_SSLv3 | ssl.OP_SINGLE_ECDH_USE |\ - ssl.OP_CIPHER_SERVER_PREFERENCE | ssl.OP_NO_TLS1_3 + ssl.OP_CIPHER_SERVER_PREFERENCE | ssl.OP_NO_TLSv1_3 else: context.options = ssl.OP_ALL | ssl.OP_NO_SSLv2 |\ ssl.OP_NO_SSLv3 | ssl.OP_SINGLE_ECDH_USE |\ From e1661162b66574abd30ddb84203c9a076cfbafa6 Mon Sep 17 00:00:00 2001 From: Kashiko Koibumi Date: Fri, 31 May 2024 01:15:20 +0900 Subject: [PATCH 02/30] disable UPnP in Python3 temporally --- src/upnp.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/upnp.py b/src/upnp.py index fcaee6f7..e3cb04bd 100644 --- a/src/upnp.py +++ b/src/upnp.py @@ -4,6 +4,7 @@ Complete UPnP port forwarding implementation in separate thread. Reference: http://mattscodecave.com/posts/using-python-and-upnp-to-forward-a-port """ +import six from six.moves import http_client as httplib import re import socket @@ -221,6 +222,10 @@ class uPnPThread(StoppableThread): def run(self): """Start the thread to manage UPnP activity""" + if six.PY3: + logger.warning("UPnP is disabled currently, due to incompleted migration to Python3.") + return + logger.debug("Starting UPnP thread") logger.debug("Local IP: %s", self.localIP) lastSent = 0 From a6e980df2ce2177fe40351049cad162d65dc20d0 Mon Sep 17 00:00:00 2001 From: Kashiko Koibumi Date: Fri, 31 May 2024 02:31:41 +0900 Subject: [PATCH 03/30] use hexlify() to display hash ID --- src/network/dandelion.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/network/dandelion.py b/src/network/dandelion.py index d78cab39..e343d5f2 100644 --- a/src/network/dandelion.py +++ b/src/network/dandelion.py @@ -7,6 +7,7 @@ from random import choice, expovariate, sample from threading import RLock from time import time import six +from binascii import hexlify import network.connectionpool # use long name to address recursive import import state @@ -76,7 +77,7 @@ class Dandelion: # pylint: disable=old-style-class if logger.isEnabledFor(logging.DEBUG): logger.debug( '%s entering fluff mode due to %s.', - ''.join('%02x' % six.byte2int(i) for i in hashId), reason) + hexlify(hashId), reason) with self.lock: try: del self.hashMap[bytes(hashId)] From a1d633dc5fd3d323ca719c6c2e9b286660384ced Mon Sep 17 00:00:00 2001 From: Kashiko Koibumi Date: Fri, 31 May 2024 02:32:58 +0900 Subject: [PATCH 04/30] workaround to invalid tag type tag sometimes contains str type, which causes an error in Python3. --- src/storage/sqlite.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/storage/sqlite.py b/src/storage/sqlite.py index 9d1f32b0..0064262d 100644 --- a/src/storage/sqlite.py +++ b/src/storage/sqlite.py @@ -1,6 +1,7 @@ """ Sqlite Inventory """ +import six import sqlite3 import time from threading import RLock @@ -110,7 +111,10 @@ class SqliteInventory(InventoryStorage): # always use the inventoryLock OUTSIDE of the sqlLock. with SqlBulkExecute() as sql: for objectHash, value in self._inventory.items(): - value = [value[0], value[1], sqlite3.Binary(value[2]), value[3], sqlite3.Binary(value[4])] + tag = value[4] + if six.PY3 and isinstance(tag, str): + tag = tag.encode("utf-8", "replace") + value = [value[0], value[1], sqlite3.Binary(value[2]), value[3], sqlite3.Binary(tag)] sql.execute( 'INSERT INTO inventory VALUES (?, ?, ?, ?, ?, ?)', sqlite3.Binary(objectHash), *value) From 08650d5e8ea4dc6566b42101bdbd27d7ebe2b274 Mon Sep 17 00:00:00 2001 From: Kashiko Koibumi Date: Fri, 31 May 2024 02:52:00 +0900 Subject: [PATCH 05/30] fix unexpected label string --- src/unqstr.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/unqstr.py b/src/unqstr.py index 7c984df9..4c2994b1 100644 --- a/src/unqstr.py +++ b/src/unqstr.py @@ -5,6 +5,8 @@ def ustr(v): if six.PY3: if isinstance(v, str): return v + elif isinstance(v, bytes): + return v.decode("utf-8", "replace") else: return str(v) # assume six.PY2 From 5af72b02e9c64526653bc439e977721f0fcc30a2 Mon Sep 17 00:00:00 2001 From: Kashiko Koibumi Date: Fri, 31 May 2024 02:59:54 +0900 Subject: [PATCH 06/30] fix user agent format on debug log --- src/network/bmproto.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/network/bmproto.py b/src/network/bmproto.py index b3e3fe3c..9fae2305 100644 --- a/src/network/bmproto.py +++ b/src/network/bmproto.py @@ -549,7 +549,7 @@ class BMProto(AdvancedDispatcher, ObjectTracker): logger.debug( 'remote node incoming address: %s:%i', self.destination.host, self.peerNode.port) - logger.debug('user agent: %s', self.userAgent) + logger.debug('user agent: %s', self.userAgent.decode("utf-8", "replace")) logger.debug('streams: [%s]', ','.join(map(str, self.streams))) if not self.peerValidityChecks(): # ABORT afterwards From 597372543bdb4800dfe02689514d0fc3d3c0815d Mon Sep 17 00:00:00 2001 From: Kashiko Koibumi Date: Fri, 31 May 2024 10:02:57 +0900 Subject: [PATCH 07/30] fix to pass tests --- start3.sh => py3start.sh | 0 src/py3bitmessage | 13 +++++++++++++ src/pybitmessage | 0 src/shared.py | 4 ++-- src/tests/test_api.py | 10 ++++++++-- src/tests/test_config_process.py | 3 --- src/tests/test_helper_inbox.py | 5 ++++- src/tests/test_helper_sent.py | 6 ++++-- src/tests/test_logger.py | 2 +- src/tests/test_network.py | 6 +++--- src/tests/test_process.py | 6 +++++- src/tests/test_shared.py | 8 +++++--- 12 files changed, 45 insertions(+), 18 deletions(-) rename start3.sh => py3start.sh (100%) create mode 100755 src/py3bitmessage mode change 100644 => 100755 src/pybitmessage diff --git a/start3.sh b/py3start.sh similarity index 100% rename from start3.sh rename to py3start.sh diff --git a/src/py3bitmessage b/src/py3bitmessage new file mode 100755 index 00000000..6277ea32 --- /dev/null +++ b/src/py3bitmessage @@ -0,0 +1,13 @@ +#!/usr/bin/python3 + +import os +import pkg_resources + +import pybitmessage + +dist = pkg_resources.get_distribution('pybitmessage') +script_file = os.path.join(dist.location, dist.key, 'bitmessagemain.py') +new_globals = globals() +new_globals.update(__file__=script_file) + +execfile(script_file, new_globals) diff --git a/src/pybitmessage b/src/pybitmessage old mode 100644 new mode 100755 diff --git a/src/shared.py b/src/shared.py index ca0b9306..f710d56d 100644 --- a/src/shared.py +++ b/src/shared.py @@ -39,7 +39,7 @@ broadcastSendersForWhichImWatching = {} def isAddressInMyAddressBook(address): """Is address in my addressbook?""" queryreturn = sqlQuery( - '''select address from addressbook where address=?''', + '''select TRUE from addressbook where address=?''', dbstr(address)) return queryreturn != [] @@ -48,7 +48,7 @@ def isAddressInMyAddressBook(address): def isAddressInMySubscriptionsList(address): """Am I subscribed to this address?""" queryreturn = sqlQuery( - '''select * from subscriptions where address=?''', + '''select TRUE from subscriptions where address=?''', dbstr(address)) return queryreturn != [] diff --git a/src/tests/test_api.py b/src/tests/test_api.py index db52cc9c..3ae547d3 100644 --- a/src/tests/test_api.py +++ b/src/tests/test_api.py @@ -23,7 +23,10 @@ from .test_process import TestProcessProto class TestAPIProto(TestProcessProto): """Test case logic for testing API""" - _process_cmd = ['pybitmessage', '-t'] + if six.PY3: + _process_cmd = ['./py3bitmessage', '-t'] + else: # assume six.PY2 + _process_cmd = ['./pybitmessage', '-t'] @classmethod def setUpClass(cls): @@ -58,7 +61,10 @@ class TestAPIShutdown(TestAPIProto): class TestAPI(TestAPIProto): """Main API test case""" - _seed = base64.encodestring(sample_seed) + if six.PY3: + _seed = base64.encodebytes(sample_seed) + else: # assume six.PY2 + _seed = base64.encodestring(sample_seed) def _add_random_address(self, label): addr = self.api.createRandomAddress(base64.encodestring(label)) diff --git a/src/tests/test_config_process.py b/src/tests/test_config_process.py index 9322a2f0..b221aa6f 100644 --- a/src/tests/test_config_process.py +++ b/src/tests/test_config_process.py @@ -6,9 +6,6 @@ import os import tempfile from pybitmessage.bmconfigparser import config from .test_process import TestProcessProto -from .common import skip_python3 - -skip_python3() class TestProcessConfig(TestProcessProto): diff --git a/src/tests/test_helper_inbox.py b/src/tests/test_helper_inbox.py index 8ff60e18..31ccc208 100644 --- a/src/tests/test_helper_inbox.py +++ b/src/tests/test_helper_inbox.py @@ -43,6 +43,7 @@ class TestHelperInbox(unittest.TestCase): @patch("pybitmessage.helper_inbox.sqlExecute") def test_trash(self, mock_sql_execute): # pylint: disable=no-self-use """Test marking a message in the `inbox` as `trash`""" + mock_sql_execute.return_value = 1 mock_msg_id = b"fefkosghsbse92" trash(msgid=mock_msg_id) mock_sql_execute.assert_called_once() @@ -50,6 +51,7 @@ class TestHelperInbox(unittest.TestCase): @patch("pybitmessage.helper_inbox.sqlExecute") def test_delete(self, mock_sql_execute): # pylint: disable=no-self-use """Test for permanent deletion of message from trash""" + mock_sql_execute.return_value = 1 mock_ack_data = genAckPayload() delete(mock_ack_data) mock_sql_execute.assert_called_once() @@ -57,6 +59,7 @@ class TestHelperInbox(unittest.TestCase): @patch("pybitmessage.helper_inbox.sqlExecute") def test_undeleteMessage(self, mock_sql_execute): # pylint: disable=no-self-use """Test for Undelete the message""" + mock_sql_execute.return_value = 1 mock_msg_id = b"fefkosghsbse92" undeleteMessage(msgid=mock_msg_id) mock_sql_execute.assert_called_once() @@ -64,7 +67,7 @@ class TestHelperInbox(unittest.TestCase): @patch("pybitmessage.helper_inbox.sqlQuery") def test_isMessageAlreadyInInbox(self, mock_sql_query): """Test for check for previous instances of this message""" - fake_sigHash = "h4dkn54546" + fake_sigHash = b"h4dkn54546" # if Message is already in Inbox mock_sql_query.return_value = [(1,)] result = isMessageAlreadyInInbox(sigHash=fake_sigHash) diff --git a/src/tests/test_helper_sent.py b/src/tests/test_helper_sent.py index 36bb8bb7..27e5d970 100644 --- a/src/tests/test_helper_sent.py +++ b/src/tests/test_helper_sent.py @@ -45,10 +45,12 @@ class TestHelperSent(unittest.TestCase): @patch("pybitmessage.helper_sent.sqlExecute") def test_delete(self, mock_sql_execute): """Test delete function""" + mock_sql_execute.return_value = 1 delete(b"ack_data") self.assertTrue(mock_sql_execute.called) + import sqlite3 mock_sql_execute.assert_called_once_with( - "DELETE FROM sent WHERE ackdata = ?", b"ack_data" + "DELETE FROM sent WHERE ackdata = ?", sqlite3.Binary(b"ack_data") ) @patch("pybitmessage.helper_sent.sqlQuery") @@ -64,7 +66,7 @@ class TestHelperSent(unittest.TestCase): ) ] mock_sql_query.return_value = return_data - result = retrieve_message_details("12345") + result = retrieve_message_details(b"12345") self.assertEqual(result, return_data) @patch("pybitmessage.helper_sent.sqlExecute") diff --git a/src/tests/test_logger.py b/src/tests/test_logger.py index 636a209f..6e4068fc 100644 --- a/src/tests/test_logger.py +++ b/src/tests/test_logger.py @@ -43,7 +43,7 @@ handlers=default cls._files = cls._files[2:] + ('logging.dat',) cls.log_file = os.path.join(cls.home, 'debug.log') - with open(os.path.join(cls.home, 'logging.dat'), 'wb') as dst: + with open(os.path.join(cls.home, 'logging.dat'), 'w') as dst: dst.write(cls.conf_template.format(cls.log_file, cls.pattern)) super(TestLogger, cls).setUpClass() diff --git a/src/tests/test_network.py b/src/tests/test_network.py index 206117e0..e54f737f 100644 --- a/src/tests/test_network.py +++ b/src/tests/test_network.py @@ -3,11 +3,8 @@ import threading import time -from .common import skip_python3 from .partial import TestPartialRun -skip_python3() - class TestNetwork(TestPartialRun): """A test case for running the network subsystem""" @@ -24,11 +21,14 @@ class TestNetwork(TestPartialRun): # config variable is still used inside of the network ): import network from network import connectionpool, stats + from network.stats import sentBytes, receivedBytes # beware of singleton connectionpool.config = cls.config cls.pool = connectionpool.pool cls.stats = stats + cls.stats.sentBytes = sentBytes + cls.stats.receivedBytes = receivedBytes network.start(cls.config, cls.state) diff --git a/src/tests/test_process.py b/src/tests/test_process.py index 37b34541..9101fe7c 100644 --- a/src/tests/test_process.py +++ b/src/tests/test_process.py @@ -11,6 +11,7 @@ import time import unittest import psutil +import six from .common import cleanup, put_signal_file, skip_python3 @@ -22,7 +23,10 @@ class TestProcessProto(unittest.TestCase): """Test case implementing common logic for external testing: it starts pybitmessage in setUpClass() and stops it in tearDownClass() """ - _process_cmd = ['pybitmessage', '-d'] + if six.PY3: + _process_cmd = ['./py3bitmessage', '-d'] + else: # assume six.PY2 + _process_cmd = ['./pybitmessage', '-d'] _threads_count_min = 15 _threads_count_max = 16 _threads_names = [ diff --git a/src/tests/test_shared.py b/src/tests/test_shared.py index 073f94e7..f53930a1 100644 --- a/src/tests/test_shared.py +++ b/src/tests/test_shared.py @@ -46,7 +46,8 @@ class TestShared(unittest.TestCase): address = sample_address # if address is in MyAddressbook - mock_sql_query.return_value = [bytes(address)] + TRUE = 1 + mock_sql_query.return_value = [TRUE] return_val = isAddressInMyAddressBook(address) mock_sql_query.assert_called_once() self.assertTrue(return_val) @@ -64,7 +65,8 @@ class TestShared(unittest.TestCase): address = sample_address # if address is in MySubscriptionsList - mock_sql_query.return_value = [bytes(address)] + TRUE = 1 + mock_sql_query.return_value = [TRUE] return_val = isAddressInMySubscriptionsList(address) self.assertTrue(return_val) @@ -78,7 +80,7 @@ class TestShared(unittest.TestCase): def test_reloadBroadcastSendersForWhichImWatching(self, mock_sql_query): """Test for reload Broadcast Senders For Which Im Watching""" mock_sql_query.return_value = [ - (bytes(sample_address),), + (bytes(sample_address.encode("utf-8", "replace")),), ] # before reload self.assertEqual(len(MyECSubscriptionCryptorObjects), 0) From ba8ccfc4887b25c087b196706a271e807550f482 Mon Sep 17 00:00:00 2001 From: Kashiko Koibumi Date: Fri, 31 May 2024 13:17:17 +0900 Subject: [PATCH 08/30] fix bug on dandelion specific with Python3 --- src/network/dandelion.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/network/dandelion.py b/src/network/dandelion.py index e343d5f2..cbc6729b 100644 --- a/src/network/dandelion.py +++ b/src/network/dandelion.py @@ -190,8 +190,8 @@ class Dandelion: # pylint: disable=old-style-class ).outboundConnections.values()), MAX_STEMS) # not enough stems available except ValueError: - self.stem = network.connectionpool.BMConnectionPool( - ).outboundConnections.values() + self.stem = list(network.connectionpool.BMConnectionPool( + ).outboundConnections.values()) self.nodeMap = {} # hashMap stays to cater for pending stems self.refresh = time() + REASSIGN_INTERVAL From f9d236444fb118df31af13452ed81fbb2351150b Mon Sep 17 00:00:00 2001 From: Kashiko Koibumi Date: Fri, 31 May 2024 18:02:23 +0900 Subject: [PATCH 09/30] fix bug in chunked database access --- src/helper_sql.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/helper_sql.py b/src/helper_sql.py index cfacfde9..c5f30851 100644 --- a/src/helper_sql.py +++ b/src/helper_sql.py @@ -81,9 +81,13 @@ def sqlExecuteChunked(sql_statement, as_text, idCount, *args): i:i + sqlExecuteChunked.chunkSize - (len(args) - idCount) ] if as_text: - sqlSubmitQueue.put( - sql_statement.format(','.join('CAST(? AS TEXT)' * len(chunk_slice))) - ) + q = "" + n = len(chunk_slice) + for i in range(n): + q += "CAST(? AS TEXT)" + if i != n - 1: + q += "," + sqlSubmitQueue.put(sql_statement.format(q)) else: sqlSubmitQueue.put( sql_statement.format(','.join('?' * len(chunk_slice))) From f8919a8f6673f15e78ac40488c5aba98ea9a06b6 Mon Sep 17 00:00:00 2001 From: Kashiko Koibumi Date: Fri, 31 May 2024 18:03:39 +0900 Subject: [PATCH 10/30] fix bug in responsibility of message list on Qt GUI --- src/bitmessageqt/__init__.py | 45 ++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/src/bitmessageqt/__init__.py b/src/bitmessageqt/__init__.py index 7f87122c..9b6461d1 100644 --- a/src/bitmessageqt/__init__.py +++ b/src/bitmessageqt/__init__.py @@ -91,6 +91,13 @@ def openKeysFile(): os.startfile(keysfile) # pylint: disable=no-member +def as_msgid(id_data): + if six.PY3: + return escape_decode(id_data)[0][2:-1] + else: # assume six.PY2 + return id_data + + class MyForm(settingsmixin.SMainWindow): # the maximum frequency of message sounds in seconds @@ -1033,7 +1040,7 @@ class MyForm(settingsmixin.SMainWindow): # related = related.findItems(msgid, QtCore.Qt.MatchExactly), # returns an empty list for rrow in range(related.rowCount()): - if related.item(rrow, 3).data() == msgid: + if as_msgid(related.item(rrow, 3).data()) == msgid: break for col in range(widget.columnCount()): @@ -1938,8 +1945,6 @@ class MyForm(settingsmixin.SMainWindow): sent.item(i, 3).setText(textToDisplay) def updateSentItemStatusByAckdata(self, ackdata, textToDisplay): - if type(ackdata) is str: - ackdata = QtCore.QByteArray(ackdata) for sent in ( self.ui.tableWidgetInbox, self.ui.tableWidgetInboxSubscriptions, @@ -1950,7 +1955,7 @@ class MyForm(settingsmixin.SMainWindow): continue for i in range(sent.rowCount()): toAddress = sent.item(i, 0).data(QtCore.Qt.UserRole) - tableAckdata = sent.item(i, 3).data() + tableAckdata = as_msgid(sent.item(i, 3).data()) status, addressVersionNumber, streamNumber, ripe = decodeAddress( toAddress) if ackdata == tableAckdata: @@ -1976,7 +1981,7 @@ class MyForm(settingsmixin.SMainWindow): ): i = None for i in range(inbox.rowCount()): - if msgid == inbox.item(i, 3).data(): + if msgid == as_msgid(inbox.item(i, 3).data()): break else: continue @@ -2702,7 +2707,7 @@ class MyForm(settingsmixin.SMainWindow): msgids = [] for i in range(0, idCount): - msgids.append(sqlite3.Binary(tableWidget.item(i, 3).data())) + msgids.append(sqlite3.Binary(as_msgid(tableWidget.item(i, 3).data()))) for col in xrange(tableWidget.columnCount()): tableWidget.item(i, col).setUnread(False) @@ -2986,8 +2991,8 @@ class MyForm(settingsmixin.SMainWindow): # modified = 0 for row in tableWidget.selectedIndexes(): currentRow = row.row() - msgid = sqlite3.Binary(tableWidget.item(currentRow, 3).data()) - msgids.add(msgid) + msgid = as_msgid(tableWidget.item(currentRow, 3).data()) + msgids.add(sqlite3.Binary(msgid)) # if not tableWidget.item(currentRow, 0).unread: # modified += 1 self.updateUnreadStatus(tableWidget, currentRow, msgid, False) @@ -3081,7 +3086,7 @@ class MyForm(settingsmixin.SMainWindow): acct = accountClass(toAddressAtCurrentInboxRow) fromAddressAtCurrentInboxRow = tableWidget.item( currentInboxRow, column_from).address - msgid = tableWidget.item(currentInboxRow, 3).data() + msgid = as_msgid(tableWidget.item(currentInboxRow, 3).data()) queryreturn = sqlQuery( "SELECT message FROM inbox WHERE msgid=?", sqlite3.Binary(msgid) ) or sqlQuery("SELECT message FROM sent WHERE ackdata=?", sqlite3.Binary(msgid)) @@ -3235,15 +3240,15 @@ class MyForm(settingsmixin.SMainWindow): messageLists = (messageLists,) for messageList in messageLists: if row is not None: - inventoryHash = messageList.item(row, 3).data() + inventoryHash = as_msgid(messageList.item(row, 3).data()) messageList.removeRow(row) elif inventoryHash is not None: for i in range(messageList.rowCount() - 1, -1, -1): - if messageList.item(i, 3).data() == inventoryHash: + if as_msgid(messageList.item(i, 3).data()) == inventoryHash: messageList.removeRow(i) elif ackData is not None: for i in range(messageList.rowCount() - 1, -1, -1): - if messageList.item(i, 3).data() == ackData: + if as_msgid(messageList.item(i, 3).data()) == ackData: messageList.removeRow(i) # Send item on the Inbox tab to trash @@ -3263,7 +3268,7 @@ class MyForm(settingsmixin.SMainWindow): )[::-1]: for i in range(r.bottomRow() - r.topRow() + 1): inventoryHashesToTrash.add( - sqlite3.Binary(tableWidget.item(r.topRow() + i, 3).data())) + sqlite3.Binary(as_msgid(tableWidget.item(r.topRow() + i, 3).data()))) currentRow = r.topRow() self.getCurrentMessageTextedit().setText("") tableWidget.model().removeRows( @@ -3296,7 +3301,7 @@ class MyForm(settingsmixin.SMainWindow): )[::-1]: for i in range(r.bottomRow() - r.topRow() + 1): inventoryHashesToTrash.add( - sqlite3.Binary(tableWidget.item(r.topRow() + i, 3).data())) + sqlite3.Binary(as_msgid(tableWidget.item(r.topRow() + i, 3).data()))) currentRow = r.topRow() self.getCurrentMessageTextedit().setText("") tableWidget.model().removeRows( @@ -3327,7 +3332,7 @@ class MyForm(settingsmixin.SMainWindow): subjectAtCurrentInboxRow = '' # Retrieve the message data out of the SQL database - msgid = tableWidget.item(currentInboxRow, 3).data() + msgid = as_msgid(tableWidget.item(currentInboxRow, 3).data()) queryreturn = sqlQuery( '''select message from inbox where msgid=?''', sqlite3.Binary(msgid)) if len(queryreturn) < 1: @@ -3363,7 +3368,7 @@ class MyForm(settingsmixin.SMainWindow): shifted = QtGui.QApplication.queryKeyboardModifiers() & QtCore.Qt.ShiftModifier while tableWidget.selectedIndexes() != []: currentRow = tableWidget.selectedIndexes()[0].row() - ackdataToTrash = tableWidget.item(currentRow, 3).data() + ackdataToTrash = as_msgid(tableWidget.item(currentRow, 3).data()) rowcount = sqlExecute( "DELETE FROM sent" if folder == "trash" or shifted else "UPDATE sent SET folder='trash'" @@ -3646,11 +3651,7 @@ class MyForm(settingsmixin.SMainWindow): if messagelist: currentRow = messagelist.currentRow() if currentRow >= 0: - msgid_str = messagelist.item(currentRow, 3).data() - if six.PY3: - return escape_decode(msgid_str)[0][2:-1] - else: # assume six.PY2 - return msgid_str + return as_msgid(messagelist.item(currentRow, 3).data()) def getCurrentMessageTextedit(self): currentIndex = self.ui.tabWidget.currentIndex() @@ -4093,7 +4094,7 @@ class MyForm(settingsmixin.SMainWindow): # Check to see if this item is toodifficult and display an additional # menu option (Force Send) if it is. if currentRow >= 0: - ackData = self.ui.tableWidgetInbox.item(currentRow, 3).data() + ackData = as_msgid(self.ui.tableWidgetInbox.item(currentRow, 3).data()) queryreturn = sqlQuery('''SELECT status FROM sent where ackdata=?''', sqlite3.Binary(ackData)) if len(queryreturn) < 1: queryreturn = sqlQuery('''SELECT status FROM sent where ackdata=CAST(? AS TEXT)''', ackData) From 9adcd1bdc980b82eeb31adb12113968bf8cc557a Mon Sep 17 00:00:00 2001 From: Kashiko Koibumi Date: Sat, 1 Jun 2024 06:01:32 +0900 Subject: [PATCH 11/30] fix SOCKS --- src/network/socks4a.py | 12 ++++++------ src/network/socks5.py | 20 ++++++++++---------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/network/socks4a.py b/src/network/socks4a.py index bf0adf29..122114b3 100644 --- a/src/network/socks4a.py +++ b/src/network/socks4a.py @@ -40,11 +40,11 @@ class Socks4a(Proxy): def state_pre_connect(self): """Handle feedback from SOCKS4a while it is connecting on our behalf""" # Get the response - if self.read_buf[0:1] != six.int2byte(0x00).encode(): + if self.read_buf[0:1] != six.int2byte(0x00): # bad data self.close() raise GeneralProxyError(1) - elif self.read_buf[1:2] != six.int2byte(0x5A).encode(): + elif self.read_buf[1:2] != six.int2byte(0x5A): # Connection failed self.close() if six.byte2int(self.read_buf[1:2]) in (91, 92, 93): @@ -103,9 +103,9 @@ class Socks4aConnection(Socks4a): self.append_write_buf(self.ipaddr) if self._auth: self.append_write_buf(self._auth[0]) - self.append_write_buf(six.int2byte(0x00).encode()) + self.append_write_buf(six.int2byte(0x00)) if rmtrslv: - self.append_write_buf(self.destination[0] + six.int2byte(0x00).encode()) + self.append_write_buf(self.destination[0].encode("utf-8", "replace") + six.int2byte(0x00)) self.set_state("pre_connect", length=0, expectBytes=8) return True @@ -133,8 +133,8 @@ class Socks4aResolver(Socks4a): self.append_write_buf(struct.pack("BBBB", 0x00, 0x00, 0x00, 0x01)) if self._auth: self.append_write_buf(self._auth[0]) - self.append_write_buf(six.int2byte(0x00).encode()) - self.append_write_buf(self.host + six.int2byte(0x00).encode()) + self.append_write_buf(six.int2byte(0x00)) + self.append_write_buf(self.host + six.int2byte(0x00)) self.set_state("pre_connect", length=0, expectBytes=8) return True diff --git a/src/network/socks5.py b/src/network/socks5.py index c295480c..ca5a52e8 100644 --- a/src/network/socks5.py +++ b/src/network/socks5.py @@ -98,10 +98,10 @@ class Socks5(Proxy): def state_pre_connect(self): """Handle feedback from socks5 while it is connecting on our behalf.""" # Get the response - if self.read_buf[0:1] != six.int2byte(0x05).encode(): + if self.read_buf[0:1] != six.int2byte(0x05): self.close() raise GeneralProxyError(1) - elif self.read_buf[1:2] != six.int2byte(0x00).encode(): + elif self.read_buf[1:2] != six.int2byte(0x00): # Connection failed self.close() if six.byte2int(self.read_buf[1:2]) <= 8: @@ -109,9 +109,9 @@ class Socks5(Proxy): else: raise Socks5Error(9) # Get the bound address/port - elif self.read_buf[3:4] == six.int2byte(0x01).encode(): + elif self.read_buf[3:4] == six.int2byte(0x01): self.set_state("proxy_addr_1", length=4, expectBytes=4) - elif self.read_buf[3:4] == six.int2byte(0x03).encode(): + elif self.read_buf[3:4] == six.int2byte(0x03): self.set_state("proxy_addr_2_1", length=4, expectBytes=1) else: self.close() @@ -172,19 +172,19 @@ class Socks5Connection(Socks5): # use the IPv4 address request even if remote resolving was specified. try: self.ipaddr = socket.inet_aton(self.destination[0]) - self.append_write_buf(six.int2byte(0x01).encode() + self.ipaddr) + self.append_write_buf(six.int2byte(0x01) + self.ipaddr) except socket.error: # may be IPv6! # Well it's not an IP number, so it's probably a DNS name. if self._remote_dns: # Resolve remotely self.ipaddr = None - self.append_write_buf(six.int2byte(0x03).encode() + six.int2byte( - len(self.destination[0])).encode() + self.destination[0]) + self.append_write_buf(six.int2byte(0x03) + six.int2byte( + len(self.destination[0])) + self.destination[0].encode("utf-8", "replace")) else: # Resolve locally self.ipaddr = socket.inet_aton( socket.gethostbyname(self.destination[0])) - self.append_write_buf(six.int2byte(0x01).encode() + self.ipaddr) + self.append_write_buf(six.int2byte(0x01) + self.ipaddr) self.append_write_buf(struct.pack(">H", self.destination[1])) self.set_state("pre_connect", length=0, expectBytes=4) return True @@ -209,8 +209,8 @@ class Socks5Resolver(Socks5): """Perform resolving""" # Now we can request the actual connection self.append_write_buf(struct.pack('BBB', 0x05, 0xF0, 0x00)) - self.append_write_buf(six.int2byte(0x03).encode() + six.int2byte( - len(self.host)).encode() + str(self.host)) + self.append_write_buf(six.int2byte(0x03) + six.int2byte( + len(self.host)) + bytes(self.host)) self.append_write_buf(struct.pack(">H", self.port)) self.set_state("pre_connect", length=0, expectBytes=4) return True From d103297d06550ef0cefd8737f3f87be072165586 Mon Sep 17 00:00:00 2001 From: Kashiko Koibumi Date: Sat, 1 Jun 2024 13:08:38 +0900 Subject: [PATCH 12/30] fix UPnP to work with Python3 --- src/upnp.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/upnp.py b/src/upnp.py index e3cb04bd..72d17343 100644 --- a/src/upnp.py +++ b/src/upnp.py @@ -94,12 +94,12 @@ class Router: # pylint: disable=old-style-class self.address = address - row = ssdpResponse.split('\r\n') + row = ssdpResponse.split(b'\r\n') header = {} for i in range(1, len(row)): - part = row[i].split(': ') + part = row[i].split(b': ') if len(part) == 2: - header[part[0].lower()] = part[1] + header[part[0].decode("utf-8", "replace").lower()] = part[1].decode("utf-8", "replace") try: self.routerPath = urlparse(header['location']) @@ -222,10 +222,6 @@ class uPnPThread(StoppableThread): def run(self): """Start the thread to manage UPnP activity""" - if six.PY3: - logger.warning("UPnP is disabled currently, due to incompleted migration to Python3.") - return - logger.debug("Starting UPnP thread") logger.debug("Local IP: %s", self.localIP) lastSent = 0 @@ -320,7 +316,7 @@ class uPnPThread(StoppableThread): try: logger.debug("Sending UPnP query") - self.sock.sendto(ssdpRequest, (uPnPThread.SSDP_ADDR, uPnPThread.SSDP_PORT)) + self.sock.sendto(ssdpRequest.encode("utf8", "replace"), (uPnPThread.SSDP_ADDR, uPnPThread.SSDP_PORT)) except: # noqa:E722 logger.exception("UPnP send query failed") From 0c110b9debf48cc2876ea35c42ac7676f287a070 Mon Sep 17 00:00:00 2001 From: Kashiko Koibumi Date: Mon, 3 Jun 2024 19:34:32 +0900 Subject: [PATCH 13/30] fix to be runnable with prctl module in Python3 --- src/threads.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/threads.py b/src/threads.py index ac8bf7a6..ce781a77 100644 --- a/src/threads.py +++ b/src/threads.py @@ -14,6 +14,7 @@ There are also other threads in the `.network` package. """ import threading +import six from class_addressGenerator import addressGenerator from class_objectProcessor import objectProcessor @@ -32,12 +33,13 @@ else: """Set the thread name for external use (visible from the OS).""" prctl.set_name(name) - def _thread_name_hack(self): - set_thread_name(self.name) - threading.Thread.__bootstrap_original__(self) - # pylint: disable=protected-access - threading.Thread.__bootstrap_original__ = threading.Thread._Thread__bootstrap - threading.Thread._Thread__bootstrap = _thread_name_hack + if six.PY2: + def _thread_name_hack(self): + set_thread_name(self.name) + threading.Thread.__bootstrap_original__(self) + # pylint: disable=protected-access + threading.Thread.__bootstrap_original__ = threading.Thread._Thread__bootstrap + threading.Thread._Thread__bootstrap = _thread_name_hack printLock = threading.Lock() From a71b44e95c8c1c0a939b494089fc185c14eecf8e Mon Sep 17 00:00:00 2001 From: anand k Date: Wed, 29 May 2024 09:18:53 +0530 Subject: [PATCH 14/30] moved dandelion_enabled from state to Dandelion class as enabled attr --- src/bitmessagemain.py | 8 +------ src/network/__init__.py | 8 ++++++- src/network/bmobject.py | 4 ++-- src/network/bmproto.py | 17 +++++++-------- src/network/dandelion.py | 41 ++++++++++++++++++++++------------- src/network/downloadthread.py | 4 ++-- src/network/invthread.py | 17 +++++++-------- src/network/objectracker.py | 8 +++---- src/network/tcp.py | 14 ++++++------ src/network/uploadthread.py | 6 ++--- src/protocol.py | 8 +++---- src/state.py | 2 -- src/tests/core.py | 7 +++--- 13 files changed, 76 insertions(+), 68 deletions(-) diff --git a/src/bitmessagemain.py b/src/bitmessagemain.py index 9acd1278..ab131a4c 100755 --- a/src/bitmessagemain.py +++ b/src/bitmessagemain.py @@ -12,6 +12,7 @@ The PyBitmessage startup script import os import sys + try: import pathmagic except ImportError: @@ -156,13 +157,6 @@ class Main(object): set_thread_name("PyBitmessage") - state.dandelion_enabled = config.safeGetInt('network', 'dandelion') - # dandelion requires outbound connections, without them, - # stem objects will get stuck forever - if state.dandelion_enabled and not config.safeGetBoolean( - 'bitmessagesettings', 'sendoutgoingconnections'): - state.dandelion_enabled = 0 - if state.testmode or config.safeGetBoolean( 'bitmessagesettings', 'extralowdifficulty'): defaults.networkDefaultProofOfWorkNonceTrialsPerByte = int( diff --git a/src/network/__init__.py b/src/network/__init__.py index 61e1d9d1..42e9d035 100644 --- a/src/network/__init__.py +++ b/src/network/__init__.py @@ -1,9 +1,10 @@ """ Network subsystem package """ - +from .dandelion import Dandelion from .threads import StoppableThread +dandelion_ins = Dandelion() __all__ = ["StoppableThread"] @@ -21,6 +22,11 @@ def start(config, state): from .receivequeuethread import ReceiveQueueThread from .uploadthread import UploadThread + # check and set dandelion enabled value at network startup + dandelion_ins.init_dandelion_enabled(config) + # pass pool instance into dandelion class instance + dandelion_ins.init_pool(connectionpool.pool) + readKnownNodes() connectionpool.pool.connectToStream(1) for thread in ( diff --git a/src/network/bmobject.py b/src/network/bmobject.py index 2f9fc0a7..83311b9b 100644 --- a/src/network/bmobject.py +++ b/src/network/bmobject.py @@ -6,8 +6,8 @@ import time import protocol import state -import dandelion import connectionpool +from network import dandelion_ins from highlevelcrypto import calculateInventoryHash logger = logging.getLogger('default') @@ -113,7 +113,7 @@ class BMObject(object): # pylint: disable=too-many-instance-attributes or advertise it unnecessarily) """ # if it's a stem duplicate, pretend we don't have it - if dandelion.instance.hasHash(self.inventoryHash): + if dandelion_ins.hasHash(self.inventoryHash): return if self.inventoryHash in state.Inventory: raise BMObjectAlreadyHaveError() diff --git a/src/network/bmproto.py b/src/network/bmproto.py index cbe39017..797dab5e 100644 --- a/src/network/bmproto.py +++ b/src/network/bmproto.py @@ -15,7 +15,6 @@ import addresses import knownnodes import protocol import state -import dandelion import connectionpool from bmconfigparser import config from queues import invQueue, objectProcessorQueue, portCheckerQueue @@ -27,7 +26,7 @@ from network.bmobject import ( BMObjectUnwantedStreamError ) from network.proxy import ProxyError - +from network import dandelion_ins from node import Node, Peer from objectracker import ObjectTracker, missingObjects @@ -351,14 +350,14 @@ class BMProto(AdvancedDispatcher, ObjectTracker): raise BMProtoExcessiveDataError() # ignore dinv if dandelion turned off - if extend_dandelion_stem and not state.dandelion_enabled: + if extend_dandelion_stem and not dandelion_ins.enabled: return True for i in map(str, items): - if i in state.Inventory and not dandelion.instance.hasHash(i): + if i in state.Inventory and not dandelion_ins.hasHash(i): continue - if extend_dandelion_stem and not dandelion.instance.hasHash(i): - dandelion.instance.addHash(i, self) + if extend_dandelion_stem and not dandelion_ins.hasHash(i): + dandelion_ins.addHash(i, self) self.handleReceivedInventory(i) return True @@ -420,9 +419,9 @@ class BMProto(AdvancedDispatcher, ObjectTracker): except KeyError: pass - if self.object.inventoryHash in state.Inventory and dandelion.instance.hasHash( + if self.object.inventoryHash in state.Inventory and dandelion_ins.hasHash( self.object.inventoryHash): - dandelion.instance.removeHash( + dandelion_ins.removeHash( self.object.inventoryHash, "cycle detection") state.Inventory[self.object.inventoryHash] = ( @@ -541,7 +540,7 @@ class BMProto(AdvancedDispatcher, ObjectTracker): if not self.isOutbound: self.append_write_buf(protocol.assembleVersionMessage( self.destination.host, self.destination.port, - connectionpool.pool.streams, True, + connectionpool.pool.streams, dandelion_ins.enabled, True, nodeid=self.nodeid)) logger.debug( '%(host)s:%(port)i sending version', diff --git a/src/network/dandelion.py b/src/network/dandelion.py index d4c51cad..564a35f9 100644 --- a/src/network/dandelion.py +++ b/src/network/dandelion.py @@ -7,9 +7,6 @@ from random import choice, expovariate, sample from threading import RLock from time import time -import connectionpool -import state -from queues import invQueue # randomise routes after 600 seconds REASSIGN_INTERVAL = 600 @@ -37,6 +34,8 @@ class Dandelion: # pylint: disable=old-style-class # when to rerandomise routes self.refresh = time() + REASSIGN_INTERVAL self.lock = RLock() + self.enabled = None + self.pool = None @staticmethod def poissonTimeout(start=None, average=0): @@ -47,10 +46,23 @@ class Dandelion: # pylint: disable=old-style-class average = FLUFF_TRIGGER_MEAN_DELAY return start + expovariate(1.0 / average) + FLUFF_TRIGGER_FIXED_DELAY + def init_pool(self, pool): + """pass pool instance""" + self.pool = pool + + def init_dandelion_enabled(self, config): + """Check if Dandelion is enabled and set value in enabled attribute""" + dandelion_enabled = config.safeGetInt('network', 'dandelion') + # dandelion requires outbound connections, without them, + # stem objects will get stuck forever + if not config.safeGetBoolean( + 'bitmessagesettings', 'sendoutgoingconnections'): + dandelion_enabled = 0 + self.enabled = dandelion_enabled + def addHash(self, hashId, source=None, stream=1): - """Add inventory vector to dandelion stem""" - if not state.dandelion_enabled: - return + """Add inventory vector to dandelion stem return status of dandelion enabled""" + assert self.enabled is not None with self.lock: self.hashMap[hashId] = Stem( self.getNodeStem(source), @@ -89,7 +101,7 @@ class Dandelion: # pylint: disable=old-style-class """Child (i.e. next) node for an inventory vector during stem mode""" return self.hashMap[hashId].child - def maybeAddStem(self, connection): + def maybeAddStem(self, connection, invQueue): """ If we had too few outbound connections, add the current one to the current stem list. Dandelion as designed by the authors should @@ -163,7 +175,7 @@ class Dandelion: # pylint: disable=old-style-class self.nodeMap[node] = self.pickStem(node) return self.nodeMap[node] - def expire(self): + def expire(self, invQueue): """Switch expired objects from stem to fluff mode""" with self.lock: deadline = time() @@ -179,19 +191,18 @@ class Dandelion: # pylint: disable=old-style-class def reRandomiseStems(self): """Re-shuffle stem mapping (parent <-> child pairs)""" + assert self.pool is not None + if self.refresh > time(): + return + with self.lock: try: # random two connections self.stem = sample( - connectionpool.BMConnectionPool( - ).outboundConnections.values(), MAX_STEMS) + self.pool.outboundConnections.values(), MAX_STEMS) # not enough stems available except ValueError: - self.stem = connectionpool.BMConnectionPool( - ).outboundConnections.values() + self.stem = self.pool.outboundConnections.values() self.nodeMap = {} # hashMap stays to cater for pending stems self.refresh = time() + REASSIGN_INTERVAL - - -instance = Dandelion() diff --git a/src/network/downloadthread.py b/src/network/downloadthread.py index 4fd1c668..30a3f2fe 100644 --- a/src/network/downloadthread.py +++ b/src/network/downloadthread.py @@ -6,8 +6,8 @@ import state import addresses import helper_random import protocol -import dandelion import connectionpool +from network import dandelion_ins from objectracker import missingObjects from threads import StoppableThread @@ -60,7 +60,7 @@ class DownloadThread(StoppableThread): payload = bytearray() chunkCount = 0 for chunk in request: - if chunk in state.Inventory and not dandelion.instance.hasHash(chunk): + if chunk in state.Inventory and not dandelion_ins.hasHash(chunk): try: del i.objectsNewToMe[chunk] except KeyError: diff --git a/src/network/invthread.py b/src/network/invthread.py index 8b90e5f7..0b79710a 100644 --- a/src/network/invthread.py +++ b/src/network/invthread.py @@ -8,8 +8,8 @@ from time import time import addresses import protocol import state -import dandelion import connectionpool +from network import dandelion_ins from queues import invQueue from threads import StoppableThread @@ -40,10 +40,10 @@ class InvThread(StoppableThread): @staticmethod def handleLocallyGenerated(stream, hashId): """Locally generated inventory items require special handling""" - dandelion.instance.addHash(hashId, stream=stream) + dandelion_ins.addHash(hashId, stream=stream) for connection in connectionpool.pool.connections(): - if state.dandelion_enabled and connection != \ - dandelion.instance.objectChildStem(hashId): + if dandelion_ins.enabled and connection != \ + dandelion_ins.objectChildStem(hashId): continue connection.objectsNewToThem[hashId] = time() @@ -52,7 +52,7 @@ class InvThread(StoppableThread): chunk = [] while True: # Dandelion fluff trigger by expiration - handleExpiredDandelion(dandelion.instance.expire()) + handleExpiredDandelion(dandelion_ins.expire(invQueue)) try: data = invQueue.get(False) chunk.append((data[0], data[1])) @@ -75,10 +75,10 @@ class InvThread(StoppableThread): except KeyError: continue try: - if connection == dandelion.instance.objectChildStem(inv[1]): + if connection == dandelion_ins.objectChildStem(inv[1]): # Fluff trigger by RNG # auto-ignore if config set to 0, i.e. dandelion is off - if random.randint(1, 100) >= state.dandelion_enabled: # nosec B311 + if random.randint(1, 100) >= dandelion_ins.enabled: # nosec B311 fluffs.append(inv[1]) # send a dinv only if the stem node supports dandelion elif connection.services & protocol.NODE_DANDELION > 0: @@ -105,7 +105,6 @@ class InvThread(StoppableThread): for _ in range(len(chunk)): invQueue.task_done() - if dandelion.instance.refresh < time(): - dandelion.instance.reRandomiseStems() + dandelion_ins.reRandomiseStems() self.stop.wait(1) diff --git a/src/network/objectracker.py b/src/network/objectracker.py index 0e8268cf..91bb0552 100644 --- a/src/network/objectracker.py +++ b/src/network/objectracker.py @@ -4,8 +4,8 @@ Module for tracking objects import time from threading import RLock -import dandelion import connectionpool +from network import dandelion_ins from randomtrackingdict import RandomTrackingDict haveBloom = False @@ -107,14 +107,14 @@ class ObjectTracker(object): del i.objectsNewToMe[hashid] except KeyError: if streamNumber in i.streams and ( - not dandelion.instance.hasHash(hashid) - or dandelion.instance.objectChildStem(hashid) == i): + not dandelion_ins.hasHash(hashid) + or dandelion_ins.objectChildStem(hashid) == i): with i.objectsNewToThemLock: i.objectsNewToThem[hashid] = time.time() # update stream number, # which we didn't have when we just received the dinv # also resets expiration of the stem mode - dandelion.instance.setHashStream(hashid, streamNumber) + dandelion_ins.setHashStream(hashid, streamNumber) if i == self: try: diff --git a/src/network/tcp.py b/src/network/tcp.py index 35937aa8..a739e256 100644 --- a/src/network/tcp.py +++ b/src/network/tcp.py @@ -15,10 +15,10 @@ import helper_random import l10n import protocol import state -import dandelion import connectionpool from bmconfigparser import config from highlevelcrypto import randomBytes +from network import dandelion_ins from queues import invQueue, receiveDataQueue, UISignalQueue from tr import _translate @@ -169,7 +169,7 @@ class TCPConnection(BMProto, TLSDispatcher): knownnodes.increaseRating(self.destination) knownnodes.addKnownNode( self.streams, self.destination, time.time()) - dandelion.instance.maybeAddStem(self) + dandelion_ins.maybeAddStem(self, invQueue) self.sendAddr() self.sendBigInv() @@ -231,7 +231,7 @@ class TCPConnection(BMProto, TLSDispatcher): with self.objectsNewToThemLock: for objHash in state.Inventory.unexpired_hashes_by_stream(stream): # don't advertise stem objects on bigInv - if dandelion.instance.hasHash(objHash): + if dandelion_ins.hasHash(objHash): continue bigInvList[objHash] = 0 objectCount = 0 @@ -268,7 +268,7 @@ class TCPConnection(BMProto, TLSDispatcher): self.append_write_buf( protocol.assembleVersionMessage( self.destination.host, self.destination.port, - connectionpool.pool.streams, + connectionpool.pool.streams, dandelion_ins.enabled, False, nodeid=self.nodeid)) self.connectedAt = time.time() receiveDataQueue.put(self.destination) @@ -293,7 +293,7 @@ class TCPConnection(BMProto, TLSDispatcher): if host_is_global: knownnodes.addKnownNode( self.streams, self.destination, time.time()) - dandelion.instance.maybeRemoveStem(self) + dandelion_ins.maybeRemoveStem(self) else: self.checkTimeOffsetNotification() if host_is_global: @@ -319,7 +319,7 @@ class Socks5BMConnection(Socks5Connection, TCPConnection): self.append_write_buf( protocol.assembleVersionMessage( self.destination.host, self.destination.port, - connectionpool.pool.streams, + connectionpool.pool.streams, dandelion_ins.enabled, False, nodeid=self.nodeid)) self.set_state("bm_header", expectBytes=protocol.Header.size) return True @@ -343,7 +343,7 @@ class Socks4aBMConnection(Socks4aConnection, TCPConnection): self.append_write_buf( protocol.assembleVersionMessage( self.destination.host, self.destination.port, - connectionpool.pool.streams, + connectionpool.pool.streams, dandelion_ins.enabled, False, nodeid=self.nodeid)) self.set_state("bm_header", expectBytes=protocol.Header.size) return True diff --git a/src/network/uploadthread.py b/src/network/uploadthread.py index 1fc3734e..e91f08fa 100644 --- a/src/network/uploadthread.py +++ b/src/network/uploadthread.py @@ -6,9 +6,9 @@ import time import helper_random import protocol import state -import dandelion import connectionpool from randomtrackingdict import RandomTrackingDict +from network import dandelion_ins from threads import StoppableThread @@ -41,8 +41,8 @@ class UploadThread(StoppableThread): chunk_count = 0 for chunk in request: del i.pendingUpload[chunk] - if dandelion.instance.hasHash(chunk) and \ - i != dandelion.instance.objectChildStem(chunk): + if dandelion_ins.hasHash(chunk) and \ + i != dandelion_ins.objectChildStem(chunk): i.antiIntersectionDelay() self.logger.info( '%s asked for a stem object we didn\'t offer to it.', diff --git a/src/protocol.py b/src/protocol.py index 7f9830e5..96c980bb 100644 --- a/src/protocol.py +++ b/src/protocol.py @@ -336,8 +336,8 @@ def assembleAddrMessage(peerList): return retval -def assembleVersionMessage( - remoteHost, remotePort, participatingStreams, server=False, nodeid=None +def assembleVersionMessage( # pylint: disable=too-many-arguments + remoteHost, remotePort, participatingStreams, dandelion_enabled=True, server=False, nodeid=None, ): """ Construct the payload of a version message, @@ -350,7 +350,7 @@ def assembleVersionMessage( '>q', NODE_NETWORK | (NODE_SSL if haveSSL(server) else 0) - | (NODE_DANDELION if state.dandelion_enabled else 0) + | (NODE_DANDELION if dandelion_enabled else 0) ) payload += pack('>q', int(time.time())) @@ -374,7 +374,7 @@ def assembleVersionMessage( '>q', NODE_NETWORK | (NODE_SSL if haveSSL(server) else 0) - | (NODE_DANDELION if state.dandelion_enabled else 0) + | (NODE_DANDELION if dandelion_enabled else 0) ) # = 127.0.0.1. This will be ignored by the remote host. # The actual remote connected IP will be used. diff --git a/src/state.py b/src/state.py index a40ebbc2..90c9cf0d 100644 --- a/src/state.py +++ b/src/state.py @@ -43,8 +43,6 @@ ownAddresses = {} discoveredPeers = {} -dandelion_enabled = 0 - kivy = False kivyapp = None diff --git a/src/tests/core.py b/src/tests/core.py index f1a11a06..fd9b0d08 100644 --- a/src/tests/core.py +++ b/src/tests/core.py @@ -319,16 +319,17 @@ class TestCore(unittest.TestCase): def test_version(self): """check encoding/decoding of the version message""" + dandelion_enabled = True # with single stream - msg = protocol.assembleVersionMessage('127.0.0.1', 8444, [1]) + msg = protocol.assembleVersionMessage('127.0.0.1', 8444, [1], dandelion_enabled) decoded = self._decode_msg(msg, "IQQiiQlsLv") peer, _, ua, streams = self._decode_msg(msg, "IQQiiQlsLv")[4:] self.assertEqual( - peer, Node(11 if state.dandelion_enabled else 3, '127.0.0.1', 8444)) + peer, Node(11 if dandelion_enabled else 3, '127.0.0.1', 8444)) self.assertEqual(ua, '/PyBitmessage:' + softwareVersion + '/') self.assertEqual(streams, [1]) # with multiple streams - msg = protocol.assembleVersionMessage('127.0.0.1', 8444, [1, 2, 3]) + msg = protocol.assembleVersionMessage('127.0.0.1', 8444, [1, 2, 3], dandelion_enabled) decoded = self._decode_msg(msg, "IQQiiQlslv") peer, _, ua = decoded[4:7] streams = decoded[7:] From 21bef1058ddd6b8f4d7f9f281ade69243e7b027c Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Tue, 14 May 2024 17:38:43 +0300 Subject: [PATCH 15/30] Improve the base class for bitmessageqt test cases --- src/bitmessageqt/tests/__init__.py | 8 ++++---- src/bitmessageqt/tests/main.py | 23 +++++++++++++++++------ 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/src/bitmessageqt/tests/__init__.py b/src/bitmessageqt/tests/__init__.py index a542abdc..a81ddb04 100644 --- a/src/bitmessageqt/tests/__init__.py +++ b/src/bitmessageqt/tests/__init__.py @@ -1,9 +1,9 @@ """bitmessageqt tests""" -from addressbook import TestAddressbook -from main import TestMain, TestUISignaler -from settings import TestSettings -from support import TestSupport +from .addressbook import TestAddressbook +from .main import TestMain, TestUISignaler +from .settings import TestSettings +from .support import TestSupport __all__ = [ "TestAddressbook", "TestMain", "TestSettings", "TestSupport", diff --git a/src/bitmessageqt/tests/main.py b/src/bitmessageqt/tests/main.py index b3aa67fa..d3fda8aa 100644 --- a/src/bitmessageqt/tests/main.py +++ b/src/bitmessageqt/tests/main.py @@ -1,19 +1,23 @@ """Common definitions for bitmessageqt tests""" -import Queue import sys import unittest from PyQt4 import QtCore, QtGui +from six.moves import queue import bitmessageqt -import queues -from tr import _translate +from bitmessageqt import _translate, config, queues class TestBase(unittest.TestCase): """Base class for bitmessageqt test case""" + @classmethod + def setUpClass(cls): + """Provide the UI test cases with common settings""" + cls.config = config + def setUp(self): self.app = ( QtGui.QApplication.instance() @@ -24,14 +28,21 @@ class TestBase(unittest.TestCase): self.window.appIndicatorInit(self.app) def tearDown(self): + """Search for exceptions in closures called by timer and fail if any""" # self.app.deleteLater() + concerning = [] while True: try: thread, exc = queues.excQueue.get(block=False) - except Queue.Empty: - return + except queue.Empty: + break if thread == 'tests': - self.fail('Exception in the main thread: %s' % exc) + concerning.append(exc) + if concerning: + self.fail( + 'Exceptions found in the main thread:\n%s' % '\n'.join(( + str(e) for e in concerning + ))) class TestMain(unittest.TestCase): From ec1b05ee90860885e585bbb9c317f5e5306d6490 Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Thu, 25 Apr 2024 22:14:50 +0300 Subject: [PATCH 16/30] Allow user to set a base Qt window style and the font (family and size only) --- src/bitmessageqt/__init__.py | 9 +++++++++ src/bitmessageqt/settings.py | 39 ++++++++++++++++++++++++++++++++++++ src/bitmessageqt/settings.ui | 32 +++++++++++++++++++++++++++++ 3 files changed, 80 insertions(+) diff --git a/src/bitmessageqt/__init__.py b/src/bitmessageqt/__init__.py index 40113b5a..7f767b03 100644 --- a/src/bitmessageqt/__init__.py +++ b/src/bitmessageqt/__init__.py @@ -4241,6 +4241,15 @@ class BitmessageQtApplication(QtGui.QApplication): QtCore.QCoreApplication.setOrganizationDomain("bitmessage.org") QtCore.QCoreApplication.setApplicationName("pybitmessageqt") + self.setStyle( + config.safeGet('bitmessagesettings', 'windowstyle', 'GTK+')) + + font = config.safeGet('bitmessagesettings', 'font') + if font: + # family, size, weight = font.split(',') + family, size = font.split(',') + self.setFont(QtGui.QFont(family, int(size))) + self.server = None self.is_running = False diff --git a/src/bitmessageqt/settings.py b/src/bitmessageqt/settings.py index 3d05db25..44cad4c9 100644 --- a/src/bitmessageqt/settings.py +++ b/src/bitmessageqt/settings.py @@ -49,6 +49,7 @@ class SettingsDialog(QtGui.QDialog): self.firstrun = firstrun self.config = config_obj self.net_restart_needed = False + self.font_setting = None self.timer = QtCore.QTimer() if self.config.safeGetBoolean('bitmessagesettings', 'dontconnect'): @@ -85,6 +86,16 @@ class SettingsDialog(QtGui.QDialog): def adjust_from_config(self, config): """Adjust all widgets state according to config settings""" # pylint: disable=too-many-branches,too-many-statements + + current_style = config.safeGet( + 'bitmessagesettings', 'windowstyle', 'GTK+') + for i, sk in enumerate(QtGui.QStyleFactory.keys()): + self.comboBoxStyle.addItem(sk) + if sk == current_style: + self.comboBoxStyle.setCurrentIndex(i) + + self.save_font_setting(QtGui.QApplication.instance().font()) + if not self.parent.tray.isSystemTrayAvailable(): self.groupBoxTray.setEnabled(False) self.groupBoxTray.setTitle(_translate( @@ -322,6 +333,18 @@ class SettingsDialog(QtGui.QDialog): if status == 'success': self.parent.namecoin = nc + def save_font_setting(self, font): + """Save user font setting and set the buttonFont text""" + font_setting = (font.family(), font.pointSize()) + self.buttonFont.setText('{} {}'.format(*font_setting)) + self.font_setting = '{},{}'.format(*font_setting) + + def choose_font(self): + """Show the font selection dialog""" + font, valid = QtGui.QFontDialog.getFont() + if valid: + self.save_font_setting(font) + def accept(self): """A callback for accepted event of buttonBox (OK button pressed)""" # pylint: disable=too-many-branches,too-many-statements @@ -348,6 +371,22 @@ class SettingsDialog(QtGui.QDialog): self.config.set('bitmessagesettings', 'replybelow', str( self.checkBoxReplyBelow.isChecked())) + window_style = str(self.comboBoxStyle.currentText()) + if self.config.safeGet( + 'bitmessagesettings', 'windowstyle', 'GTK+' + ) != window_style or self.config.safeGet( + 'bitmessagesettings', 'font' + ) != self.font_setting: + self.config.set('bitmessagesettings', 'windowstyle', window_style) + self.config.set('bitmessagesettings', 'font', self.font_setting) + queues.UISignalQueue.put(( + 'updateStatusBar', ( + _translate( + "MainWindow", + "You need to restart the application to apply" + " the window style or default font."), 1) + )) + lang = str(self.languageComboBox.itemData( self.languageComboBox.currentIndex()).toString()) self.config.set('bitmessagesettings', 'userlocale', lang) diff --git a/src/bitmessageqt/settings.ui b/src/bitmessageqt/settings.ui index 1e9a6f09..e7ce1d71 100644 --- a/src/bitmessageqt/settings.ui +++ b/src/bitmessageqt/settings.ui @@ -147,6 +147,32 @@ + + + + Custom Style + + + + + + + 100 + 0 + + + + + + + + Font + + + + + + @@ -1202,5 +1228,11 @@ + + buttonFont + clicked() + settingsDialog + choose_font + From 0ccbf6832fcff8ef0dba00ab6a57290c6bbc5939 Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Thu, 2 May 2024 18:36:15 +0300 Subject: [PATCH 17/30] Adjust the tabWidget font size relative to the base app font --- src/bitmessageqt/bitmessageui.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/bitmessageqt/bitmessageui.py b/src/bitmessageqt/bitmessageui.py index 961fc093..bee8fd57 100644 --- a/src/bitmessageqt/bitmessageui.py +++ b/src/bitmessageqt/bitmessageui.py @@ -61,7 +61,8 @@ class Ui_MainWindow(object): self.tabWidget.setMinimumSize(QtCore.QSize(0, 0)) self.tabWidget.setBaseSize(QtCore.QSize(0, 0)) font = QtGui.QFont() - font.setPointSize(9) + base_size = QtGui.QApplication.instance().font().pointSize() + font.setPointSize(int(base_size * 0.75)) self.tabWidget.setFont(font) self.tabWidget.setTabPosition(QtGui.QTabWidget.North) self.tabWidget.setTabShape(QtGui.QTabWidget.Rounded) From d46af0835a0a015042e017680c6f361a049f3aaa Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Sat, 4 May 2024 00:07:29 +0300 Subject: [PATCH 18/30] A rather rudimentary test with basic checks for the style setting --- src/bitmessageqt/tests/settings.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/bitmessageqt/tests/settings.py b/src/bitmessageqt/tests/settings.py index 0dcf8cf3..3c14183f 100644 --- a/src/bitmessageqt/tests/settings.py +++ b/src/bitmessageqt/tests/settings.py @@ -1,3 +1,4 @@ +"""Tests for PyBitmessage settings""" import threading import time @@ -14,8 +15,7 @@ class TestSettings(TestBase): def test_udp(self): """Test the effect of checkBoxUDP""" - udp_setting = config.safeGetBoolean( - 'bitmessagesettings', 'udp') + udp_setting = config.safeGetBoolean('bitmessagesettings', 'udp') self.assertEqual(udp_setting, self.dialog.checkBoxUDP.isChecked()) self.dialog.checkBoxUDP.setChecked(not udp_setting) self.dialog.accept() @@ -32,3 +32,22 @@ class TestSettings(TestBase): else: if not udp_setting: self.fail('No Announcer thread found while udp set to True') + + def test_styling(self): + """Test custom windows style and font""" + style_setting = config.safeGet('bitmessagesettings', 'windowstyle') + font_setting = config.safeGet('bitmessagesettings', 'font') + self.assertIs(style_setting, None) + self.assertIs(font_setting, None) + style_control = self.dialog.comboBoxStyle + self.assertEqual(style_control.currentText(), 'GTK+') + style_count = style_control.count() + self.assertGreater(style_count, 1) + for i in range(style_count): + if i != style_control.currentIndex(): + style_control.setCurrentIndex(i) + break + self.dialog.accept() + self.assertEqual( + config.safeGet('bitmessagesettings', 'windowstyle'), + style_control.currentText()) From c3c41902cf0f05d570d34ea12b1b97c58f5af482 Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Wed, 8 May 2024 01:01:39 +0300 Subject: [PATCH 19/30] Use QtTest and QTimer.singleShot() to check the font dialog --- src/bitmessageqt/tests/settings.py | 34 +++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/src/bitmessageqt/tests/settings.py b/src/bitmessageqt/tests/settings.py index 3c14183f..170f8f40 100644 --- a/src/bitmessageqt/tests/settings.py +++ b/src/bitmessageqt/tests/settings.py @@ -2,10 +2,13 @@ import threading import time -from main import TestBase +from PyQt4 import QtCore, QtGui, QtTest + from bmconfigparser import config from bitmessageqt import settings +from .main import TestBase + class TestSettings(TestBase): """A test case for the "Settings" dialog""" @@ -41,13 +44,34 @@ class TestSettings(TestBase): self.assertIs(font_setting, None) style_control = self.dialog.comboBoxStyle self.assertEqual(style_control.currentText(), 'GTK+') + + def call_font_dialog(): + """A function to get the open font dialog and accept it""" + font_dialog = QtGui.QApplication.activeModalWidget() + self.assertTrue(isinstance(font_dialog, QtGui.QFontDialog)) + selected_font = font_dialog.currentFont() + self.assertEqual( + config.safeGet('bitmessagesettings', 'font'), '{},{}'.format( + selected_font.family(), selected_font.pointSize())) + + font_dialog.accept() + self.dialog.accept() + self.assertEqual( + config.safeGet('bitmessagesettings', 'windowstyle'), + style_control.currentText()) + + def click_font_button(): + """Use QtTest to click the button""" + QtTest.QTest.mouseClick( + self.dialog.buttonFont, QtCore.Qt.LeftButton) + style_count = style_control.count() self.assertGreater(style_count, 1) for i in range(style_count): if i != style_control.currentIndex(): style_control.setCurrentIndex(i) break - self.dialog.accept() - self.assertEqual( - config.safeGet('bitmessagesettings', 'windowstyle'), - style_control.currentText()) + + QtCore.QTimer.singleShot(30, click_font_button) + QtCore.QTimer.singleShot(60, call_font_dialog) + time.sleep(2) From de11bc484cd31e340d70e3690d584d189bf99cc5 Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Sun, 9 Jun 2024 22:17:14 +0300 Subject: [PATCH 20/30] Make Windows the default window style on windows, not GTK+ --- src/bitmessageqt/__init__.py | 18 ++++++++++++++---- src/bitmessageqt/settings.py | 12 +++++------- src/bitmessageqt/tests/settings.py | 3 ++- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/src/bitmessageqt/__init__.py b/src/bitmessageqt/__init__.py index 7f767b03..1b1a7885 100644 --- a/src/bitmessageqt/__init__.py +++ b/src/bitmessageqt/__init__.py @@ -62,6 +62,9 @@ except ImportError: get_plugins = False +is_windows = sys.platform.startswith('win') + + # TODO: rewrite def powQueueSize(): """Returns the size of queues.workerQueue including current unfinished work""" @@ -80,7 +83,7 @@ def openKeysFile(): keysfile = os.path.join(state.appdata, 'keys.dat') if 'linux' in sys.platform: subprocess.call(["xdg-open", keysfile]) - elif sys.platform.startswith('win'): + elif is_windows: os.startfile(keysfile) # pylint: disable=no-member @@ -868,7 +871,7 @@ class MyForm(settingsmixin.SMainWindow): """ startonlogon = config.safeGetBoolean( 'bitmessagesettings', 'startonlogon') - if sys.platform.startswith('win'): # Auto-startup for Windows + if is_windows: # Auto-startup for Windows RUN_PATH = "HKEY_CURRENT_USER\\Software\\Microsoft\\Windows\\CurrentVersion\\Run" settings = QtCore.QSettings( RUN_PATH, QtCore.QSettings.NativeFormat) @@ -4233,6 +4236,14 @@ class BitmessageQtApplication(QtGui.QApplication): # Unique identifier for this application uuid = '6ec0149b-96e1-4be1-93ab-1465fb3ebf7c' + @staticmethod + def get_windowstyle(): + """Get window style set in config or default""" + return config.safeGet( + 'bitmessagesettings', 'windowstyle', + 'Windows' if is_windows else 'GTK+' + ) + def __init__(self, *argv): super(BitmessageQtApplication, self).__init__(*argv) id = BitmessageQtApplication.uuid @@ -4241,8 +4252,7 @@ class BitmessageQtApplication(QtGui.QApplication): QtCore.QCoreApplication.setOrganizationDomain("bitmessage.org") QtCore.QCoreApplication.setApplicationName("pybitmessageqt") - self.setStyle( - config.safeGet('bitmessagesettings', 'windowstyle', 'GTK+')) + self.setStyle(self.get_windowstyle()) font = config.safeGet('bitmessagesettings', 'font') if font: diff --git a/src/bitmessageqt/settings.py b/src/bitmessageqt/settings.py index 44cad4c9..eec3d69f 100644 --- a/src/bitmessageqt/settings.py +++ b/src/bitmessageqt/settings.py @@ -45,6 +45,7 @@ class SettingsDialog(QtGui.QDialog): super(SettingsDialog, self).__init__(parent) widgets.load('settings.ui', self) + self.app = QtGui.QApplication.instance() self.parent = parent self.firstrun = firstrun self.config = config_obj @@ -87,14 +88,13 @@ class SettingsDialog(QtGui.QDialog): """Adjust all widgets state according to config settings""" # pylint: disable=too-many-branches,too-many-statements - current_style = config.safeGet( - 'bitmessagesettings', 'windowstyle', 'GTK+') + current_style = self.app.get_windowstyle() for i, sk in enumerate(QtGui.QStyleFactory.keys()): self.comboBoxStyle.addItem(sk) if sk == current_style: self.comboBoxStyle.setCurrentIndex(i) - self.save_font_setting(QtGui.QApplication.instance().font()) + self.save_font_setting(self.app.font()) if not self.parent.tray.isSystemTrayAvailable(): self.groupBoxTray.setEnabled(False) @@ -148,7 +148,7 @@ class SettingsDialog(QtGui.QDialog): "MainWindow", "Tray notifications not yet supported on your OS.")) - if 'win' not in sys.platform and not self.parent.desktop: + if not sys.platform.startswith('win') and not self.parent.desktop: self.checkBoxStartOnLogon.setDisabled(True) self.checkBoxStartOnLogon.setText(_translate( "MainWindow", "Start-on-login not yet supported on your OS.")) @@ -372,9 +372,7 @@ class SettingsDialog(QtGui.QDialog): self.checkBoxReplyBelow.isChecked())) window_style = str(self.comboBoxStyle.currentText()) - if self.config.safeGet( - 'bitmessagesettings', 'windowstyle', 'GTK+' - ) != window_style or self.config.safeGet( + if self.app.get_windowstyle() != window_style or self.config.safeGet( 'bitmessagesettings', 'font' ) != self.font_setting: self.config.set('bitmessagesettings', 'windowstyle', window_style) diff --git a/src/bitmessageqt/tests/settings.py b/src/bitmessageqt/tests/settings.py index 170f8f40..bad28ed7 100644 --- a/src/bitmessageqt/tests/settings.py +++ b/src/bitmessageqt/tests/settings.py @@ -43,7 +43,8 @@ class TestSettings(TestBase): self.assertIs(style_setting, None) self.assertIs(font_setting, None) style_control = self.dialog.comboBoxStyle - self.assertEqual(style_control.currentText(), 'GTK+') + self.assertEqual( + style_control.currentText(), self.app.get_windowstyle()) def call_font_dialog(): """A function to get the open font dialog and accept it""" From de445d6bd9d18894f20dbc8ed19c509d0ac7c71d Mon Sep 17 00:00:00 2001 From: Peter Surda Date: Sun, 16 Jun 2024 11:44:18 +0800 Subject: [PATCH 21/30] Suppress pylint complaint --- src/bitmessageqt/settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/bitmessageqt/settings.py b/src/bitmessageqt/settings.py index eec3d69f..eeb507c7 100644 --- a/src/bitmessageqt/settings.py +++ b/src/bitmessageqt/settings.py @@ -41,6 +41,7 @@ def getSOCKSProxyType(config): class SettingsDialog(QtGui.QDialog): """The "Settings" dialog""" + # pylint: disable=too-many-instance-attributes def __init__(self, parent=None, firstrun=False): super(SettingsDialog, self).__init__(parent) widgets.load('settings.ui', self) From 638b5a9b1a0e3cca4da025cf1615b2e8b05f9cd4 Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Sat, 18 May 2024 23:15:29 +0300 Subject: [PATCH 22/30] Add the alpha siffix for non-merge appimage builds --- .buildbot/appimage/build.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.buildbot/appimage/build.sh b/.buildbot/appimage/build.sh index c8dc4f56..10f5ad75 100755 --- a/.buildbot/appimage/build.sh +++ b/.buildbot/appimage/build.sh @@ -4,7 +4,11 @@ export APPIMAGE_EXTRACT_AND_RUN=1 BUILDER=appimage-builder-x86_64.AppImage RECIPE=packages/AppImage/AppImageBuilder.yml +git remote add -f upstream https://github.com/Bitmessage/PyBitmessage.git +HEAD="$(git rev-parse HEAD)" +UPSTREAM="$(git merge-base --fork-point upstream/v0.6)" export APP_VERSION=$(git describe --tags | cut -d- -f1,3 | tr -d v) +[ $HEAD != $UPSTREAM ] && APP_VERSION="${APP_VERSION}-alpha" function set_sourceline { if [ ${ARCH} == amd64 ]; then From 8280e058c7c0d983b6f00b308c76c477a1635dab Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Sun, 19 May 2024 16:44:45 +0300 Subject: [PATCH 23/30] Added some notes on the appimages into the install doc --- INSTALL.md | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/INSTALL.md b/INSTALL.md index 7942a957..21629d17 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -2,11 +2,42 @@ - Binary (64bit, no separate installation of dependencies required) - Windows: https://download.bitmessage.org/snapshots/ - Linux AppImages: https://artifacts.bitmessage.at/appimage/ - - Linux snaps: https://artifacts.bitmessage.at/snap/ + - Linux snaps: https://artifacts.bitmessage.at/snap/ - Mac (not up to date): https://github.com/Bitmessage/PyBitmessage/releases/tag/v0.6.1 - Source `git clone git://github.com/Bitmessage/PyBitmessage.git` +## Notes on the AppImages + +The [AppImage](https://docs.appimage.org/introduction/index.html) +is a bundle, built by the +[appimage-builder](https://github.com/AppImageCrafters/appimage-builder) from +the Ubuntu Bionic deb files, the sources and `bitmsghash.so`, precompiled for +3 architectures, using the `packages/AppImage/AppImageBuilder.yml` recipe. + +When you run the appimage the bundle is loop mounted to a location like +`/tmp/.mount_PyBitm97wj4K` with `squashfs-tools`. + +The appimage name has several informational filds: +``` +PyBitmessage--g[-alpha]-.AppImage +``` + +E.g. `PyBitmessage-0.6.3.2-ge571ba8a-x86_64.AppImage` is an appimage, built from +the `v0.6` for x86_64 and `PyBitmessage-0.6.3.2-g9de2aaf1-alpha-aarch64.AppImage` +is one, built from some development branch for arm64. + +You can also build the appimage with local code. For that you need installed +docker: + +``` +$ docker build -t bm-appimage -f .buildbot/appimage/Dockerfile . +$ docker run -t --rm -v "$(pwd)"/dist:/out bm-appimage .buildbot/appimage/build.sh +``` + +The appimages should be in the dist dir. + + ## Helper Script for building from source Go to the directory with PyBitmessage source code and run: ``` From a69e4eebf84cff71cfaf0ab4b7dceb99a7b36a2c Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Sun, 9 Jun 2024 00:36:32 +0300 Subject: [PATCH 24/30] Update the windows exe URL --- INSTALL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/INSTALL.md b/INSTALL.md index 21629d17..4f11b199 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -1,6 +1,6 @@ # PyBitmessage Installation Instructions - Binary (64bit, no separate installation of dependencies required) - - Windows: https://download.bitmessage.org/snapshots/ + - Windows: https://artifacts.bitmessage.at/winebuild/ - Linux AppImages: https://artifacts.bitmessage.at/appimage/ - Linux snaps: https://artifacts.bitmessage.at/snap/ - Mac (not up to date): https://github.com/Bitmessage/PyBitmessage/releases/tag/v0.6.1 From 205e25337f53f6d6baa8fbcff8ae59276f75de9b Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Thu, 6 Jun 2024 16:37:52 +0300 Subject: [PATCH 25/30] Use six to avoid obsolete unittest assertions --- src/tests/test_addressgenerator.py | 5 ++-- src/tests/test_api.py | 44 +++++++++++++++--------------- src/tests/test_inventory.py | 6 ++-- src/tests/test_logger.py | 6 ++-- 4 files changed, 33 insertions(+), 28 deletions(-) diff --git a/src/tests/test_addressgenerator.py b/src/tests/test_addressgenerator.py index e48daef9..d7366fe4 100644 --- a/src/tests/test_addressgenerator.py +++ b/src/tests/test_addressgenerator.py @@ -2,6 +2,7 @@ from binascii import unhexlify +import six from six.moves import queue from .partial import TestPartialRun @@ -91,8 +92,8 @@ class TestAddressGenerator(TestPartialRun): self.command_queue.put(( 'createRandomAddress', 4, 1, 'test_random', 1, '', False, 0, 0)) addr = self.return_queue.get() - self.assertRegexpMatches(addr, r'^BM-') - self.assertRegexpMatches(addr[3:], r'[a-zA-Z1-9]+$') + six.assertRegex(self, addr, r'^BM-') + six.assertRegex(self, addr[3:], r'[a-zA-Z1-9]+$') self.assertLessEqual(len(addr[3:]), 40) self.assertEqual( diff --git a/src/tests/test_api.py b/src/tests/test_api.py index 2a4640fa..0df145bc 100644 --- a/src/tests/test_api.py +++ b/src/tests/test_api.py @@ -5,11 +5,11 @@ Tests using API. import base64 import json import time - from binascii import hexlify -from six.moves import xmlrpc_client # nosec import psutil +import six +from six.moves import xmlrpc_client # nosec from .samples import ( sample_deterministic_addr3, sample_deterministic_addr4, sample_seed, @@ -174,29 +174,29 @@ class TestAPI(TestAPIProto): self.assertEqual( self.api.getDeterministicAddress(self._seed, 3, 1), sample_deterministic_addr3) - self.assertRegexpMatches( - self.api.getDeterministicAddress(self._seed, 2, 1), + six.assertRegex( + self, self.api.getDeterministicAddress(self._seed, 2, 1), r'^API Error 0002:') # This is here until the streams will be implemented - self.assertRegexpMatches( - self.api.getDeterministicAddress(self._seed, 3, 2), + six.assertRegex( + self, self.api.getDeterministicAddress(self._seed, 3, 2), r'API Error 0003:') - self.assertRegexpMatches( - self.api.createDeterministicAddresses(self._seed, 1, 4, 2), + six.assertRegex( + self, self.api.createDeterministicAddresses(self._seed, 1, 4, 2), r'API Error 0003:') - self.assertRegexpMatches( - self.api.createDeterministicAddresses('', 1), + six.assertRegex( + self, self.api.createDeterministicAddresses('', 1), r'API Error 0001:') - self.assertRegexpMatches( - self.api.createDeterministicAddresses(self._seed, 1, 2), + six.assertRegex( + self, self.api.createDeterministicAddresses(self._seed, 1, 2), r'API Error 0002:') - self.assertRegexpMatches( - self.api.createDeterministicAddresses(self._seed, 0), + six.assertRegex( + self, self.api.createDeterministicAddresses(self._seed, 0), r'API Error 0004:') - self.assertRegexpMatches( - self.api.createDeterministicAddresses(self._seed, 1000), + six.assertRegex( + self, self.api.createDeterministicAddresses(self._seed, 1000), r'API Error 0005:') addresses = json.loads( @@ -210,8 +210,8 @@ class TestAPI(TestAPIProto): def test_create_random_address(self): """API command 'createRandomAddress': basic BM-address validation""" addr = self._add_random_address('random_1') - self.assertRegexpMatches(addr, r'^BM-') - self.assertRegexpMatches(addr[3:], r'[a-zA-Z1-9]+$') + six.assertRegex(self, addr, r'^BM-') + six.assertRegex(self, addr[3:], r'[a-zA-Z1-9]+$') # Whitepaper says "around 36 character" self.assertLessEqual(len(addr[3:]), 40) self.assertEqual(self.api.deleteAddress(addr), 'success') @@ -411,7 +411,7 @@ class TestAPI(TestAPIProto): self.assertEqual(self.api.enableAddress(addr, False), 'success') result = self.api.sendBroadcast( addr, base64.encodestring('test_subject'), msg) - self.assertRegexpMatches(result, r'^API Error 0014:') + six.assertRegex(self, result, r'^API Error 0014:') finally: self.assertEqual(self.api.deleteAddress(addr), 'success') @@ -420,7 +420,7 @@ class TestAPI(TestAPIProto): result = self.api.sendBroadcast( 'BM-GtovgYdgs7qXPkoYaRgrLFuFKz1SFpsw', base64.encodestring('test_subject'), msg) - self.assertRegexpMatches(result, r'^API Error 0013:') + six.assertRegex(self, result, r'^API Error 0013:') def test_chan(self): """Testing chan creation/joining""" @@ -435,7 +435,7 @@ class TestAPI(TestAPIProto): self.assertEqual(self.api.joinChan(self._seed, addr), 'success') self.assertEqual(self.api.leaveChan(addr), 'success') # Joining with wrong address should fail - self.assertRegexpMatches( - self.api.joinChan(self._seed, 'BM-2cWzSnwjJ7yRP3nLEW'), + six.assertRegex( + self, self.api.joinChan(self._seed, 'BM-2cWzSnwjJ7yRP3nLEW'), r'^API Error 0008:' ) diff --git a/src/tests/test_inventory.py b/src/tests/test_inventory.py index 5978f9a5..d0b9ff6d 100644 --- a/src/tests/test_inventory.py +++ b/src/tests/test_inventory.py @@ -7,6 +7,8 @@ import tempfile import time import unittest +import six + from pybitmessage import highlevelcrypto from pybitmessage.storage import storage @@ -50,8 +52,8 @@ class TestStorageAbstract(unittest.TestCase): def test_inventory_storage(self): """Check inherited abstract methods""" - with self.assertRaisesRegexp( - TypeError, "^Can't instantiate abstract class.*" + with six.assertRaisesRegex( + self, TypeError, "^Can't instantiate abstract class.*" "methods __contains__, __delitem__, __getitem__, __iter__," " __len__, __setitem__" ): # pylint: disable=abstract-class-instantiated diff --git a/src/tests/test_logger.py b/src/tests/test_logger.py index d6bf33ed..7fbb91c8 100644 --- a/src/tests/test_logger.py +++ b/src/tests/test_logger.py @@ -5,6 +5,8 @@ Testing the logger configuration import os import tempfile +import six + from .test_process import TestProcessProto @@ -52,5 +54,5 @@ handlers=default self._stop_process() data = open(self.log_file).read() - self.assertRegexpMatches(data, self.pattern) - self.assertRegexpMatches(data, 'Loaded logger configuration') + six.assertRegex(self, data, self.pattern) + six.assertRegex(self, data, 'Loaded logger configuration') From e578759a3fa5f81859be6887b63583dda42c1896 Mon Sep 17 00:00:00 2001 From: anand k Date: Thu, 13 Jun 2024 07:03:36 +0530 Subject: [PATCH 26/30] Reduced helper_randon dependency from network module --- src/network/addrthread.py | 6 +++--- src/network/asyncore_pollchoose.py | 12 ++++++------ src/network/connectionpool.py | 8 ++++---- src/network/downloadthread.py | 4 ++-- src/network/tcp.py | 3 +-- src/network/uploadthread.py | 4 ++-- 6 files changed, 18 insertions(+), 19 deletions(-) diff --git a/src/network/addrthread.py b/src/network/addrthread.py index a0e869e3..a77e609c 100644 --- a/src/network/addrthread.py +++ b/src/network/addrthread.py @@ -1,11 +1,11 @@ """ Announce addresses as they are received from other hosts """ +import random from six.moves import queue # magic imports! import connectionpool -from helper_random import randomshuffle from protocol import assembleAddrMessage from queues import addrQueue # FIXME: init with queue @@ -29,9 +29,9 @@ class AddrThread(StoppableThread): if chunk: # Choose peers randomly connections = connectionpool.pool.establishedConnections() - randomshuffle(connections) + random.shuffle(connections) for i in connections: - randomshuffle(chunk) + random.shuffle(chunk) filtered = [] for stream, peer, seen, destination in chunk: # peer's own address or address received from peer diff --git a/src/network/asyncore_pollchoose.py b/src/network/asyncore_pollchoose.py index bdd312c6..9d0ffc1b 100644 --- a/src/network/asyncore_pollchoose.py +++ b/src/network/asyncore_pollchoose.py @@ -9,6 +9,7 @@ Basic infrastructure for asynchronous socket service clients and servers. import os import select import socket +import random import sys import time import warnings @@ -19,7 +20,6 @@ from errno import ( ) from threading import current_thread -import helper_random try: from errno import WSAEWOULDBLOCK @@ -233,13 +233,13 @@ def select_poller(timeout=0.0, map=None): if err.args[0] in (WSAENOTSOCK, ): return - for fd in helper_random.randomsample(r, len(r)): + for fd in random.sample(r, len(r)): obj = map.get(fd) if obj is None: continue read(obj) - for fd in helper_random.randomsample(w, len(w)): + for fd in random.sample(w, len(w)): obj = map.get(fd) if obj is None: continue @@ -297,7 +297,7 @@ def poll_poller(timeout=0.0, map=None): except socket.error as err: if err.args[0] in (EBADF, WSAENOTSOCK, EINTR): return - for fd, flags in helper_random.randomsample(r, len(r)): + for fd, flags in random.sample(r, len(r)): obj = map.get(fd) if obj is None: continue @@ -357,7 +357,7 @@ def epoll_poller(timeout=0.0, map=None): if err.args[0] != EINTR: raise r = [] - for fd, flags in helper_random.randomsample(r, len(r)): + for fd, flags in random.sample(r, len(r)): obj = map.get(fd) if obj is None: continue @@ -420,7 +420,7 @@ def kqueue_poller(timeout=0.0, map=None): events = kqueue_poller.pollster.control(updates, selectables, timeout) if len(events) > 1: - events = helper_random.randomsample(events, len(events)) + events = random.sample(events, len(events)) for event in events: fd = event.ident diff --git a/src/network/connectionpool.py b/src/network/connectionpool.py index 36c91c18..519b7b67 100644 --- a/src/network/connectionpool.py +++ b/src/network/connectionpool.py @@ -7,9 +7,9 @@ import re import socket import sys import time +import random import asyncore_pollchoose as asyncore -import helper_random import knownnodes import protocol import state @@ -210,7 +210,7 @@ class BMConnectionPool(object): connection_base = TCPConnection elif proxy_type == 'SOCKS5': connection_base = Socks5BMConnection - hostname = helper_random.randomchoice([ + hostname = random.choice([ # nosec B311 'quzwelsuziwqgpt2.onion', None ]) elif proxy_type == 'SOCKS4a': @@ -222,7 +222,7 @@ class BMConnectionPool(object): bootstrapper = bootstrap(connection_base) if not hostname: - port = helper_random.randomchoice([8080, 8444]) + port = random.choice([8080, 8444]) # nosec B311 hostname = 'bootstrap%s.bitmessage.org' % port else: port = 8444 @@ -289,7 +289,7 @@ class BMConnectionPool(object): state.maximumNumberOfHalfOpenConnections - pending): try: chosen = self.trustedPeer or chooseConnection( - helper_random.randomchoice(self.streams)) + random.choice(self.streams)) # nosec B311 except ValueError: continue if chosen in self.outboundConnections: diff --git a/src/network/downloadthread.py b/src/network/downloadthread.py index 30a3f2fe..7c8bccb6 100644 --- a/src/network/downloadthread.py +++ b/src/network/downloadthread.py @@ -2,9 +2,9 @@ `DownloadThread` class definition """ import time +import random import state import addresses -import helper_random import protocol import connectionpool from network import dandelion_ins @@ -43,7 +43,7 @@ class DownloadThread(StoppableThread): requested = 0 # Choose downloading peers randomly connections = connectionpool.pool.establishedConnections() - helper_random.randomshuffle(connections) + random.shuffle(connections) requestChunk = max(int( min(self.maxRequestChunk, len(missingObjects)) / len(connections)), 1) if connections else 1 diff --git a/src/network/tcp.py b/src/network/tcp.py index a739e256..f2dce07d 100644 --- a/src/network/tcp.py +++ b/src/network/tcp.py @@ -11,7 +11,6 @@ import time # magic imports! import addresses -import helper_random import l10n import protocol import state @@ -201,7 +200,7 @@ class TCPConnection(BMProto, TLSDispatcher): elemCount = min( len(filtered), maxAddrCount / 2 if n else maxAddrCount) - addrs[s] = helper_random.randomsample(filtered, elemCount) + addrs[s] = random.sample(filtered, elemCount) for substream in addrs: for peer, params in addrs[substream]: templist.append((substream, peer, params["lastseen"])) diff --git a/src/network/uploadthread.py b/src/network/uploadthread.py index e91f08fa..60209832 100644 --- a/src/network/uploadthread.py +++ b/src/network/uploadthread.py @@ -3,7 +3,7 @@ """ import time -import helper_random +import random import protocol import state import connectionpool @@ -24,7 +24,7 @@ class UploadThread(StoppableThread): uploaded = 0 # Choose uploading peers randomly connections = connectionpool.pool.establishedConnections() - helper_random.randomshuffle(connections) + random.shuffle(connections) for i in connections: now = time.time() # avoid unnecessary delay From 11ba0222673ff86134f51b6766ecea5061054782 Mon Sep 17 00:00:00 2001 From: Kashiko Koibumi Date: Sun, 23 Jun 2024 00:47:51 +0900 Subject: [PATCH 27/30] fix bug in detecting file system type --- src/shared.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/shared.py b/src/shared.py index f710d56d..80147801 100644 --- a/src/shared.py +++ b/src/shared.py @@ -196,7 +196,7 @@ def checkSensitiveFilePermissions(filename): ['/usr/bin/stat', '-f', '-c', '%T', filename], stderr=subprocess.STDOUT ) # nosec B603 - if 'fuseblk' in fstype: + if b'fuseblk' in fstype: logger.info( 'Skipping file permissions check for %s.' ' Filesystem fuseblk detected.', filename) From cdcffa4b3eac18a044edeb2dc605293f0a40ca97 Mon Sep 17 00:00:00 2001 From: Kashiko Koibumi Date: Mon, 24 Jun 2024 11:04:34 +0900 Subject: [PATCH 28/30] fix timestamp type mismatch bug --- src/bitmessageqt/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bitmessageqt/__init__.py b/src/bitmessageqt/__init__.py index 9b6461d1..9a3fe89a 100644 --- a/src/bitmessageqt/__init__.py +++ b/src/bitmessageqt/__init__.py @@ -1269,8 +1269,8 @@ class MyForm(settingsmixin.SMainWindow): r.append(row[1].decode("utf-8", "replace")) # fromaddress r.append(row[2].decode("utf-8", "replace")) # subject r.append(row[3].decode("utf-8", "replace")) # status - r.append(row[3]) # ackdata - r.append(row[4]) # lastactiontime + r.append(row[4]) # ackdata + r.append(row[5]) # lastactiontime self.addMessageListItemSent(tableWidget, *r) tableWidget.horizontalHeader().setSortIndicator( From e7b5f2957e8a2b612d9b618c2c15e1d552c5b07d Mon Sep 17 00:00:00 2001 From: Kashiko Koibumi Date: Tue, 25 Jun 2024 07:48:03 +0900 Subject: [PATCH 29/30] use SafeConfigParser or ConfigParser, which is available --- src/bitmessageqt/settings.py | 7 ++++++- src/bmconfigparser.py | 5 ++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/bitmessageqt/settings.py b/src/bitmessageqt/settings.py index 5658651f..11bb55d9 100644 --- a/src/bitmessageqt/settings.py +++ b/src/bitmessageqt/settings.py @@ -27,10 +27,15 @@ from network.asyncore_pollchoose import set_rates from tr import _translate +try: + SafeConfigParser = configparser.SafeConfigParser +except AttributeError: + SafeConfigParser = configparser.ConfigParser + def getSOCKSProxyType(config): """Get user socksproxytype setting from *config*""" try: - result = configparser.SafeConfigParser.get( + result = SafeConfigParser.get( config, 'bitmessagesettings', 'socksproxytype') except (configparser.NoSectionError, configparser.NoOptionError): return None diff --git a/src/bmconfigparser.py b/src/bmconfigparser.py index ec05af3e..7a33a502 100644 --- a/src/bmconfigparser.py +++ b/src/bmconfigparser.py @@ -15,7 +15,10 @@ try: except ImportError: from pybitmessage import state -SafeConfigParser = configparser.SafeConfigParser +try: + SafeConfigParser = configparser.SafeConfigParser +except AttributeError: + SafeConfigParser = configparser.ConfigParser config_ready = Event() From 8bfcc4cf382910e40d0caa969a4a8b666738086e Mon Sep 17 00:00:00 2001 From: Kashiko Koibumi Date: Wed, 26 Jun 2024 05:10:44 +0900 Subject: [PATCH 30/30] add comments --- src/bitmessageqt/settings.py | 1 + src/bmconfigparser.py | 1 + 2 files changed, 2 insertions(+) diff --git a/src/bitmessageqt/settings.py b/src/bitmessageqt/settings.py index 11bb55d9..635d5241 100644 --- a/src/bitmessageqt/settings.py +++ b/src/bitmessageqt/settings.py @@ -30,6 +30,7 @@ from tr import _translate try: SafeConfigParser = configparser.SafeConfigParser except AttributeError: + # alpine linux, python3.12 SafeConfigParser = configparser.ConfigParser def getSOCKSProxyType(config): diff --git a/src/bmconfigparser.py b/src/bmconfigparser.py index 7a33a502..5869255f 100644 --- a/src/bmconfigparser.py +++ b/src/bmconfigparser.py @@ -18,6 +18,7 @@ except ImportError: try: SafeConfigParser = configparser.SafeConfigParser except AttributeError: + # alpine linux, python3.12 SafeConfigParser = configparser.ConfigParser config_ready = Event()