From 336808980f87b6815f3ee736432aa8769a9f8a01 Mon Sep 17 00:00:00 2001 From: Kashiko Koibumi Date: Sun, 19 May 2024 08:19:55 +0900 Subject: [PATCH] load data as 'bytes' instead of 'str' from SQLite --- .gitignore | 1 + src/addresses.py | 24 ++++++++++++------------ src/bitmessageqt/__init__.py | 28 +++++++++++++++++++++++++++- src/bitmessageqt/account.py | 14 ++++++++++---- src/bitmessageqt/blacklist.py | 18 ++++++++++-------- src/bitmessageqt/foldertree.py | 3 +++ src/class_singleCleaner.py | 2 ++ src/class_singleWorker.py | 9 +++++++++ src/class_sqlThread.py | 6 +++--- src/shared.py | 3 ++- src/tests/test_shared.py | 6 +++--- 11 files changed, 82 insertions(+), 32 deletions(-) diff --git a/.gitignore b/.gitignore index 6bc048a5..8a655762 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,4 @@ coverage.xml .buildozer .tox *.swp +typescript diff --git a/src/addresses.py b/src/addresses.py index 885c1f64..6859a9ca 100644 --- a/src/addresses.py +++ b/src/addresses.py @@ -187,7 +187,7 @@ def decodeAddress(address): integer = decodeBase58(address) if integer == 0: status = 'invalidcharacters' - return status, 0, 0, '' + return status, 0, 0, b'' # after converting to hex, the string will be prepended # with a 0x and appended with a L in python2 hexdata = hex(integer)[2:].rstrip('L') @@ -200,23 +200,23 @@ def decodeAddress(address): if checksum != double_sha512(data[:-4])[0:4]: status = 'checksumfailed' - return status, 0, 0, '' + return status, 0, 0, b'' try: addressVersionNumber, bytesUsedByVersionNumber = decodeVarint(data[:9]) except varintDecodeError as e: logger.error(str(e)) status = 'varintmalformed' - return status, 0, 0, '' + return status, 0, 0, b'' if addressVersionNumber > 4: logger.error('cannot decode address version numbers this high') status = 'versiontoohigh' - return status, 0, 0, '' + return status, 0, 0, b'' elif addressVersionNumber == 0: logger.error('cannot decode address version numbers of zero.') status = 'versiontoohigh' - return status, 0, 0, '' + return status, 0, 0, b'' try: streamNumber, bytesUsedByStreamNumber = \ @@ -224,7 +224,7 @@ def decodeAddress(address): except varintDecodeError as e: logger.error(str(e)) status = 'varintmalformed' - return status, 0, 0, '' + return status, 0, 0, b'' status = 'success' if addressVersionNumber == 1: @@ -242,21 +242,21 @@ def decodeAddress(address): return status, addressVersionNumber, streamNumber, \ b'\x00\x00' + embeddedRipeData elif len(embeddedRipeData) < 18: - return 'ripetooshort', 0, 0, '' + return 'ripetooshort', 0, 0, b'' elif len(embeddedRipeData) > 20: - return 'ripetoolong', 0, 0, '' - return 'otherproblem', 0, 0, '' + return 'ripetoolong', 0, 0, b'' + return 'otherproblem', 0, 0, b'' elif addressVersionNumber == 4: embeddedRipeData = \ data[bytesUsedByVersionNumber + bytesUsedByStreamNumber:-4] if embeddedRipeData[0:1] == b'\x00': # In order to enforce address non-malleability, encoded # RIPE data must have NULL bytes removed from the front - return 'encodingproblem', 0, 0, '' + return 'encodingproblem', 0, 0, b'' elif len(embeddedRipeData) > 20: - return 'ripetoolong', 0, 0, '' + return 'ripetoolong', 0, 0, b'' elif len(embeddedRipeData) < 4: - return 'ripetooshort', 0, 0, '' + return 'ripetooshort', 0, 0, b'' x00string = b'\x00' * (20 - len(embeddedRipeData)) return status, addressVersionNumber, streamNumber, \ x00string + embeddedRipeData diff --git a/src/bitmessageqt/__init__.py b/src/bitmessageqt/__init__.py index f454023c..5ab7a4d0 100644 --- a/src/bitmessageqt/__init__.py +++ b/src/bitmessageqt/__init__.py @@ -515,6 +515,8 @@ class MyForm(settingsmixin.SMainWindow): "GROUP BY toaddress, folder") for row in queryreturn: toaddress, folder, cnt = row + toaddress = toaddress.decode('utf-8', 'replace') + folder = folder.decode('utf-8', 'replace') total += cnt if toaddress in db and folder in db[toaddress]: db[toaddress][folder] = cnt @@ -966,6 +968,8 @@ class MyForm(settingsmixin.SMainWindow): normalUnread = {} broadcastsUnread = {} for addr, fld, count in queryReturn: + addr = addr.decode('utf-8', 'replace') + fld = fld.decode('utf-8', 'replace') try: normalUnread[addr][fld] = count except KeyError: @@ -1165,6 +1169,13 @@ class MyForm(settingsmixin.SMainWindow): xAddress, account, "sent", where, what, False) for row in queryreturn: + r = [] + r.append(row[0].decode('utf-8', 'replace')) # toaddress + r.append(row[1].decode('utf-8', 'replace')) # fromaddress + r.append(row[2].decode('utf-8', 'replace')) # subject + r.append(row[3].decode('utf-8', 'replace')) # status + r.append(row[4]) # ackdata + r.append(row[5]) # lastactiontime self.addMessageListItemSent(tableWidget, *row) tableWidget.horizontalHeader().setSortIndicator( @@ -1206,6 +1217,10 @@ class MyForm(settingsmixin.SMainWindow): for row in queryreturn: toAddress, fromAddress, subject, _, msgid, received, read = row + toAddress = toAddress.decode('utf-8', 'replace') + fromAddress = fromAddress.decode('utf-8', 'replace') + subject = subject.decode('utf-8', 'replace') + received = received.decode('utf-8', 'replace') self.addMessageListItemInbox( tableWidget, toAddress, fromAddress, subject, msgid, received, read) @@ -1289,6 +1304,7 @@ class MyForm(settingsmixin.SMainWindow): SELECT msgid, toaddress, read FROM inbox where folder='inbox' ''') for msgid, toAddress, read in queryreturn: + toAddress = toAddress.decode('utf-8', 'replace') if not read: # increment the unread subscriptions if True (1) @@ -1367,7 +1383,7 @@ class MyForm(settingsmixin.SMainWindow): # Adapters and converters for QT <-> sqlite def sqlInit(self): - register_adapter(QtCore.QByteArray, str) + register_adapter(QtCore.QByteArray, bytes) def indicatorInit(self): """ @@ -1939,6 +1955,8 @@ class MyForm(settingsmixin.SMainWindow): queryreturn = sqlQuery('SELECT label, address FROM subscriptions WHERE enabled = 1') for row in queryreturn: label, address = row + label = label.decode('utf-8', 'replace') + address = address.decode('utf-8', 'replace') newRows[address] = [label, AccountMixin.SUBSCRIPTION] # chans for address in config.addresses(True): @@ -1949,6 +1967,8 @@ class MyForm(settingsmixin.SMainWindow): queryreturn = sqlQuery('SELECT * FROM addressbook') for row in queryreturn: label, address = row + label = label.decode('utf-8', 'replace') + address = address.decode('utf-8', 'replace') newRows[address] = [label, AccountMixin.NORMAL] completerList = [] @@ -2199,6 +2219,7 @@ class MyForm(settingsmixin.SMainWindow): if queryreturn != []: for row in queryreturn: toLabel, = row + toLabel = toLabel.decode('utf-8', 'replace') self.displayNewSentMessage( toAddress, toLabel, fromAddress, subject, message, ackdata) @@ -2846,6 +2867,7 @@ class MyForm(settingsmixin.SMainWindow): if queryreturn != []: for row in queryreturn: messageText, = row + messageText = messageText.decode('utf-8', 'replace') lines = messageText.split('\n') totalLines = len(lines) @@ -2968,6 +2990,7 @@ class MyForm(settingsmixin.SMainWindow): if queryreturn != []: for row in queryreturn: messageAtCurrentInboxRow, = row + messageAtCurrentInboxRow = messageAtCurrentInboxRow.decode('utf-8', 'replace') acct.parseMessage( toAddressAtCurrentInboxRow, fromAddressAtCurrentInboxRow, tableWidget.item(currentInboxRow, 2).subject, @@ -3199,6 +3222,7 @@ class MyForm(settingsmixin.SMainWindow): if queryreturn != []: for row in queryreturn: message, = row + message = message.decode('utf-8', 'replace') defaultFilename = "".join(x for x in subjectAtCurrentInboxRow if x.isalnum()) + '.txt' filename = QtWidgets.QFileDialog.getSaveFileName( @@ -3944,6 +3968,7 @@ class MyForm(settingsmixin.SMainWindow): queryreturn = sqlQuery('''SELECT status FROM sent where ackdata=?''', ackData) for row in queryreturn: status, = row + status = status.decode('utf-8', 'replace') if status == 'toodifficult': self.popMenuSent.addAction(self.actionForceSend) @@ -4047,6 +4072,7 @@ class MyForm(settingsmixin.SMainWindow): try: message = queryreturn[-1][0] + message = message.decode('utf-8', 'replace') except NameError: message = "" except IndexError: diff --git a/src/bitmessageqt/account.py b/src/bitmessageqt/account.py index fb21ec8d..e93bb3b3 100644 --- a/src/bitmessageqt/account.py +++ b/src/bitmessageqt/account.py @@ -38,6 +38,8 @@ def getSortedSubscriptions(count=False): ret = {} for row in queryreturn: label, address, enabled = row + label = label.decode('utf-8', 'replace') + address = address.decode('utf-8', 'replace') ret[address] = {} ret[address]["inbox"] = {} ret[address]["inbox"]['label'] = label @@ -50,6 +52,8 @@ def getSortedSubscriptions(count=False): GROUP BY inbox.fromaddress, folder''', str_broadcast_subscribers) for row in queryreturn: address, folder, cnt = row + address = address.decode('utf-8', 'replace') + folder = folder.decode('utf-8', 'replace') if folder not in ret[address]: ret[address][folder] = { 'label': ret[address]['inbox']['label'], @@ -137,12 +141,14 @@ class BMAccount(object): if queryreturn != []: for row in queryreturn: label, = row + label = label.decode('utf-8', 'replace') else: queryreturn = sqlQuery( '''select label from subscriptions where address=?''', address) if queryreturn != []: for row in queryreturn: label, = row + label = label.decode('utf-8', 'replace') return label def parseMessage(self, toAddress, fromAddress, subject, message): @@ -199,11 +205,11 @@ class GatewayAccount(BMAccount): sqlExecute( '''INSERT INTO sent VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)''', '', - self.toAddress, + self.toAddress.encode(), ripe, - self.fromAddress, - self.subject, - self.message, + self.fromAddress.encode(), + self.subject.encode(), + self.message.encode(), ackdata, int(time.time()), # sentTime (this will never change) int(time.time()), # lastActionTime diff --git a/src/bitmessageqt/blacklist.py b/src/bitmessageqt/blacklist.py index 87520c92..5b3c9997 100644 --- a/src/bitmessageqt/blacklist.py +++ b/src/bitmessageqt/blacklist.py @@ -106,10 +106,10 @@ class Blacklist(QtWidgets.QWidget, RetranslateMixin): if isinstance(addressitem, QtWidgets.QTableWidgetItem): if self.radioButtonBlacklist.isChecked(): sqlExecute('''UPDATE blacklist SET label=? WHERE address=?''', - str(item.text()), str(addressitem.text())) + item.text(), addressitem.text()) else: sqlExecute('''UPDATE whitelist SET label=? WHERE address=?''', - str(item.text()), str(addressitem.text())) + item.text(), addressitem.text()) def init_blacklist_popup_menu(self, connectSignal=True): # Popup menu for the Blacklist page @@ -164,6 +164,8 @@ class Blacklist(QtWidgets.QWidget, RetranslateMixin): self.tableWidgetBlacklist.setSortingEnabled(False) for row in queryreturn: label, address, enabled = row + label = label.decode('utf-8', 'replace') + address = address.decode('utf-8', 'replace') self.tableWidgetBlacklist.insertRow(0) newItem = QtWidgets.QTableWidgetItem(label) if not enabled: @@ -191,11 +193,11 @@ class Blacklist(QtWidgets.QWidget, RetranslateMixin): if config.get('bitmessagesettings', 'blackwhitelist') == 'black': sqlExecute( '''DELETE FROM blacklist WHERE label=? AND address=?''', - str(labelAtCurrentRow), str(addressAtCurrentRow)) + labelAtCurrentRow, addressAtCurrentRow) else: sqlExecute( '''DELETE FROM whitelist WHERE label=? AND address=?''', - str(labelAtCurrentRow), str(addressAtCurrentRow)) + labelAtCurrentRow, addressAtCurrentRow) self.tableWidgetBlacklist.removeRow(currentRow) def on_action_BlacklistClipboard(self): @@ -220,11 +222,11 @@ class Blacklist(QtWidgets.QWidget, RetranslateMixin): if config.get('bitmessagesettings', 'blackwhitelist') == 'black': sqlExecute( '''UPDATE blacklist SET enabled=1 WHERE address=?''', - str(addressAtCurrentRow)) + addressAtCurrentRow) else: sqlExecute( '''UPDATE whitelist SET enabled=1 WHERE address=?''', - str(addressAtCurrentRow)) + addressAtCurrentRow) def on_action_BlacklistDisable(self): currentRow = self.tableWidgetBlacklist.currentRow() @@ -236,10 +238,10 @@ class Blacklist(QtWidgets.QWidget, RetranslateMixin): currentRow, 1).setTextColor(QtGui.QColor(128, 128, 128)) if config.get('bitmessagesettings', 'blackwhitelist') == 'black': sqlExecute( - '''UPDATE blacklist SET enabled=0 WHERE address=?''', str(addressAtCurrentRow)) + '''UPDATE blacklist SET enabled=0 WHERE address=?''', addressAtCurrentRow) else: sqlExecute( - '''UPDATE whitelist SET enabled=0 WHERE address=?''', str(addressAtCurrentRow)) + '''UPDATE whitelist SET enabled=0 WHERE address=?''', addressAtCurrentRow) def on_action_BlacklistSetAvatar(self): self.window().on_action_SetAvatar(self.tableWidgetBlacklist) diff --git a/src/bitmessageqt/foldertree.py b/src/bitmessageqt/foldertree.py index 13d7ba77..7d4ab49b 100644 --- a/src/bitmessageqt/foldertree.py +++ b/src/bitmessageqt/foldertree.py @@ -135,6 +135,7 @@ class AccountMixin(object): if queryreturn != []: for row in queryreturn: retval, = row + retval = retval.decode('utf-8', 'replace') elif self.address is None or self.type == AccountMixin.ALL: return _translate("MainWindow", "All accounts") @@ -306,6 +307,7 @@ class Ui_SubscriptionWidget(Ui_AddressWidget): if queryreturn != []: for row in queryreturn: retval, = row + retval = retval.decode('utf-8', 'replace') return retval return self.address @@ -411,6 +413,7 @@ class MessageList_AddressWidget(BMAddressWidget): if queryreturn: for row in queryreturn: newLabel = row[0] + newLabel = newLabel.decode('utf-8', 'replace') self.label = newLabel diff --git a/src/class_singleCleaner.py b/src/class_singleCleaner.py index 06153dcf..11adf935 100644 --- a/src/class_singleCleaner.py +++ b/src/class_singleCleaner.py @@ -99,6 +99,8 @@ class singleCleaner(StoppableThread): tick - state.maximumLengthOfTimeToBotherResendingMessages ) for toAddress, ackData, status in queryreturn: + toAddress = toAddress.decode('utf-8', 'replace') + status = status.decode('utf-8', 'replace') if status == 'awaitingpubkey': self.resendPubkeyRequest(toAddress) elif status == 'msgsent': diff --git a/src/class_singleWorker.py b/src/class_singleWorker.py index 846467e5..5342cd79 100644 --- a/src/class_singleWorker.py +++ b/src/class_singleWorker.py @@ -73,6 +73,7 @@ class singleWorker(StoppableThread): '''SELECT DISTINCT toaddress FROM sent''' ''' WHERE (status='awaitingpubkey' AND folder='sent')''') for toAddress, in queryreturn: + toAddress = toAddress.decode('utf-8', 'replace') toAddressVersionNumber, toStreamNumber, toRipe = \ decodeAddress(toAddress)[1:] if toAddressVersionNumber <= 3: @@ -538,6 +539,9 @@ class singleWorker(StoppableThread): for row in queryreturn: fromaddress, subject, body, ackdata, TTL, encoding = row + fromaddress = fromaddress.decode('utf-8', 'replace') + subject = subject.decode('utf-8', 'replace') + body = body.decode('utf-8', 'replace') # status _, addressVersionNumber, streamNumber, ripe = \ decodeAddress(fromaddress) @@ -726,6 +730,11 @@ class singleWorker(StoppableThread): for row in queryreturn: toaddress, fromaddress, subject, message, \ ackdata, status, TTL, retryNumber, encoding = row + toaddress = toaddress.decode('utf-8', 'replace') + fromaddress = fromaddress.decode('utf-8', 'replace') + subject = subject.decode('utf-8', 'replace') + message = message.decode('utf-8', 'replace') + status = status.decode('utf-8', 'replace') # toStatus _, toAddressVersionNumber, toStreamNumber, toRipe = \ decodeAddress(toaddress) diff --git a/src/class_sqlThread.py b/src/class_sqlThread.py index 7df9e253..49e4b98b 100644 --- a/src/class_sqlThread.py +++ b/src/class_sqlThread.py @@ -38,7 +38,7 @@ class sqlThread(threading.Thread): helper_sql.sql_available = True config_ready.wait() self.conn = sqlite3.connect(state.appdata + 'messages.dat') - self.conn.text_factory = str + self.conn.text_factory = bytes self.cur = self.conn.cursor() self.cur.execute('PRAGMA secure_delete = true') @@ -542,7 +542,7 @@ class sqlThread(threading.Thread): shutil.move( paths.lookupAppdataFolder() + 'messages.dat', paths.lookupExeFolder() + 'messages.dat') self.conn = sqlite3.connect(paths.lookupExeFolder() + 'messages.dat') - self.conn.text_factory = str + self.conn.text_factory = bytes self.cur = self.conn.cursor() elif item == 'movemessagstoappdata': logger.debug('the sqlThread is moving the messages.dat file to the Appdata folder.') @@ -568,7 +568,7 @@ class sqlThread(threading.Thread): shutil.move( paths.lookupExeFolder() + 'messages.dat', paths.lookupAppdataFolder() + 'messages.dat') self.conn = sqlite3.connect(paths.lookupAppdataFolder() + 'messages.dat') - self.conn.text_factory = str + self.conn.text_factory = bytes self.cur = self.conn.cursor() elif item == 'deleteandvacuume': self.cur.execute('''delete from inbox where folder='trash' ''') diff --git a/src/shared.py b/src/shared.py index a1541eac..ec427b2b 100644 --- a/src/shared.py +++ b/src/shared.py @@ -47,7 +47,7 @@ def isAddressInMySubscriptionsList(address): """Am I subscribed to this address?""" queryreturn = sqlQuery( '''select * from subscriptions where address=?''', - str(address)) + address) return queryreturn != [] @@ -136,6 +136,7 @@ def reloadBroadcastSendersForWhichImWatching(): logger.debug('reloading subscriptions...') for row in queryreturn: address, = row + address = address.decode('utf-8', 'replace') # status addressVersionNumber, streamNumber, hashobj = decodeAddress(address)[1:] if addressVersionNumber == 2: diff --git a/src/tests/test_shared.py b/src/tests/test_shared.py index 39bedf32..972430a3 100644 --- a/src/tests/test_shared.py +++ b/src/tests/test_shared.py @@ -46,7 +46,7 @@ class TestShared(unittest.TestCase): address = sample_address # if address is in MyAddressbook - mock_sql_query.return_value = [address] + mock_sql_query.return_value = [address.encode()] return_val = isAddressInMyAddressBook(address) mock_sql_query.assert_called_once() self.assertTrue(return_val) @@ -64,7 +64,7 @@ class TestShared(unittest.TestCase): address = sample_address # if address is in MySubscriptionsList - mock_sql_query.return_value = [address] + mock_sql_query.return_value = [address.encode()] return_val = isAddressInMySubscriptionsList(address) self.assertTrue(return_val) @@ -78,7 +78,7 @@ class TestShared(unittest.TestCase): def test_reloadBroadcastSendersForWhichImWatching(self, mock_sql_query): """Test for reload Broadcast Senders For Which Im Watching""" mock_sql_query.return_value = [ - (sample_address,), + (sample_address.encode(),), ] # before reload self.assertEqual(len(MyECSubscriptionCryptorObjects), 0)