From 0f858bca891a7405ab9f0abadfd3fa134c9df4e7 Mon Sep 17 00:00:00 2001 From: Kashiko Koibumi Date: Mon, 27 May 2024 04:45:40 +0900 Subject: [PATCH 1/2] read from and write to SQLite database in binary mode This modification is a preparation for migration to Python3. --- src/addresses.py | 24 ++++++++-------- src/bitmessageqt/__init__.py | 44 ++++++++++++++++++++++++------ src/bitmessageqt/account.py | 19 +++++++------ src/bitmessageqt/blacklist.py | 18 ++++++------ src/bitmessageqt/foldertree.py | 11 ++++---- src/bitmessageqt/safehtmlparser.py | 2 ++ src/class_singleCleaner.py | 2 ++ src/class_singleWorker.py | 9 ++++++ src/class_sqlThread.py | 6 ++-- src/helper_msgcoding.py | 3 +- src/shared.py | 3 +- src/tests/test_shared.py | 6 ++-- 12 files changed, 97 insertions(+), 50 deletions(-) diff --git a/src/addresses.py b/src/addresses.py index 885c1f64..6859a9ca 100644 --- a/src/addresses.py +++ b/src/addresses.py @@ -187,7 +187,7 @@ def decodeAddress(address): integer = decodeBase58(address) if integer == 0: status = 'invalidcharacters' - return status, 0, 0, '' + return status, 0, 0, b'' # after converting to hex, the string will be prepended # with a 0x and appended with a L in python2 hexdata = hex(integer)[2:].rstrip('L') @@ -200,23 +200,23 @@ def decodeAddress(address): if checksum != double_sha512(data[:-4])[0:4]: status = 'checksumfailed' - return status, 0, 0, '' + return status, 0, 0, b'' try: addressVersionNumber, bytesUsedByVersionNumber = decodeVarint(data[:9]) except varintDecodeError as e: logger.error(str(e)) status = 'varintmalformed' - return status, 0, 0, '' + return status, 0, 0, b'' if addressVersionNumber > 4: logger.error('cannot decode address version numbers this high') status = 'versiontoohigh' - return status, 0, 0, '' + return status, 0, 0, b'' elif addressVersionNumber == 0: logger.error('cannot decode address version numbers of zero.') status = 'versiontoohigh' - return status, 0, 0, '' + return status, 0, 0, b'' try: streamNumber, bytesUsedByStreamNumber = \ @@ -224,7 +224,7 @@ def decodeAddress(address): except varintDecodeError as e: logger.error(str(e)) status = 'varintmalformed' - return status, 0, 0, '' + return status, 0, 0, b'' status = 'success' if addressVersionNumber == 1: @@ -242,21 +242,21 @@ def decodeAddress(address): return status, addressVersionNumber, streamNumber, \ b'\x00\x00' + embeddedRipeData elif len(embeddedRipeData) < 18: - return 'ripetooshort', 0, 0, '' + return 'ripetooshort', 0, 0, b'' elif len(embeddedRipeData) > 20: - return 'ripetoolong', 0, 0, '' - return 'otherproblem', 0, 0, '' + return 'ripetoolong', 0, 0, b'' + return 'otherproblem', 0, 0, b'' elif addressVersionNumber == 4: embeddedRipeData = \ data[bytesUsedByVersionNumber + bytesUsedByStreamNumber:-4] if embeddedRipeData[0:1] == b'\x00': # In order to enforce address non-malleability, encoded # RIPE data must have NULL bytes removed from the front - return 'encodingproblem', 0, 0, '' + return 'encodingproblem', 0, 0, b'' elif len(embeddedRipeData) > 20: - return 'ripetoolong', 0, 0, '' + return 'ripetoolong', 0, 0, b'' elif len(embeddedRipeData) < 4: - return 'ripetooshort', 0, 0, '' + return 'ripetooshort', 0, 0, b'' x00string = b'\x00' * (20 - len(embeddedRipeData)) return status, addressVersionNumber, streamNumber, \ x00string + embeddedRipeData diff --git a/src/bitmessageqt/__init__.py b/src/bitmessageqt/__init__.py index 40113b5a..fd012e04 100644 --- a/src/bitmessageqt/__init__.py +++ b/src/bitmessageqt/__init__.py @@ -552,6 +552,8 @@ class MyForm(settingsmixin.SMainWindow): "GROUP BY toaddress, folder") for row in queryreturn: toaddress, folder, cnt = row + toaddress = toaddress.decode("utf-8", "replace") + folder = folder.decode("utf-8", "replace") total += cnt if toaddress in db and folder in db[toaddress]: db[toaddress][folder] = cnt @@ -1044,6 +1046,8 @@ class MyForm(settingsmixin.SMainWindow): normalUnread = {} broadcastsUnread = {} for addr, fld, count in queryReturn: + addr = addr.decode("utf-8", "replace") + fld = fld.decode("utf-8", "replace") try: normalUnread[addr][fld] = count except KeyError: @@ -1211,11 +1215,11 @@ class MyForm(settingsmixin.SMainWindow): items = [ MessageList_AddressWidget( - toAddress, unicode(acct.toLabel, 'utf-8'), not read), + toAddress, acct.toLabel, not read), MessageList_AddressWidget( - fromAddress, unicode(acct.fromLabel, 'utf-8'), not read), + fromAddress, acct.fromLabel, not read), MessageList_SubjectWidget( - str(subject), unicode(acct.subject, 'utf-8', 'replace'), + subject, acct.subject, not read), MessageList_TimeWidget( l10n.formatTimestamp(received), not read, received, msgid) @@ -1243,7 +1247,14 @@ class MyForm(settingsmixin.SMainWindow): xAddress, account, "sent", where, what, False) for row in queryreturn: - self.addMessageListItemSent(tableWidget, *row) + r = [] + r.append(row[0].decode("utf-8", "replace")) # toaddress + r.append(row[1].decode("utf-8", "replace")) # fromaddress + r.append(row[2].decode("utf-8", "replace")) # subject + r.append(row[3].decode("utf-8", "replace")) # status + r.append(row[3]) # ackdata + r.append(row[4]) # lastactiontime + self.addMessageListItemSent(tableWidget, *r) tableWidget.horizontalHeader().setSortIndicator( 3, QtCore.Qt.DescendingOrder) @@ -1284,6 +1295,10 @@ class MyForm(settingsmixin.SMainWindow): for row in queryreturn: toAddress, fromAddress, subject, _, msgid, received, read = row + toAddress = toAddress.decode("utf-8", "replace") + fromAddress = fromAddress.decode("utf-8", "replace") + subject = subject.decode("utf-8", "replace") + received = received.decode("utf-8", "replace") self.addMessageListItemInbox( tableWidget, toAddress, fromAddress, subject, msgid, received, read) @@ -1369,6 +1384,7 @@ class MyForm(settingsmixin.SMainWindow): SELECT msgid, toaddress, read FROM inbox where folder='inbox' ''') for msgid, toAddress, read in queryreturn: + toAddress = toAddress.decode("utf-8", "replace") if not read: # increment the unread subscriptions if True (1) @@ -1447,7 +1463,7 @@ class MyForm(settingsmixin.SMainWindow): # Adapters and converters for QT <-> sqlite def sqlInit(self): - register_adapter(QtCore.QByteArray, str) + register_adapter(QtCore.QByteArray, bytes) def indicatorInit(self): """ @@ -1995,9 +2011,9 @@ class MyForm(settingsmixin.SMainWindow): def rerenderAddressBook(self): def addRow (address, label, type): self.ui.tableWidgetAddressBook.insertRow(0) - newItem = Ui_AddressBookWidgetItemLabel(address, unicode(label, 'utf-8'), type) + newItem = Ui_AddressBookWidgetItemLabel(address, label, type) self.ui.tableWidgetAddressBook.setItem(0, 0, newItem) - newItem = Ui_AddressBookWidgetItemAddress(address, unicode(label, 'utf-8'), type) + newItem = Ui_AddressBookWidgetItemAddress(address, label, type) self.ui.tableWidgetAddressBook.setItem(0, 1, newItem) oldRows = {} @@ -2015,6 +2031,8 @@ class MyForm(settingsmixin.SMainWindow): queryreturn = sqlQuery('SELECT label, address FROM subscriptions WHERE enabled = 1') for row in queryreturn: label, address = row + label = label.decode("utf-8", "replace") + address = address.decode("utf-8", "replace") newRows[address] = [label, AccountMixin.SUBSCRIPTION] # chans for address in config.addresses(True): @@ -2025,6 +2043,8 @@ class MyForm(settingsmixin.SMainWindow): queryreturn = sqlQuery('SELECT * FROM addressbook') for row in queryreturn: label, address = row + label = label.decode("utf-8", "replace") + address = address.decode("utf-8", "replace") newRows[address] = [label, AccountMixin.NORMAL] completerList = [] @@ -2038,7 +2058,7 @@ class MyForm(settingsmixin.SMainWindow): self.ui.tableWidgetAddressBook.removeRow(oldRows[address][2]) for address in newRows: addRow(address, newRows[address][0], newRows[address][1]) - completerList.append(unicode(newRows[address][0], encoding="UTF-8") + " <" + address + ">") + completerList.append(newRows[address][0] + " <" + address + ">") # sort self.ui.tableWidgetAddressBook.sortByColumn( @@ -2273,6 +2293,7 @@ class MyForm(settingsmixin.SMainWindow): if queryreturn != []: for row in queryreturn: toLabel, = row + toLabel = toLabel.decode("utf-8", "replace") self.displayNewSentMessage( toAddress, toLabel, fromAddress, subject, message, ackdata) @@ -2920,6 +2941,7 @@ class MyForm(settingsmixin.SMainWindow): if queryreturn != []: for row in queryreturn: messageText, = row + messageText = messageText.decode("utf-8", "replace") lines = messageText.split('\n') totalLines = len(lines) @@ -3043,6 +3065,7 @@ class MyForm(settingsmixin.SMainWindow): if queryreturn != []: for row in queryreturn: messageAtCurrentInboxRow, = row + messageAtCurrentInboxRow = messageAtCurrentInboxRow.decode("utf-8", "replace") acct.parseMessage( toAddressAtCurrentInboxRow, fromAddressAtCurrentInboxRow, tableWidget.item(currentInboxRow, 2).subject, @@ -3274,6 +3297,7 @@ class MyForm(settingsmixin.SMainWindow): if queryreturn != []: for row in queryreturn: message, = row + message = message.decode("utf-8", "replace") defaultFilename = "".join(x for x in subjectAtCurrentInboxRow if x.isalnum()) + '.txt' filename = QtGui.QFileDialog.getSaveFileName( @@ -3285,7 +3309,7 @@ class MyForm(settingsmixin.SMainWindow): return try: f = open(filename, 'w') - f.write(message) + f.write(message.encode("utf-8", "replace")) f.close() except Exception: logger.exception('Message not saved', exc_info=True) @@ -4020,6 +4044,7 @@ class MyForm(settingsmixin.SMainWindow): queryreturn = sqlQuery('''SELECT status FROM sent where ackdata=?''', ackData) for row in queryreturn: status, = row + status = status.decode("utf-8", "replace") if status == 'toodifficult': self.popMenuSent.addAction(self.actionForceSend) @@ -4124,6 +4149,7 @@ class MyForm(settingsmixin.SMainWindow): try: message = queryreturn[-1][0] + message = message.decode("utf-8", "replace") except NameError: message = "" except IndexError: diff --git a/src/bitmessageqt/account.py b/src/bitmessageqt/account.py index 8c82c6f6..794c276e 100644 --- a/src/bitmessageqt/account.py +++ b/src/bitmessageqt/account.py @@ -38,6 +38,8 @@ def getSortedSubscriptions(count=False): ret = {} for row in queryreturn: label, address, enabled = row + label = label.decode("utf-8", "replace") + address = address.decode("utf-8", "replace") ret[address] = {} ret[address]["inbox"] = {} ret[address]["inbox"]['label'] = label @@ -50,6 +52,8 @@ def getSortedSubscriptions(count=False): GROUP BY inbox.fromaddress, folder''', str_broadcast_subscribers) for row in queryreturn: address, folder, cnt = row + address = address.decode("utf-8", "replace") + folder = folder.decode("utf-8", "replace") if folder not in ret[address]: ret[address][folder] = { 'label': ret[address]['inbox']['label'], @@ -137,12 +141,14 @@ class BMAccount(object): if queryreturn != []: for row in queryreturn: label, = row + label = label.decode("utf-8", "replace") else: queryreturn = sqlQuery( '''select label from subscriptions where address=?''', address) if queryreturn != []: for row in queryreturn: label, = row + label = label.replace("utf-8", "replace") return label def parseMessage(self, toAddress, fromAddress, subject, message): @@ -150,10 +156,7 @@ class BMAccount(object): self.toAddress = toAddress self.fromAddress = fromAddress - if isinstance(subject, unicode): - self.subject = str(subject) - else: - self.subject = subject + self.subject = subject self.message = message self.fromLabel = self.getLabel(fromAddress) self.toLabel = self.getLabel(toAddress) @@ -202,11 +205,11 @@ class GatewayAccount(BMAccount): sqlExecute( '''INSERT INTO sent VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)''', '', - self.toAddress, + self.toAddress.encode("utf-8", "replace"), ripe, - self.fromAddress, - self.subject, - self.message, + self.fromAddress.encode("utf-8", "replace"), + self.subject.encode("utf-8", "replace"), + self.message.encode("utf-8", "replace"), ackdata, int(time.time()), # sentTime (this will never change) int(time.time()), # lastActionTime diff --git a/src/bitmessageqt/blacklist.py b/src/bitmessageqt/blacklist.py index 093f23d8..41342bce 100644 --- a/src/bitmessageqt/blacklist.py +++ b/src/bitmessageqt/blacklist.py @@ -111,10 +111,10 @@ class Blacklist(QtGui.QWidget, RetranslateMixin): if isinstance(addressitem, QtGui.QTableWidgetItem): if self.radioButtonBlacklist.isChecked(): sqlExecute('''UPDATE blacklist SET label=? WHERE address=?''', - str(item.text()), str(addressitem.text())) + item.text().encode("utf-8", "replace"), addressitem.text().encode("utf-8", "replace")) else: sqlExecute('''UPDATE whitelist SET label=? WHERE address=?''', - str(item.text()), str(addressitem.text())) + item.text().encode("utf-8", "replace"), addressitem.text().encode("utf-8", "replace")) def init_blacklist_popup_menu(self, connectSignal=True): # Popup menu for the Blacklist page @@ -171,6 +171,8 @@ class Blacklist(QtGui.QWidget, RetranslateMixin): self.tableWidgetBlacklist.setSortingEnabled(False) for row in queryreturn: label, address, enabled = row + label = label.decode("utf-8", "replace") + address = address.decode("utf-8", "replace") self.tableWidgetBlacklist.insertRow(0) newItem = QtGui.QTableWidgetItem(unicode(label, 'utf-8')) if not enabled: @@ -198,11 +200,11 @@ class Blacklist(QtGui.QWidget, RetranslateMixin): if config.get('bitmessagesettings', 'blackwhitelist') == 'black': sqlExecute( '''DELETE FROM blacklist WHERE label=? AND address=?''', - str(labelAtCurrentRow), str(addressAtCurrentRow)) + labelAtCurrentRow.encode("utf-8", "replace"), addressAtCurrentRow.encode("utf-8", "replace")) else: sqlExecute( '''DELETE FROM whitelist WHERE label=? AND address=?''', - str(labelAtCurrentRow), str(addressAtCurrentRow)) + labelAtCurrentRow.encode("utf-8", "replace"), addressAtCurrentRow.encode("utf-8", "replace")) self.tableWidgetBlacklist.removeRow(currentRow) def on_action_BlacklistClipboard(self): @@ -227,11 +229,11 @@ class Blacklist(QtGui.QWidget, RetranslateMixin): if config.get('bitmessagesettings', 'blackwhitelist') == 'black': sqlExecute( '''UPDATE blacklist SET enabled=1 WHERE address=?''', - str(addressAtCurrentRow)) + addressAtCurrentRow.encode("utf-8", "replace")) else: sqlExecute( '''UPDATE whitelist SET enabled=1 WHERE address=?''', - str(addressAtCurrentRow)) + addressAtCurrentRow.encode("utf-8", "replace")) def on_action_BlacklistDisable(self): currentRow = self.tableWidgetBlacklist.currentRow() @@ -243,10 +245,10 @@ class Blacklist(QtGui.QWidget, RetranslateMixin): currentRow, 1).setTextColor(QtGui.QColor(128, 128, 128)) if config.get('bitmessagesettings', 'blackwhitelist') == 'black': sqlExecute( - '''UPDATE blacklist SET enabled=0 WHERE address=?''', str(addressAtCurrentRow)) + '''UPDATE blacklist SET enabled=0 WHERE address=?''', addressAtCurrentRow.encode("utf-8", "replace")) else: sqlExecute( - '''UPDATE whitelist SET enabled=0 WHERE address=?''', str(addressAtCurrentRow)) + '''UPDATE whitelist SET enabled=0 WHERE address=?''', addressAtCurrentRow.encode("utf-8", "replace")) def on_action_BlacklistSetAvatar(self): self.window().on_action_SetAvatar(self.tableWidgetBlacklist) diff --git a/src/bitmessageqt/foldertree.py b/src/bitmessageqt/foldertree.py index c50b7d3d..2acf215d 100644 --- a/src/bitmessageqt/foldertree.py +++ b/src/bitmessageqt/foldertree.py @@ -136,7 +136,7 @@ class AccountMixin(object): if queryreturn != []: for row in queryreturn: retval, = row - retval = unicode(retval, 'utf-8') + retval = retval.decode("utf-8", "replace") elif self.address is None or self.type == AccountMixin.ALL: return unicode( str(_translate("MainWindow", "All accounts")), 'utf-8') @@ -311,8 +311,8 @@ class Ui_SubscriptionWidget(Ui_AddressWidget): if queryreturn != []: for row in queryreturn: retval, = row - return unicode(retval, 'utf-8', 'ignore') - return unicode(self.address, 'utf-8') + return retval.decode("utf-8", "replace") + return self.address def setType(self): """Set account type""" @@ -418,7 +418,8 @@ class MessageList_AddressWidget(BMAddressWidget): '''select label from subscriptions where address=?''', self.address) if queryreturn: for row in queryreturn: - newLabel = unicode(row[0], 'utf-8', 'ignore') + newLabel = row[0] + newLabel = newLabel.decode("utf-8", "replace") self.label = newLabel @@ -456,7 +457,7 @@ class MessageList_SubjectWidget(BMTableWidgetItem): if role == QtCore.Qt.UserRole: return self.subject if role == QtCore.Qt.ToolTipRole: - return escape(unicode(self.subject, 'utf-8')) + return escape(self.subject) return super(MessageList_SubjectWidget, self).data(role) # label (or address) alphabetically, disabled at the end diff --git a/src/bitmessageqt/safehtmlparser.py b/src/bitmessageqt/safehtmlparser.py index d408d2c7..d027ec44 100644 --- a/src/bitmessageqt/safehtmlparser.py +++ b/src/bitmessageqt/safehtmlparser.py @@ -125,6 +125,8 @@ class SafeHTMLParser(HTMLParser): def feed(self, data): try: data = unicode(data, 'utf-8') + except TypeError: + pass except UnicodeDecodeError: data = unicode(data, 'utf-8', errors='replace') HTMLParser.feed(self, data) diff --git a/src/class_singleCleaner.py b/src/class_singleCleaner.py index 06153dcf..21aadfad 100644 --- a/src/class_singleCleaner.py +++ b/src/class_singleCleaner.py @@ -99,6 +99,8 @@ class singleCleaner(StoppableThread): tick - state.maximumLengthOfTimeToBotherResendingMessages ) for toAddress, ackData, status in queryreturn: + toAddress = toAddress.decode("utf-8", "replace") + status = status.decode("utf-8", "replace") if status == 'awaitingpubkey': self.resendPubkeyRequest(toAddress) elif status == 'msgsent': diff --git a/src/class_singleWorker.py b/src/class_singleWorker.py index f2821f65..71e24c2e 100644 --- a/src/class_singleWorker.py +++ b/src/class_singleWorker.py @@ -73,6 +73,7 @@ class singleWorker(StoppableThread): '''SELECT DISTINCT toaddress FROM sent''' ''' WHERE (status='awaitingpubkey' AND folder='sent')''') for toAddress, in queryreturn: + toAddress = toAddress.decode("utf-8", "replace") toAddressVersionNumber, toStreamNumber, toRipe = \ decodeAddress(toAddress)[1:] if toAddressVersionNumber <= 3: @@ -538,6 +539,9 @@ class singleWorker(StoppableThread): for row in queryreturn: fromaddress, subject, body, ackdata, TTL, encoding = row + fromaddress = fromaddress.decode("utf-8", "replace") + subject = subject.decode("utf-8", "replace") + body = body.decode("utf-8", "replace") # status _, addressVersionNumber, streamNumber, ripe = \ decodeAddress(fromaddress) @@ -726,6 +730,11 @@ class singleWorker(StoppableThread): for row in queryreturn: toaddress, fromaddress, subject, message, \ ackdata, status, TTL, retryNumber, encoding = row + toaddress = toaddress.decode("utf-8", "replace") + fromaddress = fromaddress.decode("utf-8", "replace") + subject = subject.decode("utf-8", "replace") + message = message.decode("utf-8", "replace") + status = status.decode("utf-8", "replace") # toStatus _, toAddressVersionNumber, toStreamNumber, toRipe = \ decodeAddress(toaddress) diff --git a/src/class_sqlThread.py b/src/class_sqlThread.py index 7df9e253..49e4b98b 100644 --- a/src/class_sqlThread.py +++ b/src/class_sqlThread.py @@ -38,7 +38,7 @@ class sqlThread(threading.Thread): helper_sql.sql_available = True config_ready.wait() self.conn = sqlite3.connect(state.appdata + 'messages.dat') - self.conn.text_factory = str + self.conn.text_factory = bytes self.cur = self.conn.cursor() self.cur.execute('PRAGMA secure_delete = true') @@ -542,7 +542,7 @@ class sqlThread(threading.Thread): shutil.move( paths.lookupAppdataFolder() + 'messages.dat', paths.lookupExeFolder() + 'messages.dat') self.conn = sqlite3.connect(paths.lookupExeFolder() + 'messages.dat') - self.conn.text_factory = str + self.conn.text_factory = bytes self.cur = self.conn.cursor() elif item == 'movemessagstoappdata': logger.debug('the sqlThread is moving the messages.dat file to the Appdata folder.') @@ -568,7 +568,7 @@ class sqlThread(threading.Thread): shutil.move( paths.lookupExeFolder() + 'messages.dat', paths.lookupAppdataFolder() + 'messages.dat') self.conn = sqlite3.connect(paths.lookupAppdataFolder() + 'messages.dat') - self.conn.text_factory = str + self.conn.text_factory = bytes self.cur = self.conn.cursor() elif item == 'deleteandvacuume': self.cur.execute('''delete from inbox where folder='trash' ''') diff --git a/src/helper_msgcoding.py b/src/helper_msgcoding.py index 05fa1c1b..62214525 100644 --- a/src/helper_msgcoding.py +++ b/src/helper_msgcoding.py @@ -71,7 +71,8 @@ class MsgEncode(object): def encodeSimple(self, message): """Handle simple encoding""" - self.data = 'Subject:%(subject)s\nBody:%(body)s' % message + data = 'Subject:%(subject)s\nBody:%(body)s' % message + self.data = data.encode("utf-8", "replace") self.length = len(self.data) def encodeTrivial(self, message): diff --git a/src/shared.py b/src/shared.py index b85ddb20..aa7b6bb4 100644 --- a/src/shared.py +++ b/src/shared.py @@ -47,7 +47,7 @@ def isAddressInMySubscriptionsList(address): """Am I subscribed to this address?""" queryreturn = sqlQuery( '''select * from subscriptions where address=?''', - str(address)) + address.encode("utf-8", "replace")) return queryreturn != [] @@ -136,6 +136,7 @@ def reloadBroadcastSendersForWhichImWatching(): logger.debug('reloading subscriptions...') for row in queryreturn: address, = row + address = address.decode("utf-8", "replace") # status addressVersionNumber, streamNumber, hashobj = decodeAddress(address)[1:] if addressVersionNumber == 2: diff --git a/src/tests/test_shared.py b/src/tests/test_shared.py index 39bedf32..3e9be107 100644 --- a/src/tests/test_shared.py +++ b/src/tests/test_shared.py @@ -46,7 +46,7 @@ class TestShared(unittest.TestCase): address = sample_address # if address is in MyAddressbook - mock_sql_query.return_value = [address] + mock_sql_query.return_value = [address.encode("utf-8", "replace")] return_val = isAddressInMyAddressBook(address) mock_sql_query.assert_called_once() self.assertTrue(return_val) @@ -64,7 +64,7 @@ class TestShared(unittest.TestCase): address = sample_address # if address is in MySubscriptionsList - mock_sql_query.return_value = [address] + mock_sql_query.return_value = [address.encode("utf-8", "replace")] return_val = isAddressInMySubscriptionsList(address) self.assertTrue(return_val) @@ -78,7 +78,7 @@ class TestShared(unittest.TestCase): def test_reloadBroadcastSendersForWhichImWatching(self, mock_sql_query): """Test for reload Broadcast Senders For Which Im Watching""" mock_sql_query.return_value = [ - (sample_address,), + (sample_address.encode("utf-8", "replace"),), ] # before reload self.assertEqual(len(MyECSubscriptionCryptorObjects), 0) From 96c764bd9424c0a8c573525e691b46b958034b36 Mon Sep 17 00:00:00 2001 From: Kashiko Koibumi Date: Mon, 27 May 2024 22:43:01 +0900 Subject: [PATCH 2/2] refined: read from and write to SQLite database in binary mode This modification is a preparation for migration to Python3. --- src/api.py | 34 +++++----- src/bitmessagecurses/__init__.py | 71 +++++++++++++-------- src/bitmessagekivy/baseclass/addressbook.py | 16 +++-- src/bitmessagekivy/baseclass/maildetail.py | 10 +-- src/bitmessagekivy/baseclass/popup.py | 13 ++-- src/bitmessagekivy/kivy_helper_search.py | 13 ++-- src/bitmessageqt/__init__.py | 23 ++++--- src/bitmessageqt/account.py | 25 ++++---- src/bitmessageqt/blacklist.py | 21 +++--- src/bitmessageqt/foldertree.py | 19 +++--- src/bitmessageqt/support.py | 7 +- src/class_objectProcessor.py | 33 +++++----- src/class_singleCleaner.py | 3 +- src/class_singleWorker.py | 27 ++++---- src/class_smtpServer.py | 13 ++-- src/dbcompat.py | 22 +++++++ src/helper_addressbook.py | 3 +- src/helper_inbox.py | 4 +- src/helper_search.py | 13 ++-- src/helper_sent.py | 5 +- src/protocol.py | 3 +- src/shared.py | 11 ++-- src/storage/sqlite.py | 2 +- src/tests/core.py | 9 +-- src/tests/test_helper_inbox.py | 8 +-- src/tests/test_helper_sent.py | 18 +++--- src/tests/test_helper_sql.py | 10 +-- src/tests/test_shared.py | 6 +- src/tests/test_sqlthread.py | 2 +- 29 files changed, 259 insertions(+), 185 deletions(-) create mode 100644 src/dbcompat.py diff --git a/src/api.py b/src/api.py index a4445569..d69f6506 100644 --- a/src/api.py +++ b/src/api.py @@ -70,6 +70,7 @@ from struct import pack, unpack import six from six.moves import configparser, http_client, xmlrpc_server +from dbcompat import dbstr import helper_inbox import helper_sent @@ -531,12 +532,12 @@ class BMRPCDispatcher(object): message = shared.fixPotentiallyInvalidUTF8Data(message) return { 'msgid': hexlify(msgid), - 'toAddress': toAddress, - 'fromAddress': fromAddress, + 'toAddress': toAddress.decode("utf-8", "replace"), + 'fromAddress': fromAddress.decode("utf-8", "replace"), 'subject': base64.b64encode(subject), 'message': base64.b64encode(message), 'encodingType': encodingtype, - 'receivedTime': received, + 'receivedTime': received.decode("utf-8", "replace"), 'read': read } @@ -598,11 +599,12 @@ class BMRPCDispatcher(object): """ queryreturn = sqlQuery( "SELECT label, address from addressbook WHERE label = ?", - label + dbstr(label) ) if label else sqlQuery("SELECT label, address from addressbook") data = [] for label, address in queryreturn: label = shared.fixPotentiallyInvalidUTF8Data(label) + address = address.decode("utf-8", "replace") data.append({ 'label': base64.b64encode(label), 'address': address @@ -618,12 +620,12 @@ class BMRPCDispatcher(object): self._verifyAddress(address) # TODO: add unique together constraint in the table queryreturn = sqlQuery( - "SELECT address FROM addressbook WHERE address=?", address) + "SELECT address FROM addressbook WHERE address=?", dbstr(address)) if queryreturn != []: raise APIError( 16, 'You already have this address in your address book.') - sqlExecute("INSERT INTO addressbook VALUES(?,?)", label, address) + sqlExecute("INSERT INTO addressbook VALUES(?,?)", dbstr(label), dbstr(address)) queues.UISignalQueue.put(('rerenderMessagelistFromLabels', '')) queues.UISignalQueue.put(('rerenderMessagelistToLabels', '')) queues.UISignalQueue.put(('rerenderAddressBook', '')) @@ -635,7 +637,7 @@ class BMRPCDispatcher(object): """Delete an entry from address book.""" address = addBMIfNotPresent(address) self._verifyAddress(address) - sqlExecute('DELETE FROM addressbook WHERE address=?', address) + sqlExecute('DELETE FROM addressbook WHERE address=?', dbstr(address)) queues.UISignalQueue.put(('rerenderMessagelistFromLabels', '')) queues.UISignalQueue.put(('rerenderMessagelistToLabels', '')) queues.UISignalQueue.put(('rerenderAddressBook', '')) @@ -919,6 +921,7 @@ class BMRPCDispatcher(object): " ORDER BY received" ) return {"inboxMessages": [ + self._dump_inbox_message(*data) for data in queryreturn ]} @@ -1018,7 +1021,7 @@ class BMRPCDispatcher(object): queryreturn = sqlQuery( "SELECT msgid, toaddress, fromaddress, subject, received," " message, encodingtype, read FROM inbox WHERE folder='inbox'" - " AND toAddress=?", toAddress) + " AND toAddress=?", dbstr(toAddress)) return {"inboxMessages": [ self._dump_inbox_message(*data) for data in queryreturn ]} @@ -1055,7 +1058,7 @@ class BMRPCDispatcher(object): "SELECT msgid, toaddress, fromaddress, subject, lastactiontime," " message, encodingtype, status, ackdata FROM sent" " WHERE folder='sent' AND fromAddress=? ORDER BY lastactiontime", - fromAddress + dbstr(fromAddress) ) return {"sentMessages": [ self._dump_sent_message(*data) for data in queryreturn @@ -1150,9 +1153,9 @@ class BMRPCDispatcher(object): toLabel = '' queryreturn = sqlQuery( - "SELECT label FROM addressbook WHERE address=?", toAddress) + "SELECT label FROM addressbook WHERE address=?", dbstr(toAddress)) try: - toLabel = queryreturn[0][0] + toLabel = queryreturn[0][0].decode("utf-8", "replace") except IndexError: pass @@ -1219,7 +1222,7 @@ class BMRPCDispatcher(object): queryreturn = sqlQuery( "SELECT status FROM sent where ackdata=?", ackdata) try: - return queryreturn[0][0] + return queryreturn[0][0].decode("utf-8", "replace") except IndexError: return 'notfound' @@ -1238,11 +1241,11 @@ class BMRPCDispatcher(object): # First we must check to see if the address is already in the # subscriptions list. queryreturn = sqlQuery( - "SELECT * FROM subscriptions WHERE address=?", address) + "SELECT * FROM subscriptions WHERE address=?", dbstr(address)) if queryreturn: raise APIError(16, 'You are already subscribed to that address.') sqlExecute( - "INSERT INTO subscriptions VALUES (?,?,?)", label, address, True) + "INSERT INTO subscriptions VALUES (?,?,?)", dbstr(label), dbstr(address), True) shared.reloadBroadcastSendersForWhichImWatching() queues.UISignalQueue.put(('rerenderMessagelistFromLabels', '')) queues.UISignalQueue.put(('rerenderSubscriptions', '')) @@ -1256,7 +1259,7 @@ class BMRPCDispatcher(object): """ address = addBMIfNotPresent(address) - sqlExecute("DELETE FROM subscriptions WHERE address=?", address) + sqlExecute("DELETE FROM subscriptions WHERE address=?", dbstr(address)) shared.reloadBroadcastSendersForWhichImWatching() queues.UISignalQueue.put(('rerenderMessagelistFromLabels', '')) queues.UISignalQueue.put(('rerenderSubscriptions', '')) @@ -1274,6 +1277,7 @@ class BMRPCDispatcher(object): data = [] for label, address, enabled in queryreturn: label = shared.fixPotentiallyInvalidUTF8Data(label) + address = address.decode("utf-8", "replace") data.append({ 'label': base64.b64encode(label), 'address': address, diff --git a/src/bitmessagecurses/__init__.py b/src/bitmessagecurses/__init__.py index 64fd735b..f63f820a 100644 --- a/src/bitmessagecurses/__init__.py +++ b/src/bitmessagecurses/__init__.py @@ -30,6 +30,7 @@ import state from addresses import addBMIfNotPresent, decodeAddress from bmconfigparser import config from helper_sql import sqlExecute, sqlQuery +from dbcompat import dbstr # pylint: disable=global-statement @@ -401,6 +402,7 @@ def handlech(c, stdscr): body = "\n\n------------------------------------------------------\n" for row in ret: body, = row + body = body.decode("utf-8", "replace") sendMessage(fromaddr, toaddr, ischan, subject, body, True) dialogreset(stdscr) @@ -410,7 +412,7 @@ def handlech(c, stdscr): r, t = d.inputbox("Label for address \"" + addr + "\"") if r == d.DIALOG_OK: label = t - sqlExecute("INSERT INTO addressbook VALUES (?,?)", label, addr) + sqlExecute("INSERT INTO addressbook VALUES (?,?)", dbstr(label), dbstr(addr)) # Prepend entry addrbook.reverse() addrbook.append([label, addr]) @@ -426,6 +428,7 @@ def handlech(c, stdscr): if ret != []: for row in ret: msg, = row + msg = msg.decode("utf-8", "replace") fh = open(t, "a") # Open in append mode just in case fh.write(msg) fh.close() @@ -463,7 +466,7 @@ def handlech(c, stdscr): data = "" ret = sqlQuery( "SELECT message FROM sent WHERE subject=? AND ackdata=?", - sentbox[sentcur][4], + dbstr(sentbox[sentcur][4]), sentbox[sentcur][6]) if ret != []: for row in ret: @@ -478,7 +481,7 @@ def handlech(c, stdscr): elif t == "2": # Move to trash sqlExecute( "UPDATE sent SET folder='trash' WHERE subject=? AND ackdata=?", - sentbox[sentcur][4], + dbstr(sentbox[sentcur][4]), sentbox[sentcur][6]) del sentbox[sentcur] scrollbox(d, unicode( @@ -711,29 +714,29 @@ def handlech(c, stdscr): subscriptions.append([label, addr, True]) subscriptions.reverse() - sqlExecute("INSERT INTO subscriptions VALUES (?,?,?)", label, addr, True) + sqlExecute("INSERT INTO subscriptions VALUES (?,?,?)", dbstr(label), dbstr(addr), True) shared.reloadBroadcastSendersForWhichImWatching() elif t == "2": r, t = d.inputbox("Type in \"I want to delete this subscription\"") if r == d.DIALOG_OK and t == "I want to delete this subscription": sqlExecute( "DELETE FROM subscriptions WHERE label=? AND address=?", - subscriptions[subcur][0], - subscriptions[subcur][1]) + dbstr(subscriptions[subcur][0]), + dbstr(subscriptions[subcur][1])) shared.reloadBroadcastSendersForWhichImWatching() del subscriptions[subcur] elif t == "3": sqlExecute( "UPDATE subscriptions SET enabled=1 WHERE label=? AND address=?", - subscriptions[subcur][0], - subscriptions[subcur][1]) + dbstr(subscriptions[subcur][0]), + dbstr(subscriptions[subcur][1])) shared.reloadBroadcastSendersForWhichImWatching() subscriptions[subcur][2] = True elif t == "4": sqlExecute( "UPDATE subscriptions SET enabled=0 WHERE label=? AND address=?", - subscriptions[subcur][0], - subscriptions[subcur][1]) + dbstr(subscriptions[subcur][0]), + dbstr(subscriptions[subcur][1])) shared.reloadBroadcastSendersForWhichImWatching() subscriptions[subcur][2] = False elif menutab == 6: @@ -762,7 +765,7 @@ def handlech(c, stdscr): subscriptions.append([label, addr, True]) subscriptions.reverse() - sqlExecute("INSERT INTO subscriptions VALUES (?,?,?)", label, addr, True) + sqlExecute("INSERT INTO subscriptions VALUES (?,?,?)", dbstr(label), dbstr(addr), True) shared.reloadBroadcastSendersForWhichImWatching() elif t == "3": r, t = d.inputbox("Input new address") @@ -771,7 +774,7 @@ def handlech(c, stdscr): if addr not in [item[1] for i, item in enumerate(addrbook)]: r, t = d.inputbox("Label for address \"" + addr + "\"") if r == d.DIALOG_OK: - sqlExecute("INSERT INTO addressbook VALUES (?,?)", t, addr) + sqlExecute("INSERT INTO addressbook VALUES (?,?)", dbstr(t), dbstr(addr)) # Prepend entry addrbook.reverse() addrbook.append([t, addr]) @@ -783,8 +786,8 @@ def handlech(c, stdscr): if r == d.DIALOG_OK and t == "I want to delete this Address Book entry": sqlExecute( "DELETE FROM addressbook WHERE label=? AND address=?", - addrbook[abookcur][0], - addrbook[abookcur][1]) + dbstr(addrbook[abookcur][0]), + dbstr(addrbook[abookcur][1])) del addrbook[abookcur] elif menutab == 7: set_background_title(d, "Blacklist Dialog Box") @@ -800,20 +803,20 @@ def handlech(c, stdscr): if r == d.DIALOG_OK and t == "I want to delete this Blacklist entry": sqlExecute( "DELETE FROM blacklist WHERE label=? AND address=?", - blacklist[blackcur][0], - blacklist[blackcur][1]) + dbstr(blacklist[blackcur][0]), + dbstr(blacklist[blackcur][1])) del blacklist[blackcur] elif t == "2": sqlExecute( "UPDATE blacklist SET enabled=1 WHERE label=? AND address=?", - blacklist[blackcur][0], - blacklist[blackcur][1]) + dbstr(blacklist[blackcur][0]), + dbstr(blacklist[blackcur][1])) blacklist[blackcur][2] = True elif t == "3": sqlExecute( "UPDATE blacklist SET enabled=0 WHERE label=? AND address=?", - blacklist[blackcur][0], - blacklist[blackcur][1]) + dbstr(blacklist[blackcur][0]), + dbstr(blacklist[blackcur][1])) blacklist[blackcur][2] = False dialogreset(stdscr) else: @@ -991,10 +994,13 @@ def loadInbox(): ret = sqlQuery("""SELECT msgid, toaddress, fromaddress, subject, received, read FROM inbox WHERE folder='inbox' AND %s LIKE ? ORDER BY received - """ % (where,), what) + """ % (where,), dbstr(what)) for row in ret: msgid, toaddr, fromaddr, subject, received, read = row + toaddr = toaddr.decode("utf-8", "replace") + fromaddr = fromaddr.decode("utf-8", "replace") subject = ascii(shared.fixPotentiallyInvalidUTF8Data(subject)) + received = received.decode("utf-8", "replace") # Set label for to address try: @@ -1013,18 +1019,19 @@ def loadInbox(): if config.has_section(fromaddr): fromlabel = config.get(fromaddr, "label") if fromlabel == "": # Check Address Book - qr = sqlQuery("SELECT label FROM addressbook WHERE address=?", fromaddr) + qr = sqlQuery("SELECT label FROM addressbook WHERE address=?", dbstr(fromaddr)) if qr != []: for r in qr: fromlabel, = r + fromlabel = shared.fixPotentiallyInvalidUTF8Data(fromlabel) if fromlabel == "": # Check Subscriptions - qr = sqlQuery("SELECT label FROM subscriptions WHERE address=?", fromaddr) + qr = sqlQuery("SELECT label FROM subscriptions WHERE address=?", dbstr(fromaddr)) if qr != []: for r in qr: fromlabel, = r + fromlabel = shared.fixPotentiallyInvalidUTF8Data(fromlabel) if fromlabel == "": fromlabel = fromaddr - fromlabel = shared.fixPotentiallyInvalidUTF8Data(fromlabel) # Load into array inbox.append([ @@ -1044,22 +1051,27 @@ def loadSent(): ret = sqlQuery("""SELECT toaddress, fromaddress, subject, status, ackdata, lastactiontime FROM sent WHERE folder='sent' AND %s LIKE ? ORDER BY lastactiontime - """ % (where,), what) + """ % (where,), dbstr(what)) for row in ret: toaddr, fromaddr, subject, status, ackdata, lastactiontime = row + toaddr = toaddr.decode("utf-8", "replace") + fromaddr = fromaddr.decode("utf-8", "replace") subject = ascii(shared.fixPotentiallyInvalidUTF8Data(subject)) + status = status.decode("utf-8", "replace") # Set label for to address tolabel = "" - qr = sqlQuery("SELECT label FROM addressbook WHERE address=?", toaddr) + qr = sqlQuery("SELECT label FROM addressbook WHERE address=?", dbstr(toaddr)) if qr != []: for r in qr: tolabel, = r + tolabel = tolabel.decode("utf-8", "replace") if tolabel == "": - qr = sqlQuery("SELECT label FROM subscriptions WHERE address=?", toaddr) + qr = sqlQuery("SELECT label FROM subscriptions WHERE address=?", dbstr(toaddr)) if qr != []: for r in qr: tolabel, = r + tolabel = tolabel.decode("utf-8", "replace") if tolabel == "": if config.has_section(toaddr): tolabel = config.get(toaddr, "label") @@ -1129,6 +1141,7 @@ def loadAddrBook(): for row in ret: label, addr = row label = shared.fixPotentiallyInvalidUTF8Data(label) + addr = addr.decode("utf-8", "replace") addrbook.append([label, addr]) addrbook.reverse() @@ -1138,6 +1151,8 @@ def loadSubscriptions(): ret = sqlQuery("SELECT label, address, enabled FROM subscriptions") for row in ret: label, address, enabled = row + label = label.decode("utf-8", "replace") + address = address.decode("utf-8", "replace") subscriptions.append([label, address, enabled]) subscriptions.reverse() @@ -1152,6 +1167,8 @@ def loadBlackWhiteList(): ret = sqlQuery("SELECT label, address, enabled FROM whitelist") for row in ret: label, address, enabled = row + label = label.decode("utf-8", "replace") + address = address.decode("utf-8", "replace") blacklist.append([label, address, enabled]) blacklist.reverse() diff --git a/src/bitmessagekivy/baseclass/addressbook.py b/src/bitmessagekivy/baseclass/addressbook.py index f18a0142..1000bd53 100644 --- a/src/bitmessagekivy/baseclass/addressbook.py +++ b/src/bitmessagekivy/baseclass/addressbook.py @@ -29,6 +29,7 @@ from pybitmessage.bitmessagekivy.baseclass.common import ( from pybitmessage.bitmessagekivy.baseclass.popup import SavedAddressDetailPopup from pybitmessage.bitmessagekivy.baseclass.addressbook_widgets import HelperAddressBook from pybitmessage.helper_sql import sqlExecute +from dbcompat import dbstr logger = logging.getLogger('default') @@ -59,7 +60,7 @@ class AddressBook(Screen, HelperAddressBook): self.ids.tag_label.text = '' self.queryreturn = kivy_helper_search.search_sql( xAddress, account, "addressbook", where, what, False) - self.queryreturn = [obj for obj in reversed(self.queryreturn)] + self.queryreturn = [[obj[0].decode("utf-8", "replace"), obj[1].decode("utf-8", "replace")] for obj in reversed(self.queryreturn)] if self.queryreturn: self.ids.tag_label.text = 'Address Book' self.has_refreshed = True @@ -131,7 +132,7 @@ class AddressBook(Screen, HelperAddressBook): if self.ids.ml.children is not None: self.ids.tag_label.text = '' sqlExecute( - "DELETE FROM addressbook WHERE address = ?", address) + "DELETE FROM addressbook WHERE address = ?", dbstr(address)) toast('Address Deleted') def close_pop(self, instance): @@ -142,8 +143,13 @@ class AddressBook(Screen, HelperAddressBook): def update_addbook_label(self, instance): """Updating the label of address book address""" address_list = kivy_helper_search.search_sql(folder="addressbook") - stored_labels = [labels[0] for labels in address_list] - add_dict = dict(address_list) + stored_labels = [labels[0].decode("utf-8", "replace") for labels in address_list] + add_dict = {} + for row in address_list: + label, address = row + label = label.decode("utf-8", "replace") + address = address.decode("utf-8", "replace") + add_dict[label] = address label = str(self.addbook_popup.content_cls.ids.add_label.text) if label in stored_labels and self.address == add_dict[label]: stored_labels.remove(label) @@ -151,7 +157,7 @@ class AddressBook(Screen, HelperAddressBook): sqlExecute(""" UPDATE addressbook SET label = ? - WHERE address = ?""", label, self.addbook_popup.content_cls.address) + WHERE address = ?""", dbstr(label), dbstr(self.addbook_popup.content_cls.address)) App.get_running_app().root.ids.id_addressbook.ids.ml.clear_widgets() App.get_running_app().root.ids.id_addressbook.loadAddresslist(None, 'All', '') self.addbook_popup.dismiss() diff --git a/src/bitmessagekivy/baseclass/maildetail.py b/src/bitmessagekivy/baseclass/maildetail.py index 6ddf322d..5168a522 100644 --- a/src/bitmessagekivy/baseclass/maildetail.py +++ b/src/bitmessagekivy/baseclass/maildetail.py @@ -119,16 +119,16 @@ class MailDetail(Screen): # pylint: disable=too-many-instance-attributes def assign_mail_details(self, data): """Assigning mail details""" - subject = data[0][2].decode() if isinstance(data[0][2], bytes) else data[0][2] - body = data[0][3].decode() if isinstance(data[0][2], bytes) else data[0][3] - self.to_addr = data[0][0] if len(data[0][0]) > 4 else ' ' - self.from_addr = data[0][1] + subject = data[0][2].decode("utf-8", "replace") if isinstance(data[0][2], bytes) else data[0][2] + body = data[0][3].decode("utf-8", "replace") if isinstance(data[0][2], bytes) else data[0][3] + self.to_addr = data[0][0].decode("utf-8", "replace") if len(data[0][0]) > 4 else ' ' + self.from_addr = data[0][1].decode("utf-8", "replace") self.subject = subject.capitalize( ) if subject.capitalize() else self.no_subject self.message = body if len(data[0]) == 7: - self.status = data[0][4] + self.status = data[0][4].decode("utf-8", "replace") self.time_tag = show_time_history(data[0][4]) if self.kivy_state.detail_page_type == 'inbox' \ else show_time_history(data[0][6]) self.avatarImg = os.path.join(self.kivy_state.imageDir, 'draft-icon.png') \ diff --git a/src/bitmessagekivy/baseclass/popup.py b/src/bitmessagekivy/baseclass/popup.py index d2a4c859..48868f68 100644 --- a/src/bitmessagekivy/baseclass/popup.py +++ b/src/bitmessagekivy/baseclass/popup.py @@ -59,7 +59,7 @@ class AddAddressPopup(BoxLayout): """Checking address is valid or not""" my_addresses = ( App.get_running_app().root.ids.content_drawer.ids.identity_dropdown.values) - add_book = [addr[1] for addr in kivy_helper_search.search_sql( + add_book = [addr[1].decode("utf-8", "replace") for addr in kivy_helper_search.search_sql( folder="addressbook")] entered_text = str(instance.text).strip() if entered_text in add_book: @@ -84,7 +84,7 @@ class AddAddressPopup(BoxLayout): def checkLabel_valid(self, instance): """Checking address label is unique or not""" entered_label = instance.text.strip() - addr_labels = [labels[0] for labels in kivy_helper_search.search_sql( + addr_labels = [labels[0].decode("utf-8", "replace") for labels in kivy_helper_search.search_sql( folder="addressbook")] if entered_label in addr_labels: self.ids.label.error = True @@ -125,8 +125,13 @@ class SavedAddressDetailPopup(BoxLayout): """Checking address label is unique of not""" entered_label = str(instance.text.strip()) address_list = kivy_helper_search.search_sql(folder="addressbook") - addr_labels = [labels[0] for labels in address_list] - add_dict = dict(address_list) + addr_labels = [labels[0].decode("utf-8", "replace") for labels in address_list] + add_dict = {} + for row in address_list: + label, address = row + label = label.decode("utf-8", "replace") + address = address.decode("utf-8", "replace") + add_dict[label] = address if self.address and entered_label in addr_labels \ and self.address != add_dict[entered_label]: self.ids.add_label.error = True diff --git a/src/bitmessagekivy/kivy_helper_search.py b/src/bitmessagekivy/kivy_helper_search.py index c48ca3ad..82283481 100644 --- a/src/bitmessagekivy/kivy_helper_search.py +++ b/src/bitmessagekivy/kivy_helper_search.py @@ -2,6 +2,7 @@ Sql queries for bitmessagekivy """ from pybitmessage.helper_sql import sqlQuery +from dbcompat import dbstr def search_sql( @@ -30,21 +31,21 @@ def search_sql( if account is not None: if xAddress == 'both': sqlStatementParts.append("(fromaddress = ? OR toaddress = ?)") - sqlArguments.append(account) - sqlArguments.append(account) + sqlArguments.append(dbstr(account)) + sqlArguments.append(dbstr(account)) else: sqlStatementParts.append(xAddress + " = ? ") - sqlArguments.append(account) + sqlArguments.append(dbstr(account)) if folder != "addressbook": if folder is not None: if folder == "new": folder = "inbox" unreadOnly = True sqlStatementParts.append("folder = ? ") - sqlArguments.append(folder) + sqlArguments.append(dbstr(folder)) else: sqlStatementParts.append("folder != ?") - sqlArguments.append("trash") + sqlArguments.append(dbstr("trash")) if what is not None: for colmns in where: if len(where) > 1: @@ -54,7 +55,7 @@ def search_sql( filter_col += " or %s LIKE ? )" % (colmns) else: filter_col = "%s LIKE ?" % (colmns) - sqlArguments.append(what) + sqlArguments.append(dbstr(what)) sqlStatementParts.append(filter_col) if unreadOnly: sqlStatementParts.append("read = 0") diff --git a/src/bitmessageqt/__init__.py b/src/bitmessageqt/__init__.py index fd012e04..df58e403 100644 --- a/src/bitmessageqt/__init__.py +++ b/src/bitmessageqt/__init__.py @@ -17,6 +17,7 @@ from sqlite3 import register_adapter from PyQt4 import QtCore, QtGui from PyQt4.QtNetwork import QLocalSocket, QLocalServer +from dbcompat import dbstr import shared import state @@ -1067,8 +1068,10 @@ class MyForm(settingsmixin.SMainWindow): queryReturn = sqlQuery( 'SELECT fromaddress, folder, COUNT(msgid) AS cnt' ' FROM inbox WHERE read = 0 AND toaddress = ?' - ' GROUP BY fromaddress, folder', str_broadcast_subscribers) + ' GROUP BY fromaddress, folder', dbstr(str_broadcast_subscribers)) for addr, fld, count in queryReturn: + addr = addr.decode("utf-8", "replace") + fld = fld.decode("utf-8", "replace") try: broadcastsUnread[addr][fld] = count except KeyError: @@ -2289,7 +2292,7 @@ class MyForm(settingsmixin.SMainWindow): subject=subject, message=message, encoding=encoding) toLabel = '' queryreturn = sqlQuery('''select label from addressbook where address=?''', - toAddress) + dbstr(toAddress)) if queryreturn != []: for row in queryreturn: toLabel, = row @@ -2578,7 +2581,7 @@ class MyForm(settingsmixin.SMainWindow): # Add to database (perhaps this should be separated from the MyForm class) sqlExecute( '''INSERT INTO subscriptions VALUES (?,?,?)''', - label, address, True + dbstr(label), dbstr(address), True ) self.rerenderMessagelistFromLabels() shared.reloadBroadcastSendersForWhichImWatching() @@ -3177,13 +3180,13 @@ class MyForm(settingsmixin.SMainWindow): currentInboxRow, 0).data(QtCore.Qt.UserRole) # Let's make sure that it isn't already in the address book queryreturn = sqlQuery('''select * from blacklist where address=?''', - addressAtCurrentInboxRow) + dbstr(addressAtCurrentInboxRow)) if queryreturn == []: label = "\"" + tableWidget.item(currentInboxRow, 2).subject + "\" in " + config.get( recipientAddress, "label") sqlExecute('''INSERT INTO blacklist VALUES (?,?, ?)''', - label, - addressAtCurrentInboxRow, True) + dbstr(label), + dbstr(addressAtCurrentInboxRow), True) self.ui.blackwhitelist.rerenderBlackWhiteList() self.updateStatusBar(_translate( "MainWindow", @@ -3370,7 +3373,7 @@ class MyForm(settingsmixin.SMainWindow): 0].row() item = self.ui.tableWidgetAddressBook.item(currentRow, 0) sqlExecute( - 'DELETE FROM addressbook WHERE address=?', item.address) + 'DELETE FROM addressbook WHERE address=?', dbstr(item.address)) self.ui.tableWidgetAddressBook.removeRow(currentRow) self.rerenderMessagelistFromLabels() self.rerenderMessagelistToLabels() @@ -3470,7 +3473,7 @@ class MyForm(settingsmixin.SMainWindow): return address = self.getCurrentAccount() sqlExecute('''DELETE FROM subscriptions WHERE address=?''', - address) + dbstr(address)) self.rerenderTabTreeSubscriptions() self.rerenderMessagelistFromLabels() self.rerenderAddressBook() @@ -3485,7 +3488,7 @@ class MyForm(settingsmixin.SMainWindow): address = self.getCurrentAccount() sqlExecute( '''update subscriptions set enabled=1 WHERE address=?''', - address) + dbstr(address)) account = self.getCurrentItem() account.setEnabled(True) self.rerenderAddressBook() @@ -3495,7 +3498,7 @@ class MyForm(settingsmixin.SMainWindow): address = self.getCurrentAccount() sqlExecute( '''update subscriptions set enabled=0 WHERE address=?''', - address) + dbstr(address)) account = self.getCurrentItem() account.setEnabled(False) self.rerenderAddressBook() diff --git a/src/bitmessageqt/account.py b/src/bitmessageqt/account.py index 794c276e..acb611c8 100644 --- a/src/bitmessageqt/account.py +++ b/src/bitmessageqt/account.py @@ -15,6 +15,7 @@ import sys import time from PyQt4 import QtGui +from dbcompat import dbstr import queues from addresses import decodeAddress @@ -49,7 +50,7 @@ def getSortedSubscriptions(count=False): queryreturn = sqlQuery('''SELECT fromaddress, folder, count(msgid) as cnt FROM inbox, subscriptions ON subscriptions.address = inbox.fromaddress WHERE read = 0 AND toaddress = ? - GROUP BY inbox.fromaddress, folder''', str_broadcast_subscribers) + GROUP BY inbox.fromaddress, folder''', dbstr(str_broadcast_subscribers)) for row in queryreturn: address, folder, cnt = row address = address.decode("utf-8", "replace") @@ -104,7 +105,7 @@ class AccountColor(AccountMixin): # pylint: disable=too-few-public-methods elif config.safeGetBoolean(self.address, 'chan'): self.type = AccountMixin.CHAN elif sqlQuery( - '''select label from subscriptions where address=?''', self.address): + '''select label from subscriptions where address=?''', dbstr(self.address)): self.type = AccountMixin.SUBSCRIPTION else: self.type = AccountMixin.NORMAL @@ -127,7 +128,7 @@ class BMAccount(object): self.type = AccountMixin.BROADCAST else: queryreturn = sqlQuery( - '''select label from subscriptions where address=?''', self.address) + '''select label from subscriptions where address=?''', dbstr(self.address)) if queryreturn: self.type = AccountMixin.SUBSCRIPTION @@ -137,18 +138,18 @@ class BMAccount(object): address = self.address label = config.safeGet(address, 'label', address) queryreturn = sqlQuery( - '''select label from addressbook where address=?''', address) + '''select label from addressbook where address=?''', dbstr(address)) if queryreturn != []: for row in queryreturn: label, = row label = label.decode("utf-8", "replace") else: queryreturn = sqlQuery( - '''select label from subscriptions where address=?''', address) + '''select label from subscriptions where address=?''', dbstr(address)) if queryreturn != []: for row in queryreturn: label, = row - label = label.replace("utf-8", "replace") + label = label.decode("utf-8", "replace") return label def parseMessage(self, toAddress, fromAddress, subject, message): @@ -205,18 +206,18 @@ class GatewayAccount(BMAccount): sqlExecute( '''INSERT INTO sent VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)''', '', - self.toAddress.encode("utf-8", "replace"), + dbstr(self.toAddress), ripe, - self.fromAddress.encode("utf-8", "replace"), - self.subject.encode("utf-8", "replace"), - self.message.encode("utf-8", "replace"), + dbstr(self.fromAddress), + dbstr(self.subject), + dbstr(self.message), ackdata, int(time.time()), # sentTime (this will never change) int(time.time()), # lastActionTime 0, # sleepTill time. This will get set when the POW gets done. - 'msgqueued', + dbstr('msgqueued'), 0, # retryNumber - 'sent', # folder + dbstr('sent'), # folder 2, # encodingtype # not necessary to have a TTL higher than 2 days min(config.getint('bitmessagesettings', 'ttl'), 86400 * 2) diff --git a/src/bitmessageqt/blacklist.py b/src/bitmessageqt/blacklist.py index 41342bce..0ce60541 100644 --- a/src/bitmessageqt/blacklist.py +++ b/src/bitmessageqt/blacklist.py @@ -1,4 +1,5 @@ from PyQt4 import QtCore, QtGui +from dbcompat import dbstr import widgets from addresses import addBMIfNotPresent @@ -64,7 +65,7 @@ class Blacklist(QtGui.QWidget, RetranslateMixin): # First we must check to see if the address is already in the # address book. The user cannot add it again or else it will # cause problems when updating and deleting the entry. - t = (address,) + t = (dbstr(address),) if config.get('bitmessagesettings', 'blackwhitelist') == 'black': sql = '''select * from blacklist where address=?''' else: @@ -82,7 +83,7 @@ class Blacklist(QtGui.QWidget, RetranslateMixin): QtCore.Qt.ItemIsSelectable | QtCore.Qt.ItemIsEnabled) self.tableWidgetBlacklist.setItem(0, 1, newItem) self.tableWidgetBlacklist.setSortingEnabled(True) - t = (str(self.NewBlacklistDialogInstance.lineEditLabel.text().toUtf8()), address, True) + t = (dbstr(self.NewBlacklistDialogInstance.lineEditLabel.text().toUtf8()), dbstr(address), True) if config.get('bitmessagesettings', 'blackwhitelist') == 'black': sql = '''INSERT INTO blacklist VALUES (?,?,?)''' else: @@ -111,10 +112,10 @@ class Blacklist(QtGui.QWidget, RetranslateMixin): if isinstance(addressitem, QtGui.QTableWidgetItem): if self.radioButtonBlacklist.isChecked(): sqlExecute('''UPDATE blacklist SET label=? WHERE address=?''', - item.text().encode("utf-8", "replace"), addressitem.text().encode("utf-8", "replace")) + dbstr(item.text()), dbstr(addressitem.text())) else: sqlExecute('''UPDATE whitelist SET label=? WHERE address=?''', - item.text().encode("utf-8", "replace"), addressitem.text().encode("utf-8", "replace")) + dbstr(item.text()), dbstr(addressitem.text())) def init_blacklist_popup_menu(self, connectSignal=True): # Popup menu for the Blacklist page @@ -200,11 +201,11 @@ class Blacklist(QtGui.QWidget, RetranslateMixin): if config.get('bitmessagesettings', 'blackwhitelist') == 'black': sqlExecute( '''DELETE FROM blacklist WHERE label=? AND address=?''', - labelAtCurrentRow.encode("utf-8", "replace"), addressAtCurrentRow.encode("utf-8", "replace")) + dbstr(labelAtCurrentRow), dbstr(addressAtCurrentRow)) else: sqlExecute( '''DELETE FROM whitelist WHERE label=? AND address=?''', - labelAtCurrentRow.encode("utf-8", "replace"), addressAtCurrentRow.encode("utf-8", "replace")) + dbstr(labelAtCurrentRow), dbstr(addressAtCurrentRow)) self.tableWidgetBlacklist.removeRow(currentRow) def on_action_BlacklistClipboard(self): @@ -229,11 +230,11 @@ class Blacklist(QtGui.QWidget, RetranslateMixin): if config.get('bitmessagesettings', 'blackwhitelist') == 'black': sqlExecute( '''UPDATE blacklist SET enabled=1 WHERE address=?''', - addressAtCurrentRow.encode("utf-8", "replace")) + dbstr(addressAtCurrentRow)) else: sqlExecute( '''UPDATE whitelist SET enabled=1 WHERE address=?''', - addressAtCurrentRow.encode("utf-8", "replace")) + dbstr(addressAtCurrentRow)) def on_action_BlacklistDisable(self): currentRow = self.tableWidgetBlacklist.currentRow() @@ -245,10 +246,10 @@ class Blacklist(QtGui.QWidget, RetranslateMixin): currentRow, 1).setTextColor(QtGui.QColor(128, 128, 128)) if config.get('bitmessagesettings', 'blackwhitelist') == 'black': sqlExecute( - '''UPDATE blacklist SET enabled=0 WHERE address=?''', addressAtCurrentRow.encode("utf-8", "replace")) + '''UPDATE blacklist SET enabled=0 WHERE address=?''', dbstr(addressAtCurrentRow)) else: sqlExecute( - '''UPDATE whitelist SET enabled=0 WHERE address=?''', addressAtCurrentRow.encode("utf-8", "replace")) + '''UPDATE whitelist SET enabled=0 WHERE address=?''', dbstr(addressAtCurrentRow)) def on_action_BlacklistSetAvatar(self): self.window().on_action_SetAvatar(self.tableWidgetBlacklist) diff --git a/src/bitmessageqt/foldertree.py b/src/bitmessageqt/foldertree.py index 2acf215d..b2581051 100644 --- a/src/bitmessageqt/foldertree.py +++ b/src/bitmessageqt/foldertree.py @@ -7,6 +7,7 @@ Folder tree and messagelist widgets definitions. from cgi import escape from PyQt4 import QtCore, QtGui +from dbcompat import dbstr from bmconfigparser import config from helper_sql import sqlExecute, sqlQuery @@ -111,7 +112,7 @@ class AccountMixin(object): elif config.safeGetBoolean(self.address, 'mailinglist'): self.type = self.MAILINGLIST elif sqlQuery( - '''select label from subscriptions where address=?''', self.address): + '''select label from subscriptions where address=?''', dbstr(self.address)): self.type = AccountMixin.SUBSCRIPTION else: self.type = self.NORMAL @@ -128,10 +129,10 @@ class AccountMixin(object): config.get(self.address, 'label'), 'utf-8') except Exception: queryreturn = sqlQuery( - '''select label from addressbook where address=?''', self.address) + '''select label from addressbook where address=?''', dbstr(self.address)) elif self.type == AccountMixin.SUBSCRIPTION: queryreturn = sqlQuery( - '''select label from subscriptions where address=?''', self.address) + '''select label from subscriptions where address=?''', dbstr(self.address)) if queryreturn is not None: if queryreturn != []: for row in queryreturn: @@ -307,7 +308,7 @@ class Ui_SubscriptionWidget(Ui_AddressWidget): def _getLabel(self): queryreturn = sqlQuery( - '''select label from subscriptions where address=?''', self.address) + '''select label from subscriptions where address=?''', dbstr(self.address)) if queryreturn != []: for row in queryreturn: retval, = row @@ -329,7 +330,7 @@ class Ui_SubscriptionWidget(Ui_AddressWidget): label = unicode(value, 'utf-8', 'ignore') sqlExecute( '''UPDATE subscriptions SET label=? WHERE address=?''', - label, self.address) + dbstr(label), dbstr(self.address)) return super(Ui_SubscriptionWidget, self).setData(column, role, value) @@ -412,10 +413,10 @@ class MessageList_AddressWidget(BMAddressWidget): 'utf-8', 'ignore') except: queryreturn = sqlQuery( - '''select label from addressbook where address=?''', self.address) + '''select label from addressbook where address=?''', dbstr(self.address)) elif self.type == AccountMixin.SUBSCRIPTION: queryreturn = sqlQuery( - '''select label from subscriptions where address=?''', self.address) + '''select label from subscriptions where address=?''', dbstr(self.address)) if queryreturn: for row in queryreturn: newLabel = row[0] @@ -526,9 +527,9 @@ class Ui_AddressBookWidgetItem(BMAddressWidget): config.set(self.address, 'label', self.label) config.save() except: - sqlExecute('''UPDATE addressbook set label=? WHERE address=?''', self.label, self.address) + sqlExecute('''UPDATE addressbook set label=? WHERE address=?''', dbstr(self.label), dbstr(self.address)) elif self.type == AccountMixin.SUBSCRIPTION: - sqlExecute('''UPDATE subscriptions set label=? WHERE address=?''', self.label, self.address) + sqlExecute('''UPDATE subscriptions set label=? WHERE address=?''', dbstr(self.label), dbstr(self.address)) else: pass return super(Ui_AddressBookWidgetItem, self).setData(role, value) diff --git a/src/bitmessageqt/support.py b/src/bitmessageqt/support.py index a84affa4..7f239003 100644 --- a/src/bitmessageqt/support.py +++ b/src/bitmessageqt/support.py @@ -7,6 +7,7 @@ import sys import time from PyQt4 import QtCore +from dbcompat import dbstr import account import defaults @@ -67,12 +68,12 @@ Connected hosts: {} def checkAddressBook(myapp): - sqlExecute('DELETE from addressbook WHERE address=?', OLD_SUPPORT_ADDRESS) - queryreturn = sqlQuery('SELECT * FROM addressbook WHERE address=?', SUPPORT_ADDRESS) + sqlExecute('DELETE from addressbook WHERE address=?', dbstr(OLD_SUPPORT_ADDRESS)) + queryreturn = sqlQuery('SELECT * FROM addressbook WHERE address=?', dbstr(SUPPORT_ADDRESS)) if queryreturn == []: sqlExecute( 'INSERT INTO addressbook VALUES (?,?)', - SUPPORT_LABEL.toUtf8(), SUPPORT_ADDRESS) + dbstr(SUPPORT_LABEL.toUtf8()), dbstr(SUPPORT_ADDRESS)) myapp.rerenderAddressBook() diff --git a/src/class_objectProcessor.py b/src/class_objectProcessor.py index 469ccbfa..9a0a9b9a 100644 --- a/src/class_objectProcessor.py +++ b/src/class_objectProcessor.py @@ -33,6 +33,7 @@ from helper_sql import ( from network import knownnodes from network.node import Peer from tr import _translate +from dbcompat import dbstr logger = logging.getLogger('default') @@ -327,19 +328,19 @@ class objectProcessor(threading.Thread): queryreturn = sqlQuery( "SELECT usedpersonally FROM pubkeys WHERE address=?" - " AND usedpersonally='yes'", address) + " AND usedpersonally='yes'", dbstr(address)) # if this pubkey is already in our database and if we have # used it personally: if queryreturn != []: logger.info( 'We HAVE used this pubkey personally. Updating time.') - t = (address, addressVersion, dataToStore, + t = (dbstr(address), addressVersion, dataToStore, int(time.time()), 'yes') else: logger.info( 'We have NOT used this pubkey personally. Inserting' ' in database.') - t = (address, addressVersion, dataToStore, + t = (dbstr(address), addressVersion, dataToStore, int(time.time()), 'no') sqlExecute('''INSERT INTO pubkeys VALUES (?,?,?,?,?)''', *t) self.possibleNewPubkey(address) @@ -389,20 +390,20 @@ class objectProcessor(threading.Thread): address = encodeAddress(addressVersion, streamNumber, ripe) queryreturn = sqlQuery( "SELECT usedpersonally FROM pubkeys WHERE address=?" - " AND usedpersonally='yes'", address) + " AND usedpersonally='yes'", dbstr(address)) # if this pubkey is already in our database and if we have # used it personally: if queryreturn != []: logger.info( 'We HAVE used this pubkey personally. Updating time.') - t = (address, addressVersion, dataToStore, - int(time.time()), 'yes') + t = (dbstr(address), addressVersion, dataToStore, + int(time.time()), dbstr('yes')) else: logger.info( 'We have NOT used this pubkey personally. Inserting' ' in database.') - t = (address, addressVersion, dataToStore, - int(time.time()), 'no') + t = (dbstr(address), addressVersion, dataToStore, + int(time.time()), dbstr('no')) sqlExecute('''INSERT INTO pubkeys VALUES (?,?,?,?,?)''', *t) self.possibleNewPubkey(address) @@ -590,11 +591,11 @@ class objectProcessor(threading.Thread): # person. sqlExecute( '''INSERT INTO pubkeys VALUES (?,?,?,?,?)''', - fromAddress, + dbstr(fromAddress), sendersAddressVersionNumber, decryptedData[:endOfThePublicKeyPosition], int(time.time()), - 'yes') + dbstr('yes')) # Check to see whether we happen to be awaiting this # pubkey in order to send a message. If we are, it will do the POW @@ -631,7 +632,7 @@ class objectProcessor(threading.Thread): 'bitmessagesettings', 'blackwhitelist') == 'black': queryreturn = sqlQuery( "SELECT label FROM blacklist where address=? and enabled='1'", - fromAddress) + dbstr(fromAddress)) if queryreturn != []: logger.info('Message ignored because address is in blacklist.') @@ -639,7 +640,7 @@ class objectProcessor(threading.Thread): else: # We're using a whitelist queryreturn = sqlQuery( "SELECT label FROM whitelist where address=? and enabled='1'", - fromAddress) + dbstr(fromAddress)) if queryreturn == []: logger.info( 'Message ignored because address not in whitelist.') @@ -927,11 +928,11 @@ class objectProcessor(threading.Thread): # Let's store the public key in case we want to reply to this person. sqlExecute('''INSERT INTO pubkeys VALUES (?,?,?,?,?)''', - fromAddress, - sendersAddressVersion, + dbstr(fromAddress), + dbstr(sendersAddressVersion), decryptedData[:endOfPubkeyPosition], int(time.time()), - 'yes') + dbstr('yes')) # Check to see whether we happen to be awaiting this # pubkey in order to send a message. If we are, it will do the POW @@ -1012,7 +1013,7 @@ class objectProcessor(threading.Thread): "UPDATE sent SET status='doingmsgpow', retrynumber=0" " WHERE toaddress=?" " AND (status='awaitingpubkey' OR status='doingpubkeypow')" - " AND folder='sent'", address) + " AND folder='sent'", dbstr(address)) queues.workerQueue.put(('sendmessage', '')) @staticmethod diff --git a/src/class_singleCleaner.py b/src/class_singleCleaner.py index 21aadfad..83cb48b1 100644 --- a/src/class_singleCleaner.py +++ b/src/class_singleCleaner.py @@ -29,6 +29,7 @@ from bmconfigparser import config from helper_sql import sqlExecute, sqlQuery from network import connectionpool, knownnodes, StoppableThread from tr import _translate +from dbcompat import dbstr #: Equals 4 weeks. You could make this longer if you want @@ -170,7 +171,7 @@ class singleCleaner(StoppableThread): )) sqlExecute( "UPDATE sent SET status = 'msgqueued'" - " WHERE toaddress = ? AND folder = 'sent'", address) + " WHERE toaddress = ? AND folder = 'sent'", dbstr(address)) queues.workerQueue.put(('sendmessage', '')) def resendMsg(self, ackdata): diff --git a/src/class_singleWorker.py b/src/class_singleWorker.py index 71e24c2e..7ee5138b 100644 --- a/src/class_singleWorker.py +++ b/src/class_singleWorker.py @@ -30,6 +30,7 @@ from bmconfigparser import config from helper_sql import sqlExecute, sqlQuery from network import knownnodes, StoppableThread from six.moves import configparser, queue +from dbcompat import dbstr def sizeof_fmt(num, suffix='h/s'): @@ -535,7 +536,7 @@ class singleWorker(StoppableThread): queryreturn = sqlQuery( '''SELECT fromaddress, subject, message, ''' ''' ackdata, ttl, encodingtype FROM sent ''' - ''' WHERE status=? and folder='sent' ''', 'broadcastqueued') + ''' WHERE status=? and folder='sent' ''', dbstr('broadcastqueued')) for row in queryreturn: fromaddress, subject, body, ackdata, TTL, encoding = row @@ -710,7 +711,7 @@ class singleWorker(StoppableThread): sqlExecute( '''UPDATE sent SET msgid=?, status=?, lastactiontime=? ''' ''' WHERE ackdata=? AND folder='sent' ''', - inventoryHash, 'broadcastsent', int(time.time()), ackdata + inventoryHash, dbstr('broadcastsent'), int(time.time()), ackdata ) def sendMsg(self): @@ -764,7 +765,7 @@ class singleWorker(StoppableThread): if not sqlExecute( '''UPDATE sent SET status='doingmsgpow' ''' ''' WHERE toaddress=? AND status='msgqueued' AND folder='sent' ''', - toaddress + dbstr(toaddress) ): continue status = 'doingmsgpow' @@ -772,7 +773,7 @@ class singleWorker(StoppableThread): # Let's see if we already have the pubkey in our pubkeys table queryreturn = sqlQuery( '''SELECT address FROM pubkeys WHERE address=?''', - toaddress + dbstr(toaddress) ) # If we have the needed pubkey in the pubkey table already, if queryreturn != []: @@ -780,7 +781,7 @@ class singleWorker(StoppableThread): if not sqlExecute( '''UPDATE sent SET status='doingmsgpow' ''' ''' WHERE toaddress=? AND status='msgqueued' AND folder='sent' ''', - toaddress + dbstr(toaddress) ): continue status = 'doingmsgpow' @@ -792,7 +793,7 @@ class singleWorker(StoppableThread): sqlExecute( '''UPDATE pubkeys SET usedpersonally='yes' ''' ''' WHERE address=?''', - toaddress + dbstr(toaddress) ) # We don't have the needed pubkey in the pubkeys table already. else: @@ -811,7 +812,7 @@ class singleWorker(StoppableThread): ''' sleeptill=? WHERE toaddress=? ''' ''' AND status='msgqueued' ''', int(time.time()) + 2.5 * 24 * 60 * 60, - toaddress + dbstr(toaddress) ) queues.UISignalQueue.put(( 'updateSentItemStatusByToAddress', ( @@ -867,7 +868,7 @@ class singleWorker(StoppableThread): ''' status='awaitingpubkey' or ''' ''' status='doingpubkeypow') AND ''' ''' folder='sent' ''', - toaddress) + dbstr(toaddress)) del state.neededPubkeys[tag] break # else: @@ -884,7 +885,7 @@ class singleWorker(StoppableThread): '''UPDATE sent SET ''' ''' status='doingpubkeypow' WHERE ''' ''' toaddress=? AND status='msgqueued' AND folder='sent' ''', - toaddress + dbstr(toaddress) ) queues.UISignalQueue.put(( 'updateSentItemStatusByToAddress', ( @@ -929,7 +930,7 @@ class singleWorker(StoppableThread): # is too hard then we'll abort. queryreturn = sqlQuery( 'SELECT transmitdata FROM pubkeys WHERE address=?', - toaddress) + dbstr(toaddress)) for row in queryreturn: # pylint: disable=redefined-outer-name pubkeyPayload, = row @@ -1349,7 +1350,7 @@ class singleWorker(StoppableThread): sqlExecute( '''UPDATE sent SET msgid=?, status=?, retrynumber=?, ''' ''' sleeptill=?, lastactiontime=? WHERE ackdata=? AND folder='sent' ''', - inventoryHash, newStatus, retryNumber + 1, + inventoryHash, dbstr(newStatus), retryNumber + 1, sleepTill, int(time.time()), ackdata ) @@ -1395,7 +1396,7 @@ class singleWorker(StoppableThread): '''SELECT retrynumber FROM sent WHERE toaddress=? ''' ''' AND (status='doingpubkeypow' OR status='awaitingpubkey') ''' ''' AND folder='sent' LIMIT 1''', - toAddress + dbstr(toAddress) ) if not queryReturn: self.logger.critical( @@ -1477,7 +1478,7 @@ class singleWorker(StoppableThread): ''' status='awaitingpubkey', retrynumber=?, sleeptill=? ''' ''' WHERE toaddress=? AND (status='doingpubkeypow' OR ''' ''' status='awaitingpubkey') AND folder='sent' ''', - int(time.time()), retryNumber + 1, sleeptill, toAddress) + int(time.time()), retryNumber + 1, sleeptill, dbstr(toAddress)) queues.UISignalQueue.put(( 'updateStatusBar', diff --git a/src/class_smtpServer.py b/src/class_smtpServer.py index 44ea7c9c..f753bd4e 100644 --- a/src/class_smtpServer.py +++ b/src/class_smtpServer.py @@ -20,6 +20,7 @@ from helper_ackPayload import genAckPayload from helper_sql import sqlExecute from network.threads import StoppableThread from version import softwareVersion +from dbcompat import dbstr SMTPDOMAIN = "bmaddr.lan" LISTENPORT = 8425 @@ -89,18 +90,18 @@ class smtpServerPyBitmessage(smtpd.SMTPServer): sqlExecute( '''INSERT INTO sent VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)''', '', - toAddress, + dbstr(toAddress), ripe, - fromAddress, - subject, - message, + dbstr(fromAddress), + dbstr(subject), + dbstr(message), ackdata, int(time.time()), # sentTime (this will never change) int(time.time()), # lastActionTime 0, # sleepTill time. This will get set when the POW gets done. - 'msgqueued', + dbstr('msgqueued'), 0, # retryNumber - 'sent', # folder + dbstr('sent'), # folder 2, # encodingtype # not necessary to have a TTL higher than 2 days min(config.getint('bitmessagesettings', 'ttl'), 86400 * 2) diff --git a/src/dbcompat.py b/src/dbcompat.py new file mode 100644 index 00000000..6a865630 --- /dev/null +++ b/src/dbcompat.py @@ -0,0 +1,22 @@ +import logging +import six + +logger = logging.getLogger("default") + +def dbstr(v): + if six.PY3: + if isinstance(v, str): + return v + elif isinstance(v, bytes): + return v.decode("utf-8", "replace") + logger.debug("unexpected type in dbstr(): {}".format(type(v))) + return v # hope this never happens.. + else: # assume six.PY2 + if isinstance(v, unicode): + return v.encode("utf-8", "replace") + elif isinstance(v, str): + return v + elif isinstance(v, bytes): + return str(v) + logger.debug("unexpected type in dbstr(): {}".format(type(v))) + return v # hope this never happens.. diff --git a/src/helper_addressbook.py b/src/helper_addressbook.py index 6d354113..6f1d40d2 100644 --- a/src/helper_addressbook.py +++ b/src/helper_addressbook.py @@ -4,11 +4,12 @@ Insert value into addressbook from bmconfigparser import config from helper_sql import sqlExecute +from dbcompat import dbstr def insert(address, label): """perform insert into addressbook""" if address not in config.addresses(): - return sqlExecute('''INSERT INTO addressbook VALUES (?,?)''', label, address) == 1 + return sqlExecute('''INSERT INTO addressbook VALUES (?,?)''', dbstr(label), dbstr(address)) == 1 return False diff --git a/src/helper_inbox.py b/src/helper_inbox.py index 555795df..1e490680 100644 --- a/src/helper_inbox.py +++ b/src/helper_inbox.py @@ -2,11 +2,13 @@ import queues from helper_sql import sqlExecute, sqlQuery +from dbcompat import dbstr def insert(t): """Perform an insert into the "inbox" table""" - sqlExecute('''INSERT INTO inbox VALUES (?,?,?,?,?,?,?,?,?,?)''', *t) + u = [t[0], dbstr(t[1]), dbstr(t[2]), dbstr(t[3]), dbstr(t[4]), dbstr(t[5]), dbstr(t[6]), t[7], t[8], t[9]] + sqlExecute('''INSERT INTO inbox VALUES (?,?,?,?,?,?,?,?,?,?)''', *u) # shouldn't emit changedInboxUnread and displayNewInboxMessage # at the same time # queues.UISignalQueue.put(('changedInboxUnread', None)) diff --git a/src/helper_search.py b/src/helper_search.py index 9fcb88b5..46cc4e80 100644 --- a/src/helper_search.py +++ b/src/helper_search.py @@ -5,6 +5,7 @@ Used by :mod:`.bitmessageqt`. from helper_sql import sqlQuery from tr import _translate +from dbcompat import dbstr def search_sql( @@ -52,23 +53,23 @@ def search_sql( if account is not None: if xAddress == 'both': sqlStatementParts.append('(fromaddress = ? OR toaddress = ?)') - sqlArguments.append(account) - sqlArguments.append(account) + sqlArguments.append(dbstr(account)) + sqlArguments.append(dbstr(account)) else: sqlStatementParts.append(xAddress + ' = ? ') - sqlArguments.append(account) + sqlArguments.append(dbstr(account)) if folder is not None: if folder == 'new': folder = 'inbox' unreadOnly = True sqlStatementParts.append('folder = ? ') - sqlArguments.append(folder) + sqlArguments.append(dbstr(folder)) else: sqlStatementParts.append('folder != ?') - sqlArguments.append('trash') + sqlArguments.append(dbstr('trash')) if what: sqlStatementParts.append('%s LIKE ?' % (where)) - sqlArguments.append(what) + sqlArguments.append(dbstr(what)) if unreadOnly: sqlStatementParts.append('read = 0') if sqlStatementParts: diff --git a/src/helper_sent.py b/src/helper_sent.py index aa76e756..2f24a619 100644 --- a/src/helper_sent.py +++ b/src/helper_sent.py @@ -8,6 +8,7 @@ from addresses import decodeAddress from bmconfigparser import config from helper_ackPayload import genAckPayload from helper_sql import sqlExecute, sqlQuery +from dbcompat import dbstr # pylint: disable=too-many-arguments @@ -38,8 +39,8 @@ def insert(msgid=None, toAddress='[Broadcast subscribers]', fromAddress=None, su ttl = ttl if ttl else config.getint('bitmessagesettings', 'ttl') - t = (msgid, toAddress, ripe, fromAddress, subject, message, ackdata, - sentTime, lastActionTime, sleeptill, status, retryNumber, folder, + t = (msgid, dbstr(toAddress), ripe, dbstr(fromAddress), dbstr(subject), dbstr(message), ackdata, + sentTime, lastActionTime, sleeptill, dbstr(status), retryNumber, dbstr(folder), encoding, ttl) sqlExecute('''INSERT INTO sent VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)''', *t) diff --git a/src/protocol.py b/src/protocol.py index 7f9830e5..9117cc7c 100644 --- a/src/protocol.py +++ b/src/protocol.py @@ -23,6 +23,7 @@ from debug import logger from helper_sql import sqlExecute from network.node import Peer from version import softwareVersion +from dbcompat import dbstr # Network constants magic = 0xE9BEB4D9 @@ -558,7 +559,7 @@ def decryptAndCheckPubkeyPayload(data, address): hexlify(pubSigningKey), hexlify(pubEncryptionKey) ) - t = (address, addressVersion, storedData, int(time.time()), 'yes') + t = (dbstr(address), addressVersion, storedData, int(time.time()), dbstr('yes')) sqlExecute('''INSERT INTO pubkeys VALUES (?,?,?,?,?)''', *t) return 'successful' except varintDecodeError: diff --git a/src/shared.py b/src/shared.py index aa7b6bb4..050dfa7b 100644 --- a/src/shared.py +++ b/src/shared.py @@ -22,6 +22,7 @@ from addresses import decodeAddress, encodeVarint from bmconfigparser import config from debug import logger from helper_sql import sqlQuery +from dbcompat import dbstr myECCryptorObjects = {} @@ -38,7 +39,7 @@ def isAddressInMyAddressBook(address): """Is address in my addressbook?""" queryreturn = sqlQuery( '''select address from addressbook where address=?''', - address) + dbstr(address)) return queryreturn != [] @@ -47,7 +48,7 @@ def isAddressInMySubscriptionsList(address): """Am I subscribed to this address?""" queryreturn = sqlQuery( '''select * from subscriptions where address=?''', - address.encode("utf-8", "replace")) + dbstr(address)) return queryreturn != [] @@ -61,14 +62,14 @@ def isAddressInMyAddressBookSubscriptionsListOrWhitelist(address): queryreturn = sqlQuery( '''SELECT address FROM whitelist where address=?''' ''' and enabled = '1' ''', - address) + dbstr(address)) if queryreturn != []: return True queryreturn = sqlQuery( '''select address from subscriptions where address=?''' ''' and enabled = '1' ''', - address) + dbstr(address)) if queryreturn != []: return True return False @@ -170,7 +171,7 @@ def fixPotentiallyInvalidUTF8Data(text): return text except UnicodeDecodeError: return 'Part of the message is corrupt. The message cannot be' \ - ' displayed the normal way.\n\n' + repr(text) + ' displayed the normal way.\n\n' + text.decode("utf-8", "replace") def checkSensitiveFilePermissions(filename): diff --git a/src/storage/sqlite.py b/src/storage/sqlite.py index eb5df098..2296175f 100644 --- a/src/storage/sqlite.py +++ b/src/storage/sqlite.py @@ -95,7 +95,7 @@ class SqliteInventory(InventoryStorage): t = int(time.time()) hashes = [x for x, value in self._inventory.items() if value.stream == stream and value.expires > t] - hashes += (str(payload) for payload, in sqlQuery( + hashes += (bytes(payload) for payload, in sqlQuery( 'SELECT hash FROM inventory WHERE streamnumber=?' ' AND expirestime>?', stream, t)) return hashes diff --git a/src/tests/core.py b/src/tests/core.py index f1a11a06..bf58f9f3 100644 --- a/src/tests/core.py +++ b/src/tests/core.py @@ -31,6 +31,7 @@ from network.node import Node, Peer from network.tcp import Socks4aBMConnection, Socks5BMConnection, TCPConnection from queues import excQueue from version import softwareVersion +from dbcompat import dbstr from common import cleanup @@ -346,11 +347,11 @@ class TestCore(unittest.TestCase): ) queryreturn = sqlQuery( '''select msgid from sent where ackdata=?''', result) - self.assertNotEqual(queryreturn[0][0] if queryreturn else '', '') + self.assertNotEqual(queryreturn[0][0] if queryreturn else b'', b'') column_type = sqlQuery( '''select typeof(msgid) from sent where ackdata=?''', result) - self.assertEqual(column_type[0][0] if column_type else '', 'text') + self.assertEqual(column_type[0][0] if column_type else '', 'blob') @unittest.skipIf(frozen, 'not packed test_pattern into the bundle') def test_old_knownnodes_pickle(self): @@ -368,7 +369,7 @@ class TestCore(unittest.TestCase): @staticmethod def delete_address_from_addressbook(address): """Clean up addressbook""" - sqlQuery('''delete from addressbook where address=?''', address) + sqlQuery('''delete from addressbook where address=?''', dbstr(address)) def test_add_same_address_twice_in_addressbook(self): """checking same address is added twice in addressbook""" @@ -382,7 +383,7 @@ class TestCore(unittest.TestCase): """checking is address added in addressbook or not""" helper_addressbook.insert(label='test1', address=self.addr) queryreturn = sqlQuery( - 'select count(*) from addressbook where address=?', self.addr) + 'select count(*) from addressbook where address=?', dbstr(self.addr)) self.assertEqual(queryreturn[0][0], 1) self.delete_address_from_addressbook(self.addr) diff --git a/src/tests/test_helper_inbox.py b/src/tests/test_helper_inbox.py index a0b6de1b..8ff60e18 100644 --- a/src/tests/test_helper_inbox.py +++ b/src/tests/test_helper_inbox.py @@ -26,7 +26,7 @@ class TestHelperInbox(unittest.TestCase): def test_insert(self, mock_sql_execute): # pylint: disable=no-self-use """Test to perform an insert into the "inbox" table""" mock_message_data = ( - "ruyv87bv", + b"ruyv87bv", "BM-2cUGaEcGz9Zft1SPAo8FJtfzyADTpEgU9U", "BM-2cUGaEcGz9Zft1SPAo8FJtfzyADTp5g99U", "Test subject", @@ -35,7 +35,7 @@ class TestHelperInbox(unittest.TestCase): "inbox", 2, 0, - "658gvjhtghv", + b"658gvjhtghv", ) insert(t=mock_message_data) mock_sql_execute.assert_called_once() @@ -43,7 +43,7 @@ class TestHelperInbox(unittest.TestCase): @patch("pybitmessage.helper_inbox.sqlExecute") def test_trash(self, mock_sql_execute): # pylint: disable=no-self-use """Test marking a message in the `inbox` as `trash`""" - mock_msg_id = "fefkosghsbse92" + mock_msg_id = b"fefkosghsbse92" trash(msgid=mock_msg_id) mock_sql_execute.assert_called_once() @@ -57,7 +57,7 @@ class TestHelperInbox(unittest.TestCase): @patch("pybitmessage.helper_inbox.sqlExecute") def test_undeleteMessage(self, mock_sql_execute): # pylint: disable=no-self-use """Test for Undelete the message""" - mock_msg_id = "fefkosghsbse92" + mock_msg_id = b"fefkosghsbse92" undeleteMessage(msgid=mock_msg_id) mock_sql_execute.assert_called_once() diff --git a/src/tests/test_helper_sent.py b/src/tests/test_helper_sent.py index 9227e43a..36bb8bb7 100644 --- a/src/tests/test_helper_sent.py +++ b/src/tests/test_helper_sent.py @@ -19,7 +19,7 @@ class TestHelperSent(unittest.TestCase): """Test insert with valid address""" VALID_ADDRESS = "BM-2cUGaEcGz9Zft1SPAo8FJtfzyADTpEgU9U" ackdata = insert( - msgid="123456", + msgid=b"123456", toAddress="[Broadcast subscribers]", fromAddress=VALID_ADDRESS, subject="Test Subject", @@ -45,10 +45,10 @@ class TestHelperSent(unittest.TestCase): @patch("pybitmessage.helper_sent.sqlExecute") def test_delete(self, mock_sql_execute): """Test delete function""" - delete("ack_data") + delete(b"ack_data") self.assertTrue(mock_sql_execute.called) mock_sql_execute.assert_called_once_with( - "DELETE FROM sent WHERE ackdata = ?", "ack_data" + "DELETE FROM sent WHERE ackdata = ?", b"ack_data" ) @patch("pybitmessage.helper_sent.sqlQuery") @@ -56,11 +56,11 @@ class TestHelperSent(unittest.TestCase): """Test retrieving valid message details""" return_data = [ ( - "to@example.com", - "from@example.com", - "Test Subject", - "Test Message", - "2022-01-01", + b"to@example.com", + b"from@example.com", + b"Test Subject", + b"Test Message", + b"2022-01-01", ) ] mock_sql_query.return_value = return_data @@ -70,7 +70,7 @@ class TestHelperSent(unittest.TestCase): @patch("pybitmessage.helper_sent.sqlExecute") def test_trash(self, mock_sql_execute): """Test marking a message as 'trash'""" - ackdata = "ack_data" + ackdata = b"ack_data" mock_sql_execute.return_value = 1 rowcount = trash(ackdata) self.assertEqual(rowcount, 1) diff --git a/src/tests/test_helper_sql.py b/src/tests/test_helper_sql.py index 036bd2c9..2e0a1776 100644 --- a/src/tests/test_helper_sql.py +++ b/src/tests/test_helper_sql.py @@ -23,23 +23,23 @@ class TestHelperSql(unittest.TestCase): @patch("pybitmessage.helper_sql.sqlReturnQueue.get") def test_sqlquery_no_args(self, mock_sqlreturnqueue_get, mock_sqlsubmitqueue_put): """Test sqlQuery with no additional arguments""" - mock_sqlreturnqueue_get.return_value = ("dummy_result", None) + mock_sqlreturnqueue_get.return_value = (b"dummy_result", None) result = helper_sql.sqlQuery( "SELECT msgid FROM inbox where folder='inbox' ORDER BY received" ) self.assertEqual(mock_sqlsubmitqueue_put.call_count, 2) - self.assertEqual(result, "dummy_result") + self.assertEqual(result.decode("utf-8", "replace"), "dummy_result") @patch("pybitmessage.helper_sql.sqlSubmitQueue.put") @patch("pybitmessage.helper_sql.sqlReturnQueue.get") def test_sqlquery_with_args(self, mock_sqlreturnqueue_get, mock_sqlsubmitqueue_put): """Test sqlQuery with additional arguments""" - mock_sqlreturnqueue_get.return_value = ("dummy_result", None) + mock_sqlreturnqueue_get.return_value = (b"dummy_result", None) result = helper_sql.sqlQuery( "SELECT address FROM addressbook WHERE address=?", "PB-5yfds868gbkj" ) self.assertEqual(mock_sqlsubmitqueue_put.call_count, 2) - self.assertEqual(result, "dummy_result") + self.assertEqual(result.decode("utf-8", "replace"), "dummy_result") @patch("pybitmessage.helper_sql.sqlSubmitQueue.put") @patch("pybitmessage.helper_sql.sqlReturnQueue.get") @@ -49,7 +49,7 @@ class TestHelperSql(unittest.TestCase): rowcount = helper_sql.sqlExecute( "UPDATE sent SET status = 'msgqueued'" "WHERE ackdata = ? AND folder = 'sent'", - "1710652313", + b"1710652313", ) self.assertEqual(mock_sqlsubmitqueue_put.call_count, 3) self.assertEqual(rowcount, 1) diff --git a/src/tests/test_shared.py b/src/tests/test_shared.py index 3e9be107..073f94e7 100644 --- a/src/tests/test_shared.py +++ b/src/tests/test_shared.py @@ -46,7 +46,7 @@ class TestShared(unittest.TestCase): address = sample_address # if address is in MyAddressbook - mock_sql_query.return_value = [address.encode("utf-8", "replace")] + mock_sql_query.return_value = [bytes(address)] return_val = isAddressInMyAddressBook(address) mock_sql_query.assert_called_once() self.assertTrue(return_val) @@ -64,7 +64,7 @@ class TestShared(unittest.TestCase): address = sample_address # if address is in MySubscriptionsList - mock_sql_query.return_value = [address.encode("utf-8", "replace")] + mock_sql_query.return_value = [bytes(address)] return_val = isAddressInMySubscriptionsList(address) self.assertTrue(return_val) @@ -78,7 +78,7 @@ class TestShared(unittest.TestCase): def test_reloadBroadcastSendersForWhichImWatching(self, mock_sql_query): """Test for reload Broadcast Senders For Which Im Watching""" mock_sql_query.return_value = [ - (sample_address.encode("utf-8", "replace"),), + (bytes(sample_address),), ] # before reload self.assertEqual(len(MyECSubscriptionCryptorObjects), 0) diff --git a/src/tests/test_sqlthread.py b/src/tests/test_sqlthread.py index a612df3a..7c6318e6 100644 --- a/src/tests/test_sqlthread.py +++ b/src/tests/test_sqlthread.py @@ -41,4 +41,4 @@ class TestSqlThread(unittest.TestCase): query = sqlQuery('SELECT enaddr(4, 1, "21122112211221122112")') self.assertEqual( - query[0][-1], encoded_str, "test case fail for create_function") + query[0][-1].decode("utf-8", "replace"), encoded_str, "test case fail for create_function")