diff --git a/src/addresses.py b/src/addresses.py index 885c1f64..6859a9ca 100644 --- a/src/addresses.py +++ b/src/addresses.py @@ -187,7 +187,7 @@ def decodeAddress(address): integer = decodeBase58(address) if integer == 0: status = 'invalidcharacters' - return status, 0, 0, '' + return status, 0, 0, b'' # after converting to hex, the string will be prepended # with a 0x and appended with a L in python2 hexdata = hex(integer)[2:].rstrip('L') @@ -200,23 +200,23 @@ def decodeAddress(address): if checksum != double_sha512(data[:-4])[0:4]: status = 'checksumfailed' - return status, 0, 0, '' + return status, 0, 0, b'' try: addressVersionNumber, bytesUsedByVersionNumber = decodeVarint(data[:9]) except varintDecodeError as e: logger.error(str(e)) status = 'varintmalformed' - return status, 0, 0, '' + return status, 0, 0, b'' if addressVersionNumber > 4: logger.error('cannot decode address version numbers this high') status = 'versiontoohigh' - return status, 0, 0, '' + return status, 0, 0, b'' elif addressVersionNumber == 0: logger.error('cannot decode address version numbers of zero.') status = 'versiontoohigh' - return status, 0, 0, '' + return status, 0, 0, b'' try: streamNumber, bytesUsedByStreamNumber = \ @@ -224,7 +224,7 @@ def decodeAddress(address): except varintDecodeError as e: logger.error(str(e)) status = 'varintmalformed' - return status, 0, 0, '' + return status, 0, 0, b'' status = 'success' if addressVersionNumber == 1: @@ -242,21 +242,21 @@ def decodeAddress(address): return status, addressVersionNumber, streamNumber, \ b'\x00\x00' + embeddedRipeData elif len(embeddedRipeData) < 18: - return 'ripetooshort', 0, 0, '' + return 'ripetooshort', 0, 0, b'' elif len(embeddedRipeData) > 20: - return 'ripetoolong', 0, 0, '' - return 'otherproblem', 0, 0, '' + return 'ripetoolong', 0, 0, b'' + return 'otherproblem', 0, 0, b'' elif addressVersionNumber == 4: embeddedRipeData = \ data[bytesUsedByVersionNumber + bytesUsedByStreamNumber:-4] if embeddedRipeData[0:1] == b'\x00': # In order to enforce address non-malleability, encoded # RIPE data must have NULL bytes removed from the front - return 'encodingproblem', 0, 0, '' + return 'encodingproblem', 0, 0, b'' elif len(embeddedRipeData) > 20: - return 'ripetoolong', 0, 0, '' + return 'ripetoolong', 0, 0, b'' elif len(embeddedRipeData) < 4: - return 'ripetooshort', 0, 0, '' + return 'ripetooshort', 0, 0, b'' x00string = b'\x00' * (20 - len(embeddedRipeData)) return status, addressVersionNumber, streamNumber, \ x00string + embeddedRipeData diff --git a/src/bitmessageqt/__init__.py b/src/bitmessageqt/__init__.py index 40113b5a..fd012e04 100644 --- a/src/bitmessageqt/__init__.py +++ b/src/bitmessageqt/__init__.py @@ -552,6 +552,8 @@ class MyForm(settingsmixin.SMainWindow): "GROUP BY toaddress, folder") for row in queryreturn: toaddress, folder, cnt = row + toaddress = toaddress.decode("utf-8", "replace") + folder = folder.decode("utf-8", "replace") total += cnt if toaddress in db and folder in db[toaddress]: db[toaddress][folder] = cnt @@ -1044,6 +1046,8 @@ class MyForm(settingsmixin.SMainWindow): normalUnread = {} broadcastsUnread = {} for addr, fld, count in queryReturn: + addr = addr.decode("utf-8", "replace") + fld = fld.decode("utf-8", "replace") try: normalUnread[addr][fld] = count except KeyError: @@ -1211,11 +1215,11 @@ class MyForm(settingsmixin.SMainWindow): items = [ MessageList_AddressWidget( - toAddress, unicode(acct.toLabel, 'utf-8'), not read), + toAddress, acct.toLabel, not read), MessageList_AddressWidget( - fromAddress, unicode(acct.fromLabel, 'utf-8'), not read), + fromAddress, acct.fromLabel, not read), MessageList_SubjectWidget( - str(subject), unicode(acct.subject, 'utf-8', 'replace'), + subject, acct.subject, not read), MessageList_TimeWidget( l10n.formatTimestamp(received), not read, received, msgid) @@ -1243,7 +1247,14 @@ class MyForm(settingsmixin.SMainWindow): xAddress, account, "sent", where, what, False) for row in queryreturn: - self.addMessageListItemSent(tableWidget, *row) + r = [] + r.append(row[0].decode("utf-8", "replace")) # toaddress + r.append(row[1].decode("utf-8", "replace")) # fromaddress + r.append(row[2].decode("utf-8", "replace")) # subject + r.append(row[3].decode("utf-8", "replace")) # status + r.append(row[3]) # ackdata + r.append(row[4]) # lastactiontime + self.addMessageListItemSent(tableWidget, *r) tableWidget.horizontalHeader().setSortIndicator( 3, QtCore.Qt.DescendingOrder) @@ -1284,6 +1295,10 @@ class MyForm(settingsmixin.SMainWindow): for row in queryreturn: toAddress, fromAddress, subject, _, msgid, received, read = row + toAddress = toAddress.decode("utf-8", "replace") + fromAddress = fromAddress.decode("utf-8", "replace") + subject = subject.decode("utf-8", "replace") + received = received.decode("utf-8", "replace") self.addMessageListItemInbox( tableWidget, toAddress, fromAddress, subject, msgid, received, read) @@ -1369,6 +1384,7 @@ class MyForm(settingsmixin.SMainWindow): SELECT msgid, toaddress, read FROM inbox where folder='inbox' ''') for msgid, toAddress, read in queryreturn: + toAddress = toAddress.decode("utf-8", "replace") if not read: # increment the unread subscriptions if True (1) @@ -1447,7 +1463,7 @@ class MyForm(settingsmixin.SMainWindow): # Adapters and converters for QT <-> sqlite def sqlInit(self): - register_adapter(QtCore.QByteArray, str) + register_adapter(QtCore.QByteArray, bytes) def indicatorInit(self): """ @@ -1995,9 +2011,9 @@ class MyForm(settingsmixin.SMainWindow): def rerenderAddressBook(self): def addRow (address, label, type): self.ui.tableWidgetAddressBook.insertRow(0) - newItem = Ui_AddressBookWidgetItemLabel(address, unicode(label, 'utf-8'), type) + newItem = Ui_AddressBookWidgetItemLabel(address, label, type) self.ui.tableWidgetAddressBook.setItem(0, 0, newItem) - newItem = Ui_AddressBookWidgetItemAddress(address, unicode(label, 'utf-8'), type) + newItem = Ui_AddressBookWidgetItemAddress(address, label, type) self.ui.tableWidgetAddressBook.setItem(0, 1, newItem) oldRows = {} @@ -2015,6 +2031,8 @@ class MyForm(settingsmixin.SMainWindow): queryreturn = sqlQuery('SELECT label, address FROM subscriptions WHERE enabled = 1') for row in queryreturn: label, address = row + label = label.decode("utf-8", "replace") + address = address.decode("utf-8", "replace") newRows[address] = [label, AccountMixin.SUBSCRIPTION] # chans for address in config.addresses(True): @@ -2025,6 +2043,8 @@ class MyForm(settingsmixin.SMainWindow): queryreturn = sqlQuery('SELECT * FROM addressbook') for row in queryreturn: label, address = row + label = label.decode("utf-8", "replace") + address = address.decode("utf-8", "replace") newRows[address] = [label, AccountMixin.NORMAL] completerList = [] @@ -2038,7 +2058,7 @@ class MyForm(settingsmixin.SMainWindow): self.ui.tableWidgetAddressBook.removeRow(oldRows[address][2]) for address in newRows: addRow(address, newRows[address][0], newRows[address][1]) - completerList.append(unicode(newRows[address][0], encoding="UTF-8") + " <" + address + ">") + completerList.append(newRows[address][0] + " <" + address + ">") # sort self.ui.tableWidgetAddressBook.sortByColumn( @@ -2273,6 +2293,7 @@ class MyForm(settingsmixin.SMainWindow): if queryreturn != []: for row in queryreturn: toLabel, = row + toLabel = toLabel.decode("utf-8", "replace") self.displayNewSentMessage( toAddress, toLabel, fromAddress, subject, message, ackdata) @@ -2920,6 +2941,7 @@ class MyForm(settingsmixin.SMainWindow): if queryreturn != []: for row in queryreturn: messageText, = row + messageText = messageText.decode("utf-8", "replace") lines = messageText.split('\n') totalLines = len(lines) @@ -3043,6 +3065,7 @@ class MyForm(settingsmixin.SMainWindow): if queryreturn != []: for row in queryreturn: messageAtCurrentInboxRow, = row + messageAtCurrentInboxRow = messageAtCurrentInboxRow.decode("utf-8", "replace") acct.parseMessage( toAddressAtCurrentInboxRow, fromAddressAtCurrentInboxRow, tableWidget.item(currentInboxRow, 2).subject, @@ -3274,6 +3297,7 @@ class MyForm(settingsmixin.SMainWindow): if queryreturn != []: for row in queryreturn: message, = row + message = message.decode("utf-8", "replace") defaultFilename = "".join(x for x in subjectAtCurrentInboxRow if x.isalnum()) + '.txt' filename = QtGui.QFileDialog.getSaveFileName( @@ -3285,7 +3309,7 @@ class MyForm(settingsmixin.SMainWindow): return try: f = open(filename, 'w') - f.write(message) + f.write(message.encode("utf-8", "replace")) f.close() except Exception: logger.exception('Message not saved', exc_info=True) @@ -4020,6 +4044,7 @@ class MyForm(settingsmixin.SMainWindow): queryreturn = sqlQuery('''SELECT status FROM sent where ackdata=?''', ackData) for row in queryreturn: status, = row + status = status.decode("utf-8", "replace") if status == 'toodifficult': self.popMenuSent.addAction(self.actionForceSend) @@ -4124,6 +4149,7 @@ class MyForm(settingsmixin.SMainWindow): try: message = queryreturn[-1][0] + message = message.decode("utf-8", "replace") except NameError: message = "" except IndexError: diff --git a/src/bitmessageqt/account.py b/src/bitmessageqt/account.py index 8c82c6f6..794c276e 100644 --- a/src/bitmessageqt/account.py +++ b/src/bitmessageqt/account.py @@ -38,6 +38,8 @@ def getSortedSubscriptions(count=False): ret = {} for row in queryreturn: label, address, enabled = row + label = label.decode("utf-8", "replace") + address = address.decode("utf-8", "replace") ret[address] = {} ret[address]["inbox"] = {} ret[address]["inbox"]['label'] = label @@ -50,6 +52,8 @@ def getSortedSubscriptions(count=False): GROUP BY inbox.fromaddress, folder''', str_broadcast_subscribers) for row in queryreturn: address, folder, cnt = row + address = address.decode("utf-8", "replace") + folder = folder.decode("utf-8", "replace") if folder not in ret[address]: ret[address][folder] = { 'label': ret[address]['inbox']['label'], @@ -137,12 +141,14 @@ class BMAccount(object): if queryreturn != []: for row in queryreturn: label, = row + label = label.decode("utf-8", "replace") else: queryreturn = sqlQuery( '''select label from subscriptions where address=?''', address) if queryreturn != []: for row in queryreturn: label, = row + label = label.replace("utf-8", "replace") return label def parseMessage(self, toAddress, fromAddress, subject, message): @@ -150,10 +156,7 @@ class BMAccount(object): self.toAddress = toAddress self.fromAddress = fromAddress - if isinstance(subject, unicode): - self.subject = str(subject) - else: - self.subject = subject + self.subject = subject self.message = message self.fromLabel = self.getLabel(fromAddress) self.toLabel = self.getLabel(toAddress) @@ -202,11 +205,11 @@ class GatewayAccount(BMAccount): sqlExecute( '''INSERT INTO sent VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)''', '', - self.toAddress, + self.toAddress.encode("utf-8", "replace"), ripe, - self.fromAddress, - self.subject, - self.message, + self.fromAddress.encode("utf-8", "replace"), + self.subject.encode("utf-8", "replace"), + self.message.encode("utf-8", "replace"), ackdata, int(time.time()), # sentTime (this will never change) int(time.time()), # lastActionTime diff --git a/src/bitmessageqt/blacklist.py b/src/bitmessageqt/blacklist.py index 093f23d8..41342bce 100644 --- a/src/bitmessageqt/blacklist.py +++ b/src/bitmessageqt/blacklist.py @@ -111,10 +111,10 @@ class Blacklist(QtGui.QWidget, RetranslateMixin): if isinstance(addressitem, QtGui.QTableWidgetItem): if self.radioButtonBlacklist.isChecked(): sqlExecute('''UPDATE blacklist SET label=? WHERE address=?''', - str(item.text()), str(addressitem.text())) + item.text().encode("utf-8", "replace"), addressitem.text().encode("utf-8", "replace")) else: sqlExecute('''UPDATE whitelist SET label=? WHERE address=?''', - str(item.text()), str(addressitem.text())) + item.text().encode("utf-8", "replace"), addressitem.text().encode("utf-8", "replace")) def init_blacklist_popup_menu(self, connectSignal=True): # Popup menu for the Blacklist page @@ -171,6 +171,8 @@ class Blacklist(QtGui.QWidget, RetranslateMixin): self.tableWidgetBlacklist.setSortingEnabled(False) for row in queryreturn: label, address, enabled = row + label = label.decode("utf-8", "replace") + address = address.decode("utf-8", "replace") self.tableWidgetBlacklist.insertRow(0) newItem = QtGui.QTableWidgetItem(unicode(label, 'utf-8')) if not enabled: @@ -198,11 +200,11 @@ class Blacklist(QtGui.QWidget, RetranslateMixin): if config.get('bitmessagesettings', 'blackwhitelist') == 'black': sqlExecute( '''DELETE FROM blacklist WHERE label=? AND address=?''', - str(labelAtCurrentRow), str(addressAtCurrentRow)) + labelAtCurrentRow.encode("utf-8", "replace"), addressAtCurrentRow.encode("utf-8", "replace")) else: sqlExecute( '''DELETE FROM whitelist WHERE label=? AND address=?''', - str(labelAtCurrentRow), str(addressAtCurrentRow)) + labelAtCurrentRow.encode("utf-8", "replace"), addressAtCurrentRow.encode("utf-8", "replace")) self.tableWidgetBlacklist.removeRow(currentRow) def on_action_BlacklistClipboard(self): @@ -227,11 +229,11 @@ class Blacklist(QtGui.QWidget, RetranslateMixin): if config.get('bitmessagesettings', 'blackwhitelist') == 'black': sqlExecute( '''UPDATE blacklist SET enabled=1 WHERE address=?''', - str(addressAtCurrentRow)) + addressAtCurrentRow.encode("utf-8", "replace")) else: sqlExecute( '''UPDATE whitelist SET enabled=1 WHERE address=?''', - str(addressAtCurrentRow)) + addressAtCurrentRow.encode("utf-8", "replace")) def on_action_BlacklistDisable(self): currentRow = self.tableWidgetBlacklist.currentRow() @@ -243,10 +245,10 @@ class Blacklist(QtGui.QWidget, RetranslateMixin): currentRow, 1).setTextColor(QtGui.QColor(128, 128, 128)) if config.get('bitmessagesettings', 'blackwhitelist') == 'black': sqlExecute( - '''UPDATE blacklist SET enabled=0 WHERE address=?''', str(addressAtCurrentRow)) + '''UPDATE blacklist SET enabled=0 WHERE address=?''', addressAtCurrentRow.encode("utf-8", "replace")) else: sqlExecute( - '''UPDATE whitelist SET enabled=0 WHERE address=?''', str(addressAtCurrentRow)) + '''UPDATE whitelist SET enabled=0 WHERE address=?''', addressAtCurrentRow.encode("utf-8", "replace")) def on_action_BlacklistSetAvatar(self): self.window().on_action_SetAvatar(self.tableWidgetBlacklist) diff --git a/src/bitmessageqt/foldertree.py b/src/bitmessageqt/foldertree.py index c50b7d3d..2acf215d 100644 --- a/src/bitmessageqt/foldertree.py +++ b/src/bitmessageqt/foldertree.py @@ -136,7 +136,7 @@ class AccountMixin(object): if queryreturn != []: for row in queryreturn: retval, = row - retval = unicode(retval, 'utf-8') + retval = retval.decode("utf-8", "replace") elif self.address is None or self.type == AccountMixin.ALL: return unicode( str(_translate("MainWindow", "All accounts")), 'utf-8') @@ -311,8 +311,8 @@ class Ui_SubscriptionWidget(Ui_AddressWidget): if queryreturn != []: for row in queryreturn: retval, = row - return unicode(retval, 'utf-8', 'ignore') - return unicode(self.address, 'utf-8') + return retval.decode("utf-8", "replace") + return self.address def setType(self): """Set account type""" @@ -418,7 +418,8 @@ class MessageList_AddressWidget(BMAddressWidget): '''select label from subscriptions where address=?''', self.address) if queryreturn: for row in queryreturn: - newLabel = unicode(row[0], 'utf-8', 'ignore') + newLabel = row[0] + newLabel = newLabel.decode("utf-8", "replace") self.label = newLabel @@ -456,7 +457,7 @@ class MessageList_SubjectWidget(BMTableWidgetItem): if role == QtCore.Qt.UserRole: return self.subject if role == QtCore.Qt.ToolTipRole: - return escape(unicode(self.subject, 'utf-8')) + return escape(self.subject) return super(MessageList_SubjectWidget, self).data(role) # label (or address) alphabetically, disabled at the end diff --git a/src/bitmessageqt/safehtmlparser.py b/src/bitmessageqt/safehtmlparser.py index d408d2c7..d027ec44 100644 --- a/src/bitmessageqt/safehtmlparser.py +++ b/src/bitmessageqt/safehtmlparser.py @@ -125,6 +125,8 @@ class SafeHTMLParser(HTMLParser): def feed(self, data): try: data = unicode(data, 'utf-8') + except TypeError: + pass except UnicodeDecodeError: data = unicode(data, 'utf-8', errors='replace') HTMLParser.feed(self, data) diff --git a/src/class_singleCleaner.py b/src/class_singleCleaner.py index 06153dcf..21aadfad 100644 --- a/src/class_singleCleaner.py +++ b/src/class_singleCleaner.py @@ -99,6 +99,8 @@ class singleCleaner(StoppableThread): tick - state.maximumLengthOfTimeToBotherResendingMessages ) for toAddress, ackData, status in queryreturn: + toAddress = toAddress.decode("utf-8", "replace") + status = status.decode("utf-8", "replace") if status == 'awaitingpubkey': self.resendPubkeyRequest(toAddress) elif status == 'msgsent': diff --git a/src/class_singleWorker.py b/src/class_singleWorker.py index f2821f65..71e24c2e 100644 --- a/src/class_singleWorker.py +++ b/src/class_singleWorker.py @@ -73,6 +73,7 @@ class singleWorker(StoppableThread): '''SELECT DISTINCT toaddress FROM sent''' ''' WHERE (status='awaitingpubkey' AND folder='sent')''') for toAddress, in queryreturn: + toAddress = toAddress.decode("utf-8", "replace") toAddressVersionNumber, toStreamNumber, toRipe = \ decodeAddress(toAddress)[1:] if toAddressVersionNumber <= 3: @@ -538,6 +539,9 @@ class singleWorker(StoppableThread): for row in queryreturn: fromaddress, subject, body, ackdata, TTL, encoding = row + fromaddress = fromaddress.decode("utf-8", "replace") + subject = subject.decode("utf-8", "replace") + body = body.decode("utf-8", "replace") # status _, addressVersionNumber, streamNumber, ripe = \ decodeAddress(fromaddress) @@ -726,6 +730,11 @@ class singleWorker(StoppableThread): for row in queryreturn: toaddress, fromaddress, subject, message, \ ackdata, status, TTL, retryNumber, encoding = row + toaddress = toaddress.decode("utf-8", "replace") + fromaddress = fromaddress.decode("utf-8", "replace") + subject = subject.decode("utf-8", "replace") + message = message.decode("utf-8", "replace") + status = status.decode("utf-8", "replace") # toStatus _, toAddressVersionNumber, toStreamNumber, toRipe = \ decodeAddress(toaddress) diff --git a/src/class_sqlThread.py b/src/class_sqlThread.py index 7df9e253..49e4b98b 100644 --- a/src/class_sqlThread.py +++ b/src/class_sqlThread.py @@ -38,7 +38,7 @@ class sqlThread(threading.Thread): helper_sql.sql_available = True config_ready.wait() self.conn = sqlite3.connect(state.appdata + 'messages.dat') - self.conn.text_factory = str + self.conn.text_factory = bytes self.cur = self.conn.cursor() self.cur.execute('PRAGMA secure_delete = true') @@ -542,7 +542,7 @@ class sqlThread(threading.Thread): shutil.move( paths.lookupAppdataFolder() + 'messages.dat', paths.lookupExeFolder() + 'messages.dat') self.conn = sqlite3.connect(paths.lookupExeFolder() + 'messages.dat') - self.conn.text_factory = str + self.conn.text_factory = bytes self.cur = self.conn.cursor() elif item == 'movemessagstoappdata': logger.debug('the sqlThread is moving the messages.dat file to the Appdata folder.') @@ -568,7 +568,7 @@ class sqlThread(threading.Thread): shutil.move( paths.lookupExeFolder() + 'messages.dat', paths.lookupAppdataFolder() + 'messages.dat') self.conn = sqlite3.connect(paths.lookupAppdataFolder() + 'messages.dat') - self.conn.text_factory = str + self.conn.text_factory = bytes self.cur = self.conn.cursor() elif item == 'deleteandvacuume': self.cur.execute('''delete from inbox where folder='trash' ''') diff --git a/src/helper_msgcoding.py b/src/helper_msgcoding.py index 05fa1c1b..62214525 100644 --- a/src/helper_msgcoding.py +++ b/src/helper_msgcoding.py @@ -71,7 +71,8 @@ class MsgEncode(object): def encodeSimple(self, message): """Handle simple encoding""" - self.data = 'Subject:%(subject)s\nBody:%(body)s' % message + data = 'Subject:%(subject)s\nBody:%(body)s' % message + self.data = data.encode("utf-8", "replace") self.length = len(self.data) def encodeTrivial(self, message): diff --git a/src/shared.py b/src/shared.py index b85ddb20..aa7b6bb4 100644 --- a/src/shared.py +++ b/src/shared.py @@ -47,7 +47,7 @@ def isAddressInMySubscriptionsList(address): """Am I subscribed to this address?""" queryreturn = sqlQuery( '''select * from subscriptions where address=?''', - str(address)) + address.encode("utf-8", "replace")) return queryreturn != [] @@ -136,6 +136,7 @@ def reloadBroadcastSendersForWhichImWatching(): logger.debug('reloading subscriptions...') for row in queryreturn: address, = row + address = address.decode("utf-8", "replace") # status addressVersionNumber, streamNumber, hashobj = decodeAddress(address)[1:] if addressVersionNumber == 2: diff --git a/src/tests/test_shared.py b/src/tests/test_shared.py index 39bedf32..3e9be107 100644 --- a/src/tests/test_shared.py +++ b/src/tests/test_shared.py @@ -46,7 +46,7 @@ class TestShared(unittest.TestCase): address = sample_address # if address is in MyAddressbook - mock_sql_query.return_value = [address] + mock_sql_query.return_value = [address.encode("utf-8", "replace")] return_val = isAddressInMyAddressBook(address) mock_sql_query.assert_called_once() self.assertTrue(return_val) @@ -64,7 +64,7 @@ class TestShared(unittest.TestCase): address = sample_address # if address is in MySubscriptionsList - mock_sql_query.return_value = [address] + mock_sql_query.return_value = [address.encode("utf-8", "replace")] return_val = isAddressInMySubscriptionsList(address) self.assertTrue(return_val) @@ -78,7 +78,7 @@ class TestShared(unittest.TestCase): def test_reloadBroadcastSendersForWhichImWatching(self, mock_sql_query): """Test for reload Broadcast Senders For Which Im Watching""" mock_sql_query.return_value = [ - (sample_address,), + (sample_address.encode("utf-8", "replace"),), ] # before reload self.assertEqual(len(MyECSubscriptionCryptorObjects), 0)