load data as 'bytes' instead of 'str' from SQLite

.gitignore

@@ -26,3 +26,4 @@ coverage.xml
 .buildozer
 .tox
 *.swp
+typescript

src/addresses.py

@@ -187,7 +187,7 @@ def decodeAddress(address):
         integer = decodeBase58(address)
     if integer == 0:
         status = 'invalidcharacters'
-        return status, 0, 0, ''
+        return status, 0, 0, b''
     # after converting to hex, the string will be prepended
     # with a 0x and appended with a L in python2
     hexdata = hex(integer)[2:].rstrip('L')
@@ -200,23 +200,23 @@ def decodeAddress(address):
 
     if checksum != double_sha512(data[:-4])[0:4]:
         status = 'checksumfailed'
-        return status, 0, 0, ''
+        return status, 0, 0, b''
 
     try:
         addressVersionNumber, bytesUsedByVersionNumber = decodeVarint(data[:9])
     except varintDecodeError as e:
         logger.error(str(e))
         status = 'varintmalformed'
-        return status, 0, 0, ''
+        return status, 0, 0, b''
 
     if addressVersionNumber > 4:
         logger.error('cannot decode address version numbers this high')
         status = 'versiontoohigh'
-        return status, 0, 0, ''
+        return status, 0, 0, b''
     elif addressVersionNumber == 0:
         logger.error('cannot decode address version numbers of zero.')
         status = 'versiontoohigh'
-        return status, 0, 0, ''
+        return status, 0, 0, b''
 
     try:
         streamNumber, bytesUsedByStreamNumber = \
@@ -224,7 +224,7 @@ def decodeAddress(address):
     except varintDecodeError as e:
         logger.error(str(e))
         status = 'varintmalformed'
-        return status, 0, 0, ''
+        return status, 0, 0, b''
 
     status = 'success'
     if addressVersionNumber == 1:
@@ -242,21 +242,21 @@ def decodeAddress(address):
             return status, addressVersionNumber, streamNumber, \
                 b'\x00\x00' + embeddedRipeData
         elif len(embeddedRipeData) < 18:
-            return 'ripetooshort', 0, 0, ''
+            return 'ripetooshort', 0, 0, b''
         elif len(embeddedRipeData) > 20:
-            return 'ripetoolong', 0, 0, ''
+            return 'ripetoolong', 0, 0, b''
-        return 'otherproblem', 0, 0, ''
+        return 'otherproblem', 0, 0, b''
     elif addressVersionNumber == 4:
         embeddedRipeData = \
             data[bytesUsedByVersionNumber + bytesUsedByStreamNumber:-4]
         if embeddedRipeData[0:1] == b'\x00':
             # In order to enforce address non-malleability, encoded
             # RIPE data must have NULL bytes removed from the front
-            return 'encodingproblem', 0, 0, ''
+            return 'encodingproblem', 0, 0, b''
         elif len(embeddedRipeData) > 20:
-            return 'ripetoolong', 0, 0, ''
+            return 'ripetoolong', 0, 0, b''
         elif len(embeddedRipeData) < 4:
-            return 'ripetooshort', 0, 0, ''
+            return 'ripetooshort', 0, 0, b''
         x00string = b'\x00' * (20 - len(embeddedRipeData))
         return status, addressVersionNumber, streamNumber, \
             x00string + embeddedRipeData

src/bitmessageqt/__init__.py

@@ -515,6 +515,8 @@ class MyForm(settingsmixin.SMainWindow):
             "GROUP BY toaddress, folder")
         for row in queryreturn:
             toaddress, folder, cnt = row
+            toaddress = toaddress.decode('utf-8', 'replace')
+            folder = folder.decode('utf-8', 'replace')
             total += cnt
             if toaddress in db and folder in db[toaddress]:
                 db[toaddress][folder] = cnt
@@ -966,6 +968,8 @@ class MyForm(settingsmixin.SMainWindow):
         normalUnread = {}
         broadcastsUnread = {}
         for addr, fld, count in queryReturn:
+            addr = addr.decode('utf-8', 'replace')
+            fld = fld.decode('utf-8', 'replace')
             try:
                 normalUnread[addr][fld] = count
             except KeyError:
@@ -1165,6 +1169,13 @@ class MyForm(settingsmixin.SMainWindow):
             xAddress, account, "sent", where, what, False)
 
         for row in queryreturn:
+            r = []
+            r.append(row[0].decode('utf-8', 'replace'))  # toaddress
+            r.append(row[1].decode('utf-8', 'replace'))  # fromaddress
+            r.append(row[2].decode('utf-8', 'replace'))  # subject
+            r.append(row[3].decode('utf-8', 'replace'))  # status
+            r.append(row[4])  # ackdata
+            r.append(row[5])  # lastactiontime
             self.addMessageListItemSent(tableWidget, *row)
 
         tableWidget.horizontalHeader().setSortIndicator(
@@ -1206,6 +1217,10 @@ class MyForm(settingsmixin.SMainWindow):
 
         for row in queryreturn:
             toAddress, fromAddress, subject, _, msgid, received, read = row
+            toAddress = toAddress.decode('utf-8', 'replace')
+            fromAddress = fromAddress.decode('utf-8', 'replace')
+            subject = subject.decode('utf-8', 'replace')
+            received = received.decode('utf-8', 'replace')
             self.addMessageListItemInbox(
                 tableWidget, toAddress, fromAddress, subject,
                 msgid, received, read)
@@ -1289,6 +1304,7 @@ class MyForm(settingsmixin.SMainWindow):
         SELECT msgid, toaddress, read FROM inbox where folder='inbox'
         ''')
         for msgid, toAddress, read in queryreturn:
+            toAddress = toAddress.decode('utf-8', 'replace')
 
             if not read:
                 # increment the unread subscriptions if True (1)
@@ -1367,7 +1383,7 @@ class MyForm(settingsmixin.SMainWindow):
 
     # Adapters and converters for QT <-> sqlite
     def sqlInit(self):
-        register_adapter(QtCore.QByteArray, str)
+        register_adapter(QtCore.QByteArray, bytes)
 
     def indicatorInit(self):
         """
@@ -1939,6 +1955,8 @@ class MyForm(settingsmixin.SMainWindow):
         queryreturn = sqlQuery('SELECT label, address FROM subscriptions WHERE enabled = 1')
         for row in queryreturn:
             label, address = row
+            label = label.decode('utf-8', 'replace')
+            address = address.decode('utf-8', 'replace')
             newRows[address] = [label, AccountMixin.SUBSCRIPTION]
         # chans
         for address in config.addresses(True):
@@ -1949,6 +1967,8 @@ class MyForm(settingsmixin.SMainWindow):
         queryreturn = sqlQuery('SELECT * FROM addressbook')
         for row in queryreturn:
             label, address = row
+            label = label.decode('utf-8', 'replace')
+            address = address.decode('utf-8', 'replace')
             newRows[address] = [label, AccountMixin.NORMAL]
 
         completerList = []
@@ -2199,6 +2219,7 @@ class MyForm(settingsmixin.SMainWindow):
                         if queryreturn != []:
                             for row in queryreturn:
                                 toLabel, = row
+                                toLabel = toLabel.decode('utf-8', 'replace')
 
                         self.displayNewSentMessage(
                             toAddress, toLabel, fromAddress, subject, message, ackdata)
@@ -2846,6 +2867,7 @@ class MyForm(settingsmixin.SMainWindow):
         if queryreturn != []:
             for row in queryreturn:
                 messageText, = row
+                messageText = messageText.decode('utf-8', 'replace')
 
         lines = messageText.split('\n')
         totalLines = len(lines)
@@ -2968,6 +2990,7 @@ class MyForm(settingsmixin.SMainWindow):
         if queryreturn != []:
             for row in queryreturn:
                 messageAtCurrentInboxRow, = row
+                messageAtCurrentInboxRow = messageAtCurrentInboxRow.decode('utf-8', 'replace')
         acct.parseMessage(
             toAddressAtCurrentInboxRow, fromAddressAtCurrentInboxRow,
             tableWidget.item(currentInboxRow, 2).subject,
@@ -3199,6 +3222,7 @@ class MyForm(settingsmixin.SMainWindow):
         if queryreturn != []:
             for row in queryreturn:
                 message, = row
+                message = message.decode('utf-8', 'replace')
 
         defaultFilename = "".join(x for x in subjectAtCurrentInboxRow if x.isalnum()) + '.txt'
         filename = QtWidgets.QFileDialog.getSaveFileName(
@@ -3944,6 +3968,7 @@ class MyForm(settingsmixin.SMainWindow):
             queryreturn = sqlQuery('''SELECT status FROM sent where ackdata=?''', ackData)
             for row in queryreturn:
                 status, = row
+                status = status.decode('utf-8', 'replace')
             if status == 'toodifficult':
                 self.popMenuSent.addAction(self.actionForceSend)
 
@@ -4047,6 +4072,7 @@ class MyForm(settingsmixin.SMainWindow):
 
         try:
             message = queryreturn[-1][0]
+            message = message.decode('utf-8', 'replace')
         except NameError:
             message = ""
         except IndexError:

src/bitmessageqt/account.py

@@ -38,6 +38,8 @@ def getSortedSubscriptions(count=False):
     ret = {}
     for row in queryreturn:
         label, address, enabled = row
+        label = label.decode('utf-8', 'replace')
+        address = address.decode('utf-8', 'replace')
         ret[address] = {}
         ret[address]["inbox"] = {}
         ret[address]["inbox"]['label'] = label
@@ -50,6 +52,8 @@ def getSortedSubscriptions(count=False):
             GROUP BY inbox.fromaddress, folder''', str_broadcast_subscribers)
         for row in queryreturn:
             address, folder, cnt = row
+            address = address.decode('utf-8', 'replace')
+            folder = folder.decode('utf-8', 'replace')
             if folder not in ret[address]:
                 ret[address][folder] = {
                     'label': ret[address]['inbox']['label'],
@@ -137,12 +141,14 @@ class BMAccount(object):
         if queryreturn != []:
             for row in queryreturn:
                 label, = row
+                label = label.decode('utf-8', 'replace')
         else:
             queryreturn = sqlQuery(
                 '''select label from subscriptions where address=?''', address)
             if queryreturn != []:
                 for row in queryreturn:
                     label, = row
+                    label = label.decode('utf-8', 'replace')
         return label
 
     def parseMessage(self, toAddress, fromAddress, subject, message):
@@ -199,11 +205,11 @@ class GatewayAccount(BMAccount):
         sqlExecute(
             '''INSERT INTO sent VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)''',
             '',
-            self.toAddress,
+            self.toAddress.encode(),
             ripe,
-            self.fromAddress,
+            self.fromAddress.encode(),
-            self.subject,
+            self.subject.encode(),
-            self.message,
+            self.message.encode(),
             ackdata,
             int(time.time()),  # sentTime (this will never change)
             int(time.time()),  # lastActionTime

src/bitmessageqt/blacklist.py

@@ -106,10 +106,10 @@ class Blacklist(QtWidgets.QWidget, RetranslateMixin):
             if isinstance(addressitem, QtWidgets.QTableWidgetItem):
                 if self.radioButtonBlacklist.isChecked():
                     sqlExecute('''UPDATE blacklist SET label=? WHERE address=?''',
-                            str(item.text()), str(addressitem.text()))
+                            item.text(), addressitem.text())
                 else:
                     sqlExecute('''UPDATE whitelist SET label=? WHERE address=?''',
-                            str(item.text()), str(addressitem.text()))
+                            item.text(), addressitem.text())
 
     def init_blacklist_popup_menu(self, connectSignal=True):
         # Popup menu for the Blacklist page
@@ -164,6 +164,8 @@ class Blacklist(QtWidgets.QWidget, RetranslateMixin):
         self.tableWidgetBlacklist.setSortingEnabled(False)
         for row in queryreturn:
             label, address, enabled = row
+            label = label.decode('utf-8', 'replace')
+            address = address.decode('utf-8', 'replace')
             self.tableWidgetBlacklist.insertRow(0)
             newItem = QtWidgets.QTableWidgetItem(label)
             if not enabled:
@@ -191,11 +193,11 @@ class Blacklist(QtWidgets.QWidget, RetranslateMixin):
         if config.get('bitmessagesettings', 'blackwhitelist') == 'black':
             sqlExecute(
                 '''DELETE FROM blacklist WHERE label=? AND address=?''',
-                str(labelAtCurrentRow), str(addressAtCurrentRow))
+                labelAtCurrentRow, addressAtCurrentRow)
         else:
             sqlExecute(
                 '''DELETE FROM whitelist WHERE label=? AND address=?''',
-                str(labelAtCurrentRow), str(addressAtCurrentRow))
+                labelAtCurrentRow, addressAtCurrentRow)
         self.tableWidgetBlacklist.removeRow(currentRow)
 
     def on_action_BlacklistClipboard(self):
@@ -220,11 +222,11 @@ class Blacklist(QtWidgets.QWidget, RetranslateMixin):
         if config.get('bitmessagesettings', 'blackwhitelist') == 'black':
             sqlExecute(
                 '''UPDATE blacklist SET enabled=1 WHERE address=?''',
-                str(addressAtCurrentRow))
+                addressAtCurrentRow)
         else:
             sqlExecute(
                 '''UPDATE whitelist SET enabled=1 WHERE address=?''',
-                str(addressAtCurrentRow))
+                addressAtCurrentRow)
 
     def on_action_BlacklistDisable(self):
         currentRow = self.tableWidgetBlacklist.currentRow()
@@ -236,10 +238,10 @@ class Blacklist(QtWidgets.QWidget, RetranslateMixin):
             currentRow, 1).setTextColor(QtGui.QColor(128, 128, 128))
         if config.get('bitmessagesettings', 'blackwhitelist') == 'black':
             sqlExecute(
-                '''UPDATE blacklist SET enabled=0 WHERE address=?''', str(addressAtCurrentRow))
+                '''UPDATE blacklist SET enabled=0 WHERE address=?''', addressAtCurrentRow)
         else:
             sqlExecute(
-                '''UPDATE whitelist SET enabled=0 WHERE address=?''', str(addressAtCurrentRow))
+                '''UPDATE whitelist SET enabled=0 WHERE address=?''', addressAtCurrentRow)
 
     def on_action_BlacklistSetAvatar(self):
         self.window().on_action_SetAvatar(self.tableWidgetBlacklist)

src/bitmessageqt/foldertree.py

@@ -135,6 +135,7 @@ class AccountMixin(object):
             if queryreturn != []:
                 for row in queryreturn:
                     retval, = row
+                    retval = retval.decode('utf-8', 'replace')
         elif self.address is None or self.type == AccountMixin.ALL:
             return _translate("MainWindow", "All accounts")
 
@@ -306,6 +307,7 @@ class Ui_SubscriptionWidget(Ui_AddressWidget):
         if queryreturn != []:
             for row in queryreturn:
                 retval, = row
+                retval = retval.decode('utf-8', 'replace')
             return retval
         return self.address
 
@@ -411,6 +413,7 @@ class MessageList_AddressWidget(BMAddressWidget):
         if queryreturn:
             for row in queryreturn:
                 newLabel = row[0]
+                newLabel = newLabel.decode('utf-8', 'replace')
 
         self.label = newLabel
 

src/class_singleCleaner.py

@@ -99,6 +99,8 @@ class singleCleaner(StoppableThread):
                     tick - state.maximumLengthOfTimeToBotherResendingMessages
                 )
                 for toAddress, ackData, status in queryreturn:
+                    toAddress = toAddress.decode('utf-8', 'replace')
+                    status = status.decode('utf-8', 'replace')
                     if status == 'awaitingpubkey':
                         self.resendPubkeyRequest(toAddress)
                     elif status == 'msgsent':

src/class_singleWorker.py

@@ -73,6 +73,7 @@ class singleWorker(StoppableThread):
             '''SELECT DISTINCT toaddress FROM sent'''
             ''' WHERE (status='awaitingpubkey' AND folder='sent')''')
         for toAddress, in queryreturn:
+            toAddress = toAddress.decode('utf-8', 'replace')
             toAddressVersionNumber, toStreamNumber, toRipe = \
                 decodeAddress(toAddress)[1:]
             if toAddressVersionNumber <= 3:
@@ -538,6 +539,9 @@ class singleWorker(StoppableThread):
 
         for row in queryreturn:
             fromaddress, subject, body, ackdata, TTL, encoding = row
+            fromaddress = fromaddress.decode('utf-8', 'replace')
+            subject = subject.decode('utf-8', 'replace')
+            body = body.decode('utf-8', 'replace')
             # status
             _, addressVersionNumber, streamNumber, ripe = \
                 decodeAddress(fromaddress)
@@ -726,6 +730,11 @@ class singleWorker(StoppableThread):
         for row in queryreturn:
             toaddress, fromaddress, subject, message, \
                 ackdata, status, TTL, retryNumber, encoding = row
+            toaddress = toaddress.decode('utf-8', 'replace')
+            fromaddress = fromaddress.decode('utf-8', 'replace')
+            subject = subject.decode('utf-8', 'replace')
+            message = message.decode('utf-8', 'replace')
+            status = status.decode('utf-8', 'replace')
             # toStatus
             _, toAddressVersionNumber, toStreamNumber, toRipe = \
                 decodeAddress(toaddress)

src/class_sqlThread.py

@@ -38,7 +38,7 @@ class sqlThread(threading.Thread):
         helper_sql.sql_available = True
         config_ready.wait()
         self.conn = sqlite3.connect(state.appdata + 'messages.dat')
-        self.conn.text_factory = str
+        self.conn.text_factory = bytes
         self.cur = self.conn.cursor()
 
         self.cur.execute('PRAGMA secure_delete = true')
@@ -542,7 +542,7 @@ class sqlThread(threading.Thread):
                 shutil.move(
                     paths.lookupAppdataFolder() + 'messages.dat', paths.lookupExeFolder() + 'messages.dat')
                 self.conn = sqlite3.connect(paths.lookupExeFolder() + 'messages.dat')
-                self.conn.text_factory = str
+                self.conn.text_factory = bytes
                 self.cur = self.conn.cursor()
             elif item == 'movemessagstoappdata':
                 logger.debug('the sqlThread is moving the messages.dat file to the Appdata folder.')
@@ -568,7 +568,7 @@ class sqlThread(threading.Thread):
                 shutil.move(
                     paths.lookupExeFolder() + 'messages.dat', paths.lookupAppdataFolder() + 'messages.dat')
                 self.conn = sqlite3.connect(paths.lookupAppdataFolder() + 'messages.dat')
-                self.conn.text_factory = str
+                self.conn.text_factory = bytes
                 self.cur = self.conn.cursor()
             elif item == 'deleteandvacuume':
                 self.cur.execute('''delete from inbox where folder='trash' ''')

src/shared.py

@@ -47,7 +47,7 @@ def isAddressInMySubscriptionsList(address):
     """Am I subscribed to this address?"""
     queryreturn = sqlQuery(
         '''select * from subscriptions where address=?''',
-        str(address))
+        address)
     return queryreturn != []
 
 
@@ -136,6 +136,7 @@ def reloadBroadcastSendersForWhichImWatching():
     logger.debug('reloading subscriptions...')
     for row in queryreturn:
         address, = row
+        address = address.decode('utf-8', 'replace')
         # status
         addressVersionNumber, streamNumber, hashobj = decodeAddress(address)[1:]
         if addressVersionNumber == 2:

src/tests/test_shared.py

@@ -46,7 +46,7 @@ class TestShared(unittest.TestCase):
         address = sample_address
 
         # if address is in MyAddressbook
-        mock_sql_query.return_value = [address]
+        mock_sql_query.return_value = [address.encode()]
         return_val = isAddressInMyAddressBook(address)
         mock_sql_query.assert_called_once()
         self.assertTrue(return_val)
@@ -64,7 +64,7 @@ class TestShared(unittest.TestCase):
         address = sample_address
 
         # if address is in MySubscriptionsList
-        mock_sql_query.return_value = [address]
+        mock_sql_query.return_value = [address.encode()]
         return_val = isAddressInMySubscriptionsList(address)
         self.assertTrue(return_val)
 
@@ -78,7 +78,7 @@ class TestShared(unittest.TestCase):
     def test_reloadBroadcastSendersForWhichImWatching(self, mock_sql_query):
         """Test for reload Broadcast Senders For Which Im Watching"""
         mock_sql_query.return_value = [
-            (sample_address,),
+            (sample_address.encode(),),
         ]
         # before reload
         self.assertEqual(len(MyECSubscriptionCryptorObjects), 0)