refined: read from and write to SQLite database in binary mode

This modification is a preparation for migration to Python3.

src/api.py

@@ -70,6 +70,7 @@ from struct import pack, unpack
 
 import six
 from six.moves import configparser, http_client, xmlrpc_server
+from dbcompat import dbstr
 
 import helper_inbox
 import helper_sent
@@ -531,12 +532,12 @@ class BMRPCDispatcher(object):
         message = shared.fixPotentiallyInvalidUTF8Data(message)
         return {
             'msgid': hexlify(msgid),
-            'toAddress': toAddress,
+            'toAddress': toAddress.decode("utf-8", "replace"),
-            'fromAddress': fromAddress,
+            'fromAddress': fromAddress.decode("utf-8", "replace"),
             'subject': base64.b64encode(subject),
             'message': base64.b64encode(message),
             'encodingType': encodingtype,
-            'receivedTime': received,
+            'receivedTime': received.decode("utf-8", "replace"),
             'read': read
         }
 
@@ -598,11 +599,12 @@ class BMRPCDispatcher(object):
         """
         queryreturn = sqlQuery(
             "SELECT label, address from addressbook WHERE label = ?",
-            label
+            dbstr(label)
         ) if label else sqlQuery("SELECT label, address from addressbook")
         data = []
         for label, address in queryreturn:
             label = shared.fixPotentiallyInvalidUTF8Data(label)
+            address = address.decode("utf-8", "replace")
             data.append({
                 'label': base64.b64encode(label),
                 'address': address
@@ -618,12 +620,12 @@ class BMRPCDispatcher(object):
         self._verifyAddress(address)
         # TODO: add unique together constraint in the table
         queryreturn = sqlQuery(
-            "SELECT address FROM addressbook WHERE address=?", address)
+            "SELECT address FROM addressbook WHERE address=?", dbstr(address))
         if queryreturn != []:
             raise APIError(
                 16, 'You already have this address in your address book.')
 
-        sqlExecute("INSERT INTO addressbook VALUES(?,?)", label, address)
+        sqlExecute("INSERT INTO addressbook VALUES(?,?)", dbstr(label), dbstr(address))
         queues.UISignalQueue.put(('rerenderMessagelistFromLabels', ''))
         queues.UISignalQueue.put(('rerenderMessagelistToLabels', ''))
         queues.UISignalQueue.put(('rerenderAddressBook', ''))
@@ -635,7 +637,7 @@ class BMRPCDispatcher(object):
         """Delete an entry from address book."""
         address = addBMIfNotPresent(address)
         self._verifyAddress(address)
-        sqlExecute('DELETE FROM addressbook WHERE address=?', address)
+        sqlExecute('DELETE FROM addressbook WHERE address=?', dbstr(address))
         queues.UISignalQueue.put(('rerenderMessagelistFromLabels', ''))
         queues.UISignalQueue.put(('rerenderMessagelistToLabels', ''))
         queues.UISignalQueue.put(('rerenderAddressBook', ''))
@@ -919,6 +921,7 @@ class BMRPCDispatcher(object):
             " ORDER BY received"
         )
         return {"inboxMessages": [
+
             self._dump_inbox_message(*data) for data in queryreturn
         ]}
 
@@ -1018,7 +1021,7 @@ class BMRPCDispatcher(object):
         queryreturn = sqlQuery(
             "SELECT msgid, toaddress, fromaddress, subject, received,"
             " message, encodingtype, read FROM inbox WHERE folder='inbox'"
-            " AND toAddress=?", toAddress)
+            " AND toAddress=?", dbstr(toAddress))
         return {"inboxMessages": [
             self._dump_inbox_message(*data) for data in queryreturn
         ]}
@@ -1055,7 +1058,7 @@ class BMRPCDispatcher(object):
             "SELECT msgid, toaddress, fromaddress, subject, lastactiontime,"
             " message, encodingtype, status, ackdata FROM sent"
             " WHERE folder='sent' AND fromAddress=? ORDER BY lastactiontime",
-            fromAddress
+            dbstr(fromAddress)
         )
         return {"sentMessages": [
             self._dump_sent_message(*data) for data in queryreturn
@@ -1150,9 +1153,9 @@ class BMRPCDispatcher(object):
 
         toLabel = ''
         queryreturn = sqlQuery(
-            "SELECT label FROM addressbook WHERE address=?", toAddress)
+            "SELECT label FROM addressbook WHERE address=?", dbstr(toAddress))
         try:
-            toLabel = queryreturn[0][0]
+            toLabel = queryreturn[0][0].decode("utf-8", "replace")
         except IndexError:
             pass
 
@@ -1219,7 +1222,7 @@ class BMRPCDispatcher(object):
         queryreturn = sqlQuery(
             "SELECT status FROM sent where ackdata=?", ackdata)
         try:
-            return queryreturn[0][0]
+            return queryreturn[0][0].decode("utf-8", "replace")
         except IndexError:
             return 'notfound'
 
@@ -1238,11 +1241,11 @@ class BMRPCDispatcher(object):
         # First we must check to see if the address is already in the
         # subscriptions list.
         queryreturn = sqlQuery(
-            "SELECT * FROM subscriptions WHERE address=?", address)
+            "SELECT * FROM subscriptions WHERE address=?", dbstr(address))
         if queryreturn:
             raise APIError(16, 'You are already subscribed to that address.')
         sqlExecute(
-            "INSERT INTO subscriptions VALUES (?,?,?)", label, address, True)
+            "INSERT INTO subscriptions VALUES (?,?,?)", dbstr(label), dbstr(address), True)
         shared.reloadBroadcastSendersForWhichImWatching()
         queues.UISignalQueue.put(('rerenderMessagelistFromLabels', ''))
         queues.UISignalQueue.put(('rerenderSubscriptions', ''))
@@ -1256,7 +1259,7 @@ class BMRPCDispatcher(object):
         """
 
         address = addBMIfNotPresent(address)
-        sqlExecute("DELETE FROM subscriptions WHERE address=?", address)
+        sqlExecute("DELETE FROM subscriptions WHERE address=?", dbstr(address))
         shared.reloadBroadcastSendersForWhichImWatching()
         queues.UISignalQueue.put(('rerenderMessagelistFromLabels', ''))
         queues.UISignalQueue.put(('rerenderSubscriptions', ''))
@@ -1274,6 +1277,7 @@ class BMRPCDispatcher(object):
         data = []
         for label, address, enabled in queryreturn:
             label = shared.fixPotentiallyInvalidUTF8Data(label)
+            address = address.decode("utf-8", "replace")
             data.append({
                 'label': base64.b64encode(label),
                 'address': address,

src/bitmessagecurses/__init__.py

@@ -30,6 +30,7 @@ import state
 from addresses import addBMIfNotPresent, decodeAddress
 from bmconfigparser import config
 from helper_sql import sqlExecute, sqlQuery
+from dbcompat import dbstr
 
 # pylint: disable=global-statement
 
@@ -401,6 +402,7 @@ def handlech(c, stdscr):
                                 body = "\n\n------------------------------------------------------\n"
                                 for row in ret:
                                     body, = row
+                                body = body.decode("utf-8", "replace")
 
                             sendMessage(fromaddr, toaddr, ischan, subject, body, True)
                             dialogreset(stdscr)
@@ -410,7 +412,7 @@ def handlech(c, stdscr):
                                 r, t = d.inputbox("Label for address \"" + addr + "\"")
                                 if r == d.DIALOG_OK:
                                     label = t
-                                    sqlExecute("INSERT INTO addressbook VALUES (?,?)", label, addr)
+                                    sqlExecute("INSERT INTO addressbook VALUES (?,?)", dbstr(label), dbstr(addr))
                                     # Prepend entry
                                     addrbook.reverse()
                                     addrbook.append([label, addr])
@@ -426,6 +428,7 @@ def handlech(c, stdscr):
                                 if ret != []:
                                     for row in ret:
                                         msg, = row
+                                    msg = msg.decode("utf-8", "replace")
                                     fh = open(t, "a")       # Open in append mode just in case
                                     fh.write(msg)
                                     fh.close()
@@ -463,7 +466,7 @@ def handlech(c, stdscr):
                             data = ""
                             ret = sqlQuery(
                                 "SELECT message FROM sent WHERE subject=? AND ackdata=?",
-                                sentbox[sentcur][4],
+                                dbstr(sentbox[sentcur][4]),
                                 sentbox[sentcur][6])
                             if ret != []:
                                 for row in ret:
@@ -478,7 +481,7 @@ def handlech(c, stdscr):
                         elif t == "2":       # Move to trash
                             sqlExecute(
                                 "UPDATE sent SET folder='trash' WHERE subject=? AND ackdata=?",
-                                sentbox[sentcur][4],
+                                dbstr(sentbox[sentcur][4]),
                                 sentbox[sentcur][6])
                             del sentbox[sentcur]
                             scrollbox(d, unicode(
@@ -711,29 +714,29 @@ def handlech(c, stdscr):
                                         subscriptions.append([label, addr, True])
                                         subscriptions.reverse()
 
-                                        sqlExecute("INSERT INTO subscriptions VALUES (?,?,?)", label, addr, True)
+                                        sqlExecute("INSERT INTO subscriptions VALUES (?,?,?)", dbstr(label), dbstr(addr), True)
                                         shared.reloadBroadcastSendersForWhichImWatching()
                         elif t == "2":
                             r, t = d.inputbox("Type in \"I want to delete this subscription\"")
                             if r == d.DIALOG_OK and t == "I want to delete this subscription":
                                 sqlExecute(
                                     "DELETE FROM subscriptions WHERE label=? AND address=?",
-                                    subscriptions[subcur][0],
+                                    dbstr(subscriptions[subcur][0]),
-                                    subscriptions[subcur][1])
+                                    dbstr(subscriptions[subcur][1]))
                                 shared.reloadBroadcastSendersForWhichImWatching()
                                 del subscriptions[subcur]
                         elif t == "3":
                             sqlExecute(
                                 "UPDATE subscriptions SET enabled=1 WHERE label=? AND address=?",
-                                subscriptions[subcur][0],
+                                dbstr(subscriptions[subcur][0]),
-                                subscriptions[subcur][1])
+                                dbstr(subscriptions[subcur][1]))
                             shared.reloadBroadcastSendersForWhichImWatching()
                             subscriptions[subcur][2] = True
                         elif t == "4":
                             sqlExecute(
                                 "UPDATE subscriptions SET enabled=0 WHERE label=? AND address=?",
-                                subscriptions[subcur][0],
+                                dbstr(subscriptions[subcur][0]),
-                                subscriptions[subcur][1])
+                                dbstr(subscriptions[subcur][1]))
                             shared.reloadBroadcastSendersForWhichImWatching()
                             subscriptions[subcur][2] = False
                 elif menutab == 6:
@@ -762,7 +765,7 @@ def handlech(c, stdscr):
                                 subscriptions.append([label, addr, True])
                                 subscriptions.reverse()
 
-                                sqlExecute("INSERT INTO subscriptions VALUES (?,?,?)", label, addr, True)
+                                sqlExecute("INSERT INTO subscriptions VALUES (?,?,?)", dbstr(label), dbstr(addr), True)
                                 shared.reloadBroadcastSendersForWhichImWatching()
                         elif t == "3":
                             r, t = d.inputbox("Input new address")
@@ -771,7 +774,7 @@ def handlech(c, stdscr):
                                 if addr not in [item[1] for i, item in enumerate(addrbook)]:
                                     r, t = d.inputbox("Label for address \"" + addr + "\"")
                                     if r == d.DIALOG_OK:
-                                        sqlExecute("INSERT INTO addressbook VALUES (?,?)", t, addr)
+                                        sqlExecute("INSERT INTO addressbook VALUES (?,?)", dbstr(t), dbstr(addr))
                                         # Prepend entry
                                         addrbook.reverse()
                                         addrbook.append([t, addr])
@@ -783,8 +786,8 @@ def handlech(c, stdscr):
                             if r == d.DIALOG_OK and t == "I want to delete this Address Book entry":
                                 sqlExecute(
                                     "DELETE FROM addressbook WHERE label=? AND address=?",
-                                    addrbook[abookcur][0],
+                                    dbstr(addrbook[abookcur][0]),
-                                    addrbook[abookcur][1])
+                                    dbstr(addrbook[abookcur][1]))
                                 del addrbook[abookcur]
                 elif menutab == 7:
                     set_background_title(d, "Blacklist Dialog Box")
@@ -800,20 +803,20 @@ def handlech(c, stdscr):
                             if r == d.DIALOG_OK and t == "I want to delete this Blacklist entry":
                                 sqlExecute(
                                     "DELETE FROM blacklist WHERE label=? AND address=?",
-                                    blacklist[blackcur][0],
+                                    dbstr(blacklist[blackcur][0]),
-                                    blacklist[blackcur][1])
+                                    dbstr(blacklist[blackcur][1]))
                                 del blacklist[blackcur]
                         elif t == "2":
                             sqlExecute(
                                 "UPDATE blacklist SET enabled=1 WHERE label=? AND address=?",
-                                blacklist[blackcur][0],
+                                dbstr(blacklist[blackcur][0]),
-                                blacklist[blackcur][1])
+                                dbstr(blacklist[blackcur][1]))
                             blacklist[blackcur][2] = True
                         elif t == "3":
                             sqlExecute(
                                 "UPDATE blacklist SET enabled=0 WHERE label=? AND address=?",
-                                blacklist[blackcur][0],
+                                dbstr(blacklist[blackcur][0]),
-                                blacklist[blackcur][1])
+                                dbstr(blacklist[blackcur][1]))
                             blacklist[blackcur][2] = False
                 dialogreset(stdscr)
         else:
@@ -991,10 +994,13 @@ def loadInbox():
     ret = sqlQuery("""SELECT msgid, toaddress, fromaddress, subject, received, read
         FROM inbox WHERE folder='inbox' AND %s LIKE ?
         ORDER BY received
-        """ % (where,), what)
+        """ % (where,), dbstr(what))
     for row in ret:
         msgid, toaddr, fromaddr, subject, received, read = row
+        toaddr = toaddr.decode("utf-8", "replace")
+        fromaddr = fromaddr.decode("utf-8", "replace")
         subject = ascii(shared.fixPotentiallyInvalidUTF8Data(subject))
+        received = received.decode("utf-8", "replace")
 
         # Set label for to address
         try:
@@ -1013,18 +1019,19 @@ def loadInbox():
         if config.has_section(fromaddr):
             fromlabel = config.get(fromaddr, "label")
         if fromlabel == "":         # Check Address Book
-            qr = sqlQuery("SELECT label FROM addressbook WHERE address=?", fromaddr)
+            qr = sqlQuery("SELECT label FROM addressbook WHERE address=?", dbstr(fromaddr))
             if qr != []:
                 for r in qr:
                     fromlabel, = r
+                fromlabel = shared.fixPotentiallyInvalidUTF8Data(fromlabel)
         if fromlabel == "":         # Check Subscriptions
-            qr = sqlQuery("SELECT label FROM subscriptions WHERE address=?", fromaddr)
+            qr = sqlQuery("SELECT label FROM subscriptions WHERE address=?", dbstr(fromaddr))
             if qr != []:
                 for r in qr:
                     fromlabel, = r
+                fromlabel = shared.fixPotentiallyInvalidUTF8Data(fromlabel)
         if fromlabel == "":
             fromlabel = fromaddr
-        fromlabel = shared.fixPotentiallyInvalidUTF8Data(fromlabel)
 
         # Load into array
         inbox.append([
@@ -1044,22 +1051,27 @@ def loadSent():
     ret = sqlQuery("""SELECT toaddress, fromaddress, subject, status, ackdata, lastactiontime
         FROM sent WHERE folder='sent' AND %s LIKE ?
         ORDER BY lastactiontime
-        """ % (where,), what)
+        """ % (where,), dbstr(what))
     for row in ret:
         toaddr, fromaddr, subject, status, ackdata, lastactiontime = row
+        toaddr = toaddr.decode("utf-8", "replace")
+        fromaddr = fromaddr.decode("utf-8", "replace")
         subject = ascii(shared.fixPotentiallyInvalidUTF8Data(subject))
+        status = status.decode("utf-8", "replace")
 
         # Set label for to address
         tolabel = ""
-        qr = sqlQuery("SELECT label FROM addressbook WHERE address=?", toaddr)
+        qr = sqlQuery("SELECT label FROM addressbook WHERE address=?", dbstr(toaddr))
         if qr != []:
             for r in qr:
                 tolabel, = r
+            tolabel = tolabel.decode("utf-8", "replace")
         if tolabel == "":
-            qr = sqlQuery("SELECT label FROM subscriptions WHERE address=?", toaddr)
+            qr = sqlQuery("SELECT label FROM subscriptions WHERE address=?", dbstr(toaddr))
             if qr != []:
                 for r in qr:
                     tolabel, = r
+                tolabel = tolabel.decode("utf-8", "replace")
         if tolabel == "":
             if config.has_section(toaddr):
                 tolabel = config.get(toaddr, "label")
@@ -1129,6 +1141,7 @@ def loadAddrBook():
     for row in ret:
         label, addr = row
         label = shared.fixPotentiallyInvalidUTF8Data(label)
+        addr = addr.decode("utf-8", "replace")
         addrbook.append([label, addr])
     addrbook.reverse()
 
@@ -1138,6 +1151,8 @@ def loadSubscriptions():
     ret = sqlQuery("SELECT label, address, enabled FROM subscriptions")
     for row in ret:
         label, address, enabled = row
+        label = label.decode("utf-8", "replace")
+        address = address.decode("utf-8", "replace")
         subscriptions.append([label, address, enabled])
     subscriptions.reverse()
 
@@ -1152,6 +1167,8 @@ def loadBlackWhiteList():
         ret = sqlQuery("SELECT label, address, enabled FROM whitelist")
     for row in ret:
         label, address, enabled = row
+        label = label.decode("utf-8", "replace")
+        address = address.decode("utf-8", "replace")
         blacklist.append([label, address, enabled])
     blacklist.reverse()
 

src/bitmessagekivy/baseclass/addressbook.py

@@ -29,6 +29,7 @@ from pybitmessage.bitmessagekivy.baseclass.common import (
 from pybitmessage.bitmessagekivy.baseclass.popup import SavedAddressDetailPopup
 from pybitmessage.bitmessagekivy.baseclass.addressbook_widgets import HelperAddressBook
 from pybitmessage.helper_sql import sqlExecute
+from dbcompat import dbstr
 
 logger = logging.getLogger('default')
 
@@ -59,7 +60,7 @@ class AddressBook(Screen, HelperAddressBook):
         self.ids.tag_label.text = ''
         self.queryreturn = kivy_helper_search.search_sql(
             xAddress, account, "addressbook", where, what, False)
-        self.queryreturn = [obj for obj in reversed(self.queryreturn)]
+        self.queryreturn = [[obj[0].decode("utf-8", "replace"), obj[1].decode("utf-8", "replace")] for obj in reversed(self.queryreturn)]
         if self.queryreturn:
             self.ids.tag_label.text = 'Address Book'
             self.has_refreshed = True
@@ -131,7 +132,7 @@ class AddressBook(Screen, HelperAddressBook):
         if self.ids.ml.children is not None:
             self.ids.tag_label.text = ''
         sqlExecute(
-            "DELETE FROM  addressbook WHERE address = ?", address)
+            "DELETE FROM  addressbook WHERE address = ?", dbstr(address))
         toast('Address Deleted')
 
     def close_pop(self, instance):
@@ -142,8 +143,13 @@ class AddressBook(Screen, HelperAddressBook):
     def update_addbook_label(self, instance):
         """Updating the label of address book address"""
         address_list = kivy_helper_search.search_sql(folder="addressbook")
-        stored_labels = [labels[0] for labels in address_list]
+        stored_labels = [labels[0].decode("utf-8", "replace") for labels in address_list]
-        add_dict = dict(address_list)
+        add_dict = {}
+        for row in address_list:
+            label, address = row
+            label = label.decode("utf-8", "replace")
+            address = address.decode("utf-8", "replace")
+            add_dict[label] = address
         label = str(self.addbook_popup.content_cls.ids.add_label.text)
         if label in stored_labels and self.address == add_dict[label]:
             stored_labels.remove(label)
@@ -151,7 +157,7 @@ class AddressBook(Screen, HelperAddressBook):
             sqlExecute("""
                 UPDATE addressbook
                 SET label = ?
-                WHERE address = ?""", label, self.addbook_popup.content_cls.address)
+                WHERE address = ?""", dbstr(label), dbstr(self.addbook_popup.content_cls.address))
             App.get_running_app().root.ids.id_addressbook.ids.ml.clear_widgets()
             App.get_running_app().root.ids.id_addressbook.loadAddresslist(None, 'All', '')
             self.addbook_popup.dismiss()

src/bitmessagekivy/baseclass/maildetail.py

@@ -119,16 +119,16 @@ class MailDetail(Screen):  # pylint: disable=too-many-instance-attributes
 
     def assign_mail_details(self, data):
         """Assigning mail details"""
-        subject = data[0][2].decode() if isinstance(data[0][2], bytes) else data[0][2]
+        subject = data[0][2].decode("utf-8", "replace") if isinstance(data[0][2], bytes) else data[0][2]
-        body = data[0][3].decode() if isinstance(data[0][2], bytes) else data[0][3]
+        body = data[0][3].decode("utf-8", "replace") if isinstance(data[0][2], bytes) else data[0][3]
-        self.to_addr = data[0][0] if len(data[0][0]) > 4 else ' '
+        self.to_addr = data[0][0].decode("utf-8", "replace") if len(data[0][0]) > 4 else ' '
-        self.from_addr = data[0][1]
+        self.from_addr = data[0][1].decode("utf-8", "replace")
 
         self.subject = subject.capitalize(
         ) if subject.capitalize() else self.no_subject
         self.message = body
         if len(data[0]) == 7:
-            self.status = data[0][4]
+            self.status = data[0][4].decode("utf-8", "replace")
         self.time_tag = show_time_history(data[0][4]) if self.kivy_state.detail_page_type == 'inbox' \
             else show_time_history(data[0][6])
         self.avatarImg = os.path.join(self.kivy_state.imageDir, 'draft-icon.png') \

src/bitmessagekivy/baseclass/popup.py

@@ -59,7 +59,7 @@ class AddAddressPopup(BoxLayout):
         """Checking address is valid or not"""
         my_addresses = (
             App.get_running_app().root.ids.content_drawer.ids.identity_dropdown.values)
-        add_book = [addr[1] for addr in kivy_helper_search.search_sql(
+        add_book = [addr[1].decode("utf-8", "replace") for addr in kivy_helper_search.search_sql(
             folder="addressbook")]
         entered_text = str(instance.text).strip()
         if entered_text in add_book:
@@ -84,7 +84,7 @@ class AddAddressPopup(BoxLayout):
     def checkLabel_valid(self, instance):
         """Checking address label is unique or not"""
         entered_label = instance.text.strip()
-        addr_labels = [labels[0] for labels in kivy_helper_search.search_sql(
+        addr_labels = [labels[0].decode("utf-8", "replace") for labels in kivy_helper_search.search_sql(
             folder="addressbook")]
         if entered_label in addr_labels:
             self.ids.label.error = True
@@ -125,8 +125,13 @@ class SavedAddressDetailPopup(BoxLayout):
         """Checking address label is unique of not"""
         entered_label = str(instance.text.strip())
         address_list = kivy_helper_search.search_sql(folder="addressbook")
-        addr_labels = [labels[0] for labels in address_list]
+        addr_labels = [labels[0].decode("utf-8", "replace") for labels in address_list]
-        add_dict = dict(address_list)
+        add_dict = {}
+        for row in address_list:
+            label, address = row
+            label = label.decode("utf-8", "replace")
+            address = address.decode("utf-8", "replace")
+            add_dict[label] = address
         if self.address and entered_label in addr_labels \
                 and self.address != add_dict[entered_label]:
             self.ids.add_label.error = True

src/bitmessagekivy/kivy_helper_search.py

@@ -2,6 +2,7 @@
 Sql queries for bitmessagekivy
 """
 from pybitmessage.helper_sql import sqlQuery
+from dbcompat import dbstr
 
 
 def search_sql(
@@ -30,21 +31,21 @@ def search_sql(
     if account is not None:
         if xAddress == 'both':
             sqlStatementParts.append("(fromaddress = ? OR toaddress = ?)")
-            sqlArguments.append(account)
+            sqlArguments.append(dbstr(account))
-            sqlArguments.append(account)
+            sqlArguments.append(dbstr(account))
         else:
             sqlStatementParts.append(xAddress + " = ? ")
-            sqlArguments.append(account)
+            sqlArguments.append(dbstr(account))
     if folder != "addressbook":
         if folder is not None:
             if folder == "new":
                 folder = "inbox"
                 unreadOnly = True
             sqlStatementParts.append("folder = ? ")
-            sqlArguments.append(folder)
+            sqlArguments.append(dbstr(folder))
         else:
             sqlStatementParts.append("folder != ?")
-            sqlArguments.append("trash")
+            sqlArguments.append(dbstr("trash"))
     if what is not None:
         for colmns in where:
             if len(where) > 1:
@@ -54,7 +55,7 @@ def search_sql(
                     filter_col += " or %s LIKE ? )" % (colmns)
             else:
                 filter_col = "%s LIKE ?" % (colmns)
-            sqlArguments.append(what)
+            sqlArguments.append(dbstr(what))
         sqlStatementParts.append(filter_col)
     if unreadOnly:
         sqlStatementParts.append("read = 0")

src/bitmessageqt/__init__.py

@@ -17,6 +17,7 @@ from sqlite3 import register_adapter
 
 from PyQt4 import QtCore, QtGui
 from PyQt4.QtNetwork import QLocalSocket, QLocalServer
+from dbcompat import dbstr
 
 import shared
 import state
@@ -1067,8 +1068,10 @@ class MyForm(settingsmixin.SMainWindow):
             queryReturn = sqlQuery(
                 'SELECT fromaddress, folder, COUNT(msgid) AS cnt'
                 ' FROM inbox WHERE read = 0 AND toaddress = ?'
-                ' GROUP BY fromaddress, folder', str_broadcast_subscribers)
+                ' GROUP BY fromaddress, folder', dbstr(str_broadcast_subscribers))
             for addr, fld, count in queryReturn:
+                addr = addr.decode("utf-8", "replace")
+                fld = fld.decode("utf-8", "replace")
                 try:
                     broadcastsUnread[addr][fld] = count
                 except KeyError:
@@ -2289,7 +2292,7 @@ class MyForm(settingsmixin.SMainWindow):
                             subject=subject, message=message, encoding=encoding)
                         toLabel = ''
                         queryreturn = sqlQuery('''select label from addressbook where address=?''',
-                                               toAddress)
+                                               dbstr(toAddress))
                         if queryreturn != []:
                             for row in queryreturn:
                                 toLabel, = row
@@ -2578,7 +2581,7 @@ class MyForm(settingsmixin.SMainWindow):
         # Add to database (perhaps this should be separated from the MyForm class)
         sqlExecute(
             '''INSERT INTO subscriptions VALUES (?,?,?)''',
-            label, address, True
+            dbstr(label), dbstr(address), True
         )
         self.rerenderMessagelistFromLabels()
         shared.reloadBroadcastSendersForWhichImWatching()
@@ -3177,13 +3180,13 @@ class MyForm(settingsmixin.SMainWindow):
             currentInboxRow, 0).data(QtCore.Qt.UserRole)
         # Let's make sure that it isn't already in the address book
         queryreturn = sqlQuery('''select * from blacklist where address=?''',
-                               addressAtCurrentInboxRow)
+                               dbstr(addressAtCurrentInboxRow))
         if queryreturn == []:
             label = "\"" + tableWidget.item(currentInboxRow, 2).subject + "\" in " + config.get(
                 recipientAddress, "label")
             sqlExecute('''INSERT INTO blacklist VALUES (?,?, ?)''',
-                       label,
+                       dbstr(label),
-                       addressAtCurrentInboxRow, True)
+                       dbstr(addressAtCurrentInboxRow), True)
             self.ui.blackwhitelist.rerenderBlackWhiteList()
             self.updateStatusBar(_translate(
                 "MainWindow",
@@ -3370,7 +3373,7 @@ class MyForm(settingsmixin.SMainWindow):
                 0].row()
             item = self.ui.tableWidgetAddressBook.item(currentRow, 0)
             sqlExecute(
-                'DELETE FROM addressbook WHERE address=?', item.address)
+                'DELETE FROM addressbook WHERE address=?', dbstr(item.address))
             self.ui.tableWidgetAddressBook.removeRow(currentRow)
         self.rerenderMessagelistFromLabels()
         self.rerenderMessagelistToLabels()
@@ -3470,7 +3473,7 @@ class MyForm(settingsmixin.SMainWindow):
             return
         address = self.getCurrentAccount()
         sqlExecute('''DELETE FROM subscriptions WHERE address=?''',
-                   address)
+                   dbstr(address))
         self.rerenderTabTreeSubscriptions()
         self.rerenderMessagelistFromLabels()
         self.rerenderAddressBook()
@@ -3485,7 +3488,7 @@ class MyForm(settingsmixin.SMainWindow):
         address = self.getCurrentAccount()
         sqlExecute(
             '''update subscriptions set enabled=1 WHERE address=?''',
-            address)
+            dbstr(address))
         account = self.getCurrentItem()
         account.setEnabled(True)
         self.rerenderAddressBook()
@@ -3495,7 +3498,7 @@ class MyForm(settingsmixin.SMainWindow):
         address = self.getCurrentAccount()
         sqlExecute(
             '''update subscriptions set enabled=0 WHERE address=?''',
-            address)
+            dbstr(address))
         account = self.getCurrentItem()
         account.setEnabled(False)
         self.rerenderAddressBook()

src/bitmessageqt/account.py

@@ -15,6 +15,7 @@ import sys
 import time
 
 from PyQt4 import QtGui
+from dbcompat import dbstr
 
 import queues
 from addresses import decodeAddress
@@ -49,7 +50,7 @@ def getSortedSubscriptions(count=False):
         queryreturn = sqlQuery('''SELECT fromaddress, folder, count(msgid) as cnt
             FROM inbox, subscriptions ON subscriptions.address = inbox.fromaddress
             WHERE read = 0 AND toaddress = ?
-            GROUP BY inbox.fromaddress, folder''', str_broadcast_subscribers)
+            GROUP BY inbox.fromaddress, folder''', dbstr(str_broadcast_subscribers))
         for row in queryreturn:
             address, folder, cnt = row
             address = address.decode("utf-8", "replace")
@@ -104,7 +105,7 @@ class AccountColor(AccountMixin):  # pylint: disable=too-few-public-methods
             elif config.safeGetBoolean(self.address, 'chan'):
                 self.type = AccountMixin.CHAN
             elif sqlQuery(
-                    '''select label from subscriptions where address=?''', self.address):
+                    '''select label from subscriptions where address=?''', dbstr(self.address)):
                 self.type = AccountMixin.SUBSCRIPTION
             else:
                 self.type = AccountMixin.NORMAL
@@ -127,7 +128,7 @@ class BMAccount(object):
             self.type = AccountMixin.BROADCAST
         else:
             queryreturn = sqlQuery(
-                '''select label from subscriptions where address=?''', self.address)
+                '''select label from subscriptions where address=?''', dbstr(self.address))
             if queryreturn:
                 self.type = AccountMixin.SUBSCRIPTION
 
@@ -137,18 +138,18 @@ class BMAccount(object):
             address = self.address
         label = config.safeGet(address, 'label', address)
         queryreturn = sqlQuery(
-            '''select label from addressbook where address=?''', address)
+            '''select label from addressbook where address=?''', dbstr(address))
         if queryreturn != []:
             for row in queryreturn:
                 label, = row
             label = label.decode("utf-8", "replace")
         else:
             queryreturn = sqlQuery(
-                '''select label from subscriptions where address=?''', address)
+                '''select label from subscriptions where address=?''', dbstr(address))
             if queryreturn != []:
                 for row in queryreturn:
                     label, = row
-                label = label.replace("utf-8", "replace")
+                label = label.decode("utf-8", "replace")
         return label
 
     def parseMessage(self, toAddress, fromAddress, subject, message):
@@ -205,18 +206,18 @@ class GatewayAccount(BMAccount):
         sqlExecute(
             '''INSERT INTO sent VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)''',
             '',
-            self.toAddress.encode("utf-8", "replace"),
+            dbstr(self.toAddress),
             ripe,
-            self.fromAddress.encode("utf-8", "replace"),
+            dbstr(self.fromAddress),
-            self.subject.encode("utf-8", "replace"),
+            dbstr(self.subject),
-            self.message.encode("utf-8", "replace"),
+            dbstr(self.message),
             ackdata,
             int(time.time()),  # sentTime (this will never change)
             int(time.time()),  # lastActionTime
             0,  # sleepTill time. This will get set when the POW gets done.
-            'msgqueued',
+            dbstr('msgqueued'),
             0,  # retryNumber
-            'sent',  # folder
+            dbstr('sent'),  # folder
             2,  # encodingtype
             # not necessary to have a TTL higher than 2 days
             min(config.getint('bitmessagesettings', 'ttl'), 86400 * 2)

src/bitmessageqt/blacklist.py

@@ -1,4 +1,5 @@
 from PyQt4 import QtCore, QtGui
+from dbcompat import dbstr
 
 import widgets
 from addresses import addBMIfNotPresent
@@ -64,7 +65,7 @@ class Blacklist(QtGui.QWidget, RetranslateMixin):
                 # First we must check to see if the address is already in the
                 # address book. The user cannot add it again or else it will
                 # cause problems when updating and deleting the entry.
-                t = (address,)
+                t = (dbstr(address),)
                 if config.get('bitmessagesettings', 'blackwhitelist') == 'black':
                     sql = '''select * from blacklist where address=?'''
                 else:
@@ -82,7 +83,7 @@ class Blacklist(QtGui.QWidget, RetranslateMixin):
                         QtCore.Qt.ItemIsSelectable | QtCore.Qt.ItemIsEnabled)
                     self.tableWidgetBlacklist.setItem(0, 1, newItem)
                     self.tableWidgetBlacklist.setSortingEnabled(True)
-                    t = (str(self.NewBlacklistDialogInstance.lineEditLabel.text().toUtf8()), address, True)
+                    t = (dbstr(self.NewBlacklistDialogInstance.lineEditLabel.text().toUtf8()), dbstr(address), True)
                     if config.get('bitmessagesettings', 'blackwhitelist') == 'black':
                         sql = '''INSERT INTO blacklist VALUES (?,?,?)'''
                     else:
@@ -111,10 +112,10 @@ class Blacklist(QtGui.QWidget, RetranslateMixin):
             if isinstance(addressitem, QtGui.QTableWidgetItem):
                 if self.radioButtonBlacklist.isChecked():
                     sqlExecute('''UPDATE blacklist SET label=? WHERE address=?''',
-                            item.text().encode("utf-8", "replace"), addressitem.text().encode("utf-8", "replace"))
+                            dbstr(item.text()), dbstr(addressitem.text()))
                 else:
                     sqlExecute('''UPDATE whitelist SET label=? WHERE address=?''',
-                            item.text().encode("utf-8", "replace"), addressitem.text().encode("utf-8", "replace"))
+                            dbstr(item.text()), dbstr(addressitem.text()))
 
     def init_blacklist_popup_menu(self, connectSignal=True):
         # Popup menu for the Blacklist page
@@ -200,11 +201,11 @@ class Blacklist(QtGui.QWidget, RetranslateMixin):
         if config.get('bitmessagesettings', 'blackwhitelist') == 'black':
             sqlExecute(
                 '''DELETE FROM blacklist WHERE label=? AND address=?''',
-                labelAtCurrentRow.encode("utf-8", "replace"), addressAtCurrentRow.encode("utf-8", "replace"))
+                dbstr(labelAtCurrentRow), dbstr(addressAtCurrentRow))
         else:
             sqlExecute(
                 '''DELETE FROM whitelist WHERE label=? AND address=?''',
-                labelAtCurrentRow.encode("utf-8", "replace"), addressAtCurrentRow.encode("utf-8", "replace"))
+                dbstr(labelAtCurrentRow), dbstr(addressAtCurrentRow))
         self.tableWidgetBlacklist.removeRow(currentRow)
 
     def on_action_BlacklistClipboard(self):
@@ -229,11 +230,11 @@ class Blacklist(QtGui.QWidget, RetranslateMixin):
         if config.get('bitmessagesettings', 'blackwhitelist') == 'black':
             sqlExecute(
                 '''UPDATE blacklist SET enabled=1 WHERE address=?''',
-                addressAtCurrentRow.encode("utf-8", "replace"))
+                dbstr(addressAtCurrentRow))
         else:
             sqlExecute(
                 '''UPDATE whitelist SET enabled=1 WHERE address=?''',
-                addressAtCurrentRow.encode("utf-8", "replace"))
+                dbstr(addressAtCurrentRow))
 
     def on_action_BlacklistDisable(self):
         currentRow = self.tableWidgetBlacklist.currentRow()
@@ -245,10 +246,10 @@ class Blacklist(QtGui.QWidget, RetranslateMixin):
             currentRow, 1).setTextColor(QtGui.QColor(128, 128, 128))
         if config.get('bitmessagesettings', 'blackwhitelist') == 'black':
             sqlExecute(
-                '''UPDATE blacklist SET enabled=0 WHERE address=?''', addressAtCurrentRow.encode("utf-8", "replace"))
+                '''UPDATE blacklist SET enabled=0 WHERE address=?''', dbstr(addressAtCurrentRow))
         else:
             sqlExecute(
-                '''UPDATE whitelist SET enabled=0 WHERE address=?''', addressAtCurrentRow.encode("utf-8", "replace"))
+                '''UPDATE whitelist SET enabled=0 WHERE address=?''', dbstr(addressAtCurrentRow))
 
     def on_action_BlacklistSetAvatar(self):
         self.window().on_action_SetAvatar(self.tableWidgetBlacklist)

src/bitmessageqt/foldertree.py

@@ -7,6 +7,7 @@ Folder tree and messagelist widgets definitions.
 from cgi import escape
 
 from PyQt4 import QtCore, QtGui
+from dbcompat import dbstr
 
 from bmconfigparser import config
 from helper_sql import sqlExecute, sqlQuery
@@ -111,7 +112,7 @@ class AccountMixin(object):
         elif config.safeGetBoolean(self.address, 'mailinglist'):
             self.type = self.MAILINGLIST
         elif sqlQuery(
-                '''select label from subscriptions where address=?''', self.address):
+                '''select label from subscriptions where address=?''', dbstr(self.address)):
             self.type = AccountMixin.SUBSCRIPTION
         else:
             self.type = self.NORMAL
@@ -128,10 +129,10 @@ class AccountMixin(object):
                     config.get(self.address, 'label'), 'utf-8')
             except Exception:
                 queryreturn = sqlQuery(
-                    '''select label from addressbook where address=?''', self.address)
+                    '''select label from addressbook where address=?''', dbstr(self.address))
         elif self.type == AccountMixin.SUBSCRIPTION:
             queryreturn = sqlQuery(
-                '''select label from subscriptions where address=?''', self.address)
+                '''select label from subscriptions where address=?''', dbstr(self.address))
         if queryreturn is not None:
             if queryreturn != []:
                 for row in queryreturn:
@@ -307,7 +308,7 @@ class Ui_SubscriptionWidget(Ui_AddressWidget):
 
     def _getLabel(self):
         queryreturn = sqlQuery(
-            '''select label from subscriptions where address=?''', self.address)
+            '''select label from subscriptions where address=?''', dbstr(self.address))
         if queryreturn != []:
             for row in queryreturn:
                 retval, = row
@@ -329,7 +330,7 @@ class Ui_SubscriptionWidget(Ui_AddressWidget):
                 label = unicode(value, 'utf-8', 'ignore')
             sqlExecute(
                 '''UPDATE subscriptions SET label=? WHERE address=?''',
-                label, self.address)
+                dbstr(label), dbstr(self.address))
         return super(Ui_SubscriptionWidget, self).setData(column, role, value)
 
 
@@ -412,10 +413,10 @@ class MessageList_AddressWidget(BMAddressWidget):
                     'utf-8', 'ignore')
             except:
                 queryreturn = sqlQuery(
-                    '''select label from addressbook where address=?''', self.address)
+                    '''select label from addressbook where address=?''', dbstr(self.address))
         elif self.type == AccountMixin.SUBSCRIPTION:
             queryreturn = sqlQuery(
-                '''select label from subscriptions where address=?''', self.address)
+                '''select label from subscriptions where address=?''', dbstr(self.address))
         if queryreturn:
             for row in queryreturn:
                 newLabel = row[0]
@@ -526,9 +527,9 @@ class Ui_AddressBookWidgetItem(BMAddressWidget):
                     config.set(self.address, 'label', self.label)
                     config.save()
                 except:
-                    sqlExecute('''UPDATE addressbook set label=? WHERE address=?''', self.label, self.address)
+                    sqlExecute('''UPDATE addressbook set label=? WHERE address=?''', dbstr(self.label), dbstr(self.address))
             elif self.type == AccountMixin.SUBSCRIPTION:
-                sqlExecute('''UPDATE subscriptions set label=? WHERE address=?''', self.label, self.address)
+                sqlExecute('''UPDATE subscriptions set label=? WHERE address=?''', dbstr(self.label), dbstr(self.address))
             else:
                 pass
         return super(Ui_AddressBookWidgetItem, self).setData(role, value)

src/bitmessageqt/support.py

@@ -7,6 +7,7 @@ import sys
 import time
 
 from PyQt4 import QtCore
+from dbcompat import dbstr
 
 import account
 import defaults
@@ -67,12 +68,12 @@ Connected hosts: {}
 
 
 def checkAddressBook(myapp):
-    sqlExecute('DELETE from addressbook WHERE address=?', OLD_SUPPORT_ADDRESS)
+    sqlExecute('DELETE from addressbook WHERE address=?', dbstr(OLD_SUPPORT_ADDRESS))
-    queryreturn = sqlQuery('SELECT * FROM addressbook WHERE address=?', SUPPORT_ADDRESS)
+    queryreturn = sqlQuery('SELECT * FROM addressbook WHERE address=?', dbstr(SUPPORT_ADDRESS))
     if queryreturn == []:
         sqlExecute(
             'INSERT INTO addressbook VALUES (?,?)',
-            SUPPORT_LABEL.toUtf8(), SUPPORT_ADDRESS)
+            dbstr(SUPPORT_LABEL.toUtf8()), dbstr(SUPPORT_ADDRESS))
         myapp.rerenderAddressBook()
 
 

src/class_objectProcessor.py

@@ -33,6 +33,7 @@ from helper_sql import (
 from network import knownnodes
 from network.node import Peer
 from tr import _translate
+from dbcompat import dbstr
 
 logger = logging.getLogger('default')
 
@@ -327,19 +328,19 @@ class objectProcessor(threading.Thread):
 
             queryreturn = sqlQuery(
                 "SELECT usedpersonally FROM pubkeys WHERE address=?"
-                " AND usedpersonally='yes'", address)
+                " AND usedpersonally='yes'", dbstr(address))
             # if this pubkey is already in our database and if we have
             # used it personally:
             if queryreturn != []:
                 logger.info(
                     'We HAVE used this pubkey personally. Updating time.')
-                t = (address, addressVersion, dataToStore,
+                t = (dbstr(address), addressVersion, dataToStore,
                      int(time.time()), 'yes')
             else:
                 logger.info(
                     'We have NOT used this pubkey personally. Inserting'
                     ' in database.')
-                t = (address, addressVersion, dataToStore,
+                t = (dbstr(address), addressVersion, dataToStore,
                      int(time.time()), 'no')
             sqlExecute('''INSERT INTO pubkeys VALUES (?,?,?,?,?)''', *t)
             self.possibleNewPubkey(address)
@@ -389,20 +390,20 @@ class objectProcessor(threading.Thread):
             address = encodeAddress(addressVersion, streamNumber, ripe)
             queryreturn = sqlQuery(
                 "SELECT usedpersonally FROM pubkeys WHERE address=?"
-                " AND usedpersonally='yes'", address)
+                " AND usedpersonally='yes'", dbstr(address))
             # if this pubkey is already in our database and if we have
             # used it personally:
             if queryreturn != []:
                 logger.info(
                     'We HAVE used this pubkey personally. Updating time.')
-                t = (address, addressVersion, dataToStore,
+                t = (dbstr(address), addressVersion, dataToStore,
-                     int(time.time()), 'yes')
+                     int(time.time()), dbstr('yes'))
             else:
                 logger.info(
                     'We have NOT used this pubkey personally. Inserting'
                     ' in database.')
-                t = (address, addressVersion, dataToStore,
+                t = (dbstr(address), addressVersion, dataToStore,
-                     int(time.time()), 'no')
+                     int(time.time()), dbstr('no'))
             sqlExecute('''INSERT INTO pubkeys VALUES (?,?,?,?,?)''', *t)
             self.possibleNewPubkey(address)
 
@@ -590,11 +591,11 @@ class objectProcessor(threading.Thread):
         # person.
         sqlExecute(
             '''INSERT INTO pubkeys VALUES (?,?,?,?,?)''',
-            fromAddress,
+            dbstr(fromAddress),
             sendersAddressVersionNumber,
             decryptedData[:endOfThePublicKeyPosition],
             int(time.time()),
-            'yes')
+            dbstr('yes'))
 
         # Check to see whether we happen to be awaiting this
         # pubkey in order to send a message. If we are, it will do the POW
@@ -631,7 +632,7 @@ class objectProcessor(threading.Thread):
                 'bitmessagesettings', 'blackwhitelist') == 'black':
             queryreturn = sqlQuery(
                 "SELECT label FROM blacklist where address=? and enabled='1'",
-                fromAddress)
+                dbstr(fromAddress))
             if queryreturn != []:
                 logger.info('Message ignored because address is in blacklist.')
 
@@ -639,7 +640,7 @@ class objectProcessor(threading.Thread):
         else:  # We're using a whitelist
             queryreturn = sqlQuery(
                 "SELECT label FROM whitelist where address=? and enabled='1'",
-                fromAddress)
+                dbstr(fromAddress))
             if queryreturn == []:
                 logger.info(
                     'Message ignored because address not in whitelist.')
@@ -927,11 +928,11 @@ class objectProcessor(threading.Thread):
 
         # Let's store the public key in case we want to reply to this person.
         sqlExecute('''INSERT INTO pubkeys VALUES (?,?,?,?,?)''',
-                   fromAddress,
+                   dbstr(fromAddress),
-                   sendersAddressVersion,
+                   dbstr(sendersAddressVersion),
                    decryptedData[:endOfPubkeyPosition],
                    int(time.time()),
-                   'yes')
+                   dbstr('yes'))
 
         # Check to see whether we happen to be awaiting this
         # pubkey in order to send a message. If we are, it will do the POW
@@ -1012,7 +1013,7 @@ class objectProcessor(threading.Thread):
             "UPDATE sent SET status='doingmsgpow', retrynumber=0"
             " WHERE toaddress=?"
             " AND (status='awaitingpubkey' OR status='doingpubkeypow')"
-            " AND folder='sent'", address)
+            " AND folder='sent'", dbstr(address))
         queues.workerQueue.put(('sendmessage', ''))
 
     @staticmethod

src/class_singleCleaner.py

@@ -29,6 +29,7 @@ from bmconfigparser import config
 from helper_sql import sqlExecute, sqlQuery
 from network import connectionpool, knownnodes, StoppableThread
 from tr import _translate
+from dbcompat import dbstr
 
 
 #: Equals 4 weeks. You could make this longer if you want
@@ -170,7 +171,7 @@ class singleCleaner(StoppableThread):
         ))
         sqlExecute(
             "UPDATE sent SET status = 'msgqueued'"
-            " WHERE toaddress = ? AND folder = 'sent'", address)
+            " WHERE toaddress = ? AND folder = 'sent'", dbstr(address))
         queues.workerQueue.put(('sendmessage', ''))
 
     def resendMsg(self, ackdata):

src/class_singleWorker.py

@@ -30,6 +30,7 @@ from bmconfigparser import config
 from helper_sql import sqlExecute, sqlQuery
 from network import knownnodes, StoppableThread
 from six.moves import configparser, queue
+from dbcompat import dbstr
 
 
 def sizeof_fmt(num, suffix='h/s'):
@@ -535,7 +536,7 @@ class singleWorker(StoppableThread):
         queryreturn = sqlQuery(
             '''SELECT fromaddress, subject, message, '''
             ''' ackdata, ttl, encodingtype FROM sent '''
-            ''' WHERE status=? and folder='sent' ''', 'broadcastqueued')
+            ''' WHERE status=? and folder='sent' ''', dbstr('broadcastqueued'))
 
         for row in queryreturn:
             fromaddress, subject, body, ackdata, TTL, encoding = row
@@ -710,7 +711,7 @@ class singleWorker(StoppableThread):
             sqlExecute(
                 '''UPDATE sent SET msgid=?, status=?, lastactiontime=? '''
                 ''' WHERE ackdata=? AND folder='sent' ''',
-                inventoryHash, 'broadcastsent', int(time.time()), ackdata
+                inventoryHash, dbstr('broadcastsent'), int(time.time()), ackdata
             )
 
     def sendMsg(self):
@@ -764,7 +765,7 @@ class singleWorker(StoppableThread):
                 if not sqlExecute(
                     '''UPDATE sent SET status='doingmsgpow' '''
                     ''' WHERE toaddress=? AND status='msgqueued' AND folder='sent' ''',
-                    toaddress
+                    dbstr(toaddress)
                 ):
                     continue
                 status = 'doingmsgpow'
@@ -772,7 +773,7 @@ class singleWorker(StoppableThread):
                 # Let's see if we already have the pubkey in our pubkeys table
                 queryreturn = sqlQuery(
                     '''SELECT address FROM pubkeys WHERE address=?''',
-                    toaddress
+                    dbstr(toaddress)
                 )
                 # If we have the needed pubkey in the pubkey table already,
                 if queryreturn != []:
@@ -780,7 +781,7 @@ class singleWorker(StoppableThread):
                     if not sqlExecute(
                         '''UPDATE sent SET status='doingmsgpow' '''
                         ''' WHERE toaddress=? AND status='msgqueued' AND folder='sent' ''',
-                        toaddress
+                        dbstr(toaddress)
                     ):
                         continue
                     status = 'doingmsgpow'
@@ -792,7 +793,7 @@ class singleWorker(StoppableThread):
                     sqlExecute(
                         '''UPDATE pubkeys SET usedpersonally='yes' '''
                         ''' WHERE address=?''',
-                        toaddress
+                        dbstr(toaddress)
                     )
                 # We don't have the needed pubkey in the pubkeys table already.
                 else:
@@ -811,7 +812,7 @@ class singleWorker(StoppableThread):
                             ''' sleeptill=? WHERE toaddress=? '''
                             ''' AND status='msgqueued' ''',
                             int(time.time()) + 2.5 * 24 * 60 * 60,
-                            toaddress
+                            dbstr(toaddress)
                         )
                         queues.UISignalQueue.put((
                             'updateSentItemStatusByToAddress', (
@@ -867,7 +868,7 @@ class singleWorker(StoppableThread):
                                         ''' status='awaitingpubkey' or '''
                                         ''' status='doingpubkeypow') AND '''
                                         ''' folder='sent' ''',
-                                        toaddress)
+                                        dbstr(toaddress))
                                     del state.neededPubkeys[tag]
                                     break
                                 # else:
@@ -884,7 +885,7 @@ class singleWorker(StoppableThread):
                                 '''UPDATE sent SET '''
                                 ''' status='doingpubkeypow' WHERE '''
                                 ''' toaddress=? AND status='msgqueued' AND folder='sent' ''',
-                                toaddress
+                                dbstr(toaddress)
                             )
                             queues.UISignalQueue.put((
                                 'updateSentItemStatusByToAddress', (
@@ -929,7 +930,7 @@ class singleWorker(StoppableThread):
                 # is too hard then we'll abort.
                 queryreturn = sqlQuery(
                     'SELECT transmitdata FROM pubkeys WHERE address=?',
-                    toaddress)
+                    dbstr(toaddress))
                 for row in queryreturn:  # pylint: disable=redefined-outer-name
                     pubkeyPayload, = row
 
@@ -1349,7 +1350,7 @@ class singleWorker(StoppableThread):
             sqlExecute(
                 '''UPDATE sent SET msgid=?, status=?, retrynumber=?, '''
                 ''' sleeptill=?, lastactiontime=? WHERE ackdata=? AND folder='sent' ''',
-                inventoryHash, newStatus, retryNumber + 1,
+                inventoryHash, dbstr(newStatus), retryNumber + 1,
                 sleepTill, int(time.time()), ackdata
             )
 
@@ -1395,7 +1396,7 @@ class singleWorker(StoppableThread):
             '''SELECT retrynumber FROM sent WHERE toaddress=? '''
             ''' AND (status='doingpubkeypow' OR status='awaitingpubkey') '''
             ''' AND folder='sent' LIMIT 1''',
-            toAddress
+            dbstr(toAddress)
         )
         if not queryReturn:
             self.logger.critical(
@@ -1477,7 +1478,7 @@ class singleWorker(StoppableThread):
             ''' status='awaitingpubkey', retrynumber=?, sleeptill=? '''
             ''' WHERE toaddress=? AND (status='doingpubkeypow' OR '''
             ''' status='awaitingpubkey') AND folder='sent' ''',
-            int(time.time()), retryNumber + 1, sleeptill, toAddress)
+            int(time.time()), retryNumber + 1, sleeptill, dbstr(toAddress))
 
         queues.UISignalQueue.put((
             'updateStatusBar',

src/class_smtpServer.py

@@ -20,6 +20,7 @@ from helper_ackPayload import genAckPayload
 from helper_sql import sqlExecute
 from network.threads import StoppableThread
 from version import softwareVersion
+from dbcompat import dbstr
 
 SMTPDOMAIN = "bmaddr.lan"
 LISTENPORT = 8425
@@ -89,18 +90,18 @@ class smtpServerPyBitmessage(smtpd.SMTPServer):
         sqlExecute(
             '''INSERT INTO sent VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)''',
             '',
-            toAddress,
+            dbstr(toAddress),
             ripe,
-            fromAddress,
+            dbstr(fromAddress),
-            subject,
+            dbstr(subject),
-            message,
+            dbstr(message),
             ackdata,
             int(time.time()),  # sentTime (this will never change)
             int(time.time()),  # lastActionTime
             0,  # sleepTill time. This will get set when the POW gets done.
-            'msgqueued',
+            dbstr('msgqueued'),
             0,  # retryNumber
-            'sent',  # folder
+            dbstr('sent'),  # folder
             2,  # encodingtype
             # not necessary to have a TTL higher than 2 days
             min(config.getint('bitmessagesettings', 'ttl'), 86400 * 2)

src/dbcompat.py

@@ -0,0 +1,22 @@
+import logging
+import six
+
+logger = logging.getLogger("default")
+
+def dbstr(v):
+    if six.PY3:
+        if isinstance(v, str):
+            return v
+        elif isinstance(v, bytes):
+            return v.decode("utf-8", "replace")
+        logger.debug("unexpected type in dbstr(): {}".format(type(v)))
+        return v  # hope this never happens..
+    else:  # assume six.PY2
+        if isinstance(v, unicode):
+            return v.encode("utf-8", "replace")
+        elif isinstance(v, str):
+            return v
+        elif isinstance(v, bytes):
+            return str(v)
+        logger.debug("unexpected type in dbstr(): {}".format(type(v)))
+        return v  # hope this never happens..

src/helper_addressbook.py

@@ -4,11 +4,12 @@ Insert value into addressbook
 
 from bmconfigparser import config
 from helper_sql import sqlExecute
+from dbcompat import dbstr
 
 
 def insert(address, label):
     """perform insert into addressbook"""
 
     if address not in config.addresses():
-        return sqlExecute('''INSERT INTO addressbook VALUES (?,?)''', label, address) == 1
+        return sqlExecute('''INSERT INTO addressbook VALUES (?,?)''', dbstr(label), dbstr(address)) == 1
     return False

src/helper_inbox.py

@@ -2,11 +2,13 @@
 
 import queues
 from helper_sql import sqlExecute, sqlQuery
+from dbcompat import dbstr
 
 
 def insert(t):
     """Perform an insert into the "inbox" table"""
-    sqlExecute('''INSERT INTO inbox VALUES (?,?,?,?,?,?,?,?,?,?)''', *t)
+    u = [t[0], dbstr(t[1]), dbstr(t[2]), dbstr(t[3]), dbstr(t[4]), dbstr(t[5]), dbstr(t[6]), t[7], t[8], t[9]]
+    sqlExecute('''INSERT INTO inbox VALUES (?,?,?,?,?,?,?,?,?,?)''', *u)
     # shouldn't emit changedInboxUnread and displayNewInboxMessage
     # at the same time
     # queues.UISignalQueue.put(('changedInboxUnread', None))

src/helper_search.py

@@ -5,6 +5,7 @@ Used by :mod:`.bitmessageqt`.
 
 from helper_sql import sqlQuery
 from tr import _translate
+from dbcompat import dbstr
 
 
 def search_sql(
@@ -52,23 +53,23 @@ def search_sql(
     if account is not None:
         if xAddress == 'both':
             sqlStatementParts.append('(fromaddress = ? OR toaddress = ?)')
-            sqlArguments.append(account)
+            sqlArguments.append(dbstr(account))
-            sqlArguments.append(account)
+            sqlArguments.append(dbstr(account))
         else:
             sqlStatementParts.append(xAddress + ' = ? ')
-            sqlArguments.append(account)
+            sqlArguments.append(dbstr(account))
     if folder is not None:
         if folder == 'new':
             folder = 'inbox'
             unreadOnly = True
         sqlStatementParts.append('folder = ? ')
-        sqlArguments.append(folder)
+        sqlArguments.append(dbstr(folder))
     else:
         sqlStatementParts.append('folder != ?')
-        sqlArguments.append('trash')
+        sqlArguments.append(dbstr('trash'))
     if what:
         sqlStatementParts.append('%s LIKE ?' % (where))
-        sqlArguments.append(what)
+        sqlArguments.append(dbstr(what))
     if unreadOnly:
         sqlStatementParts.append('read = 0')
     if sqlStatementParts:

src/helper_sent.py

@@ -8,6 +8,7 @@ from addresses import decodeAddress
 from bmconfigparser import config
 from helper_ackPayload import genAckPayload
 from helper_sql import sqlExecute, sqlQuery
+from dbcompat import dbstr
 
 
 # pylint: disable=too-many-arguments
@@ -38,8 +39,8 @@ def insert(msgid=None, toAddress='[Broadcast subscribers]', fromAddress=None, su
 
         ttl = ttl if ttl else config.getint('bitmessagesettings', 'ttl')
 
-        t = (msgid, toAddress, ripe, fromAddress, subject, message, ackdata,
+        t = (msgid, dbstr(toAddress), ripe, dbstr(fromAddress), dbstr(subject), dbstr(message), ackdata,
-             sentTime, lastActionTime, sleeptill, status, retryNumber, folder,
+             sentTime, lastActionTime, sleeptill, dbstr(status), retryNumber, dbstr(folder),
              encoding, ttl)
 
         sqlExecute('''INSERT INTO sent VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)''', *t)

src/protocol.py

@@ -23,6 +23,7 @@ from debug import logger
 from helper_sql import sqlExecute
 from network.node import Peer
 from version import softwareVersion
+from dbcompat import dbstr
 
 # Network constants
 magic = 0xE9BEB4D9
@@ -558,7 +559,7 @@ def decryptAndCheckPubkeyPayload(data, address):
             hexlify(pubSigningKey), hexlify(pubEncryptionKey)
         )
 
-        t = (address, addressVersion, storedData, int(time.time()), 'yes')
+        t = (dbstr(address), addressVersion, storedData, int(time.time()), dbstr('yes'))
         sqlExecute('''INSERT INTO pubkeys VALUES (?,?,?,?,?)''', *t)
         return 'successful'
     except varintDecodeError:

src/shared.py

@@ -22,6 +22,7 @@ from addresses import decodeAddress, encodeVarint
 from bmconfigparser import config
 from debug import logger
 from helper_sql import sqlQuery
+from dbcompat import dbstr
 
 
 myECCryptorObjects = {}
@@ -38,7 +39,7 @@ def isAddressInMyAddressBook(address):
     """Is address in my addressbook?"""
     queryreturn = sqlQuery(
         '''select address from addressbook where address=?''',
-        address)
+        dbstr(address))
     return queryreturn != []
 
 
@@ -47,7 +48,7 @@ def isAddressInMySubscriptionsList(address):
     """Am I subscribed to this address?"""
     queryreturn = sqlQuery(
         '''select * from subscriptions where address=?''',
-        address.encode("utf-8", "replace"))
+        dbstr(address))
     return queryreturn != []
 
 
@@ -61,14 +62,14 @@ def isAddressInMyAddressBookSubscriptionsListOrWhitelist(address):
     queryreturn = sqlQuery(
         '''SELECT address FROM whitelist where address=?'''
         ''' and enabled = '1' ''',
-        address)
+        dbstr(address))
     if queryreturn != []:
         return True
 
     queryreturn = sqlQuery(
         '''select address from subscriptions where address=?'''
         ''' and enabled = '1' ''',
-        address)
+        dbstr(address))
     if queryreturn != []:
         return True
     return False
@@ -170,7 +171,7 @@ def fixPotentiallyInvalidUTF8Data(text):
         return text
     except UnicodeDecodeError:
         return 'Part of the message is corrupt. The message cannot be' \
-            ' displayed the normal way.\n\n' + repr(text)
+            ' displayed the normal way.\n\n' + text.decode("utf-8", "replace")
 
 
 def checkSensitiveFilePermissions(filename):

src/storage/sqlite.py

@@ -95,7 +95,7 @@ class SqliteInventory(InventoryStorage):
             t = int(time.time())
             hashes = [x for x, value in self._inventory.items()
                       if value.stream == stream and value.expires > t]
-            hashes += (str(payload) for payload, in sqlQuery(
+            hashes += (bytes(payload) for payload, in sqlQuery(
                 'SELECT hash FROM inventory WHERE streamnumber=?'
                 ' AND expirestime>?', stream, t))
             return hashes

src/tests/core.py

@@ -31,6 +31,7 @@ from network.node import Node, Peer
 from network.tcp import Socks4aBMConnection, Socks5BMConnection, TCPConnection
 from queues import excQueue
 from version import softwareVersion
+from dbcompat import dbstr
 
 from common import cleanup
 
@@ -346,11 +347,11 @@ class TestCore(unittest.TestCase):
         )
         queryreturn = sqlQuery(
             '''select msgid from sent where ackdata=?''', result)
-        self.assertNotEqual(queryreturn[0][0] if queryreturn else '', '')
+        self.assertNotEqual(queryreturn[0][0] if queryreturn else b'', b'')
 
         column_type = sqlQuery(
             '''select typeof(msgid) from sent where ackdata=?''', result)
-        self.assertEqual(column_type[0][0] if column_type else '', 'text')
+        self.assertEqual(column_type[0][0] if column_type else '', 'blob')
 
     @unittest.skipIf(frozen, 'not packed test_pattern into the bundle')
     def test_old_knownnodes_pickle(self):
@@ -368,7 +369,7 @@ class TestCore(unittest.TestCase):
     @staticmethod
     def delete_address_from_addressbook(address):
         """Clean up addressbook"""
-        sqlQuery('''delete from addressbook where address=?''', address)
+        sqlQuery('''delete from addressbook where address=?''', dbstr(address))
 
     def test_add_same_address_twice_in_addressbook(self):
         """checking same address is added twice in addressbook"""
@@ -382,7 +383,7 @@ class TestCore(unittest.TestCase):
         """checking is address added in addressbook or not"""
         helper_addressbook.insert(label='test1', address=self.addr)
         queryreturn = sqlQuery(
-            'select count(*) from addressbook where address=?', self.addr)
+            'select count(*) from addressbook where address=?', dbstr(self.addr))
         self.assertEqual(queryreturn[0][0], 1)
         self.delete_address_from_addressbook(self.addr)
 

src/tests/test_helper_inbox.py

@@ -26,7 +26,7 @@ class TestHelperInbox(unittest.TestCase):
     def test_insert(self, mock_sql_execute):  # pylint: disable=no-self-use
         """Test to perform an insert into the "inbox" table"""
         mock_message_data = (
-            "ruyv87bv",
+            b"ruyv87bv",
             "BM-2cUGaEcGz9Zft1SPAo8FJtfzyADTpEgU9U",
             "BM-2cUGaEcGz9Zft1SPAo8FJtfzyADTp5g99U",
             "Test subject",
@@ -35,7 +35,7 @@ class TestHelperInbox(unittest.TestCase):
             "inbox",
             2,
             0,
-            "658gvjhtghv",
+            b"658gvjhtghv",
         )
         insert(t=mock_message_data)
         mock_sql_execute.assert_called_once()
@@ -43,7 +43,7 @@ class TestHelperInbox(unittest.TestCase):
     @patch("pybitmessage.helper_inbox.sqlExecute")
     def test_trash(self, mock_sql_execute):  # pylint: disable=no-self-use
         """Test marking a message in the `inbox` as `trash`"""
-        mock_msg_id = "fefkosghsbse92"
+        mock_msg_id = b"fefkosghsbse92"
         trash(msgid=mock_msg_id)
         mock_sql_execute.assert_called_once()
 
@@ -57,7 +57,7 @@ class TestHelperInbox(unittest.TestCase):
     @patch("pybitmessage.helper_inbox.sqlExecute")
     def test_undeleteMessage(self, mock_sql_execute):  # pylint: disable=no-self-use
         """Test for Undelete the message"""
-        mock_msg_id = "fefkosghsbse92"
+        mock_msg_id = b"fefkosghsbse92"
         undeleteMessage(msgid=mock_msg_id)
         mock_sql_execute.assert_called_once()
 

src/tests/test_helper_sent.py

@@ -19,7 +19,7 @@ class TestHelperSent(unittest.TestCase):
         """Test insert with valid address"""
         VALID_ADDRESS = "BM-2cUGaEcGz9Zft1SPAo8FJtfzyADTpEgU9U"
         ackdata = insert(
-            msgid="123456",
+            msgid=b"123456",
             toAddress="[Broadcast subscribers]",
             fromAddress=VALID_ADDRESS,
             subject="Test Subject",
@@ -45,10 +45,10 @@ class TestHelperSent(unittest.TestCase):
     @patch("pybitmessage.helper_sent.sqlExecute")
     def test_delete(self, mock_sql_execute):
         """Test delete function"""
-        delete("ack_data")
+        delete(b"ack_data")
         self.assertTrue(mock_sql_execute.called)
         mock_sql_execute.assert_called_once_with(
-            "DELETE FROM sent WHERE ackdata = ?", "ack_data"
+            "DELETE FROM sent WHERE ackdata = ?", b"ack_data"
         )
 
     @patch("pybitmessage.helper_sent.sqlQuery")
@@ -56,11 +56,11 @@ class TestHelperSent(unittest.TestCase):
         """Test retrieving valid message details"""
         return_data = [
             (
-                "to@example.com",
+                b"to@example.com",
-                "from@example.com",
+                b"from@example.com",
-                "Test Subject",
+                b"Test Subject",
-                "Test Message",
+                b"Test Message",
-                "2022-01-01",
+                b"2022-01-01",
             )
         ]
         mock_sql_query.return_value = return_data
@@ -70,7 +70,7 @@ class TestHelperSent(unittest.TestCase):
     @patch("pybitmessage.helper_sent.sqlExecute")
     def test_trash(self, mock_sql_execute):
         """Test marking a message as 'trash'"""
-        ackdata = "ack_data"
+        ackdata = b"ack_data"
         mock_sql_execute.return_value = 1
         rowcount = trash(ackdata)
         self.assertEqual(rowcount, 1)

src/tests/test_helper_sql.py

@@ -23,23 +23,23 @@ class TestHelperSql(unittest.TestCase):
     @patch("pybitmessage.helper_sql.sqlReturnQueue.get")
     def test_sqlquery_no_args(self, mock_sqlreturnqueue_get, mock_sqlsubmitqueue_put):
         """Test sqlQuery with no additional arguments"""
-        mock_sqlreturnqueue_get.return_value = ("dummy_result", None)
+        mock_sqlreturnqueue_get.return_value = (b"dummy_result", None)
         result = helper_sql.sqlQuery(
             "SELECT msgid FROM inbox where folder='inbox' ORDER BY received"
         )
         self.assertEqual(mock_sqlsubmitqueue_put.call_count, 2)
-        self.assertEqual(result, "dummy_result")
+        self.assertEqual(result.decode("utf-8", "replace"), "dummy_result")
 
     @patch("pybitmessage.helper_sql.sqlSubmitQueue.put")
     @patch("pybitmessage.helper_sql.sqlReturnQueue.get")
     def test_sqlquery_with_args(self, mock_sqlreturnqueue_get, mock_sqlsubmitqueue_put):
         """Test sqlQuery with additional arguments"""
-        mock_sqlreturnqueue_get.return_value = ("dummy_result", None)
+        mock_sqlreturnqueue_get.return_value = (b"dummy_result", None)
         result = helper_sql.sqlQuery(
             "SELECT address FROM addressbook WHERE address=?", "PB-5yfds868gbkj"
         )
         self.assertEqual(mock_sqlsubmitqueue_put.call_count, 2)
-        self.assertEqual(result, "dummy_result")
+        self.assertEqual(result.decode("utf-8", "replace"), "dummy_result")
 
     @patch("pybitmessage.helper_sql.sqlSubmitQueue.put")
     @patch("pybitmessage.helper_sql.sqlReturnQueue.get")
@@ -49,7 +49,7 @@ class TestHelperSql(unittest.TestCase):
         rowcount = helper_sql.sqlExecute(
             "UPDATE sent SET status = 'msgqueued'"
             "WHERE ackdata = ? AND folder = 'sent'",
-            "1710652313",
+            b"1710652313",
         )
         self.assertEqual(mock_sqlsubmitqueue_put.call_count, 3)
         self.assertEqual(rowcount, 1)

src/tests/test_shared.py

@@ -46,7 +46,7 @@ class TestShared(unittest.TestCase):
         address = sample_address
 
         # if address is in MyAddressbook
-        mock_sql_query.return_value = [address.encode("utf-8", "replace")]
+        mock_sql_query.return_value = [bytes(address)]
         return_val = isAddressInMyAddressBook(address)
         mock_sql_query.assert_called_once()
         self.assertTrue(return_val)
@@ -64,7 +64,7 @@ class TestShared(unittest.TestCase):
         address = sample_address
 
         # if address is in MySubscriptionsList
-        mock_sql_query.return_value = [address.encode("utf-8", "replace")]
+        mock_sql_query.return_value = [bytes(address)]
         return_val = isAddressInMySubscriptionsList(address)
         self.assertTrue(return_val)
 
@@ -78,7 +78,7 @@ class TestShared(unittest.TestCase):
     def test_reloadBroadcastSendersForWhichImWatching(self, mock_sql_query):
         """Test for reload Broadcast Senders For Which Im Watching"""
         mock_sql_query.return_value = [
-            (sample_address.encode("utf-8", "replace"),),
+            (bytes(sample_address),),
         ]
         # before reload
         self.assertEqual(len(MyECSubscriptionCryptorObjects), 0)

src/tests/test_sqlthread.py

@@ -41,4 +41,4 @@ class TestSqlThread(unittest.TestCase):
 
         query = sqlQuery('SELECT enaddr(4, 1, "21122112211221122112")')
         self.assertEqual(
-            query[0][-1], encoded_str, "test case fail for create_function")
+            query[0][-1].decode("utf-8", "replace"), encoded_str, "test case fail for create_function")