packages/collectd/pybitmessagestatus.py

@@ -2,7 +2,7 @@
 
 import collectd
 import json
-import xmlrpclib
+from six.moves import xmlrpc_client as xmlrpclib
 
 pybmurl = ""
 api = ""

py3start.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+python3 pybitmessage/bitmessagemain.py "$@"

revert_blob_to_text.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+python3 pybitmessage/revert_blob_to_text.py "$@"

setup.py

@@ -4,6 +4,7 @@ import os
 import platform
 import shutil
 import sys
+import six
 
 from importlib import import_module
 from setuptools import setup, Extension
@@ -83,7 +84,7 @@ if __name__ == "__main__":
         'images/kivy/text_images*.png'
     ]}
 
-    if sys.version_info[0] == 3:
+    if six.PY3:
         packages.extend(
             [
                 'pybitmessage.bitmessagekivy',

src/addresses.py

@@ -187,7 +187,7 @@ def decodeAddress(address):
         integer = decodeBase58(address)
     if integer == 0:
         status = 'invalidcharacters'
-        return status, 0, 0, ''
+        return status, 0, 0, b''
     # after converting to hex, the string will be prepended
     # with a 0x and appended with a L in python2
     hexdata = hex(integer)[2:].rstrip('L')
@@ -200,23 +200,23 @@ def decodeAddress(address):
 
     if checksum != double_sha512(data[:-4])[0:4]:
         status = 'checksumfailed'
-        return status, 0, 0, ''
+        return status, 0, 0, b''
 
     try:
         addressVersionNumber, bytesUsedByVersionNumber = decodeVarint(data[:9])
     except varintDecodeError as e:
         logger.error(str(e))
         status = 'varintmalformed'
-        return status, 0, 0, ''
+        return status, 0, 0, b''
 
     if addressVersionNumber > 4:
         logger.error('cannot decode address version numbers this high')
         status = 'versiontoohigh'
-        return status, 0, 0, ''
+        return status, 0, 0, b''
     elif addressVersionNumber == 0:
         logger.error('cannot decode address version numbers of zero.')
         status = 'versiontoohigh'
-        return status, 0, 0, ''
+        return status, 0, 0, b''
 
     try:
         streamNumber, bytesUsedByStreamNumber = \
@@ -224,7 +224,7 @@ def decodeAddress(address):
     except varintDecodeError as e:
         logger.error(str(e))
         status = 'varintmalformed'
-        return status, 0, 0, ''
+        return status, 0, 0, b''
 
     status = 'success'
     if addressVersionNumber == 1:
@@ -242,21 +242,21 @@ def decodeAddress(address):
             return status, addressVersionNumber, streamNumber, \
                 b'\x00\x00' + embeddedRipeData
         elif len(embeddedRipeData) < 18:
-            return 'ripetooshort', 0, 0, ''
+            return 'ripetooshort', 0, 0, b''
         elif len(embeddedRipeData) > 20:
-            return 'ripetoolong', 0, 0, ''
+            return 'ripetoolong', 0, 0, b''
-        return 'otherproblem', 0, 0, ''
+        return 'otherproblem', 0, 0, b''
     elif addressVersionNumber == 4:
         embeddedRipeData = \
             data[bytesUsedByVersionNumber + bytesUsedByStreamNumber:-4]
         if embeddedRipeData[0:1] == b'\x00':
             # In order to enforce address non-malleability, encoded
             # RIPE data must have NULL bytes removed from the front
-            return 'encodingproblem', 0, 0, ''
+            return 'encodingproblem', 0, 0, b''
         elif len(embeddedRipeData) > 20:
-            return 'ripetoolong', 0, 0, ''
+            return 'ripetoolong', 0, 0, b''
         elif len(embeddedRipeData) < 4:
-            return 'ripetooshort', 0, 0, ''
+            return 'ripetooshort', 0, 0, b''
         x00string = b'\x00' * (20 - len(embeddedRipeData))
         return status, addressVersionNumber, streamNumber, \
             x00string + embeddedRipeData

src/api.py

@@ -67,9 +67,12 @@ import subprocess  # nosec B404
 import time
 from binascii import hexlify, unhexlify
 from struct import pack, unpack
+import sqlite3
 
 import six
 from six.moves import configparser, http_client, xmlrpc_server
+from six.moves.reprlib import repr
+from dbcompat import dbstr
 
 import helper_inbox
 import helper_sent
@@ -531,12 +534,12 @@ class BMRPCDispatcher(object):
         message = shared.fixPotentiallyInvalidUTF8Data(message)
         return {
             'msgid': hexlify(msgid),
-            'toAddress': toAddress,
+            'toAddress': toAddress.decode("utf-8", "replace"),
-            'fromAddress': fromAddress,
+            'fromAddress': fromAddress.decode("utf-8", "replace"),
             'subject': base64.b64encode(subject),
             'message': base64.b64encode(message),
             'encodingType': encodingtype,
-            'receivedTime': received,
+            'receivedTime': received.decode("utf-8", "replace"),
             'read': read
         }
 
@@ -598,11 +601,12 @@ class BMRPCDispatcher(object):
         """
         queryreturn = sqlQuery(
             "SELECT label, address from addressbook WHERE label = ?",
-            label
+            dbstr(label)
         ) if label else sqlQuery("SELECT label, address from addressbook")
         data = []
         for label, address in queryreturn:
             label = shared.fixPotentiallyInvalidUTF8Data(label)
+            address = address.decode("utf-8", "replace")
             data.append({
                 'label': base64.b64encode(label),
                 'address': address
@@ -618,12 +622,12 @@ class BMRPCDispatcher(object):
         self._verifyAddress(address)
         # TODO: add unique together constraint in the table
         queryreturn = sqlQuery(
-            "SELECT address FROM addressbook WHERE address=?", address)
+            "SELECT address FROM addressbook WHERE address=?", dbstr(address))
         if queryreturn != []:
             raise APIError(
                 16, 'You already have this address in your address book.')
 
-        sqlExecute("INSERT INTO addressbook VALUES(?,?)", label, address)
+        sqlExecute("INSERT INTO addressbook VALUES(?,?)", dbstr(label), dbstr(address))
         queues.UISignalQueue.put(('rerenderMessagelistFromLabels', ''))
         queues.UISignalQueue.put(('rerenderMessagelistToLabels', ''))
         queues.UISignalQueue.put(('rerenderAddressBook', ''))
@@ -635,7 +639,7 @@ class BMRPCDispatcher(object):
         """Delete an entry from address book."""
         address = addBMIfNotPresent(address)
         self._verifyAddress(address)
-        sqlExecute('DELETE FROM addressbook WHERE address=?', address)
+        sqlExecute('DELETE FROM addressbook WHERE address=?', dbstr(address))
         queues.UISignalQueue.put(('rerenderMessagelistFromLabels', ''))
         queues.UISignalQueue.put(('rerenderMessagelistToLabels', ''))
         queues.UISignalQueue.put(('rerenderAddressBook', ''))
@@ -919,6 +923,7 @@ class BMRPCDispatcher(object):
             " ORDER BY received"
         )
         return {"inboxMessages": [
+
             self._dump_inbox_message(*data) for data in queryreturn
         ]}
 
@@ -953,19 +958,31 @@ class BMRPCDispatcher(object):
                     23, 'Bool expected in readStatus, saw %s instead.'
                     % type(readStatus))
             queryreturn = sqlQuery(
-                "SELECT read FROM inbox WHERE msgid=?", msgid)
+                "SELECT read FROM inbox WHERE msgid=?", sqlite3.Binary(msgid))
+            if len(queryreturn) < 1:
+                queryreturn = sqlQuery(
+                    "SELECT read FROM inbox WHERE msgid=CAST(? AS TEXT)", msgid)
             # UPDATE is slow, only update if status is different
             try:
                 if (queryreturn[0][0] == 1) != readStatus:
-                    sqlExecute(
+                    rowcount = sqlExecute(
                         "UPDATE inbox set read = ? WHERE msgid=?",
+                        readStatus, sqlite3.Binary(msgid))
+                    if rowcount < 1:
+                        rowcount = sqlExecute(
+                            "UPDATE inbox set read = ? WHERE msgid=CAST(? AS TEXT)",
                             readStatus, msgid)
                     queues.UISignalQueue.put(('changedInboxUnread', None))
             except IndexError:
                 pass
         queryreturn = sqlQuery(
             "SELECT msgid, toaddress, fromaddress, subject, received, message,"
-            " encodingtype, read FROM inbox WHERE msgid=?", msgid
+            " encodingtype, read FROM inbox WHERE msgid=?", sqlite3.Binary(msgid)
+        )
+        if len(queryreturn) < 1:
+            queryreturn = sqlQuery(
+                "SELECT msgid, toaddress, fromaddress, subject, received, message,"
+                " encodingtype, read FROM inbox WHERE msgid=CAST(? AS TEXT)", msgid
             )
         try:
             return {"inboxMessage": [
@@ -1018,7 +1035,7 @@ class BMRPCDispatcher(object):
         queryreturn = sqlQuery(
             "SELECT msgid, toaddress, fromaddress, subject, received,"
             " message, encodingtype, read FROM inbox WHERE folder='inbox'"
-            " AND toAddress=?", toAddress)
+            " AND toAddress=?", dbstr(toAddress))
         return {"inboxMessages": [
             self._dump_inbox_message(*data) for data in queryreturn
         ]}
@@ -1035,6 +1052,12 @@ class BMRPCDispatcher(object):
         queryreturn = sqlQuery(
             "SELECT msgid, toaddress, fromaddress, subject, lastactiontime,"
             " message, encodingtype, status, ackdata FROM sent WHERE msgid=?",
+            sqlite3.Binary(msgid)
+        )
+        if len(queryreturn) < 1:
+            queryreturn = sqlQuery(
+                "SELECT msgid, toaddress, fromaddress, subject, lastactiontime,"
+                " message, encodingtype, status, ackdata FROM sent WHERE msgid=CAST(? AS TEXT)",
                 msgid
             )
         try:
@@ -1055,7 +1078,7 @@ class BMRPCDispatcher(object):
             "SELECT msgid, toaddress, fromaddress, subject, lastactiontime,"
             " message, encodingtype, status, ackdata FROM sent"
             " WHERE folder='sent' AND fromAddress=? ORDER BY lastactiontime",
-            fromAddress
+            dbstr(fromAddress)
         )
         return {"sentMessages": [
             self._dump_sent_message(*data) for data in queryreturn
@@ -1072,7 +1095,13 @@ class BMRPCDispatcher(object):
         queryreturn = sqlQuery(
             "SELECT msgid, toaddress, fromaddress, subject, lastactiontime,"
             " message, encodingtype, status, ackdata FROM sent"
-            " WHERE ackdata=?", ackData
+            " WHERE ackdata=?", sqlite3.Binary(ackData)
+        )
+        if len(queryreturn) < 1:
+            queryreturn = sqlQuery(
+                "SELECT msgid, toaddress, fromaddress, subject, lastactiontime,"
+                " message, encodingtype, status, ackdata FROM sent"
+                " WHERE ackdata=CAST(? AS TEXT)", ackData
             )
 
         try:
@@ -1093,7 +1122,9 @@ class BMRPCDispatcher(object):
         # Trash if in inbox table
         helper_inbox.trash(msgid)
         # Trash if in sent table
-        sqlExecute("UPDATE sent SET folder='trash' WHERE msgid=?", msgid)
+        rowcount = sqlExecute("UPDATE sent SET folder='trash' WHERE msgid=?", sqlite3.Binary(msgid))
+        if rowcount < 1:
+            sqlExecute("UPDATE sent SET folder='trash' WHERE msgid=CAST(? AS TEXT)", msgid)
         return 'Trashed message (assuming message existed).'
 
     @command('trashInboxMessage')
@@ -1107,7 +1138,9 @@ class BMRPCDispatcher(object):
     def HandleTrashSentMessage(self, msgid):
         """Trash sent message by msgid (encoded in hex)."""
         msgid = self._decode(msgid, "hex")
-        sqlExecute('''UPDATE sent SET folder='trash' WHERE msgid=?''', msgid)
+        rowcount = sqlExecute('''UPDATE sent SET folder='trash' WHERE msgid=?''', sqlite3.Binary(msgid))
+        if rowcount < 1:
+            sqlExecute('''UPDATE sent SET folder='trash' WHERE msgid=CAST(? AS TEXT)''', msgid)
         return 'Trashed sent message (assuming message existed).'
 
     @command('sendMessage')
@@ -1150,9 +1183,9 @@ class BMRPCDispatcher(object):
 
         toLabel = ''
         queryreturn = sqlQuery(
-            "SELECT label FROM addressbook WHERE address=?", toAddress)
+            "SELECT label FROM addressbook WHERE address=?", dbstr(toAddress))
         try:
-            toLabel = queryreturn[0][0]
+            toLabel = queryreturn[0][0].decode("utf-8", "replace")
         except IndexError:
             pass
 
@@ -1217,9 +1250,12 @@ class BMRPCDispatcher(object):
             raise APIError(15, 'Invalid ackData object size.')
         ackdata = self._decode(ackdata, "hex")
         queryreturn = sqlQuery(
-            "SELECT status FROM sent where ackdata=?", ackdata)
+            "SELECT status FROM sent where ackdata=?", sqlite3.Binary(ackdata))
+        if len(queryreturn) < 1:
+            queryreturn = sqlQuery(
+                "SELECT status FROM sent where ackdata=CAST(? AS TEXT)", ackdata)
         try:
-            return queryreturn[0][0]
+            return queryreturn[0][0].decode("utf-8", "replace")
         except IndexError:
             return 'notfound'
 
@@ -1238,11 +1274,11 @@ class BMRPCDispatcher(object):
         # First we must check to see if the address is already in the
         # subscriptions list.
         queryreturn = sqlQuery(
-            "SELECT * FROM subscriptions WHERE address=?", address)
+            "SELECT * FROM subscriptions WHERE address=?", dbstr(address))
         if queryreturn:
             raise APIError(16, 'You are already subscribed to that address.')
         sqlExecute(
-            "INSERT INTO subscriptions VALUES (?,?,?)", label, address, True)
+            "INSERT INTO subscriptions VALUES (?,?,?)", dbstr(label), dbstr(address), True)
         shared.reloadBroadcastSendersForWhichImWatching()
         queues.UISignalQueue.put(('rerenderMessagelistFromLabels', ''))
         queues.UISignalQueue.put(('rerenderSubscriptions', ''))
@@ -1256,7 +1292,7 @@ class BMRPCDispatcher(object):
         """
 
         address = addBMIfNotPresent(address)
-        sqlExecute("DELETE FROM subscriptions WHERE address=?", address)
+        sqlExecute("DELETE FROM subscriptions WHERE address=?", dbstr(address))
         shared.reloadBroadcastSendersForWhichImWatching()
         queues.UISignalQueue.put(('rerenderMessagelistFromLabels', ''))
         queues.UISignalQueue.put(('rerenderSubscriptions', ''))
@@ -1274,6 +1310,7 @@ class BMRPCDispatcher(object):
         data = []
         for label, address, enabled in queryreturn:
             label = shared.fixPotentiallyInvalidUTF8Data(label)
+            address = address.decode("utf-8", "replace")
             data.append({
                 'label': base64.b64encode(label),
                 'address': address,
@@ -1354,7 +1391,9 @@ class BMRPCDispatcher(object):
         """Trash a sent message by ackdata (hex encoded)"""
         # This API method should only be used when msgid is not available
         ackdata = self._decode(ackdata, "hex")
-        sqlExecute("UPDATE sent SET folder='trash' WHERE ackdata=?", ackdata)
+        rowcount = sqlExecute("UPDATE sent SET folder='trash' WHERE ackdata=?", sqlite3.Binary(ackdata))
+        if rowcount < 1:
+            sqlExecute("UPDATE sent SET folder='trash' WHERE ackdata=CAST(? AS TEXT)", ackdata)
         return 'Trashed sent message (assuming message existed).'
 
     @command('disseminatePubkey')
@@ -1421,19 +1460,29 @@ class BMRPCDispatcher(object):
         # use it we'll need to fill out a field in our inventory database
         # which is blank by default (first20bytesofencryptedmessage).
         queryreturn = sqlQuery(
-            "SELECT hash, payload FROM inventory WHERE tag = ''"
+            "SELECT hash, payload FROM inventory WHERE tag = ?"
-            " and objecttype = 2")
+            " and objecttype = 2", sqlite3.Binary(b""))
+        if len(queryreturn) < 1:
+            queryreturn = sqlQuery(
+                "SELECT hash, payload FROM inventory WHERE tag = CAST(? AS TEXT)"
+                " and objecttype = 2", b"")
         with SqlBulkExecute() as sql:
             for hash01, payload in queryreturn:
                 readPosition = 16  # Nonce length + time length
                 # Stream Number length
                 readPosition += decodeVarint(
                     payload[readPosition:readPosition + 10])[1]
-                t = (payload[readPosition:readPosition + 32], hash01)
+                t = (sqlite3.Binary(payload[readPosition:readPosition + 32]), sqlite3.Binary(hash01))
-                sql.execute("UPDATE inventory SET tag=? WHERE hash=?", *t)
+                _, rowcount = sql.execute("UPDATE inventory SET tag=? WHERE hash=?", *t)
+                if rowcount < 1:
+                    t = (sqlite3.Binary(payload[readPosition:readPosition + 32]), hash01)
+                    sql.execute("UPDATE inventory SET tag=? WHERE hash=CAST(? AS TEXT)", *t)
 
         queryreturn = sqlQuery(
-            "SELECT payload FROM inventory WHERE tag = ?", requestedHash)
+            "SELECT payload FROM inventory WHERE tag = ?", sqlite3.Binary(requestedHash))
+        if len(queryreturn) < 1:
+            queryreturn = sqlQuery(
+                "SELECT payload FROM inventory WHERE tag = CAST(? AS TEXT)", requestedHash)
         return {"receivedMessageDatas": [
             {'data': hexlify(payload)} for payload, in queryreturn
         ]}

src/bitmessagecli.py

@@ -21,7 +21,8 @@ import os
 import socket
 import sys
 import time
-import xmlrpclib
+from six.moves import xmlrpc_client as xmlrpclib
+from six.moves import input as raw_input
 
 from bmconfigparser import config
 

src/bitmessagecurses/__init__.py

@@ -10,13 +10,15 @@ Bitmessage commandline interface
 #     * python2-pythondialog
 #  * dialog
 
-import ConfigParser
+from six.moves import configparser
 import curses
 import os
 import sys
 import time
 from textwrap import fill
 from threading import Timer
+import six
+import sqlite3
 
 from dialog import Dialog
 import helper_sent
@@ -30,6 +32,7 @@ import state
 from addresses import addBMIfNotPresent, decodeAddress
 from bmconfigparser import config
 from helper_sql import sqlExecute, sqlQuery
+from dbcompat import dbstr
 
 # pylint: disable=global-statement
 
@@ -105,7 +108,7 @@ def ascii(s):
     """ASCII values"""
     r = ""
     for c in s:
-        if ord(c) in range(128):
+        if six.byte2int(c) in range(128):
             r += c
     return r
 
@@ -326,13 +329,13 @@ def handlech(c, stdscr):
     if c != curses.ERR:
         global inboxcur, addrcur, sentcur, subcur, abookcur, blackcur
         if c in range(256):
-            if chr(c) in '12345678':
+            if six.int2byte(c) in '12345678':
                 global menutab
-                menutab = int(chr(c))
+                menutab = int(six.int2byte(c))
-            elif chr(c) == 'q':
+            elif six.int2byte(c) == 'q':
                 global quit_
                 quit_ = True
-            elif chr(c) == '\n':
+            elif six.int2byte(c) == '\n':
                 curses.curs_set(1)
                 d = Dialog(dialog="dialog")
                 if menutab == 1:
@@ -358,7 +361,9 @@ def handlech(c, stdscr):
                                 inbox[inboxcur][1] +
                                 "\"")
                             data = ""       # pyint: disable=redefined-outer-name
-                            ret = sqlQuery("SELECT message FROM inbox WHERE msgid=?", inbox[inboxcur][0])
+                            ret = sqlQuery("SELECT message FROM inbox WHERE msgid=?", sqlite3.Binary(inbox[inboxcur][0]))
+                            if len(ret) < 1:
+                                ret = sqlQuery("SELECT message FROM inbox WHERE msgid=CAST(? AS TEXT)", inbox[inboxcur][0])
                             if ret != []:
                                 for row in ret:
                                     data, = row
@@ -367,12 +372,16 @@ def handlech(c, stdscr):
                                 for i, item in enumerate(data.split("\n")):
                                     msg += fill(item, replace_whitespace=False) + "\n"
                                 scrollbox(d, unicode(ascii(msg)), 30, 80)
-                                sqlExecute("UPDATE inbox SET read=1 WHERE msgid=?", inbox[inboxcur][0])
+                                rowcount = sqlExecute("UPDATE inbox SET read=1 WHERE msgid=?", sqlite3.Binary(inbox[inboxcur][0]))
+                                if rowcount < 1:
+                                    sqlExecute("UPDATE inbox SET read=1 WHERE msgid=CAST(? AS TEXT)", inbox[inboxcur][0])
                                 inbox[inboxcur][7] = 1
                             else:
                                 scrollbox(d, unicode("Could not fetch message."))
                         elif t == "2":       # Mark unread
-                            sqlExecute("UPDATE inbox SET read=0 WHERE msgid=?", inbox[inboxcur][0])
+                            rowcount = sqlExecute("UPDATE inbox SET read=0 WHERE msgid=?", sqlite3.Binary(inbox[inboxcur][0]))
+                            if rowcount < 1:
+                                sqlExecute("UPDATE inbox SET read=0 WHERE msgid=CAST(? AS TEXT)", inbox[inboxcur][0])
                             inbox[inboxcur][7] = 0
                         elif t == "3":       # Reply
                             curses.curs_set(1)
@@ -396,11 +405,14 @@ def handlech(c, stdscr):
                             if not m[5][:4] == "Re: ":
                                 subject = "Re: " + m[5]
                             body = ""
-                            ret = sqlQuery("SELECT message FROM inbox WHERE msgid=?", m[0])
+                            ret = sqlQuery("SELECT message FROM inbox WHERE msgid=?", sqlite3.Binary(m[0]))
+                            if len(ret) < 1:
+                                ret = sqlQuery("SELECT message FROM inbox WHERE msgid=CAST(? AS TEXT)", m[0])
                             if ret != []:
                                 body = "\n\n------------------------------------------------------\n"
                                 for row in ret:
                                     body, = row
+                                body = body.decode("utf-8", "replace")
 
                             sendMessage(fromaddr, toaddr, ischan, subject, body, True)
                             dialogreset(stdscr)
@@ -410,7 +422,7 @@ def handlech(c, stdscr):
                                 r, t = d.inputbox("Label for address \"" + addr + "\"")
                                 if r == d.DIALOG_OK:
                                     label = t
-                                    sqlExecute("INSERT INTO addressbook VALUES (?,?)", label, addr)
+                                    sqlExecute("INSERT INTO addressbook VALUES (?,?)", dbstr(label), dbstr(addr))
                                     # Prepend entry
                                     addrbook.reverse()
                                     addrbook.append([label, addr])
@@ -422,17 +434,22 @@ def handlech(c, stdscr):
                             r, t = d.inputbox("Filename", init=inbox[inboxcur][5] + ".txt")
                             if r == d.DIALOG_OK:
                                 msg = ""
-                                ret = sqlQuery("SELECT message FROM inbox WHERE msgid=?", inbox[inboxcur][0])
+                                ret = sqlQuery("SELECT message FROM inbox WHERE msgid=?", sqlite3.Binary(inbox[inboxcur][0]))
+                                if len(ret) < 1:
+                                    ret = sqlQuery("SELECT message FROM inbox WHERE msgid=CAST(? AS TEXT)", inbox[inboxcur][0])
                                 if ret != []:
                                     for row in ret:
                                         msg, = row
+                                    msg = msg.decode("utf-8", "replace")
                                     fh = open(t, "a")       # Open in append mode just in case
                                     fh.write(msg)
                                     fh.close()
                                 else:
                                     scrollbox(d, unicode("Could not fetch message."))
                         elif t == "6":       # Move to trash
-                            sqlExecute("UPDATE inbox SET folder='trash' WHERE msgid=?", inbox[inboxcur][0])
+                            rowcount = sqlExecute("UPDATE inbox SET folder='trash' WHERE msgid=?", sqlite3.Binary(inbox[inboxcur][0]))
+                            if rowcount < 1:
+                                sqlExecute("UPDATE inbox SET folder='trash' WHERE msgid=CAST(? AS TEXT)", inbox[inboxcur][0])
                             del inbox[inboxcur]
                             scrollbox(d, unicode(
                                 "Message moved to trash. There is no interface to view your trash,"
@@ -463,7 +480,12 @@ def handlech(c, stdscr):
                             data = ""
                             ret = sqlQuery(
                                 "SELECT message FROM sent WHERE subject=? AND ackdata=?",
-                                sentbox[sentcur][4],
+                                dbstr(sentbox[sentcur][4]),
+                                sqlite3.Binary(sentbox[sentcur][6]))
+                            if len(ret) < 1:
+                                ret = sqlQuery(
+                                    "SELECT message FROM sent WHERE subject=? AND ackdata=CAST(? AS TEXT)",
+                                    dbstr(sentbox[sentcur][4]),
                                     sentbox[sentcur][6])
                             if ret != []:
                                 for row in ret:
@@ -476,9 +498,14 @@ def handlech(c, stdscr):
                             else:
                                 scrollbox(d, unicode("Could not fetch message."))
                         elif t == "2":       # Move to trash
-                            sqlExecute(
+                            rowcount = sqlExecute(
                                 "UPDATE sent SET folder='trash' WHERE subject=? AND ackdata=?",
-                                sentbox[sentcur][4],
+                                dbstr(sentbox[sentcur][4]),
+                                sqlite3.Binary(sentbox[sentcur][6]))
+                            if rowcount < 1:
+                                rowcount = sqlExecute(
+                                    "UPDATE sent SET folder='trash' WHERE subject=? AND ackdata=CAST(? AS TEXT)",
+                                    dbstr(sentbox[sentcur][4]),
                                     sentbox[sentcur][6])
                             del sentbox[sentcur]
                             scrollbox(d, unicode(
@@ -672,7 +699,7 @@ def handlech(c, stdscr):
                                     elif t == "2" and m is False:
                                         try:
                                             mn = config.get(a, "mailinglistname")
-                                        except ConfigParser.NoOptionError:
+                                        except configparser.NoOptionError:
                                             mn = ""
                                         r, t = d.inputbox("Mailing list name", init=mn)
                                         if r == d.DIALOG_OK:
@@ -711,29 +738,29 @@ def handlech(c, stdscr):
                                         subscriptions.append([label, addr, True])
                                         subscriptions.reverse()
 
-                                        sqlExecute("INSERT INTO subscriptions VALUES (?,?,?)", label, addr, True)
+                                        sqlExecute("INSERT INTO subscriptions VALUES (?,?,?)", dbstr(label), dbstr(addr), True)
                                         shared.reloadBroadcastSendersForWhichImWatching()
                         elif t == "2":
                             r, t = d.inputbox("Type in \"I want to delete this subscription\"")
                             if r == d.DIALOG_OK and t == "I want to delete this subscription":
                                 sqlExecute(
                                     "DELETE FROM subscriptions WHERE label=? AND address=?",
-                                    subscriptions[subcur][0],
+                                    dbstr(subscriptions[subcur][0]),
-                                    subscriptions[subcur][1])
+                                    dbstr(subscriptions[subcur][1]))
                                 shared.reloadBroadcastSendersForWhichImWatching()
                                 del subscriptions[subcur]
                         elif t == "3":
                             sqlExecute(
                                 "UPDATE subscriptions SET enabled=1 WHERE label=? AND address=?",
-                                subscriptions[subcur][0],
+                                dbstr(subscriptions[subcur][0]),
-                                subscriptions[subcur][1])
+                                dbstr(subscriptions[subcur][1]))
                             shared.reloadBroadcastSendersForWhichImWatching()
                             subscriptions[subcur][2] = True
                         elif t == "4":
                             sqlExecute(
                                 "UPDATE subscriptions SET enabled=0 WHERE label=? AND address=?",
-                                subscriptions[subcur][0],
+                                dbstr(subscriptions[subcur][0]),
-                                subscriptions[subcur][1])
+                                dbstr(subscriptions[subcur][1]))
                             shared.reloadBroadcastSendersForWhichImWatching()
                             subscriptions[subcur][2] = False
                 elif menutab == 6:
@@ -762,7 +789,7 @@ def handlech(c, stdscr):
                                 subscriptions.append([label, addr, True])
                                 subscriptions.reverse()
 
-                                sqlExecute("INSERT INTO subscriptions VALUES (?,?,?)", label, addr, True)
+                                sqlExecute("INSERT INTO subscriptions VALUES (?,?,?)", dbstr(label), dbstr(addr), True)
                                 shared.reloadBroadcastSendersForWhichImWatching()
                         elif t == "3":
                             r, t = d.inputbox("Input new address")
@@ -771,7 +798,7 @@ def handlech(c, stdscr):
                                 if addr not in [item[1] for i, item in enumerate(addrbook)]:
                                     r, t = d.inputbox("Label for address \"" + addr + "\"")
                                     if r == d.DIALOG_OK:
-                                        sqlExecute("INSERT INTO addressbook VALUES (?,?)", t, addr)
+                                        sqlExecute("INSERT INTO addressbook VALUES (?,?)", dbstr(t), dbstr(addr))
                                         # Prepend entry
                                         addrbook.reverse()
                                         addrbook.append([t, addr])
@@ -783,8 +810,8 @@ def handlech(c, stdscr):
                             if r == d.DIALOG_OK and t == "I want to delete this Address Book entry":
                                 sqlExecute(
                                     "DELETE FROM addressbook WHERE label=? AND address=?",
-                                    addrbook[abookcur][0],
+                                    dbstr(addrbook[abookcur][0]),
-                                    addrbook[abookcur][1])
+                                    dbstr(addrbook[abookcur][1]))
                                 del addrbook[abookcur]
                 elif menutab == 7:
                     set_background_title(d, "Blacklist Dialog Box")
@@ -800,20 +827,20 @@ def handlech(c, stdscr):
                             if r == d.DIALOG_OK and t == "I want to delete this Blacklist entry":
                                 sqlExecute(
                                     "DELETE FROM blacklist WHERE label=? AND address=?",
-                                    blacklist[blackcur][0],
+                                    dbstr(blacklist[blackcur][0]),
-                                    blacklist[blackcur][1])
+                                    dbstr(blacklist[blackcur][1]))
                                 del blacklist[blackcur]
                         elif t == "2":
                             sqlExecute(
                                 "UPDATE blacklist SET enabled=1 WHERE label=? AND address=?",
-                                blacklist[blackcur][0],
+                                dbstr(blacklist[blackcur][0]),
-                                blacklist[blackcur][1])
+                                dbstr(blacklist[blackcur][1]))
                             blacklist[blackcur][2] = True
                         elif t == "3":
                             sqlExecute(
                                 "UPDATE blacklist SET enabled=0 WHERE label=? AND address=?",
-                                blacklist[blackcur][0],
+                                dbstr(blacklist[blackcur][0]),
-                                blacklist[blackcur][1])
+                                dbstr(blacklist[blackcur][1]))
                             blacklist[blackcur][2] = False
                 dialogreset(stdscr)
         else:
@@ -991,10 +1018,13 @@ def loadInbox():
     ret = sqlQuery("""SELECT msgid, toaddress, fromaddress, subject, received, read
         FROM inbox WHERE folder='inbox' AND %s LIKE ?
         ORDER BY received
-        """ % (where,), what)
+        """ % (where,), dbstr(what))
     for row in ret:
         msgid, toaddr, fromaddr, subject, received, read = row
+        toaddr = toaddr.decode("utf-8", "replace")
+        fromaddr = fromaddr.decode("utf-8", "replace")
         subject = ascii(shared.fixPotentiallyInvalidUTF8Data(subject))
+        received = received.decode("utf-8", "replace")
 
         # Set label for to address
         try:
@@ -1013,18 +1043,19 @@ def loadInbox():
         if config.has_section(fromaddr):
             fromlabel = config.get(fromaddr, "label")
         if fromlabel == "":         # Check Address Book
-            qr = sqlQuery("SELECT label FROM addressbook WHERE address=?", fromaddr)
+            qr = sqlQuery("SELECT label FROM addressbook WHERE address=?", dbstr(fromaddr))
             if qr != []:
                 for r in qr:
                     fromlabel, = r
+                fromlabel = shared.fixPotentiallyInvalidUTF8Data(fromlabel)
         if fromlabel == "":         # Check Subscriptions
-            qr = sqlQuery("SELECT label FROM subscriptions WHERE address=?", fromaddr)
+            qr = sqlQuery("SELECT label FROM subscriptions WHERE address=?", dbstr(fromaddr))
             if qr != []:
                 for r in qr:
                     fromlabel, = r
+                fromlabel = shared.fixPotentiallyInvalidUTF8Data(fromlabel)
         if fromlabel == "":
             fromlabel = fromaddr
-        fromlabel = shared.fixPotentiallyInvalidUTF8Data(fromlabel)
 
         # Load into array
         inbox.append([
@@ -1044,22 +1075,27 @@ def loadSent():
     ret = sqlQuery("""SELECT toaddress, fromaddress, subject, status, ackdata, lastactiontime
         FROM sent WHERE folder='sent' AND %s LIKE ?
         ORDER BY lastactiontime
-        """ % (where,), what)
+        """ % (where,), dbstr(what))
     for row in ret:
         toaddr, fromaddr, subject, status, ackdata, lastactiontime = row
+        toaddr = toaddr.decode("utf-8", "replace")
+        fromaddr = fromaddr.decode("utf-8", "replace")
         subject = ascii(shared.fixPotentiallyInvalidUTF8Data(subject))
+        status = status.decode("utf-8", "replace")
 
         # Set label for to address
         tolabel = ""
-        qr = sqlQuery("SELECT label FROM addressbook WHERE address=?", toaddr)
+        qr = sqlQuery("SELECT label FROM addressbook WHERE address=?", dbstr(toaddr))
         if qr != []:
             for r in qr:
                 tolabel, = r
+            tolabel = tolabel.decode("utf-8", "replace")
         if tolabel == "":
-            qr = sqlQuery("SELECT label FROM subscriptions WHERE address=?", toaddr)
+            qr = sqlQuery("SELECT label FROM subscriptions WHERE address=?", dbstr(toaddr))
             if qr != []:
                 for r in qr:
                     tolabel, = r
+                tolabel = tolabel.decode("utf-8", "replace")
         if tolabel == "":
             if config.has_section(toaddr):
                 tolabel = config.get(toaddr, "label")
@@ -1129,6 +1165,7 @@ def loadAddrBook():
     for row in ret:
         label, addr = row
         label = shared.fixPotentiallyInvalidUTF8Data(label)
+        addr = addr.decode("utf-8", "replace")
         addrbook.append([label, addr])
     addrbook.reverse()
 
@@ -1138,6 +1175,8 @@ def loadSubscriptions():
     ret = sqlQuery("SELECT label, address, enabled FROM subscriptions")
     for row in ret:
         label, address, enabled = row
+        label = label.decode("utf-8", "replace")
+        address = address.decode("utf-8", "replace")
         subscriptions.append([label, address, enabled])
     subscriptions.reverse()
 
@@ -1152,6 +1191,8 @@ def loadBlackWhiteList():
         ret = sqlQuery("SELECT label, address, enabled FROM whitelist")
     for row in ret:
         label, address, enabled = row
+        label = label.decode("utf-8", "replace")
+        address = address.decode("utf-8", "replace")
         blacklist.append([label, address, enabled])
     blacklist.reverse()
 

src/bitmessagekivy/baseclass/addressbook.py

@@ -29,6 +29,7 @@ from pybitmessage.bitmessagekivy.baseclass.common import (
 from pybitmessage.bitmessagekivy.baseclass.popup import SavedAddressDetailPopup
 from pybitmessage.bitmessagekivy.baseclass.addressbook_widgets import HelperAddressBook
 from pybitmessage.helper_sql import sqlExecute
+from dbcompat import dbstr
 
 logger = logging.getLogger('default')
 
@@ -59,7 +60,7 @@ class AddressBook(Screen, HelperAddressBook):
         self.ids.tag_label.text = ''
         self.queryreturn = kivy_helper_search.search_sql(
             xAddress, account, "addressbook", where, what, False)
-        self.queryreturn = [obj for obj in reversed(self.queryreturn)]
+        self.queryreturn = [[obj[0].decode("utf-8", "replace"), obj[1].decode("utf-8", "replace")] for obj in reversed(self.queryreturn)]
         if self.queryreturn:
             self.ids.tag_label.text = 'Address Book'
             self.has_refreshed = True
@@ -131,7 +132,7 @@ class AddressBook(Screen, HelperAddressBook):
         if self.ids.ml.children is not None:
             self.ids.tag_label.text = ''
         sqlExecute(
-            "DELETE FROM  addressbook WHERE address = ?", address)
+            "DELETE FROM  addressbook WHERE address = ?", dbstr(address))
         toast('Address Deleted')
 
     def close_pop(self, instance):
@@ -142,8 +143,13 @@ class AddressBook(Screen, HelperAddressBook):
     def update_addbook_label(self, instance):
         """Updating the label of address book address"""
         address_list = kivy_helper_search.search_sql(folder="addressbook")
-        stored_labels = [labels[0] for labels in address_list]
+        stored_labels = [labels[0].decode("utf-8", "replace") for labels in address_list]
-        add_dict = dict(address_list)
+        add_dict = {}
+        for row in address_list:
+            label, address = row
+            label = label.decode("utf-8", "replace")
+            address = address.decode("utf-8", "replace")
+            add_dict[label] = address
         label = str(self.addbook_popup.content_cls.ids.add_label.text)
         if label in stored_labels and self.address == add_dict[label]:
             stored_labels.remove(label)
@@ -151,7 +157,7 @@ class AddressBook(Screen, HelperAddressBook):
             sqlExecute("""
                 UPDATE addressbook
                 SET label = ?
-                WHERE address = ?""", label, self.addbook_popup.content_cls.address)
+                WHERE address = ?""", dbstr(label), dbstr(self.addbook_popup.content_cls.address))
             App.get_running_app().root.ids.id_addressbook.ids.ml.clear_widgets()
             App.get_running_app().root.ids.id_addressbook.loadAddresslist(None, 'All', '')
             self.addbook_popup.dismiss()

src/bitmessagekivy/baseclass/maildetail.py

@@ -7,6 +7,7 @@ Maildetail screen for inbox, sent, draft and trash.
 
 import os
 from datetime import datetime
+import sqlite3
 
 from kivy.core.clipboard import Clipboard
 from kivy.clock import Clock
@@ -111,7 +112,11 @@ class MailDetail(Screen):  # pylint: disable=too-many-instance-attributes
             elif self.kivy_state.detail_page_type == 'inbox':
                 data = sqlQuery(
                     "select toaddress, fromaddress, subject, message, received from inbox"
-                    " where msgid = ?", self.kivy_state.mail_id)
+                    " where msgid = ?", sqlite3.Binary(self.kivy_state.mail_id))
+                if len(data) < 1:
+                    data = sqlQuery(
+                        "select toaddress, fromaddress, subject, message, received from inbox"
+                        " where msgid = CAST(? AS TEXT)", self.kivy_state.mail_id)
                 self.assign_mail_details(data)
                 App.get_running_app().set_mail_detail_header()
         except Exception as e:  # pylint: disable=unused-variable
@@ -119,16 +124,16 @@ class MailDetail(Screen):  # pylint: disable=too-many-instance-attributes
 
     def assign_mail_details(self, data):
         """Assigning mail details"""
-        subject = data[0][2].decode() if isinstance(data[0][2], bytes) else data[0][2]
+        subject = data[0][2].decode("utf-8", "replace") if isinstance(data[0][2], bytes) else data[0][2]
-        body = data[0][3].decode() if isinstance(data[0][2], bytes) else data[0][3]
+        body = data[0][3].decode("utf-8", "replace") if isinstance(data[0][2], bytes) else data[0][3]
-        self.to_addr = data[0][0] if len(data[0][0]) > 4 else ' '
+        self.to_addr = data[0][0].decode("utf-8", "replace") if len(data[0][0]) > 4 else ' '
-        self.from_addr = data[0][1]
+        self.from_addr = data[0][1].decode("utf-8", "replace")
 
         self.subject = subject.capitalize(
         ) if subject.capitalize() else self.no_subject
         self.message = body
         if len(data[0]) == 7:
-            self.status = data[0][4]
+            self.status = data[0][4].decode("utf-8", "replace")
         self.time_tag = show_time_history(data[0][4]) if self.kivy_state.detail_page_type == 'inbox' \
             else show_time_history(data[0][6])
         self.avatarImg = os.path.join(self.kivy_state.imageDir, 'draft-icon.png') \

src/bitmessagekivy/baseclass/popup.py

@@ -59,7 +59,7 @@ class AddAddressPopup(BoxLayout):
         """Checking address is valid or not"""
         my_addresses = (
             App.get_running_app().root.ids.content_drawer.ids.identity_dropdown.values)
-        add_book = [addr[1] for addr in kivy_helper_search.search_sql(
+        add_book = [addr[1].decode("utf-8", "replace") for addr in kivy_helper_search.search_sql(
             folder="addressbook")]
         entered_text = str(instance.text).strip()
         if entered_text in add_book:
@@ -84,7 +84,7 @@ class AddAddressPopup(BoxLayout):
     def checkLabel_valid(self, instance):
         """Checking address label is unique or not"""
         entered_label = instance.text.strip()
-        addr_labels = [labels[0] for labels in kivy_helper_search.search_sql(
+        addr_labels = [labels[0].decode("utf-8", "replace") for labels in kivy_helper_search.search_sql(
             folder="addressbook")]
         if entered_label in addr_labels:
             self.ids.label.error = True
@@ -125,8 +125,13 @@ class SavedAddressDetailPopup(BoxLayout):
         """Checking address label is unique of not"""
         entered_label = str(instance.text.strip())
         address_list = kivy_helper_search.search_sql(folder="addressbook")
-        addr_labels = [labels[0] for labels in address_list]
+        addr_labels = [labels[0].decode("utf-8", "replace") for labels in address_list]
-        add_dict = dict(address_list)
+        add_dict = {}
+        for row in address_list:
+            label, address = row
+            label = label.decode("utf-8", "replace")
+            address = address.decode("utf-8", "replace")
+            add_dict[label] = address
         if self.address and entered_label in addr_labels \
                 and self.address != add_dict[entered_label]:
             self.ids.add_label.error = True

src/bitmessagekivy/identiconGeneration.py

@@ -3,7 +3,7 @@ Core classes for loading images and converting them to a Texture.
 The raw image data can be keep in memory for further access
 """
 import hashlib
-from io import BytesIO
+from six import BytesIO
 
 from PIL import Image
 from kivy.core.image import Image as CoreImage

src/bitmessagekivy/kivy_helper_search.py

@@ -2,6 +2,7 @@
 Sql queries for bitmessagekivy
 """
 from pybitmessage.helper_sql import sqlQuery
+from dbcompat import dbstr
 
 
 def search_sql(
@@ -30,21 +31,21 @@ def search_sql(
     if account is not None:
         if xAddress == 'both':
             sqlStatementParts.append("(fromaddress = ? OR toaddress = ?)")
-            sqlArguments.append(account)
+            sqlArguments.append(dbstr(account))
-            sqlArguments.append(account)
+            sqlArguments.append(dbstr(account))
         else:
             sqlStatementParts.append(xAddress + " = ? ")
-            sqlArguments.append(account)
+            sqlArguments.append(dbstr(account))
     if folder != "addressbook":
         if folder is not None:
             if folder == "new":
                 folder = "inbox"
                 unreadOnly = True
             sqlStatementParts.append("folder = ? ")
-            sqlArguments.append(folder)
+            sqlArguments.append(dbstr(folder))
         else:
             sqlStatementParts.append("folder != ?")
-            sqlArguments.append("trash")
+            sqlArguments.append(dbstr("trash"))
     if what is not None:
         for colmns in where:
             if len(where) > 1:
@@ -54,7 +55,7 @@ def search_sql(
                     filter_col += " or %s LIKE ? )" % (colmns)
             else:
                 filter_col = "%s LIKE ?" % (colmns)
-            sqlArguments.append(what)
+            sqlArguments.append(dbstr(what))
         sqlStatementParts.append(filter_col)
     if unreadOnly:
         sqlStatementParts.append("read = 0")

src/bitmessagekivy/tests/telenium_process.py

@@ -6,6 +6,7 @@ import os
 import shutil
 import tempfile
 from time import time, sleep
+from six.moves import getcwdb
 
 from requests.exceptions import ChunkedEncodingError
 
@@ -34,7 +35,7 @@ def cleanup(files=_files):
 
 class TeleniumTestProcess(TeleniumTestCase):
     """Setting Screen Functionality Testing"""
-    cmd_entrypoint = [os.path.join(os.path.abspath(os.getcwd()), 'src', 'mockbm', 'kivy_main.py')]
+    cmd_entrypoint = [os.path.join(os.path.abspath(getcwdb()), 'src', 'mockbm', 'kivy_main.py')]
 
     @classmethod
     def setUpClass(cls):

src/bitmessageqt/__init__.py

@@ -14,46 +14,53 @@ import threading
 import time
 from datetime import datetime, timedelta
 from sqlite3 import register_adapter
+import sqlite3
+import six
+from six.moves import range as xrange
+if six.PY3:
+    from codecs import escape_decode
 
+from unqstr import ustr, unic
 from PyQt4 import QtCore, QtGui
 from PyQt4.QtNetwork import QLocalSocket, QLocalServer
+from dbcompat import dbstr
 
 import shared
 import state
 from debug import logger
 from tr import _translate
-from account import (
+from .account import (
     accountClass, getSortedSubscriptions,
     BMAccount, GatewayAccount, MailchuckAccount, AccountColor)
 from addresses import decodeAddress, addBMIfNotPresent
-from bitmessageui import Ui_MainWindow
+from bitmessageqt.bitmessageui import Ui_MainWindow
 from bmconfigparser import config
 import namecoin
-from messageview import MessageView
+from .messageview import MessageView
-from migrationwizard import Ui_MigrationWizard
+from .migrationwizard import Ui_MigrationWizard
-from foldertree import (
+from .foldertree import (
     AccountMixin, Ui_FolderWidget, Ui_AddressWidget, Ui_SubscriptionWidget,
     MessageList_AddressWidget, MessageList_SubjectWidget,
     Ui_AddressBookWidgetItemLabel, Ui_AddressBookWidgetItemAddress,
     MessageList_TimeWidget)
-import settingsmixin
+from bitmessageqt import settingsmixin
-import support
+from bitmessageqt import support
 from helper_sql import sqlQuery, sqlExecute, sqlExecuteChunked, sqlStoredProcedure
 import helper_addressbook
 import helper_search
 import l10n
-from utils import str_broadcast_subscribers, avatarize
+from .utils import str_broadcast_subscribers, avatarize
-import dialogs
+from bitmessageqt import dialogs
 from network.stats import pendingDownload, pendingUpload
-from uisignaler import UISignaler
+from .uisignaler import UISignaler
 import paths
 from proofofwork import getPowType
 import queues
 import shutdown
-from statusbar import BMStatusBar
+from .statusbar import BMStatusBar
-import sound
+from bitmessageqt import sound
 # This is needed for tray icon
-import bitmessage_icons_rc  # noqa:F401 pylint: disable=unused-import
+from bitmessageqt import bitmessage_icons_rc  # noqa:F401 pylint: disable=unused-import
 import helper_sent
 
 try:
@@ -87,6 +94,13 @@ def openKeysFile():
         os.startfile(keysfile)  # pylint: disable=no-member
 
 
+def as_msgid(id_data):
+    if six.PY3:
+        return escape_decode(id_data)[0][2:-1]
+    else:  # assume six.PY2
+        return id_data
+
+
 class MyForm(settingsmixin.SMainWindow):
 
     # the maximum frequency of message sounds in seconds
@@ -123,7 +137,7 @@ class MyForm(settingsmixin.SMainWindow):
                 paths.codePath(), 'translations', 'qt_' + newlocale)
         else:
             translationpath = os.path.join(
-                str(QtCore.QLibraryInfo.location(
+                ustr(QtCore.QLibraryInfo.location(
                     QtCore.QLibraryInfo.TranslationsPath)), 'qt_' + newlocale)
         self.qsystranslator.load(translationpath)
         QtGui.QApplication.installTranslator(self.qsystranslator)
@@ -424,7 +438,7 @@ class MyForm(settingsmixin.SMainWindow):
     def rerenderTabTreeSubscriptions(self):
         treeWidget = self.ui.treeWidgetSubscriptions
         folders = Ui_FolderWidget.folderWeight.keys()
-        folders.remove("new")
+        Ui_FolderWidget.folderWeight.pop("new", None)
 
         # sort ascending when creating
         if treeWidget.topLevelItemCount() == 0:
@@ -471,7 +485,7 @@ class MyForm(settingsmixin.SMainWindow):
             # add missing folders
             if len(db[toAddress]) > 0:
                 j = 0
-                for f, c in db[toAddress].iteritems():
+                for f, c in six.iteritems(db[toAddress]):
                     try:
                         subwidget = Ui_FolderWidget(widget, j, toAddress, f, c['count'])
                     except KeyError:
@@ -555,6 +569,8 @@ class MyForm(settingsmixin.SMainWindow):
             "GROUP BY toaddress, folder")
         for row in queryreturn:
             toaddress, folder, cnt = row
+            toaddress = toaddress.decode("utf-8", "replace")
+            folder = folder.decode("utf-8", "replace")
             total += cnt
             if toaddress in db and folder in db[toaddress]:
                 db[toaddress][folder] = cnt
@@ -601,7 +617,7 @@ class MyForm(settingsmixin.SMainWindow):
             # add missing folders
             if len(db[toAddress]) > 0:
                 j = 0
-                for f, c in db[toAddress].iteritems():
+                for f, c in six.iteritems(db[toAddress]):
                     if toAddress is not None and tab == 'messages' and folder == "new":
                         continue
                     subwidget = Ui_FolderWidget(widget, j, toAddress, f, c)
@@ -649,9 +665,9 @@ class MyForm(settingsmixin.SMainWindow):
             if addressVersionNumber == 1:
                 displayMsg = _translate(
                     "MainWindow",
-                    "One of your addresses, %1, is an old version 1 address. "
+                    "One of your addresses, {0}, is an old version 1 address. "
                     "Version 1 addresses are no longer supported. "
-                    "May we delete it now?").arg(addressInKeysFile)
+                    "May we delete it now?").format(addressInKeysFile)
                 reply = QtGui.QMessageBox.question(
                     self, 'Message', displayMsg, QtGui.QMessageBox.Yes, QtGui.QMessageBox.No)
                 if reply == QtGui.QMessageBox.Yes:
@@ -833,9 +849,10 @@ class MyForm(settingsmixin.SMainWindow):
         TTL = config.getint('bitmessagesettings', 'ttl')
         if TTL < 3600: # an hour
             TTL = 3600
-        elif TTL > 28*24*60*60: # 28 days
+        elif TTL > 28 * 24 * 60 * 60:  # 28 days
-            TTL = 28*24*60*60
+            TTL = 28 * 24 * 60 * 60
-        self.ui.horizontalSliderTTL.setSliderPosition((TTL - 3600) ** (1/3.199))
+        self.ui.horizontalSliderTTL.setSliderPosition(
+            int((TTL - 3600) ** (1 / 3.199)))
         self.updateHumanFriendlyTTLDescription(TTL)
 
         QtCore.QObject.connect(self.ui.horizontalSliderTTL, QtCore.SIGNAL(
@@ -1026,7 +1043,7 @@ class MyForm(settingsmixin.SMainWindow):
             # related = related.findItems(msgid, QtCore.Qt.MatchExactly),
             # returns an empty list
             for rrow in range(related.rowCount()):
-                if related.item(rrow, 3).data() == msgid:
+                if as_msgid(related.item(rrow, 3).data()) == msgid:
                     break
 
         for col in range(widget.columnCount()):
@@ -1047,6 +1064,8 @@ class MyForm(settingsmixin.SMainWindow):
         normalUnread = {}
         broadcastsUnread = {}
         for addr, fld, count in queryReturn:
+            addr = addr.decode("utf-8", "replace")
+            fld = fld.decode("utf-8", "replace")
             try:
                 normalUnread[addr][fld] = count
             except KeyError:
@@ -1066,8 +1085,10 @@ class MyForm(settingsmixin.SMainWindow):
             queryReturn = sqlQuery(
                 'SELECT fromaddress, folder, COUNT(msgid) AS cnt'
                 ' FROM inbox WHERE read = 0 AND toaddress = ?'
-                ' GROUP BY fromaddress, folder', str_broadcast_subscribers)
+                ' GROUP BY fromaddress, folder', dbstr(str_broadcast_subscribers))
             for addr, fld, count in queryReturn:
+                addr = addr.decode("utf-8", "replace")
+                fld = fld.decode("utf-8", "replace")
                 try:
                     broadcastsUnread[addr][fld] = count
                 except KeyError:
@@ -1078,15 +1099,15 @@ class MyForm(settingsmixin.SMainWindow):
             for i in range(root.childCount()):
                 addressItem = root.child(i)
                 if addressItem.type == AccountMixin.ALL:
-                    newCount = sum(totalUnread.itervalues())
+                    newCount = sum(six.itervalues(totalUnread))
                     self.drawTrayIcon(self.currentTrayIconFileName, newCount)
                 else:
                     try:
-                        newCount = sum((
+                        newCount = sum(six.itervalues((
                             broadcastsUnread
                             if addressItem.type == AccountMixin.SUBSCRIPTION
                             else normalUnread
-                        )[addressItem.address].itervalues())
+                        )[addressItem.address]))
                     except KeyError:
                         newCount = 0
                 if newCount != addressItem.unreadCount:
@@ -1143,20 +1164,20 @@ class MyForm(settingsmixin.SMainWindow):
         elif status == 'msgsent':
             statusText = _translate(
                 "MainWindow",
-                "Message sent. Waiting for acknowledgement. Sent at %1"
+                "Message sent. Waiting for acknowledgement. Sent at {0}"
-            ).arg(l10n.formatTimestamp(lastactiontime))
+            ).format(l10n.formatTimestamp(lastactiontime))
         elif status == 'msgsentnoackexpected':
             statusText = _translate(
-                "MainWindow", "Message sent. Sent at %1"
+                "MainWindow", "Message sent. Sent at {0}"
-            ).arg(l10n.formatTimestamp(lastactiontime))
+            ).format(l10n.formatTimestamp(lastactiontime))
         elif status == 'doingmsgpow':
             statusText = _translate(
                 "MainWindow", "Doing work necessary to send message.")
         elif status == 'ackreceived':
             statusText = _translate(
                 "MainWindow",
-                "Acknowledgement of the message received %1"
+                "Acknowledgement of the message received {0}"
-            ).arg(l10n.formatTimestamp(lastactiontime))
+            ).format(l10n.formatTimestamp(lastactiontime))
         elif status == 'broadcastqueued':
             statusText = _translate(
                 "MainWindow", "Broadcast queued.")
@@ -1164,36 +1185,36 @@ class MyForm(settingsmixin.SMainWindow):
             statusText = _translate(
                 "MainWindow", "Doing work necessary to send broadcast.")
         elif status == 'broadcastsent':
-            statusText = _translate("MainWindow", "Broadcast on %1").arg(
+            statusText = _translate("MainWindow", "Broadcast on {0}").format(
                 l10n.formatTimestamp(lastactiontime))
         elif status == 'toodifficult':
             statusText = _translate(
                 "MainWindow",
                 "Problem: The work demanded by the recipient is more"
-                " difficult than you are willing to do. %1"
+                " difficult than you are willing to do. {0}"
-            ).arg(l10n.formatTimestamp(lastactiontime))
+            ).format(l10n.formatTimestamp(lastactiontime))
         elif status == 'badkey':
             statusText = _translate(
                 "MainWindow",
                 "Problem: The recipient\'s encryption key is no good."
-                " Could not encrypt message. %1"
+                " Could not encrypt message. {0}"
-            ).arg(l10n.formatTimestamp(lastactiontime))
+            ).format(l10n.formatTimestamp(lastactiontime))
         elif status == 'forcepow':
             statusText = _translate(
                 "MainWindow",
                 "Forced difficulty override. Send should start soon.")
         else:
             statusText = _translate(
-                "MainWindow", "Unknown status: %1 %2").arg(status).arg(
+                "MainWindow", "Unknown status: {0} {1}").format(status,
                 l10n.formatTimestamp(lastactiontime))
 
         items = [
             MessageList_AddressWidget(
-                toAddress, unicode(acct.toLabel, 'utf-8')),
+                toAddress, unic(ustr(acct.toLabel))),
             MessageList_AddressWidget(
-                fromAddress, unicode(acct.fromLabel, 'utf-8')),
+                fromAddress, unic(ustr(acct.fromLabel))),
             MessageList_SubjectWidget(
-                str(subject), unicode(acct.subject, 'utf-8', 'replace')),
+                ustr(subject), unic(ustr(acct.subject))),
             MessageList_TimeWidget(
                 statusText, False, lastactiontime, ackdata)]
         self.addMessageListItem(tableWidget, items)
@@ -1214,11 +1235,11 @@ class MyForm(settingsmixin.SMainWindow):
 
         items = [
             MessageList_AddressWidget(
-                toAddress, unicode(acct.toLabel, 'utf-8'), not read),
+                toAddress, unic(ustr(acct.toLabel)), not read),
             MessageList_AddressWidget(
-                fromAddress, unicode(acct.fromLabel, 'utf-8'), not read),
+                fromAddress, unic(ustr(acct.fromLabel)), not read),
             MessageList_SubjectWidget(
-                str(subject), unicode(acct.subject, 'utf-8', 'replace'),
+                ustr(subject), unic(ustr(acct.subject)),
                 not read),
             MessageList_TimeWidget(
                 l10n.formatTimestamp(received), not read, received, msgid)
@@ -1246,7 +1267,14 @@ class MyForm(settingsmixin.SMainWindow):
             xAddress, account, "sent", where, what, False)
 
         for row in queryreturn:
-            self.addMessageListItemSent(tableWidget, *row)
+            r = []
+            r.append(row[0].decode("utf-8", "replace"))  # toaddress
+            r.append(row[1].decode("utf-8", "replace"))  # fromaddress
+            r.append(row[2].decode("utf-8", "replace"))  # subject
+            r.append(row[3].decode("utf-8", "replace"))  # status
+            r.append(row[4])  # ackdata
+            r.append(row[5])  # lastactiontime
+            self.addMessageListItemSent(tableWidget, *r)
 
         tableWidget.horizontalHeader().setSortIndicator(
             3, QtCore.Qt.DescendingOrder)
@@ -1287,6 +1315,10 @@ class MyForm(settingsmixin.SMainWindow):
 
         for row in queryreturn:
             toAddress, fromAddress, subject, _, msgid, received, read = row
+            toAddress = toAddress.decode("utf-8", "replace")
+            fromAddress = fromAddress.decode("utf-8", "replace")
+            subject = subject.decode("utf-8", "replace")
+            received = received.decode("utf-8", "replace")
             self.addMessageListItemInbox(
                 tableWidget, toAddress, fromAddress, subject,
                 msgid, received, read)
@@ -1372,6 +1404,7 @@ class MyForm(settingsmixin.SMainWindow):
         SELECT msgid, toaddress, read FROM inbox where folder='inbox'
         ''')
         for msgid, toAddress, read in queryreturn:
+            toAddress = toAddress.decode("utf-8", "replace")
 
             if not read:
                 # increment the unread subscriptions if True (1)
@@ -1450,7 +1483,7 @@ class MyForm(settingsmixin.SMainWindow):
 
     # Adapters and converters for QT <-> sqlite
     def sqlInit(self):
-        register_adapter(QtCore.QByteArray, str)
+        register_adapter(QtCore.QByteArray, bytes)
 
     def indicatorInit(self):
         """
@@ -1499,7 +1532,7 @@ class MyForm(settingsmixin.SMainWindow):
             self, title, subtitle, category, label=None, icon=None):
         self.playSound(category, label)
         self._notifier(
-            unicode(title), unicode(subtitle), category, label, icon)
+            unic(ustr(title)), unic(ustr(subtitle)), category, label, icon)
 
     # tree
     def treeWidgetKeyPressEvent(self, event):
@@ -1602,9 +1635,9 @@ class MyForm(settingsmixin.SMainWindow):
                     _translate(
                         "MainWindow",
                         "You may manage your keys by editing the keys.dat file stored in"
-                        "\n %1 \n"
+                        "\n {0} \n"
                         "It is important that you back up this file."
-                    ).arg(state.appdata),
+                    ).format(state.appdata),
                     QtGui.QMessageBox.Ok)
         elif sys.platform == 'win32' or sys.platform == 'win64':
             if state.appdata == '':
@@ -1625,9 +1658,9 @@ class MyForm(settingsmixin.SMainWindow):
                     _translate("MainWindow", "Open keys.dat?"),
                     _translate(
                         "MainWindow",
-                        "You may manage your keys by editing the keys.dat file stored in\n %1 \n"
+                        "You may manage your keys by editing the keys.dat file stored in\n {0} \n"
                         "It is important that you back up this file. Would you like to open the file now?"
-                        "(Be sure to close Bitmessage before making any changes.)").arg(state.appdata),
+                        "(Be sure to close Bitmessage before making any changes.)").format(state.appdata),
                     QtGui.QMessageBox.Yes, QtGui.QMessageBox.No)
             if reply == QtGui.QMessageBox.Yes:
                 openKeysFile()
@@ -1702,7 +1735,7 @@ class MyForm(settingsmixin.SMainWindow):
                 addressVersionNumber, streamNumberForAddress,
                 "regenerated deterministic address",
                 dialog.spinBoxNumberOfAddressesToMake.value(),
-                dialog.lineEditPassphrase.text().toUtf8(),
+                ustr(dialog.lineEditPassphrase.text()).encode("utf-8", "replace"),
                 dialog.checkBoxEighteenByteRipe.isChecked()
             ))
             self.ui.tabWidget.setCurrentIndex(
@@ -1915,8 +1948,6 @@ class MyForm(settingsmixin.SMainWindow):
                         sent.item(i, 3).setText(textToDisplay)
 
     def updateSentItemStatusByAckdata(self, ackdata, textToDisplay):
-        if type(ackdata) is str:
-            ackdata = QtCore.QByteArray(ackdata)
         for sent in (
             self.ui.tableWidgetInbox,
             self.ui.tableWidgetInboxSubscriptions,
@@ -1927,7 +1958,7 @@ class MyForm(settingsmixin.SMainWindow):
                 continue
             for i in range(sent.rowCount()):
                 toAddress = sent.item(i, 0).data(QtCore.Qt.UserRole)
-                tableAckdata = sent.item(i, 3).data()
+                tableAckdata = as_msgid(sent.item(i, 3).data())
                 status, addressVersionNumber, streamNumber, ripe = decodeAddress(
                     toAddress)
                 if ackdata == tableAckdata:
@@ -1953,7 +1984,7 @@ class MyForm(settingsmixin.SMainWindow):
         ):
             i = None
             for i in range(inbox.rowCount()):
-                if msgid == inbox.item(i, 3).data():
+                if msgid == as_msgid(inbox.item(i, 3).data()):
                     break
             else:
                 continue
@@ -1970,9 +2001,9 @@ class MyForm(settingsmixin.SMainWindow):
         self.notifiedNewVersion = ".".join(str(n) for n in version)
         self.updateStatusBar(_translate(
             "MainWindow",
-            "New version of PyBitmessage is available: %1. Download it"
+            "New version of PyBitmessage is available: {0}. Download it"
             " from https://github.com/Bitmessage/PyBitmessage/releases/latest"
-            ).arg(self.notifiedNewVersion)
+            ).format(self.notifiedNewVersion)
         )
 
     def displayAlert(self, title, text, exitAfterUserClicksOk):
@@ -1998,9 +2029,9 @@ class MyForm(settingsmixin.SMainWindow):
     def rerenderAddressBook(self):
         def addRow (address, label, type):
             self.ui.tableWidgetAddressBook.insertRow(0)
-            newItem = Ui_AddressBookWidgetItemLabel(address, unicode(label, 'utf-8'), type)
+            newItem = Ui_AddressBookWidgetItemLabel(address, unic(ustr(label)), type)
             self.ui.tableWidgetAddressBook.setItem(0, 0, newItem)
-            newItem = Ui_AddressBookWidgetItemAddress(address, unicode(label, 'utf-8'), type)
+            newItem = Ui_AddressBookWidgetItemAddress(address, unic(ustr(label)), type)
             self.ui.tableWidgetAddressBook.setItem(0, 1, newItem)
 
         oldRows = {}
@@ -2018,6 +2049,8 @@ class MyForm(settingsmixin.SMainWindow):
         queryreturn = sqlQuery('SELECT label, address FROM subscriptions WHERE enabled = 1')
         for row in queryreturn:
             label, address = row
+            label = label.decode("utf-8", "replace")
+            address = address.decode("utf-8", "replace")
             newRows[address] = [label, AccountMixin.SUBSCRIPTION]
         # chans
         for address in config.addresses(True):
@@ -2028,6 +2061,8 @@ class MyForm(settingsmixin.SMainWindow):
         queryreturn = sqlQuery('SELECT * FROM addressbook')
         for row in queryreturn:
             label, address = row
+            label = label.decode("utf-8", "replace")
+            address = address.decode("utf-8", "replace")
             newRows[address] = [label, AccountMixin.NORMAL]
 
         completerList = []
@@ -2041,7 +2076,7 @@ class MyForm(settingsmixin.SMainWindow):
                 self.ui.tableWidgetAddressBook.removeRow(oldRows[address][2])
         for address in newRows:
             addRow(address, newRows[address][0], newRows[address][1])
-            completerList.append(unicode(newRows[address][0], encoding="UTF-8") + " <" + address + ">")
+            completerList.append(unic(ustr(newRows[address][0]) + " <" + ustr(address) + ">"))
 
         # sort
         self.ui.tableWidgetAddressBook.sortByColumn(
@@ -2079,22 +2114,22 @@ class MyForm(settingsmixin.SMainWindow):
                 self.ui.tabWidgetSend.indexOf(self.ui.sendDirect):
             # message to specific people
             sendMessageToPeople = True
-            fromAddress = str(self.ui.comboBoxSendFrom.itemData(
+            fromAddress = ustr(self.ui.comboBoxSendFrom.itemData(
                 self.ui.comboBoxSendFrom.currentIndex(),
-                QtCore.Qt.UserRole).toString())
+                QtCore.Qt.UserRole))
-            toAddresses = str(self.ui.lineEditTo.text().toUtf8())
+            toAddresses = ustr(self.ui.lineEditTo.text())
-            subject = str(self.ui.lineEditSubject.text().toUtf8())
+            subject = ustr(self.ui.lineEditSubject.text())
-            message = str(
+            message = ustr(
-                self.ui.textEditMessage.document().toPlainText().toUtf8())
+                self.ui.textEditMessage.document().toPlainText())
         else:
             # broadcast message
             sendMessageToPeople = False
-            fromAddress = str(self.ui.comboBoxSendFromBroadcast.itemData(
+            fromAddress = ustr(self.ui.comboBoxSendFromBroadcast.itemData(
                 self.ui.comboBoxSendFromBroadcast.currentIndex(),
-                QtCore.Qt.UserRole).toString())
+                QtCore.Qt.UserRole))
-            subject = str(self.ui.lineEditSubjectBroadcast.text().toUtf8())
+            subject = ustr(self.ui.lineEditSubjectBroadcast.text())
-            message = str(
+            message = ustr(
-                self.ui.textEditMessageBroadcast.document().toPlainText().toUtf8())
+                self.ui.textEditMessageBroadcast.document().toPlainText())
         """
         The whole network message must fit in 2^18 bytes.
         Let's assume 500 bytes of overhead. If someone wants to get that
@@ -2108,9 +2143,9 @@ class MyForm(settingsmixin.SMainWindow):
                 _translate(
                     "MainWindow",
                     "The message that you are trying to send is too long"
-                    " by %1 bytes. (The maximum is 261644 bytes). Please"
+                    " by {0} bytes. (The maximum is 261644 bytes). Please"
                     " cut it down before sending."
-                ).arg(len(message) - (2 ** 18 - 500)))
+                ).format(len(message) - (2 ** 18 - 500)))
             return
 
         acct = accountClass(fromAddress)
@@ -2159,15 +2194,15 @@ class MyForm(settingsmixin.SMainWindow):
                                 "MainWindow",
                                 "Error: Your account wasn't registered at"
                                 " an email gateway. Sending registration"
-                                " now as %1, please wait for the registration"
+                                " now as {0}, please wait for the registration"
                                 " to be processed before retrying sending."
-                                ).arg(email)
+                                ).format(email)
                             )
                             return
                     status, addressVersionNumber, streamNumber = decodeAddress(toAddress)[:3]
                     if status != 'success':
                         try:
-                            toAddress = unicode(toAddress, 'utf-8', 'ignore')
+                            toAddress = unic(ustr(toAddress))
                         except:
                             pass
                         logger.error('Error: Could not decode recipient address ' + toAddress + ':' + status)
@@ -2176,58 +2211,58 @@ class MyForm(settingsmixin.SMainWindow):
                             self.updateStatusBar(_translate(
                                 "MainWindow",
                                 "Error: Bitmessage addresses start with"
-                                " BM-   Please check the recipient address %1"
+                                " BM-   Please check the recipient address {0}"
-                                ).arg(toAddress))
+                                ).format(toAddress))
                         elif status == 'checksumfailed':
                             self.updateStatusBar(_translate(
                                 "MainWindow",
-                                "Error: The recipient address %1 is not"
+                                "Error: The recipient address {0} is not"
                                 " typed or copied correctly. Please check it."
-                                ).arg(toAddress))
+                                ).format(toAddress))
                         elif status == 'invalidcharacters':
                             self.updateStatusBar(_translate(
                                 "MainWindow",
-                                "Error: The recipient address %1 contains"
+                                "Error: The recipient address {0} contains"
                                 " invalid characters. Please check it."
-                                ).arg(toAddress))
+                                ).format(toAddress))
                         elif status == 'versiontoohigh':
                             self.updateStatusBar(_translate(
                                 "MainWindow",
                                 "Error: The version of the recipient address"
-                                " %1 is too high. Either you need to upgrade"
+                                " {0} is too high. Either you need to upgrade"
                                 " your Bitmessage software or your"
                                 " acquaintance is being clever."
-                                ).arg(toAddress))
+                                ).format(toAddress))
                         elif status == 'ripetooshort':
                             self.updateStatusBar(_translate(
                                 "MainWindow",
                                 "Error: Some data encoded in the recipient"
-                                " address %1 is too short. There might be"
+                                " address {0} is too short. There might be"
                                 " something wrong with the software of"
                                 " your acquaintance."
-                                ).arg(toAddress))
+                                ).format(toAddress))
                         elif status == 'ripetoolong':
                             self.updateStatusBar(_translate(
                                 "MainWindow",
                                 "Error: Some data encoded in the recipient"
-                                " address %1 is too long. There might be"
+                                " address {0} is too long. There might be"
                                 " something wrong with the software of"
                                 " your acquaintance."
-                                ).arg(toAddress))
+                                ).format(toAddress))
                         elif status == 'varintmalformed':
                             self.updateStatusBar(_translate(
                                 "MainWindow",
                                 "Error: Some data encoded in the recipient"
-                                " address %1 is malformed. There might be"
+                                " address {0} is malformed. There might be"
                                 " something wrong with the software of"
                                 " your acquaintance."
-                                ).arg(toAddress))
+                                ).format(toAddress))
                         else:
                             self.updateStatusBar(_translate(
                                 "MainWindow",
                                 "Error: Something is wrong with the"
-                                " recipient address %1."
+                                " recipient address {0}."
-                                ).arg(toAddress))
+                                ).format(toAddress))
                     elif fromAddress == '':
                         self.updateStatusBar(_translate(
                             "MainWindow",
@@ -2244,9 +2279,9 @@ class MyForm(settingsmixin.SMainWindow):
                                 _translate("MainWindow", "Address version number"),
                                 _translate(
                                     "MainWindow",
-                                    "Concerning the address %1, Bitmessage cannot understand address version numbers"
+                                    "Concerning the address {0}, Bitmessage cannot understand address version numbers"
-                                    " of %2. Perhaps upgrade Bitmessage to the latest version."
+                                    " of {1}. Perhaps upgrade Bitmessage to the latest version."
-                                ).arg(toAddress).arg(str(addressVersionNumber)))
+                                ).format(toAddress, str(addressVersionNumber)))
                             continue
                         if streamNumber > 1 or streamNumber == 0:
                             QtGui.QMessageBox.about(
@@ -2254,9 +2289,9 @@ class MyForm(settingsmixin.SMainWindow):
                                 _translate("MainWindow", "Stream number"),
                                 _translate(
                                     "MainWindow",
-                                    "Concerning the address %1, Bitmessage cannot handle stream numbers of %2."
+                                    "Concerning the address {0}, Bitmessage cannot handle stream numbers of {1}."
                                     " Perhaps upgrade Bitmessage to the latest version."
-                                ).arg(toAddress).arg(str(streamNumber)))
+                                ).format(toAddress, str(streamNumber)))
                             continue
                         self.statusbar.clearMessage()
                         if state.statusIconColor == 'red':
@@ -2272,10 +2307,11 @@ class MyForm(settingsmixin.SMainWindow):
                             subject=subject, message=message, encoding=encoding)
                         toLabel = ''
                         queryreturn = sqlQuery('''select label from addressbook where address=?''',
-                                               toAddress)
+                                               dbstr(toAddress))
                         if queryreturn != []:
                             for row in queryreturn:
                                 toLabel, = row
+                            toLabel = toLabel.decode("utf-8", "replace")
 
                         self.displayNewSentMessage(
                             toAddress, toLabel, fromAddress, subject, message, ackdata)
@@ -2341,11 +2377,11 @@ class MyForm(settingsmixin.SMainWindow):
             ))
 
     def click_pushButtonFetchNamecoinID(self):
-        identities = str(self.ui.lineEditTo.text().toUtf8()).split(";")
+        identities = ustr(self.ui.lineEditTo.text()).split(";")
         err, addr = self.namecoin.query(identities[-1].strip())
         if err is not None:
             self.updateStatusBar(
-                _translate("MainWindow", "Error: %1").arg(err))
+                _translate("MainWindow", "Error: {0}").format(err))
         else:
             identities[-1] = addr
             self.ui.lineEditTo.setText("; ".join(identities))
@@ -2358,7 +2394,7 @@ class MyForm(settingsmixin.SMainWindow):
         self.ui.tabWidgetSend.setCurrentIndex(
             self.ui.tabWidgetSend.indexOf(
                 self.ui.sendBroadcast
-                if config.safeGetBoolean(str(address), 'mailinglist')
+                if config.safeGetBoolean(ustr(address), 'mailinglist')
                 else self.ui.sendDirect
             ))
 
@@ -2371,14 +2407,14 @@ class MyForm(settingsmixin.SMainWindow):
                 addressInKeysFile, 'enabled')
             isMaillinglist = config.safeGetBoolean(addressInKeysFile, 'mailinglist')
             if isEnabled and not isMaillinglist:
-                label = unicode(config.get(addressInKeysFile, 'label'), 'utf-8', 'ignore').strip()
+                label = unic(ustr(config.get(addressInKeysFile, 'label')).strip())
                 if label == "":
                     label = addressInKeysFile
                 self.ui.comboBoxSendFrom.addItem(avatarize(addressInKeysFile), label, addressInKeysFile)
 #        self.ui.comboBoxSendFrom.model().sort(1, Qt.AscendingOrder)
         for i in range(self.ui.comboBoxSendFrom.count()):
-            address = str(self.ui.comboBoxSendFrom.itemData(
+            address = ustr(self.ui.comboBoxSendFrom.itemData(
-                i, QtCore.Qt.UserRole).toString())
+                i, QtCore.Qt.UserRole))
             self.ui.comboBoxSendFrom.setItemData(
                 i, AccountColor(address).accountColor(),
                 QtCore.Qt.ForegroundRole)
@@ -2395,13 +2431,13 @@ class MyForm(settingsmixin.SMainWindow):
                 addressInKeysFile, 'enabled')
             isChan = config.safeGetBoolean(addressInKeysFile, 'chan')
             if isEnabled and not isChan:
-                label = unicode(config.get(addressInKeysFile, 'label'), 'utf-8', 'ignore').strip()
+                label = unic(ustr(config.get(addressInKeysFile, 'label')).strip())
                 if label == "":
                     label = addressInKeysFile
                 self.ui.comboBoxSendFromBroadcast.addItem(avatarize(addressInKeysFile), label, addressInKeysFile)
         for i in range(self.ui.comboBoxSendFromBroadcast.count()):
-            address = str(self.ui.comboBoxSendFromBroadcast.itemData(
+            address = ustr(self.ui.comboBoxSendFromBroadcast.itemData(
-                i, QtCore.Qt.UserRole).toString())
+                i, QtCore.Qt.UserRole))
             self.ui.comboBoxSendFromBroadcast.setItemData(
                 i, AccountColor(address).accountColor(),
                 QtCore.Qt.ForegroundRole)
@@ -2500,8 +2536,8 @@ class MyForm(settingsmixin.SMainWindow):
                 'bitmessagesettings', 'showtraynotifications'):
             self.notifierShow(
                 _translate("MainWindow", "New Message"),
-                _translate("MainWindow", "From %1").arg(
+                _translate("MainWindow", "From {0}").format(
-                    unicode(acct.fromLabel, 'utf-8')),
+                    unic(ustr(acct.fromLabel))),
                 sound.SOUND_UNKNOWN
             )
         if self.getCurrentAccount() is not None and (
@@ -2560,7 +2596,7 @@ class MyForm(settingsmixin.SMainWindow):
         # Add to database (perhaps this should be separated from the MyForm class)
         sqlExecute(
             '''INSERT INTO subscriptions VALUES (?,?,?)''',
-            label, address, True
+            dbstr(label), dbstr(address), True
         )
         self.rerenderMessagelistFromLabels()
         shared.reloadBroadcastSendersForWhichImWatching()
@@ -2639,7 +2675,7 @@ class MyForm(settingsmixin.SMainWindow):
         # Only settings remain here
         acct.settings()
         for i in range(self.ui.comboBoxSendFrom.count()):
-            if str(self.ui.comboBoxSendFrom.itemData(i).toPyObject()) \
+            if ustr(self.ui.comboBoxSendFrom.itemData(i)) \
                     == acct.fromAddress:
                 self.ui.comboBoxSendFrom.setCurrentIndex(i)
                 break
@@ -2674,13 +2710,18 @@ class MyForm(settingsmixin.SMainWindow):
 
         msgids = []
         for i in range(0, idCount):
-            msgids.append(tableWidget.item(i, 3).data())
+            msgids.append(sqlite3.Binary(as_msgid(tableWidget.item(i, 3).data())))
             for col in xrange(tableWidget.columnCount()):
                 tableWidget.item(i, col).setUnread(False)
 
         markread = sqlExecuteChunked(
             "UPDATE inbox SET read = 1 WHERE msgid IN({0}) AND read=0",
-            idCount, *msgids
+            False, idCount, *msgids
+        )
+        if markread < 1:
+            markread = sqlExecuteChunked(
+                "UPDATE inbox SET read = 1 WHERE msgid IN({0}) AND read=0",
+                True, idCount, *msgids
             )
 
         if markread > 0:
@@ -2706,7 +2747,7 @@ class MyForm(settingsmixin.SMainWindow):
         if reply != QtGui.QMessageBox.Yes:
             return
         config.set(
-            'bitmessagesettings', 'dontconnect', str(dontconnect_option))
+            'bitmessagesettings', 'dontconnect', ustr(dontconnect_option))
         config.save()
         self.ui.updateNetworkSwitchMenuLabel(dontconnect_option)
 
@@ -2790,7 +2831,7 @@ class MyForm(settingsmixin.SMainWindow):
         self.quitAccepted = True
 
         self.updateStatusBar(_translate(
-            "MainWindow", "Shutting down PyBitmessage... %1%").arg(0))
+            "MainWindow", "Shutting down PyBitmessage... {0}%").format(0))
 
         if waitForConnection:
             self.updateStatusBar(_translate(
@@ -2824,8 +2865,8 @@ class MyForm(settingsmixin.SMainWindow):
                     maxWorkerQueue = curWorkerQueue
                 if curWorkerQueue > 0:
                     self.updateStatusBar(_translate(
-                        "MainWindow", "Waiting for PoW to finish... %1%"
+                        "MainWindow", "Waiting for PoW to finish... {0}%"
-                    ).arg(50 * (maxWorkerQueue - curWorkerQueue) /
+                    ).format(50 * (maxWorkerQueue - curWorkerQueue) /
                           maxWorkerQueue))
                     time.sleep(0.5)
                     QtCore.QCoreApplication.processEvents(
@@ -2833,7 +2874,7 @@ class MyForm(settingsmixin.SMainWindow):
                     )
 
             self.updateStatusBar(_translate(
-                "MainWindow", "Shutting down Pybitmessage... %1%").arg(50))
+                "MainWindow", "Shutting down Pybitmessage... {0}%").format(50))
 
             QtCore.QCoreApplication.processEvents(
                 QtCore.QEventLoop.AllEvents, 1000
@@ -2847,14 +2888,14 @@ class MyForm(settingsmixin.SMainWindow):
 
             # check if upload (of objects created locally) pending
             self.updateStatusBar(_translate(
-                "MainWindow", "Waiting for objects to be sent... %1%").arg(50))
+                "MainWindow", "Waiting for objects to be sent... {0}%").format(50))
             maxPendingUpload = max(1, pendingUpload())
 
             while pendingUpload() > 1:
                 self.updateStatusBar(_translate(
                     "MainWindow",
-                    "Waiting for objects to be sent... %1%"
+                    "Waiting for objects to be sent... {0}%"
-                ).arg(int(50 + 20 * (pendingUpload() / maxPendingUpload))))
+                ).format(int(50 + 20 * (pendingUpload() / maxPendingUpload))))
                 time.sleep(0.5)
                 QtCore.QCoreApplication.processEvents(
                     QtCore.QEventLoop.AllEvents, 1000
@@ -2869,12 +2910,12 @@ class MyForm(settingsmixin.SMainWindow):
 
         # save state and geometry self and all widgets
         self.updateStatusBar(_translate(
-            "MainWindow", "Saving settings... %1%").arg(70))
+            "MainWindow", "Saving settings... {0}%").format(70))
         QtCore.QCoreApplication.processEvents(
             QtCore.QEventLoop.AllEvents, 1000
         )
         self.saveSettings()
-        for attr, obj in self.ui.__dict__.iteritems():
+        for attr, obj in six.iteritems(self.ui.__dict__):
             if hasattr(obj, "__class__") \
                     and isinstance(obj, settingsmixin.SettingsMixin):
                 saveMethod = getattr(obj, "saveSettings", None)
@@ -2882,18 +2923,18 @@ class MyForm(settingsmixin.SMainWindow):
                     obj.saveSettings()
 
         self.updateStatusBar(_translate(
-            "MainWindow", "Shutting down core... %1%").arg(80))
+            "MainWindow", "Shutting down core... {0}%").format(80))
         QtCore.QCoreApplication.processEvents(
             QtCore.QEventLoop.AllEvents, 1000
         )
         shutdown.doCleanShutdown()
 
         self.updateStatusBar(_translate(
-            "MainWindow", "Stopping notifications... %1%").arg(90))
+            "MainWindow", "Stopping notifications... {0}%").format(90))
         self.tray.hide()
 
         self.updateStatusBar(_translate(
-            "MainWindow", "Shutdown imminent... %1%").arg(100))
+            "MainWindow", "Shutdown imminent... {0}%").format(100))
 
         logger.info("Shutdown complete")
         self.close()
@@ -2919,10 +2960,14 @@ class MyForm(settingsmixin.SMainWindow):
         if not msgid:
             return
         queryreturn = sqlQuery(
-            '''select message from inbox where msgid=?''', msgid)
+            'SELECT message FROM inbox WHERE msgid=?', sqlite3.Binary(msgid))
+        if len(queryreturn) < 1:
+            queryreturn = sqlQuery(
+                'SELECT message FROM inbox WHERE msgid=CAST(? AS TEXT)', msgid)
         if queryreturn != []:
             for row in queryreturn:
                 messageText, = row
+            messageText = messageText.decode("utf-8", "replace")
 
         lines = messageText.split('\n')
         totalLines = len(lines)
@@ -2937,8 +2982,8 @@ class MyForm(settingsmixin.SMainWindow):
                 lines[i] = '<br><br>'
         content = ' '.join(lines) # To keep the whitespace between lines
         content = shared.fixPotentiallyInvalidUTF8Data(content)
-        content = unicode(content, 'utf-8)')
+        content = unic(ustr(content))
-        textEdit.setHtml(QtCore.QString(content))
+        textEdit.setHtml(content)
 
     def on_action_InboxMarkUnread(self):
         tableWidget = self.getCurrentMessagelist()
@@ -2949,8 +2994,8 @@ class MyForm(settingsmixin.SMainWindow):
         # modified = 0
         for row in tableWidget.selectedIndexes():
             currentRow = row.row()
-            msgid = tableWidget.item(currentRow, 3).data()
+            msgid = as_msgid(tableWidget.item(currentRow, 3).data())
-            msgids.add(msgid)
+            msgids.add(sqlite3.Binary(msgid))
             # if not tableWidget.item(currentRow, 0).unread:
             #     modified += 1
             self.updateUnreadStatus(tableWidget, currentRow, msgid, False)
@@ -2958,9 +3003,14 @@ class MyForm(settingsmixin.SMainWindow):
         # for 1081
         idCount = len(msgids)
         # rowcount =
+        total_row_count = sqlExecuteChunked(
+            '''UPDATE inbox SET read=0 WHERE msgid IN ({0}) AND read=1''',
+            False, idCount, *msgids
+        )
+        if total_row_count < 1:
             sqlExecuteChunked(
                 '''UPDATE inbox SET read=0 WHERE msgid IN ({0}) AND read=1''',
-            idCount, *msgids
+                True, idCount, *msgids
             )
 
         self.propagateUnreadCount()
@@ -3005,7 +3055,7 @@ class MyForm(settingsmixin.SMainWindow):
             self.ui.comboBoxSendFrom, self.ui.comboBoxSendFromBroadcast
         ):
             for i in range(box.count()):
-                if str(box.itemData(i).toPyObject()) == address:
+                if ustr(box.itemData(i)) == ustr(address):
                     box.setCurrentIndex(i)
                     break
             else:
@@ -3039,13 +3089,18 @@ class MyForm(settingsmixin.SMainWindow):
         acct = accountClass(toAddressAtCurrentInboxRow)
         fromAddressAtCurrentInboxRow = tableWidget.item(
             currentInboxRow, column_from).address
-        msgid = tableWidget.item(currentInboxRow, 3).data()
+        msgid = as_msgid(tableWidget.item(currentInboxRow, 3).data())
         queryreturn = sqlQuery(
-            "SELECT message FROM inbox WHERE msgid=?", msgid
+            "SELECT message FROM inbox WHERE msgid=?", sqlite3.Binary(msgid)
-        ) or sqlQuery("SELECT message FROM sent WHERE ackdata=?", msgid)
+        ) or sqlQuery("SELECT message FROM sent WHERE ackdata=?", sqlite3.Binary(msgid))
+        if len(queryreturn) < 1:
+            queryreturn = sqlQuery(
+                "SELECT message FROM inbox WHERE msgid=CAST(? AS TEXT)", msgid
+            ) or sqlQuery("SELECT message FROM sent WHERE ackdata=CAST(? AS TEXT)", msgid)
         if queryreturn != []:
             for row in queryreturn:
                 messageAtCurrentInboxRow, = row
+            messageAtCurrentInboxRow = messageAtCurrentInboxRow.decode("utf-8", "replace")
         acct.parseMessage(
             toAddressAtCurrentInboxRow, fromAddressAtCurrentInboxRow,
             tableWidget.item(currentInboxRow, 2).subject,
@@ -3066,9 +3121,9 @@ class MyForm(settingsmixin.SMainWindow):
                 self, _translate("MainWindow", "Address is gone"),
                 _translate(
                     "MainWindow",
-                    "Bitmessage cannot find your address %1. Perhaps you"
+                    "Bitmessage cannot find your address {0}. Perhaps you"
                     " removed it?"
-                ).arg(toAddressAtCurrentInboxRow), QtGui.QMessageBox.Ok)
+                ).format(toAddressAtCurrentInboxRow), QtGui.QMessageBox.Ok)
         elif not config.getboolean(
                 toAddressAtCurrentInboxRow, 'enabled'):
             QtGui.QMessageBox.information(
@@ -3096,7 +3151,7 @@ class MyForm(settingsmixin.SMainWindow):
             tableWidget.item(currentInboxRow, column_from).label or (
                 isinstance(acct, GatewayAccount) and
                 fromAddressAtCurrentInboxRow == acct.relayAddress):
-            self.ui.lineEditTo.setText(str(acct.fromAddress))
+            self.ui.lineEditTo.setText(ustr(acct.fromAddress))
         else:
             self.ui.lineEditTo.setText(
                 tableWidget.item(currentInboxRow, column_from).accountString()
@@ -3111,7 +3166,7 @@ class MyForm(settingsmixin.SMainWindow):
                 ' reply to the chan address.')
             if toAddressAtCurrentInboxRow == \
                     tableWidget.item(currentInboxRow, column_to).label:
-                self.ui.lineEditTo.setText(str(toAddressAtCurrentInboxRow))
+                self.ui.lineEditTo.setText(ustr(toAddressAtCurrentInboxRow))
             else:
                 self.ui.lineEditTo.setText(
                     tableWidget.item(currentInboxRow, column_to).accountString()
@@ -3120,7 +3175,7 @@ class MyForm(settingsmixin.SMainWindow):
         self.setSendFromComboBox(toAddressAtCurrentInboxRow)
 
         quotedText = self.quoted_text(
-            unicode(messageAtCurrentInboxRow, 'utf-8', 'replace'))
+            unic(ustr(messageAtCurrentInboxRow)))
         widget['message'].setPlainText(quotedText)
         if acct.subject[0:3] in ('Re:', 'RE:'):
             widget['subject'].setText(
@@ -3157,13 +3212,13 @@ class MyForm(settingsmixin.SMainWindow):
             currentInboxRow, 0).data(QtCore.Qt.UserRole)
         # Let's make sure that it isn't already in the address book
         queryreturn = sqlQuery('''select * from blacklist where address=?''',
-                               addressAtCurrentInboxRow)
+                               dbstr(addressAtCurrentInboxRow))
         if queryreturn == []:
             label = "\"" + tableWidget.item(currentInboxRow, 2).subject + "\" in " + config.get(
                 recipientAddress, "label")
             sqlExecute('''INSERT INTO blacklist VALUES (?,?, ?)''',
-                       label,
+                       dbstr(label),
-                       addressAtCurrentInboxRow, True)
+                       dbstr(addressAtCurrentInboxRow), True)
             self.ui.blackwhitelist.rerenderBlackWhiteList()
             self.updateStatusBar(_translate(
                 "MainWindow",
@@ -3188,15 +3243,15 @@ class MyForm(settingsmixin.SMainWindow):
             messageLists = (messageLists,)
         for messageList in messageLists:
             if row is not None:
-                inventoryHash = messageList.item(row, 3).data()
+                inventoryHash = as_msgid(messageList.item(row, 3).data())
                 messageList.removeRow(row)
             elif inventoryHash is not None:
                 for i in range(messageList.rowCount() - 1, -1, -1):
-                    if messageList.item(i, 3).data() == inventoryHash:
+                    if as_msgid(messageList.item(i, 3).data()) == inventoryHash:
                         messageList.removeRow(i)
             elif ackData is not None:
                 for i in range(messageList.rowCount() - 1, -1, -1):
-                    if messageList.item(i, 3).data() == ackData:
+                    if as_msgid(messageList.item(i, 3).data()) == ackData:
                         messageList.removeRow(i)
 
     # Send item on the Inbox tab to trash
@@ -3216,16 +3271,21 @@ class MyForm(settingsmixin.SMainWindow):
         )[::-1]:
             for i in range(r.bottomRow() - r.topRow() + 1):
                 inventoryHashesToTrash.add(
-                    tableWidget.item(r.topRow() + i, 3).data())
+                    sqlite3.Binary(as_msgid(tableWidget.item(r.topRow() + i, 3).data())))
             currentRow = r.topRow()
             self.getCurrentMessageTextedit().setText("")
             tableWidget.model().removeRows(
                 r.topRow(), r.bottomRow() - r.topRow() + 1)
         idCount = len(inventoryHashesToTrash)
+        total_row_count = sqlExecuteChunked(
+            ("DELETE FROM inbox" if folder == "trash" or shifted else
+             "UPDATE inbox SET folder='trash', read=1") +
+            " WHERE msgid IN ({0})", False, idCount, *inventoryHashesToTrash)
+        if total_row_count < 1:
             sqlExecuteChunked(
                 ("DELETE FROM inbox" if folder == "trash" or shifted else
                  "UPDATE inbox SET folder='trash', read=1") +
-            " WHERE msgid IN ({0})", idCount, *inventoryHashesToTrash)
+                " WHERE msgid IN ({0})", True, idCount, *inventoryHashesToTrash)
         tableWidget.selectRow(0 if currentRow == 0 else currentRow - 1)
         tableWidget.setUpdatesEnabled(True)
         self.propagateUnreadCount(folder)
@@ -3244,16 +3304,20 @@ class MyForm(settingsmixin.SMainWindow):
         )[::-1]:
             for i in range(r.bottomRow() - r.topRow() + 1):
                 inventoryHashesToTrash.add(
-                    tableWidget.item(r.topRow() + i, 3).data())
+                    sqlite3.Binary(as_msgid(tableWidget.item(r.topRow() + i, 3).data())))
             currentRow = r.topRow()
             self.getCurrentMessageTextedit().setText("")
             tableWidget.model().removeRows(
                 r.topRow(), r.bottomRow() - r.topRow() + 1)
         tableWidget.selectRow(0 if currentRow == 0 else currentRow - 1)
         idCount = len(inventoryHashesToTrash)
+        total_row_count = sqlExecuteChunked(
+            "UPDATE inbox SET folder='inbox' WHERE msgid IN({0})",
+            False, idCount, *inventoryHashesToTrash)
+        if total_row_count < 1:
             sqlExecuteChunked(
                 "UPDATE inbox SET folder='inbox' WHERE msgid IN({0})",
-            idCount, *inventoryHashesToTrash)
+                True, idCount, *inventoryHashesToTrash)
         tableWidget.selectRow(0 if currentRow == 0 else currentRow - 1)
         tableWidget.setUpdatesEnabled(True)
         self.propagateUnreadCount()
@@ -3265,18 +3329,22 @@ class MyForm(settingsmixin.SMainWindow):
             return
         currentInboxRow = tableWidget.currentRow()
         try:
-            subjectAtCurrentInboxRow = str(tableWidget.item(
+            subjectAtCurrentInboxRow = ustr(tableWidget.item(
                 currentInboxRow, 2).data(QtCore.Qt.UserRole))
         except:
             subjectAtCurrentInboxRow = ''
 
         # Retrieve the message data out of the SQL database
-        msgid = tableWidget.item(currentInboxRow, 3).data()
+        msgid = as_msgid(tableWidget.item(currentInboxRow, 3).data())
         queryreturn = sqlQuery(
-            '''select message from inbox where msgid=?''', msgid)
+            '''select message from inbox where msgid=?''', sqlite3.Binary(msgid))
+        if len(queryreturn) < 1:
+            queryreturn = sqlQuery(
+                '''select message from inbox where msgid=CAST(? AS TEXT)''', msgid)
         if queryreturn != []:
             for row in queryreturn:
                 message, = row
+            message = message.decode("utf-8", "replace")
 
         defaultFilename = "".join(x for x in subjectAtCurrentInboxRow if x.isalnum()) + '.txt'
         filename = QtGui.QFileDialog.getSaveFileName(
@@ -3288,7 +3356,7 @@ class MyForm(settingsmixin.SMainWindow):
             return
         try:
             f = open(filename, 'w')
-            f.write(message)
+            f.write(message.encode("utf-8", "replace"))
             f.close()
         except Exception:
             logger.exception('Message not saved', exc_info=True)
@@ -3303,11 +3371,17 @@ class MyForm(settingsmixin.SMainWindow):
         shifted = QtGui.QApplication.queryKeyboardModifiers() & QtCore.Qt.ShiftModifier
         while tableWidget.selectedIndexes() != []:
             currentRow = tableWidget.selectedIndexes()[0].row()
-            ackdataToTrash = tableWidget.item(currentRow, 3).data()
+            ackdataToTrash = as_msgid(tableWidget.item(currentRow, 3).data())
+            rowcount = sqlExecute(
+                "DELETE FROM sent" if folder == "trash" or shifted else
+                "UPDATE sent SET folder='trash'"
+                " WHERE ackdata = ?", sqlite3.Binary(ackdataToTrash)
+            )
+            if rowcount < 1:
                 sqlExecute(
                     "DELETE FROM sent" if folder == "trash" or shifted else
                     "UPDATE sent SET folder='trash'"
-                " WHERE ackdata = ?", ackdataToTrash
+                    " WHERE ackdata = CAST(? AS TEXT)", ackdataToTrash
                 )
             self.getCurrentMessageTextedit().setPlainText("")
             tableWidget.removeRow(currentRow)
@@ -3322,8 +3396,12 @@ class MyForm(settingsmixin.SMainWindow):
         addressAtCurrentRow = self.ui.tableWidgetInbox.item(
             currentRow, 0).data(QtCore.Qt.UserRole)
         toRipe = decodeAddress(addressAtCurrentRow)[3]
-        sqlExecute(
+        rowcount = sqlExecute(
             '''UPDATE sent SET status='forcepow' WHERE toripe=? AND status='toodifficult' and folder='sent' ''',
+            sqlite3.Binary(toRipe))
+        if rowcount < 1:
+            sqlExecute(
+                '''UPDATE sent SET status='forcepow' WHERE toripe=CAST(? AS TEXT) AND status='toodifficult' and folder='sent' ''',
                 toRipe)
         queryreturn = sqlQuery('''select ackdata FROM sent WHERE status='forcepow' ''')
         for row in queryreturn:
@@ -3337,7 +3415,7 @@ class MyForm(settingsmixin.SMainWindow):
         addressAtCurrentRow = self.ui.tableWidgetInbox.item(
             currentRow, 0).data(QtCore.Qt.UserRole)
         clipboard = QtGui.QApplication.clipboard()
-        clipboard.setText(str(addressAtCurrentRow))
+        clipboard.setText(ustr(addressAtCurrentRow))
 
     # Group of functions for the Address Book dialog box
     def on_action_AddressBookNew(self):
@@ -3349,7 +3427,7 @@ class MyForm(settingsmixin.SMainWindow):
                 0].row()
             item = self.ui.tableWidgetAddressBook.item(currentRow, 0)
             sqlExecute(
-                'DELETE FROM addressbook WHERE address=?', item.address)
+                'DELETE FROM addressbook WHERE address=?', dbstr(item.address))
             self.ui.tableWidgetAddressBook.removeRow(currentRow)
         self.rerenderMessagelistFromLabels()
         self.rerenderMessagelistToLabels()
@@ -3371,8 +3449,8 @@ class MyForm(settingsmixin.SMainWindow):
             return self.updateStatusBar(_translate(
                 "MainWindow", "No addresses selected."))
 
-        addresses_string = unicode(
+        addresses_string = unic(ustr(
-            self.ui.lineEditTo.text().toUtf8(), 'utf-8')
+            self.ui.lineEditTo.text()))
         for item in selected_items:
             address_string = item.accountString()
             if not addresses_string:
@@ -3449,7 +3527,7 @@ class MyForm(settingsmixin.SMainWindow):
             return
         address = self.getCurrentAccount()
         sqlExecute('''DELETE FROM subscriptions WHERE address=?''',
-                   address)
+                   dbstr(address))
         self.rerenderTabTreeSubscriptions()
         self.rerenderMessagelistFromLabels()
         self.rerenderAddressBook()
@@ -3458,13 +3536,13 @@ class MyForm(settingsmixin.SMainWindow):
     def on_action_SubscriptionsClipboard(self):
         address = self.getCurrentAccount()
         clipboard = QtGui.QApplication.clipboard()
-        clipboard.setText(str(address))
+        clipboard.setText(ustr(address))
 
     def on_action_SubscriptionsEnable(self):
         address = self.getCurrentAccount()
         sqlExecute(
             '''update subscriptions set enabled=1 WHERE address=?''',
-            address)
+            dbstr(address))
         account = self.getCurrentItem()
         account.setEnabled(True)
         self.rerenderAddressBook()
@@ -3474,7 +3552,7 @@ class MyForm(settingsmixin.SMainWindow):
         address = self.getCurrentAccount()
         sqlExecute(
             '''update subscriptions set enabled=0 WHERE address=?''',
-            address)
+            dbstr(address))
         account = self.getCurrentItem()
         account.setEnabled(False)
         self.rerenderAddressBook()
@@ -3576,7 +3654,7 @@ class MyForm(settingsmixin.SMainWindow):
         if messagelist:
             currentRow = messagelist.currentRow()
             if currentRow >= 0:
-                return messagelist.item(currentRow, 3).data()
+                return as_msgid(messagelist.item(currentRow, 3).data())
 
     def getCurrentMessageTextedit(self):
         currentIndex = self.ui.tabWidget.currentIndex()
@@ -3610,9 +3688,9 @@ class MyForm(settingsmixin.SMainWindow):
             self.ui.inboxSearchLineEditChans,
         )
         if currentIndex >= 0 and currentIndex < len(messagelistList):
-            return (
+            return ustr(
                 messagelistList[currentIndex] if retObj
-                else messagelistList[currentIndex].text().toUtf8().data())
+                else ustr(messagelistList[currentIndex].text()))
 
     def getCurrentSearchOption(self, currentIndex=None):
         if currentIndex is None:
@@ -3681,7 +3759,7 @@ class MyForm(settingsmixin.SMainWindow):
                         " delete the channel?"
                     ), QtGui.QMessageBox.Yes | QtGui.QMessageBox.No
             ) == QtGui.QMessageBox.Yes:
-                config.remove_section(str(account.address))
+                config.remove_section(ustr(account.address))
             else:
                 return
         else:
@@ -3714,7 +3792,7 @@ class MyForm(settingsmixin.SMainWindow):
         account.setEnabled(False)
 
     def disableIdentity(self, address):
-        config.set(str(address), 'enabled', 'false')
+        config.set(ustr(address), 'enabled', 'false')
         config.save()
         shared.reloadMyAddressHashes()
         self.rerenderAddressBook()
@@ -3722,7 +3800,7 @@ class MyForm(settingsmixin.SMainWindow):
     def on_action_Clipboard(self):
         address = self.getCurrentAccount()
         clipboard = QtGui.QApplication.clipboard()
-        clipboard.setText(str(address))
+        clipboard.setText(ustr(address))
 
     def on_action_ClipboardMessagelist(self):
         tableWidget = self.getCurrentMessagelist()
@@ -3742,7 +3820,7 @@ class MyForm(settingsmixin.SMainWindow):
         if isinstance(account, GatewayAccount) and otherAddress == account.relayAddress and (
                 (currentColumn in [0, 2] and self.getCurrentFolder() == "sent") or
                 (currentColumn in [1, 2] and self.getCurrentFolder() != "sent")):
-            text = str(tableWidget.item(currentRow, currentColumn).label)
+            text = ustr(tableWidget.item(currentRow, currentColumn).label)
         else:
             text = tableWidget.item(currentRow, currentColumn).data(QtCore.Qt.UserRole)
 
@@ -3759,8 +3837,8 @@ class MyForm(settingsmixin.SMainWindow):
 
     def on_action_SetAvatar(self, thisTableWidget):
         currentRow = thisTableWidget.currentRow()
-        addressAtCurrentRow = thisTableWidget.item(
+        addressAtCurrentRow = ustr(thisTableWidget.item(
-            currentRow, 1).text()
+            currentRow, 1).text())
         setToIdenticon = not self.setAvatar(addressAtCurrentRow)
         if setToIdenticon:
             thisTableWidget.item(
@@ -3770,7 +3848,7 @@ class MyForm(settingsmixin.SMainWindow):
     def setAvatar(self, addressAtCurrentRow):
         if not os.path.exists(state.appdata + 'avatars/'):
             os.makedirs(state.appdata + 'avatars/')
-        hash = hashlib.md5(addBMIfNotPresent(addressAtCurrentRow)).hexdigest()
+        hash = hashlib.md5(addBMIfNotPresent(addressAtCurrentRow).encode("utf-8", "replace")).hexdigest()
         extensions = [
             'PNG', 'GIF', 'JPG', 'JPEG', 'SVG', 'BMP', 'MNG', 'PBM',
             'PGM', 'PPM', 'TIFF', 'XBM', 'XPM', 'TGA']
@@ -3861,23 +3939,23 @@ class MyForm(settingsmixin.SMainWindow):
 
     def on_action_AddressBookSetSound(self):
         widget = self.ui.tableWidgetAddressBook
-        self.setAddressSound(widget.item(widget.currentRow(), 0).text())
+        self.setAddressSound(ustr(widget.item(widget.currentRow(), 0).text()))
 
     def setAddressSound(self, addr):
-        filters = [unicode(_translate(
+        filters = [unic(_translate(
             "MainWindow", "Sound files (%s)" %
             ' '.join(['*%s%s' % (os.extsep, ext) for ext in sound.extensions])
         ))]
-        sourcefile = unicode(QtGui.QFileDialog.getOpenFileName(
+        sourcefile = unic(ustr(QtGui.QFileDialog.getOpenFileName(
             self, _translate("MainWindow", "Set notification sound..."),
             filter=';;'.join(filters)
-        ))
+        )))
 
         if not sourcefile:
             return
 
         destdir = os.path.join(state.appdata, 'sounds')
-        destfile = unicode(addr) + os.path.splitext(sourcefile)[-1]
+        destfile = unic(ustr(addr) + os.path.splitext(sourcefile)[-1])
         destination = os.path.join(destdir, destfile)
 
         if sourcefile == destination:
@@ -4019,10 +4097,13 @@ class MyForm(settingsmixin.SMainWindow):
         # Check to see if this item is toodifficult and display an additional
         # menu option (Force Send) if it is.
         if currentRow >= 0:
-            ackData = self.ui.tableWidgetInbox.item(currentRow, 3).data()
+            ackData = as_msgid(self.ui.tableWidgetInbox.item(currentRow, 3).data())
-            queryreturn = sqlQuery('''SELECT status FROM sent where ackdata=?''', ackData)
+            queryreturn = sqlQuery('''SELECT status FROM sent where ackdata=?''', sqlite3.Binary(ackData))
+            if len(queryreturn) < 1:
+                queryreturn = sqlQuery('''SELECT status FROM sent where ackdata=CAST(? AS TEXT)''', ackData)
             for row in queryreturn:
                 status, = row
+            status = status.decode("utf-8", "replace")
             if status == 'toodifficult':
                 self.popMenuSent.addAction(self.actionForceSend)
 
@@ -4030,7 +4111,7 @@ class MyForm(settingsmixin.SMainWindow):
 
     def inboxSearchLineEditUpdated(self, text):
         # dynamic search for too short text is slow
-        text = text.toUtf8()
+        text = ustr(text)
         if 0 < len(text) < 3:
             return
         messagelist = self.getCurrentMessagelist()
@@ -4045,7 +4126,7 @@ class MyForm(settingsmixin.SMainWindow):
         logger.debug("Search return pressed")
         searchLine = self.getCurrentSearchLine()
         messagelist = self.getCurrentMessagelist()
-        if messagelist and len(str(searchLine)) < 3:
+        if messagelist and len(ustr(searchLine)) < 3:
             searchOption = self.getCurrentSearchOption()
             account = self.getCurrentAccount()
             folder = self.getCurrentFolder()
@@ -4087,7 +4168,7 @@ class MyForm(settingsmixin.SMainWindow):
         if item.type == AccountMixin.ALL:
             return
 
-        newLabel = unicode(item.text(0), 'utf-8', 'ignore')
+        newLabel = unic(ustr(item.text(0)))
         oldLabel = item.defaultLabel()
 
         # unchanged, do not do anything either
@@ -4119,7 +4200,14 @@ class MyForm(settingsmixin.SMainWindow):
         folder = self.getCurrentFolder()
         if msgid:
             queryreturn = sqlQuery(
-                '''SELECT message FROM %s WHERE %s=?''' % (
+                'SELECT message FROM %s WHERE %s=?' % (
+                    ('sent', 'ackdata') if folder == 'sent'
+                    else ('inbox', 'msgid')
+                ), sqlite3.Binary(msgid)
+            )
+            if len(queryreturn) < 1:
+                queryreturn = sqlQuery(
+                    'SELECT message FROM %s WHERE %s=CAST(? AS TEXT)' % (
                         ('sent', 'ackdata') if folder == 'sent'
                         else ('inbox', 'msgid')
                     ), msgid
@@ -4127,6 +4215,7 @@ class MyForm(settingsmixin.SMainWindow):
 
         try:
             message = queryreturn[-1][0]
+            message = message.decode("utf-8", "replace")
         except NameError:
             message = ""
         except IndexError:
@@ -4141,10 +4230,16 @@ class MyForm(settingsmixin.SMainWindow):
             if tableWidget.item(currentRow, 0).unread is True:
                 self.updateUnreadStatus(tableWidget, currentRow, msgid)
             # propagate
-            if folder != 'sent' and sqlExecute(
+            rowcount = sqlExecute(
                 '''UPDATE inbox SET read=1 WHERE msgid=? AND read=0''',
+                sqlite3.Binary(msgid)
+            )
+            if rowcount < 1:
+                rowcount = sqlExecute(
+                    '''UPDATE inbox SET read=1 WHERE msgid=CAST(? AS TEXT) AND read=0''',
                     msgid
-            ) > 0:
+                )
+            if folder != 'sent' and rowcount > 0:
                 self.propagateUnreadCount()
 
         messageTextedit.setCurrentFont(QtGui.QFont())
@@ -4158,8 +4253,8 @@ class MyForm(settingsmixin.SMainWindow):
         self.rerenderMessagelistToLabels()
         completerList = self.ui.lineEditTo.completer().model().stringList()
         for i in range(len(completerList)):
-            if unicode(completerList[i]).endswith(" <" + item.address + ">"):
+            if unic(ustr(completerList[i])).endswith(" <" + ustr(item.address) + ">"):
-                completerList[i] = item.label + " <" + item.address + ">"
+                completerList[i] = ustr(item.label) + " <" + ustr(item.address) + ">"
         self.ui.lineEditTo.completer().model().setStringList(completerList)
 
     def tabWidgetCurrentChanged(self, n):
@@ -4212,7 +4307,7 @@ class MyForm(settingsmixin.SMainWindow):
 
     def initSettings(self):
         self.loadSettings()
-        for attr, obj in self.ui.__dict__.iteritems():
+        for attr, obj in six.iteritems(self.ui.__dict__):
             if hasattr(obj, "__class__") and \
                     isinstance(obj, settingsmixin.SettingsMixin):
                 loadMethod = getattr(obj, "loadSettings", None)

src/bitmessageqt/account.py

@@ -13,8 +13,11 @@ import inspect
 import re
 import sys
 import time
+import sqlite3
 
+from unqstr import ustr
 from PyQt4 import QtGui
+from dbcompat import dbstr
 
 import queues
 from addresses import decodeAddress
@@ -38,6 +41,8 @@ def getSortedSubscriptions(count=False):
     ret = {}
     for row in queryreturn:
         label, address, enabled = row
+        label = label.decode("utf-8", "replace")
+        address = address.decode("utf-8", "replace")
         ret[address] = {}
         ret[address]["inbox"] = {}
         ret[address]["inbox"]['label'] = label
@@ -47,9 +52,11 @@ def getSortedSubscriptions(count=False):
         queryreturn = sqlQuery('''SELECT fromaddress, folder, count(msgid) as cnt
             FROM inbox, subscriptions ON subscriptions.address = inbox.fromaddress
             WHERE read = 0 AND toaddress = ?
-            GROUP BY inbox.fromaddress, folder''', str_broadcast_subscribers)
+            GROUP BY inbox.fromaddress, folder''', dbstr(str_broadcast_subscribers))
         for row in queryreturn:
             address, folder, cnt = row
+            address = address.decode("utf-8", "replace")
+            folder = folder.decode("utf-8", "replace")
             if folder not in ret[address]:
                 ret[address][folder] = {
                     'label': ret[address]['inbox']['label'],
@@ -100,7 +107,7 @@ class AccountColor(AccountMixin):  # pylint: disable=too-few-public-methods
             elif config.safeGetBoolean(self.address, 'chan'):
                 self.type = AccountMixin.CHAN
             elif sqlQuery(
-                    '''select label from subscriptions where address=?''', self.address):
+                    '''select label from subscriptions where address=?''', dbstr(self.address)):
                 self.type = AccountMixin.SUBSCRIPTION
             else:
                 self.type = AccountMixin.NORMAL
@@ -123,7 +130,7 @@ class BMAccount(object):
             self.type = AccountMixin.BROADCAST
         else:
             queryreturn = sqlQuery(
-                '''select label from subscriptions where address=?''', self.address)
+                '''select label from subscriptions where address=?''', dbstr(self.address))
             if queryreturn:
                 self.type = AccountMixin.SUBSCRIPTION
 
@@ -131,32 +138,33 @@ class BMAccount(object):
         """Get a label for this bitmessage account"""
         if address is None:
             address = self.address
+        else:
+            address = ustr(address)
         label = config.safeGet(address, 'label', address)
         queryreturn = sqlQuery(
-            '''select label from addressbook where address=?''', address)
+            '''select label from addressbook where address=?''', dbstr(address))
         if queryreturn != []:
             for row in queryreturn:
                 label, = row
+            label = label.decode("utf-8", "replace")
         else:
             queryreturn = sqlQuery(
-                '''select label from subscriptions where address=?''', address)
+                '''select label from subscriptions where address=?''', dbstr(address))
             if queryreturn != []:
                 for row in queryreturn:
                     label, = row
+                label = label.decode("utf-8", "replace")
         return label
 
     def parseMessage(self, toAddress, fromAddress, subject, message):
         """Set metadata and address labels on self"""
 
-        self.toAddress = toAddress
+        self.toAddress = ustr(toAddress)
-        self.fromAddress = fromAddress
+        self.fromAddress = ustr(fromAddress)
-        if isinstance(subject, unicode):
+        self.subject = ustr(subject)
-            self.subject = str(subject)
+        self.message = ustr(message)
-        else:
+        self.fromLabel = ustr(self.getLabel(fromAddress))
-            self.subject = subject
+        self.toLabel = ustr(self.getLabel(toAddress))
-        self.message = message
-        self.fromLabel = self.getLabel(fromAddress)
-        self.toLabel = self.getLabel(toAddress)
 
 
 class NoAccount(BMAccount):
@@ -201,19 +209,19 @@ class GatewayAccount(BMAccount):
         ackdata = genAckPayload(streamNumber, stealthLevel)
         sqlExecute(
             '''INSERT INTO sent VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)''',
-            '',
+            sqlite3.Binary(b''),
-            self.toAddress,
+            dbstr(self.toAddress),
-            ripe,
+            sqlite3.Binary(ripe),
-            self.fromAddress,
+            dbstr(self.fromAddress),
-            self.subject,
+            dbstr(self.subject),
-            self.message,
+            dbstr(self.message),
-            ackdata,
+            sqlite3.Binary(ackdata),
             int(time.time()),  # sentTime (this will never change)
             int(time.time()),  # lastActionTime
             0,  # sleepTill time. This will get set when the POW gets done.
-            'msgqueued',
+            dbstr('msgqueued'),
             0,  # retryNumber
-            'sent',  # folder
+            dbstr('sent'),  # folder
             2,  # encodingtype
             # not necessary to have a TTL higher than 2 days
             min(config.getint('bitmessagesettings', 'ttl'), 86400 * 2)

src/bitmessageqt/address_dialogs.py

@@ -5,12 +5,13 @@ Dialogs that work with BM address.
 
 import hashlib
 
+from unqstr import ustr, unic
 from PyQt4 import QtCore, QtGui
 
 import queues
-import widgets
+from bitmessageqt import widgets
 import state
-from account import AccountMixin, GatewayAccount, MailchuckAccount, accountClass
+from .account import AccountMixin, GatewayAccount, MailchuckAccount, accountClass
 from addresses import addBMIfNotPresent, decodeAddress, encodeVarint
 from bmconfigparser import config as global_config
 from tr import _translate
@@ -29,12 +30,12 @@ class AddressCheckMixin(object):
     def _onSuccess(self, addressVersion, streamNumber, ripe):
         pass
 
-    def addressChanged(self, QString):
+    def addressChanged(self, addr):
         """
         Address validation callback, performs validation and gives feedback
         """
         status, addressVersion, streamNumber, ripe = decodeAddress(
-            str(QString))
+            ustr(addr))
         self.valid = status == 'success'
         if self.valid:
             self.labelAddressCheck.setText(
@@ -90,8 +91,8 @@ class AddressDataDialog(QtGui.QDialog, AddressCheckMixin):
         """Callback for QDIalog accepting value"""
         if self.valid:
             self.data = (
-                addBMIfNotPresent(str(self.lineEditAddress.text())),
+                addBMIfNotPresent(ustr(self.lineEditAddress.text())),
-                str(self.lineEditLabel.text().toUtf8())
+                ustr(self.lineEditLabel.text())
             )
         else:
             queues.UISignalQueue.put(('updateStatusBar', _translate(
@@ -142,12 +143,12 @@ class NewAddressDialog(QtGui.QDialog):
                     self.comboBoxExisting.currentText())[2]
             queues.addressGeneratorQueue.put((
                 'createRandomAddress', 4, streamNumberForAddress,
-                str(self.newaddresslabel.text().toUtf8()), 1, "",
+                ustr(self.newaddresslabel.text()), 1, "",
                 self.checkBoxEighteenByteRipe.isChecked()
             ))
         else:
-            if self.lineEditPassphrase.text() != \
+            if ustr(self.lineEditPassphrase.text()) != \
-                    self.lineEditPassphraseAgain.text():
+                    ustr(self.lineEditPassphraseAgain.text()):
                 QtGui.QMessageBox.about(
                     self, _translate("MainWindow", "Passphrase mismatch"),
                     _translate(
@@ -169,7 +170,7 @@ class NewAddressDialog(QtGui.QDialog):
                     'createDeterministicAddresses', 4, streamNumberForAddress,
                     "unused deterministic address",
                     self.spinBoxNumberOfAddressesToMake.value(),
-                    self.lineEditPassphrase.text().toUtf8(),
+                    ustr(self.lineEditPassphrase.text()),
                     self.checkBoxEighteenByteRipe.isChecked()
                 ))
 
@@ -234,7 +235,7 @@ class SpecialAddressBehaviorDialog(QtGui.QDialog):
     def __init__(self, parent=None, config=global_config):
         super(SpecialAddressBehaviorDialog, self).__init__(parent)
         widgets.load('specialaddressbehavior.ui', self)
-        self.address = parent.getCurrentAccount()
+        self.address = ustr(parent.getCurrentAccount())
         self.parent = parent
         self.config = config
 
@@ -259,7 +260,7 @@ class SpecialAddressBehaviorDialog(QtGui.QDialog):
                     self.radioButtonBehaveNormalAddress.click()
                 mailingListName = config.safeGet(self.address, 'mailinglistname', '')
                 self.lineEditMailingListName.setText(
-                    unicode(mailingListName, 'utf-8')
+                    unic(ustr(mailingListName))
                 )
 
         QtGui.QWidget.resize(self, QtGui.QWidget.sizeHint(self))
@@ -271,7 +272,7 @@ class SpecialAddressBehaviorDialog(QtGui.QDialog):
         if self.address_is_chan:
             return
         if self.radioButtonBehaveNormalAddress.isChecked():
-            self.config.set(str(self.address), 'mailinglist', 'false')
+            self.config.set(self.address, 'mailinglist', 'false')
             # Set the color to either black or grey
             if self.config.getboolean(self.address, 'enabled'):
                 self.parent.setCurrentItemColor(
@@ -280,9 +281,9 @@ class SpecialAddressBehaviorDialog(QtGui.QDialog):
             else:
                 self.parent.setCurrentItemColor(QtGui.QColor(128, 128, 128))
         else:
-            self.config.set(str(self.address), 'mailinglist', 'true')
+            self.config.set(self.address, 'mailinglist', 'true')
-            self.config.set(str(self.address), 'mailinglistname', str(
+            self.config.set(self.address, 'mailinglistname', ustr(
-                self.lineEditMailingListName.text().toUtf8()))
+                self.lineEditMailingListName.text()))
             self.parent.setCurrentItemColor(
                 QtGui.QColor(137, 4, 177))  # magenta
         self.parent.rerenderComboBoxSendFrom()
@@ -344,7 +345,7 @@ class EmailGatewayDialog(QtGui.QDialog):
 
         if self.radioButtonRegister.isChecked() \
                 or self.radioButtonRegister.isHidden():
-            email = str(self.lineEditEmail.text().toUtf8())
+            email = ustr(self.lineEditEmail.text())
             self.acct.register(email)
             self.config.set(self.acct.fromAddress, 'label', email)
             self.config.set(self.acct.fromAddress, 'gateway', 'mailchuck')

src/bitmessageqt/addressvalidator.py

@@ -3,15 +3,16 @@ Address validator module.
 """
 # pylint: disable=too-many-branches,too-many-arguments
 
-from Queue import Empty
+from six.moves.queue import Empty
 
+from unqstr import ustr
 from PyQt4 import QtGui
 
 from addresses import decodeAddress, addBMIfNotPresent
 from bmconfigparser import config
 from queues import apiAddressGeneratorReturnQueue, addressGeneratorQueue
 from tr import _translate
-from utils import str_chan
+from .utils import str_chan
 
 
 class AddressPassPhraseValidatorMixin(object):
@@ -108,26 +109,26 @@ class AddressPassPhraseValidatorMixin(object):
         if self.addressObject is None:
             address = None
         else:
-            address = str(self.addressObject.text().toUtf8())
+            address = ustr(self.addressObject.text())
             if address == "":
                 address = None
         if self.passPhraseObject is None:
             passPhrase = ""
         else:
-            passPhrase = str(self.passPhraseObject.text().toUtf8())
+            passPhrase = ustr(self.passPhraseObject.text())
             if passPhrase == "":
                 passPhrase = None
 
         # no chan name
         if passPhrase is None:
             self.setError(_translate("AddressValidator", "Chan name/passphrase needed. You didn't enter a chan name."))
-            return (QtGui.QValidator.Intermediate, pos)
+            return (QtGui.QValidator.Intermediate, s, pos)
 
         if self.addressMandatory or address is not None:
             # check if address already exists:
             if address in config.addresses():
                 self.setError(_translate("AddressValidator", "Address already present as one of your identities."))
-                return (QtGui.QValidator.Intermediate, pos)
+                return (QtGui.QValidator.Intermediate, s, pos)
 
             # version too high
             if decodeAddress(address)[0] == 'versiontoohigh':
@@ -138,12 +139,12 @@ class AddressPassPhraseValidatorMixin(object):
                         " address might be valid, its version number"
                         " is too new for us to handle. Perhaps you need"
                         " to upgrade Bitmessage."))
-                return (QtGui.QValidator.Intermediate, pos)
+                return (QtGui.QValidator.Intermediate, s, pos)
 
             # invalid
             if decodeAddress(address)[0] != 'success':
                 self.setError(_translate("AddressValidator", "The Bitmessage address is not valid."))
-                return (QtGui.QValidator.Intermediate, pos)
+                return (QtGui.QValidator.Intermediate, s, pos)
 
         # this just disables the OK button without changing the feedback text
         # but only if triggered by textEdited, not by clicking the Ok button
@@ -152,15 +153,15 @@ class AddressPassPhraseValidatorMixin(object):
 
         # check through generator
         if address is None:
-            addressGeneratorQueue.put(('createChan', 4, 1, str_chan + ' ' + str(passPhrase), passPhrase, False))
+            addressGeneratorQueue.put(('createChan', 4, 1, str_chan + ' ' + ustr(passPhrase), passPhrase.encode("utf-8", "replace"), False))
         else:
             addressGeneratorQueue.put(
                 ('joinChan', addBMIfNotPresent(address),
-                 "{} {}".format(str_chan, passPhrase), passPhrase, False))
+                 "{} {}".format(str_chan, passPhrase), passPhrase.encode("utf-8", "replace"), False))
 
         if self.buttonBox.button(QtGui.QDialogButtonBox.Ok).hasFocus():
-            return (self.returnValid(), pos)
+            return (self.returnValid(), s, pos)
-        return (QtGui.QValidator.Intermediate, pos)
+        return (QtGui.QValidator.Intermediate, s, pos)
 
     def checkData(self):
         """Validator Qt signal interface"""

src/bitmessageqt/bitmessage_icons_rc.py

@@ -9,7 +9,7 @@
 
 from PyQt4 import QtCore
 
-qt_resource_data = "\
+qt_resource_data = b"\
 \x00\x00\x03\x66\
 \x89\
 \x50\x4e\x47\x0d\x0a\x1a\x0a\x00\x00\x00\x0d\x49\x48\x44\x52\x00\
@@ -1534,7 +1534,7 @@ qt_resource_data = "\
 \x82\
 "
 
-qt_resource_name = "\
+qt_resource_name = b"\
 \x00\x09\
 \x0c\x78\x54\x88\
 \x00\x6e\
@@ -1639,7 +1639,7 @@ qt_resource_name = "\
 \x00\x70\x00\x6e\x00\x67\
 "
 
-qt_resource_struct = "\
+qt_resource_struct = b"\
 \x00\x00\x00\x00\x00\x02\x00\x00\x00\x01\x00\x00\x00\x01\
 \x00\x00\x00\x00\x00\x02\x00\x00\x00\x01\x00\x00\x00\x02\
 \x00\x00\x00\x18\x00\x02\x00\x00\x00\x15\x00\x00\x00\x03\

src/bitmessageqt/bitmessageui.py

@@ -9,12 +9,12 @@
 
 from PyQt4 import QtCore, QtGui
 from bmconfigparser import config
-from foldertree import AddressBookCompleter
+from .foldertree import AddressBookCompleter
-from messageview import MessageView
+from .messageview import MessageView
-from messagecompose import MessageCompose
+from .messagecompose import MessageCompose
-import settingsmixin
+from bitmessageqt import settingsmixin
-from networkstatus import NetworkStatus
+from .networkstatus import NetworkStatus
-from blacklist import Blacklist
+from .blacklist import Blacklist
 
 try:
     _fromUtf8 = QtCore.QString.fromUtf8

src/bitmessageqt/blacklist.py

@@ -1,15 +1,17 @@
+from unqstr import ustr, unic
 from PyQt4 import QtCore, QtGui
+from dbcompat import dbstr
 
-import widgets
+from bitmessageqt import widgets
 from addresses import addBMIfNotPresent
 from bmconfigparser import config
-from dialogs import AddAddressDialog
+from .dialogs import AddAddressDialog
 from helper_sql import sqlExecute, sqlQuery
 from queues import UISignalQueue
-from retranslateui import RetranslateMixin
+from .retranslateui import RetranslateMixin
 from tr import _translate
-from uisignaler import UISignaler
+from .uisignaler import UISignaler
-from utils import avatarize
+from .utils import avatarize
 
 
 class Blacklist(QtGui.QWidget, RetranslateMixin):
@@ -59,12 +61,12 @@ class Blacklist(QtGui.QWidget, RetranslateMixin):
         if self.NewBlacklistDialogInstance.exec_():
             if self.NewBlacklistDialogInstance.labelAddressCheck.text() == \
                     _translate("MainWindow", "Address is valid."):
-                address = addBMIfNotPresent(str(
+                address = addBMIfNotPresent(ustr(
                     self.NewBlacklistDialogInstance.lineEditAddress.text()))
                 # First we must check to see if the address is already in the
                 # address book. The user cannot add it again or else it will
                 # cause problems when updating and deleting the entry.
-                t = (address,)
+                t = (dbstr(address),)
                 if config.get('bitmessagesettings', 'blackwhitelist') == 'black':
                     sql = '''select * from blacklist where address=?'''
                 else:
@@ -73,8 +75,8 @@ class Blacklist(QtGui.QWidget, RetranslateMixin):
                 if queryreturn == []:
                     self.tableWidgetBlacklist.setSortingEnabled(False)
                     self.tableWidgetBlacklist.insertRow(0)
-                    newItem = QtGui.QTableWidgetItem(unicode(
+                    newItem = QtGui.QTableWidgetItem(unic(ustr(
-                        self.NewBlacklistDialogInstance.lineEditLabel.text().toUtf8(), 'utf-8'))
+                        self.NewBlacklistDialogInstance.lineEditLabel.text())))
                     newItem.setIcon(avatarize(address))
                     self.tableWidgetBlacklist.setItem(0, 0, newItem)
                     newItem = QtGui.QTableWidgetItem(address)
@@ -82,7 +84,7 @@ class Blacklist(QtGui.QWidget, RetranslateMixin):
                         QtCore.Qt.ItemIsSelectable | QtCore.Qt.ItemIsEnabled)
                     self.tableWidgetBlacklist.setItem(0, 1, newItem)
                     self.tableWidgetBlacklist.setSortingEnabled(True)
-                    t = (str(self.NewBlacklistDialogInstance.lineEditLabel.text().toUtf8()), address, True)
+                    t = (dbstr(self.NewBlacklistDialogInstance.lineEditLabel.text()), dbstr(address), True)
                     if config.get('bitmessagesettings', 'blackwhitelist') == 'black':
                         sql = '''INSERT INTO blacklist VALUES (?,?,?)'''
                     else:
@@ -111,10 +113,10 @@ class Blacklist(QtGui.QWidget, RetranslateMixin):
             if isinstance(addressitem, QtGui.QTableWidgetItem):
                 if self.radioButtonBlacklist.isChecked():
                     sqlExecute('''UPDATE blacklist SET label=? WHERE address=?''',
-                            str(item.text()), str(addressitem.text()))
+                            dbstr(item.text()), dbstr(addressitem.text()))
                 else:
                     sqlExecute('''UPDATE whitelist SET label=? WHERE address=?''',
-                            str(item.text()), str(addressitem.text()))
+                            dbstr(item.text()), dbstr(addressitem.text()))
 
     def init_blacklist_popup_menu(self, connectSignal=True):
         # Popup menu for the Blacklist page
@@ -171,8 +173,10 @@ class Blacklist(QtGui.QWidget, RetranslateMixin):
         self.tableWidgetBlacklist.setSortingEnabled(False)
         for row in queryreturn:
             label, address, enabled = row
+            label = label.decode("utf-8", "replace")
+            address = address.decode("utf-8", "replace")
             self.tableWidgetBlacklist.insertRow(0)
-            newItem = QtGui.QTableWidgetItem(unicode(label, 'utf-8'))
+            newItem = QtGui.QTableWidgetItem(unic(ustr(label)))
             if not enabled:
                 newItem.setTextColor(QtGui.QColor(128, 128, 128))
             newItem.setIcon(avatarize(address))
@@ -191,18 +195,18 @@ class Blacklist(QtGui.QWidget, RetranslateMixin):
 
     def on_action_BlacklistDelete(self):
         currentRow = self.tableWidgetBlacklist.currentRow()
-        labelAtCurrentRow = self.tableWidgetBlacklist.item(
+        labelAtCurrentRow = ustr(self.tableWidgetBlacklist.item(
-            currentRow, 0).text().toUtf8()
+            currentRow, 0).text())
         addressAtCurrentRow = self.tableWidgetBlacklist.item(
             currentRow, 1).text()
         if config.get('bitmessagesettings', 'blackwhitelist') == 'black':
             sqlExecute(
                 '''DELETE FROM blacklist WHERE label=? AND address=?''',
-                str(labelAtCurrentRow), str(addressAtCurrentRow))
+                dbstr(labelAtCurrentRow), dbstr(addressAtCurrentRow))
         else:
             sqlExecute(
                 '''DELETE FROM whitelist WHERE label=? AND address=?''',
-                str(labelAtCurrentRow), str(addressAtCurrentRow))
+                dbstr(labelAtCurrentRow), dbstr(addressAtCurrentRow))
         self.tableWidgetBlacklist.removeRow(currentRow)
 
     def on_action_BlacklistClipboard(self):
@@ -210,7 +214,7 @@ class Blacklist(QtGui.QWidget, RetranslateMixin):
         addressAtCurrentRow = self.tableWidgetBlacklist.item(
             currentRow, 1).text()
         clipboard = QtGui.QApplication.clipboard()
-        clipboard.setText(str(addressAtCurrentRow))
+        clipboard.setText(ustr(addressAtCurrentRow))
 
     def on_context_menuBlacklist(self, point):
         self.popMenuBlacklist.exec_(
@@ -227,11 +231,11 @@ class Blacklist(QtGui.QWidget, RetranslateMixin):
         if config.get('bitmessagesettings', 'blackwhitelist') == 'black':
             sqlExecute(
                 '''UPDATE blacklist SET enabled=1 WHERE address=?''',
-                str(addressAtCurrentRow))
+                dbstr(addressAtCurrentRow))
         else:
             sqlExecute(
                 '''UPDATE whitelist SET enabled=1 WHERE address=?''',
-                str(addressAtCurrentRow))
+                dbstr(addressAtCurrentRow))
 
     def on_action_BlacklistDisable(self):
         currentRow = self.tableWidgetBlacklist.currentRow()
@@ -243,10 +247,10 @@ class Blacklist(QtGui.QWidget, RetranslateMixin):
             currentRow, 1).setTextColor(QtGui.QColor(128, 128, 128))
         if config.get('bitmessagesettings', 'blackwhitelist') == 'black':
             sqlExecute(
-                '''UPDATE blacklist SET enabled=0 WHERE address=?''', str(addressAtCurrentRow))
+                '''UPDATE blacklist SET enabled=0 WHERE address=?''', dbstr(addressAtCurrentRow))
         else:
             sqlExecute(
-                '''UPDATE whitelist SET enabled=0 WHERE address=?''', str(addressAtCurrentRow))
+                '''UPDATE whitelist SET enabled=0 WHERE address=?''', dbstr(addressAtCurrentRow))
 
     def on_action_BlacklistSetAvatar(self):
         self.window().on_action_SetAvatar(self.tableWidgetBlacklist)

src/bitmessageqt/dialogs.py

@@ -2,17 +2,18 @@
 Custom dialog classes
 """
 # pylint: disable=too-few-public-methods
+from unqstr import ustr
 from PyQt4 import QtGui
 
 import paths
-import widgets
+from bitmessageqt import widgets
-from address_dialogs import (
+from .address_dialogs import (
     AddAddressDialog, EmailGatewayDialog, NewAddressDialog,
     NewSubscriptionDialog, RegenerateAddressesDialog,
     SpecialAddressBehaviorDialog
 )
-from newchandialog import NewChanDialog
+from .newchandialog import NewChanDialog
-from settings import SettingsDialog
+from .settings import SettingsDialog
 from tr import _translate
 from version import softwareVersion
 
@@ -36,7 +37,7 @@ class AboutDialog(QtGui.QDialog):
         if commit:
             version += '-' + commit[:7]
         self.labelVersion.setText(
-            self.labelVersion.text().replace(
+            ustr(self.labelVersion.text()).replace(
                 ':version:', version
             ).replace(':branch:', commit or 'v%s' % version)
         )
@@ -44,8 +45,8 @@ class AboutDialog(QtGui.QDialog):
 
         try:
             self.label_2.setText(
-                self.label_2.text().replace(
+                ustr(self.label_2.text()).replace(
-                    '2022', str(last_commit.get('time').year)
+                    '2022', ustr(last_commit.get('time').year)
                 ))
         except AttributeError:
             pass
@@ -64,8 +65,8 @@ class IconGlossaryDialog(QtGui.QDialog):
 
         self.labelPortNumber.setText(_translate(
             "iconGlossaryDialog",
-            "You are using TCP port %1. (This can be changed in the settings)."
+            "You are using TCP port {0}. (This can be changed in the settings)."
-        ).arg(config.getint('bitmessagesettings', 'port')))
+        ).format(config.getint('bitmessagesettings', 'port')))
         self.setFixedSize(QtGui.QWidget.sizeHint(self))
 
 

src/bitmessageqt/foldertree.py

@@ -4,15 +4,20 @@ Folder tree and messagelist widgets definitions.
 # pylint: disable=too-many-arguments,bad-super-call
 # pylint: disable=attribute-defined-outside-init
 
-from cgi import escape
+try:
+    from cgi import escape
+except ImportError:
+    from html import escape
 
+from unqstr import ustr, unic
 from PyQt4 import QtCore, QtGui
+from dbcompat import dbstr
 
 from bmconfigparser import config
 from helper_sql import sqlExecute, sqlQuery
-from settingsmixin import SettingsMixin
+from .settingsmixin import SettingsMixin
 from tr import _translate
-from utils import avatarize
+from .utils import avatarize
 
 # for pylupdate
 _translate("MainWindow", "inbox")
@@ -62,7 +67,7 @@ class AccountMixin(object):
 
     def accountString(self):
         """Account string suitable for use in To: field: label <address>"""
-        label = self._getLabel()
+        label = ustr(self._getLabel())
         return (
             self.address if label == self.address
             else '%s <%s>' % (label, self.address)
@@ -73,7 +78,7 @@ class AccountMixin(object):
         if address is None:
             self.address = None
         else:
-            self.address = str(address)
+            self.address = ustr(address)
 
     def setUnreadCount(self, cnt):
         """Set number of unread messages"""
@@ -111,7 +116,7 @@ class AccountMixin(object):
         elif config.safeGetBoolean(self.address, 'mailinglist'):
             self.type = self.MAILINGLIST
         elif sqlQuery(
-                '''select label from subscriptions where address=?''', self.address):
+                '''select label from subscriptions where address=?''', dbstr(self.address)):
             self.type = AccountMixin.SUBSCRIPTION
         else:
             self.type = self.NORMAL
@@ -124,24 +129,23 @@ class AccountMixin(object):
                 AccountMixin.NORMAL,
                 AccountMixin.CHAN, AccountMixin.MAILINGLIST):
             try:
-                retval = unicode(
+                retval = unic(ustr(
-                    config.get(self.address, 'label'), 'utf-8')
+                    config.get(self.address, 'label')))
             except Exception:
                 queryreturn = sqlQuery(
-                    '''select label from addressbook where address=?''', self.address)
+                    '''select label from addressbook where address=?''', dbstr(self.address))
         elif self.type == AccountMixin.SUBSCRIPTION:
             queryreturn = sqlQuery(
-                '''select label from subscriptions where address=?''', self.address)
+                '''select label from subscriptions where address=?''', dbstr(self.address))
         if queryreturn is not None:
             if queryreturn != []:
                 for row in queryreturn:
                     retval, = row
-                    retval = unicode(retval, 'utf-8')
+                retval = unic(ustr(retval))
         elif self.address is None or self.type == AccountMixin.ALL:
-            return unicode(
+            return unic(_translate("MainWindow", "All accounts"))
-                str(_translate("MainWindow", "All accounts")), 'utf-8')
 
-        return retval or unicode(self.address, 'utf-8')
+        return retval or unic(self.address)
 
 
 class BMTreeWidgetItem(QtGui.QTreeWidgetItem, AccountMixin):
@@ -154,7 +158,7 @@ class BMTreeWidgetItem(QtGui.QTreeWidgetItem, AccountMixin):
         self._setup(parent, pos)
 
     def _getAddressBracket(self, unreadCount=False):
-        return " (" + str(self.unreadCount) + ")" if unreadCount else ""
+        return " (" + ustr(self.unreadCount) + ")" if unreadCount else ""
 
     def data(self, column, role):
         """Override internal QT method for returning object data"""
@@ -191,7 +195,7 @@ class Ui_FolderWidget(BMTreeWidgetItem):
 
     def setFolderName(self, fname):
         """Set folder name (for QT UI)"""
-        self.folderName = str(fname)
+        self.folderName = ustr(fname)
 
     def data(self, column, role):
         """Override internal QT method for returning object data"""
@@ -232,15 +236,14 @@ class Ui_AddressWidget(BMTreeWidgetItem, SettingsMixin):
 
     def _getLabel(self):
         if self.address is None:
-            return unicode(_translate(
+            return unic(_translate(
-                "MainWindow", "All accounts").toUtf8(), 'utf-8', 'ignore')
+                "MainWindow", "All accounts"))
         else:
             try:
-                return unicode(
+                return unic(ustr(
-                    config.get(self.address, 'label'),
+                    config.get(self.address, 'label')))
-                    'utf-8', 'ignore')
             except:
-                return unicode(self.address, 'utf-8')
+                return unic(self.address)
 
     def _getAddressBracket(self, unreadCount=False):
         ret = "" if self.isExpanded() \
@@ -264,8 +267,8 @@ class Ui_AddressWidget(BMTreeWidgetItem, SettingsMixin):
         if role == QtCore.Qt.EditRole \
                 and self.type != AccountMixin.SUBSCRIPTION:
             config.set(
-                str(self.address), 'label',
+                self.address, 'label',
-                str(value.toString().toUtf8())
+                ustr(value)
                 if isinstance(value, QtCore.QVariant)
                 else value.encode('utf-8')
             )
@@ -307,12 +310,12 @@ class Ui_SubscriptionWidget(Ui_AddressWidget):
 
     def _getLabel(self):
         queryreturn = sqlQuery(
-            '''select label from subscriptions where address=?''', self.address)
+            '''select label from subscriptions where address=?''', dbstr(self.address))
         if queryreturn != []:
             for row in queryreturn:
                 retval, = row
-            return unicode(retval, 'utf-8', 'ignore')
+            return unic(ustr(retval))
-        return unicode(self.address, 'utf-8')
+        return unic(self.address)
 
     def setType(self):
         """Set account type"""
@@ -323,13 +326,13 @@ class Ui_SubscriptionWidget(Ui_AddressWidget):
         """Save subscription label to database"""
         if role == QtCore.Qt.EditRole:
             if isinstance(value, QtCore.QVariant):
-                label = str(
+                label = ustr(
-                    value.toString().toUtf8()).decode('utf-8', 'ignore')
+                    value)
             else:
-                label = unicode(value, 'utf-8', 'ignore')
+                label = unic(ustr(value))
             sqlExecute(
                 '''UPDATE subscriptions SET label=? WHERE address=?''',
-                label, self.address)
+                dbstr(label), dbstr(self.address))
         return super(Ui_SubscriptionWidget, self).setData(column, role, value)
 
 
@@ -407,18 +410,18 @@ class MessageList_AddressWidget(BMAddressWidget):
                 AccountMixin.NORMAL,
                 AccountMixin.CHAN, AccountMixin.MAILINGLIST):
             try:
-                newLabel = unicode(
+                newLabel = unic(ustr(
-                    config.get(self.address, 'label'),
+                    config.get(self.address, 'label')))
-                    'utf-8', 'ignore')
             except:
                 queryreturn = sqlQuery(
-                    '''select label from addressbook where address=?''', self.address)
+                    '''select label from addressbook where address=?''', dbstr(self.address))
         elif self.type == AccountMixin.SUBSCRIPTION:
             queryreturn = sqlQuery(
-                '''select label from subscriptions where address=?''', self.address)
+                '''select label from subscriptions where address=?''', dbstr(self.address))
         if queryreturn:
             for row in queryreturn:
-                newLabel = unicode(row[0], 'utf-8', 'ignore')
+                newLabel = row[0]
+            newLabel = unic(ustr(newLabel))
 
         self.label = newLabel
 
@@ -454,9 +457,9 @@ class MessageList_SubjectWidget(BMTableWidgetItem):
     def data(self, role):
         """Return object data (QT UI)"""
         if role == QtCore.Qt.UserRole:
-            return self.subject
+            return ustr(self.subject)
         if role == QtCore.Qt.ToolTipRole:
-            return escape(unicode(self.subject, 'utf-8'))
+            return escape(unic(ustr(self.subject)))
         return super(MessageList_SubjectWidget, self).data(role)
 
     # label (or address) alphabetically, disabled at the end
@@ -476,9 +479,9 @@ class MessageList_TimeWidget(BMTableWidgetItem):
     msgid is available by QtCore.Qt.UserRole
     """
 
-    def __init__(self, label=None, unread=False, timestamp=None, msgid=''):
+    def __init__(self, label=None, unread=False, timestamp=None, msgid=b''):
         super(MessageList_TimeWidget, self).__init__(label, unread)
-        self.setData(QtCore.Qt.UserRole, QtCore.QByteArray(msgid))
+        self.setData(QtCore.Qt.UserRole, QtCore.QByteArray(bytes(msgid)))
         self.setData(TimestampRole, int(timestamp))
 
     def __lt__(self, other):
@@ -491,9 +494,9 @@ class MessageList_TimeWidget(BMTableWidgetItem):
         """
         data = super(MessageList_TimeWidget, self).data(role)
         if role == TimestampRole:
-            return int(data.toPyObject())
+            return int(data)
         if role == QtCore.Qt.UserRole:
-            return str(data.toPyObject())
+            return ustr(data)
         return data
 
 
@@ -513,8 +516,8 @@ class Ui_AddressBookWidgetItem(BMAddressWidget):
     def setData(self, role, value):
         """Set data"""
         if role == QtCore.Qt.EditRole:
-            self.label = str(
+            self.label = ustr(
-                value.toString().toUtf8()
+                value
                 if isinstance(value, QtCore.QVariant) else value
             )
             if self.type in (
@@ -525,9 +528,9 @@ class Ui_AddressBookWidgetItem(BMAddressWidget):
                     config.set(self.address, 'label', self.label)
                     config.save()
                 except:
-                    sqlExecute('''UPDATE addressbook set label=? WHERE address=?''', self.label, self.address)
+                    sqlExecute('''UPDATE addressbook set label=? WHERE address=?''', dbstr(self.label), dbstr(self.address))
             elif self.type == AccountMixin.SUBSCRIPTION:
-                sqlExecute('''UPDATE subscriptions set label=? WHERE address=?''', self.label, self.address)
+                sqlExecute('''UPDATE subscriptions set label=? WHERE address=?''', dbstr(self.label), dbstr(self.address))
             else:
                 pass
         return super(Ui_AddressBookWidgetItem, self).setData(role, value)
@@ -546,7 +549,7 @@ class Ui_AddressBookWidgetItem(BMAddressWidget):
 class Ui_AddressBookWidgetItemLabel(Ui_AddressBookWidgetItem):
     """Addressbook label item"""
     def __init__(self, address, label, acc_type):
-        self.address = address
+        self.address = ustr(address)
         super(Ui_AddressBookWidgetItemLabel, self).__init__(label, acc_type)
 
     def data(self, role):
@@ -558,7 +561,7 @@ class Ui_AddressBookWidgetItemLabel(Ui_AddressBookWidgetItem):
 class Ui_AddressBookWidgetItemAddress(Ui_AddressBookWidgetItem):
     """Addressbook address item"""
     def __init__(self, address, label, acc_type):
-        self.address = address
+        self.address = ustr(address)
         super(Ui_AddressBookWidgetItemAddress, self).__init__(address, acc_type)
 
     def data(self, role):
@@ -584,14 +587,14 @@ class AddressBookCompleter(QtGui.QCompleter):
 
     def splitPath(self, path):
         """Split on semicolon"""
-        text = unicode(path.toUtf8(), 'utf-8')
+        text = unic(ustr(path))
         return [text[:self.widget().cursorPosition()].split(';')[-1].strip()]
 
     def pathFromIndex(self, index):
         """Perform autocompletion (reimplemented QCompleter method)"""
-        autoString = unicode(
+        autoString = unic(ustr(
-            index.data(QtCore.Qt.EditRole).toString().toUtf8(), 'utf-8')
+            index.data(QtCore.Qt.EditRole)))
-        text = unicode(self.widget().text().toUtf8(), 'utf-8')
+        text = unic(ustr(self.widget().text()))
 
         # If cursor position was saved, restore it, else save it
         if self.cursorPos != -1:

src/bitmessageqt/messagecompose.py

@@ -24,7 +24,7 @@ class MessageCompose(QtGui.QTextEdit):
                 self.zoomOut(1)
             zoom = self.currentFont().pointSize() * 100 / self.defaultFontPointSize
             QtGui.QApplication.activeWindow().statusBar().showMessage(
-                QtGui.QApplication.translate("MainWindow", "Zoom level %1%").arg(
+                QtGui.QApplication.translate("MainWindow", "Zoom level {0}%").format(
                     str(zoom)
                 )
             )

src/bitmessageqt/messageview.py

@@ -5,9 +5,10 @@ zoom and URL click warning popup
 
 """
 
+from unqstr import ustr, unic
 from PyQt4 import QtCore, QtGui
 
-from safehtmlparser import SafeHTMLParser
+from .safehtmlparser import SafeHTMLParser
 from tr import _translate
 
 
@@ -56,7 +57,7 @@ class MessageView(QtGui.QTextBrowser):
         ) == QtCore.Qt.ControlModifier and event.orientation() == QtCore.Qt.Vertical:
             zoom = self.currentFont().pointSize() * 100 / self.defaultFontPointSize
             QtGui.QApplication.activeWindow().statusBar().showMessage(_translate(
-                "MainWindow", "Zoom level %1%").arg(str(zoom)))
+                "MainWindow", "Zoom level {0}%").format(ustr(zoom)))
 
     def setWrappingWidth(self, width=None):
         """Set word-wrapping width"""
@@ -90,8 +91,8 @@ class MessageView(QtGui.QTextBrowser):
                 "Follow external link"),
             QtGui.QApplication.translate(
                 "MessageView",
-                "The link \"%1\" will open in a browser. It may be a security risk, it could de-anonymise you"
+                "The link \"{0}\" will open in a browser. It may be a security risk, it could de-anonymise you"
-                " or download malicious data. Are you sure?").arg(unicode(link.toString())),
+                " or download malicious data. Are you sure?").format(unic(ustr(link))),
             QtGui.QMessageBox.Yes,
             QtGui.QMessageBox.No)
         if reply == QtGui.QMessageBox.Yes:
@@ -124,7 +125,7 @@ class MessageView(QtGui.QTextBrowser):
                 if pos > self.outpos:
                     self.outpos = pos + 1
             cursor.movePosition(QtGui.QTextCursor.End, QtGui.QTextCursor.MoveAnchor)
-            cursor.insertHtml(QtCore.QString(self.out[startpos:self.outpos]))
+            cursor.insertHtml(unic(self.out[startpos:self.outpos]))
         self.verticalScrollBar().setValue(position)
         self.rendering = False
 
@@ -133,9 +134,9 @@ class MessageView(QtGui.QTextBrowser):
         self.mode = MessageView.MODE_PLAIN
         out = self.html.raw
         if self.html.has_html:
-            out = "<div align=\"center\" style=\"text-decoration: underline;\"><b>" + unicode(
+            out = "<div align=\"center\" style=\"text-decoration: underline;\"><b>" + unic(ustr(
                 QtGui.QApplication.translate(
-                    "MessageView", "HTML detected, click here to display")) + "</b></div><br/>" + out
+                    "MessageView", "HTML detected, click here to display")) + "</b></div><br/>" + out)
         self.out = out
         self.outpos = 0
         self.setHtml("")
@@ -145,8 +146,8 @@ class MessageView(QtGui.QTextBrowser):
         """Render message as HTML"""
         self.mode = MessageView.MODE_HTML
         out = self.html.sanitised
-        out = "<div align=\"center\" style=\"text-decoration: underline;\"><b>" + unicode(
+        out = "<div align=\"center\" style=\"text-decoration: underline;\"><b>" + unic(ustr(
-            QtGui.QApplication.translate("MessageView", "Click here to disable HTML")) + "</b></div><br/>" + out
+            QtGui.QApplication.translate("MessageView", "Click here to disable HTML")) + "</b></div><br/>" + out)
         self.out = out
         self.outpos = 0
         self.setHtml("")

src/bitmessageqt/networkstatus.py

@@ -9,11 +9,11 @@ from PyQt4 import QtCore, QtGui
 import l10n
 import network.stats
 import state
-import widgets
+from bitmessageqt import widgets
 from network import connectionpool, knownnodes
-from retranslateui import RetranslateMixin
+from .retranslateui import RetranslateMixin
 from tr import _translate
-from uisignaler import UISignaler
+from .uisignaler import UISignaler
 
 
 class NetworkStatus(QtGui.QWidget, RetranslateMixin):
@@ -134,12 +134,12 @@ class NetworkStatus(QtGui.QWidget, RetranslateMixin):
         self.labelBytesRecvCount.setText(
             _translate(
                 "networkstatus",
-                "Down: %1/s  Total: %2").arg(
+                "Down: {0}/s  Total: {1}").format(
                     self.formatByteRate(network.stats.downloadSpeed()),
                     self.formatBytes(network.stats.receivedBytes())))
         self.labelBytesSentCount.setText(
             _translate(
-                "networkstatus", "Up: %1/s  Total: %2").arg(
+                "networkstatus", "Up: {0}/s  Total: {1}").format(
                     self.formatByteRate(network.stats.uploadSpeed()),
                     self.formatBytes(network.stats.sentBytes())))
 
@@ -204,9 +204,9 @@ class NetworkStatus(QtGui.QWidget, RetranslateMixin):
             if not connectionpool.pool.inboundConnections:
                 self.window().setStatusIcon('yellow')
             for i in range(self.tableWidgetConnectionCount.rowCount()):
-                if self.tableWidgetConnectionCount.item(i, 0).data(QtCore.Qt.UserRole).toPyObject() != destination:
+                if self.tableWidgetConnectionCount.item(i, 0).data(QtCore.Qt.UserRole) != destination:
                     continue
-                if self.tableWidgetConnectionCount.item(i, 1).data(QtCore.Qt.UserRole).toPyObject() == outbound:
+                if self.tableWidgetConnectionCount.item(i, 1).data(QtCore.Qt.UserRole) == outbound:
                     self.tableWidgetConnectionCount.removeRow(i)
                     break
 
@@ -214,7 +214,7 @@ class NetworkStatus(QtGui.QWidget, RetranslateMixin):
         self.tableWidgetConnectionCount.setSortingEnabled(True)
         self.labelTotalConnections.setText(
             _translate(
-                "networkstatus", "Total Connections: %1").arg(
+                "networkstatus", "Total Connections: {0}").format(
                     str(self.tableWidgetConnectionCount.rowCount())))
         # FYI: The 'singlelistener' thread sets the icon color to green when it
         # receives an incoming connection, meaning that the user's firewall is
@@ -227,7 +227,7 @@ class NetworkStatus(QtGui.QWidget, RetranslateMixin):
     # timer driven
     def runEveryTwoSeconds(self):
         """Updates counters, runs every 2 seconds if the timer is running"""
-        self.labelLookupsPerSecond.setText(_translate("networkstatus", "Inventory lookups per second: %1").arg(
+        self.labelLookupsPerSecond.setText(_translate("networkstatus", "Inventory lookups per second: {0}").format(
             str(state.Inventory.numberOfInventoryLookupsPerformed / 2)))
         state.Inventory.numberOfInventoryLookupsPerformed = 0
         self.updateNumberOfBytes()
@@ -238,11 +238,11 @@ class NetworkStatus(QtGui.QWidget, RetranslateMixin):
         super(NetworkStatus, self).retranslateUi()
         self.labelTotalConnections.setText(
             _translate(
-                "networkstatus", "Total Connections: %1").arg(
+                "networkstatus", "Total Connections: {0}").format(
                     str(self.tableWidgetConnectionCount.rowCount())))
         self.labelStartupTime.setText(_translate(
-            "networkstatus", "Since startup on %1"
+            "networkstatus", "Since startup on {0}"
-        ).arg(l10n.formatTimestamp(self.startup)))
+        ).format(l10n.formatTimestamp(self.startup)))
         self.updateNumberOfMessagesProcessed()
         self.updateNumberOfBroadcastsProcessed()
         self.updateNumberOfPubkeysProcessed()

src/bitmessageqt/newchandialog.py

@@ -4,15 +4,16 @@ src/bitmessageqt/newchandialog.py
 
 """
 
+from unqstr import ustr, unic
 from PyQt4 import QtCore, QtGui
 
-import widgets
+from bitmessageqt import widgets
 from addresses import addBMIfNotPresent
-from addressvalidator import AddressValidator, PassPhraseValidator
+from .addressvalidator import AddressValidator, PassPhraseValidator
 from queues import (
     addressGeneratorQueue, apiAddressGeneratorReturnQueue, UISignalQueue)
 from tr import _translate
-from utils import str_chan
+from .utils import str_chan
 
 
 class NewChanDialog(QtGui.QDialog):
@@ -52,21 +53,21 @@ class NewChanDialog(QtGui.QDialog):
         self.timer.stop()
         self.hide()
         apiAddressGeneratorReturnQueue.queue.clear()
-        if self.chanAddress.text().toUtf8() == "":
+        if ustr(self.chanAddress.text()) == "":
             addressGeneratorQueue.put(
-                ('createChan', 4, 1, str_chan + ' ' + str(self.chanPassPhrase.text().toUtf8()),
+                ('createChan', 4, 1, str_chan + ' ' + ustr(self.chanPassPhrase.text()),
-                 self.chanPassPhrase.text().toUtf8(),
+                 ustr(self.chanPassPhrase.text()).encode("utf-8", "replace"),
                  True))
         else:
             addressGeneratorQueue.put(
-                ('joinChan', addBMIfNotPresent(self.chanAddress.text().toUtf8()),
+                ('joinChan', addBMIfNotPresent(ustr(self.chanAddress.text())),
-                 str_chan + ' ' + str(self.chanPassPhrase.text().toUtf8()),
+                 str_chan + ' ' + ustr(self.chanPassPhrase.text()),
-                 self.chanPassPhrase.text().toUtf8(),
+                 ustr(self.chanPassPhrase.text()).encode("utf-8", "replace"),
                  True))
         addressGeneratorReturnValue = apiAddressGeneratorReturnQueue.get(True)
         if addressGeneratorReturnValue and addressGeneratorReturnValue[0] != 'chan name does not match address':
             UISignalQueue.put(('updateStatusBar', _translate(
-                "newchandialog", "Successfully created / joined chan %1").arg(unicode(self.chanPassPhrase.text()))))
+                "newchandialog", "Successfully created / joined chan {0}").format(unic(ustr(self.chanPassPhrase.text())))))
             self.parent.ui.tabWidget.setCurrentIndex(
                 self.parent.ui.tabWidget.indexOf(self.parent.ui.chans)
             )

src/bitmessageqt/retranslateui.py

@@ -1,20 +1,22 @@
 from os import path
+from unqstr import ustr
+import six
 from PyQt4 import QtGui
 from debug import logger
-import widgets
+from bitmessageqt import widgets
 
 class RetranslateMixin(object):
     def retranslateUi(self):
         defaults = QtGui.QWidget()
         widgets.load(self.__class__.__name__.lower() + '.ui', defaults)
-        for attr, value in defaults.__dict__.iteritems():
+        for attr, value in six.iteritems(defaults.__dict__):
             setTextMethod = getattr(value, "setText", None)
             if callable(setTextMethod):
-                getattr(self, attr).setText(getattr(defaults, attr).text())
+                getattr(self, attr).setText(ustr(getattr(defaults, attr).text()))
             elif isinstance(value, QtGui.QTableWidget):
                 for i in range (value.columnCount()):
                     getattr(self, attr).horizontalHeaderItem(i).setText(
-                        getattr(defaults, attr).horizontalHeaderItem(i).text())
+                        ustr(getattr(defaults, attr).horizontalHeaderItem(i).text()))
                 for i in range (value.rowCount()):
                     getattr(self, attr).verticalHeaderItem(i).setText(
-                        getattr(defaults, attr).verticalHeaderItem(i).text())
+                        ustr(getattr(defaults, attr).verticalHeaderItem(i).text()))

src/bitmessageqt/safehtmlparser.py

@@ -2,11 +2,11 @@
 
 import inspect
 import re
-from HTMLParser import HTMLParser
+from six.moves.html_parser import HTMLParser
 
-from urllib import quote_plus
+from six.moves.urllib.parse import quote_plus, urlparse
-from urlparse import urlparse
 
+from unqstr import ustr, unic
 
 class SafeHTMLParser(HTMLParser):
     """HTML parser with sanitisation"""
@@ -124,9 +124,9 @@ class SafeHTMLParser(HTMLParser):
 
     def feed(self, data):
         try:
-            data = unicode(data, 'utf-8')
+            data = unic(ustr(data))
-        except UnicodeDecodeError:
+        except TypeError:
-            data = unicode(data, 'utf-8', errors='replace')
+            pass
         HTMLParser.feed(self, data)
         tmp = SafeHTMLParser.replace_pre(data)
         tmp = self.uriregex1.sub(r'<a href="\1">\1</a>', tmp)

src/bitmessageqt/settings.py

@@ -1,12 +1,13 @@
 """
 This module setting file is for settings
 """
-import ConfigParser
+from six.moves import configparser
 import os
 import sys
 import tempfile
 
 import six
+from unqstr import ustr
 from PyQt4 import QtCore, QtGui
 
 import debug
@@ -16,7 +17,7 @@ import openclpow
 import paths
 import queues
 import state
-import widgets
+from bitmessageqt import widgets
 from bmconfigparser import config as config_obj
 from helper_sql import sqlExecute, sqlStoredProcedure
 from helper_startup import start_proxyconfig
@@ -26,12 +27,18 @@ from network.asyncore_pollchoose import set_rates
 from tr import _translate
 
 
+try:
+    SafeConfigParser = configparser.SafeConfigParser
+except AttributeError:
+    # alpine linux, python3.12
+    SafeConfigParser = configparser.ConfigParser
+
 def getSOCKSProxyType(config):
     """Get user socksproxytype setting from *config*"""
     try:
-        result = ConfigParser.SafeConfigParser.get(
+        result = SafeConfigParser.get(
             config, 'bitmessagesettings', 'socksproxytype')
-    except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
+    except (configparser.NoSectionError, configparser.NoOptionError):
         return None
     else:
         if result.lower() in ('', 'none', 'false'):
@@ -187,7 +194,7 @@ class SettingsDialog(QtGui.QDialog):
             else:
                 if self.checkBoxOnionOnly.isChecked():
                     self.checkBoxOnionOnly.setText(
-                        self.checkBoxOnionOnly.text() + ", " + _translate(
+                        ustr(self.checkBoxOnionOnly.text()) + ", " + _translate(
                             "MainWindow", "may cause connection problems!"))
                     self.checkBoxOnionOnly.setStyleSheet(
                         "QCheckBox { color : red; }")
@@ -324,10 +331,10 @@ class SettingsDialog(QtGui.QDialog):
             _translate("MainWindow", "Testing..."))
         nc = namecoin.namecoinConnection({
             'type': self.getNamecoinType(),
-            'host': str(self.lineEditNamecoinHost.text().toUtf8()),
+            'host': ustr(self.lineEditNamecoinHost.text()),
-            'port': str(self.lineEditNamecoinPort.text().toUtf8()),
+            'port': ustr(self.lineEditNamecoinPort.text()),
-            'user': str(self.lineEditNamecoinUser.text().toUtf8()),
+            'user': ustr(self.lineEditNamecoinUser.text()),
-            'password': str(self.lineEditNamecoinPassword.text().toUtf8())
+            'password': ustr(self.lineEditNamecoinPassword.text())
         })
         status, text = nc.test()
         self.labelNamecoinTestResult.setText(text)
@@ -387,7 +394,7 @@ class SettingsDialog(QtGui.QDialog):
             ))
 
         lang = str(self.languageComboBox.itemData(
-            self.languageComboBox.currentIndex()).toString())
+            self.languageComboBox.currentIndex()))
         self.config.set('bitmessagesettings', 'userlocale', lang)
         self.parent.change_translation()
 
@@ -486,13 +493,13 @@ class SettingsDialog(QtGui.QDialog):
 
         self.config.set(
             'bitmessagesettings', 'namecoinrpctype', self.getNamecoinType())
-        self.config.set('bitmessagesettings', 'namecoinrpchost', str(
+        self.config.set('bitmessagesettings', 'namecoinrpchost', ustr(
             self.lineEditNamecoinHost.text()))
-        self.config.set('bitmessagesettings', 'namecoinrpcport', str(
+        self.config.set('bitmessagesettings', 'namecoinrpcport', ustr(
             self.lineEditNamecoinPort.text()))
-        self.config.set('bitmessagesettings', 'namecoinrpcuser', str(
+        self.config.set('bitmessagesettings', 'namecoinrpcuser', ustr(
             self.lineEditNamecoinUser.text()))
-        self.config.set('bitmessagesettings', 'namecoinrpcpassword', str(
+        self.config.set('bitmessagesettings', 'namecoinrpcpassword', ustr(
             self.lineEditNamecoinPassword.text()))
         self.parent.resetNamecoinConnection()
 
@@ -510,11 +517,11 @@ class SettingsDialog(QtGui.QDialog):
                     float(self.lineEditSmallMessageDifficulty.text())
                     * defaults.networkDefaultPayloadLengthExtraBytes)))
 
-        if self.comboBoxOpenCL.currentText().toUtf8() != self.config.safeGet(
+        if ustr(self.comboBoxOpenCL.currentText()) != ustr(self.config.safeGet(
-                'bitmessagesettings', 'opencl'):
+                'bitmessagesettings', 'opencl')):
             self.config.set(
                 'bitmessagesettings', 'opencl',
-                str(self.comboBoxOpenCL.currentText()))
+                ustr(self.comboBoxOpenCL.currentText()))
             queues.workerQueue.put(('resetPoW', ''))
 
         acceptableDifficultyChanged = False

src/bitmessageqt/settingsmixin.py

@@ -5,6 +5,7 @@ src/settingsmixin.py
 
 """
 
+from unqstr import ustr
 from PyQt4 import QtCore, QtGui
 
 
@@ -40,7 +41,7 @@ class SettingsMixin(object):
         self.warnIfNoObjectName()
         settings = QtCore.QSettings()
         try:
-            geom = settings.value("/".join([str(self.objectName()), "geometry"]))
+            geom = settings.value("/".join([ustr(self.objectName()), "geometry"]))
             target.restoreGeometry(geom.toByteArray() if hasattr(geom, 'toByteArray') else geom)
         except Exception:
             pass
@@ -50,7 +51,7 @@ class SettingsMixin(object):
         self.warnIfNoObjectName()
         settings = QtCore.QSettings()
         try:
-            state = settings.value("/".join([str(self.objectName()), "state"]))
+            state = settings.value("/".join([ustr(self.objectName()), "state"]))
             target.restoreState(state.toByteArray() if hasattr(state, 'toByteArray') else state)
         except Exception:
             pass

src/bitmessageqt/support.py

@@ -6,9 +6,11 @@ import ssl
 import sys
 import time
 
+from unqstr import ustr, unic
 from PyQt4 import QtCore
+from dbcompat import dbstr
 
-import account
+from bitmessageqt import account
 import defaults
 import network.stats
 import paths
@@ -16,12 +18,12 @@ import proofofwork
 import queues
 import state
 from bmconfigparser import config
-from foldertree import AccountMixin
+from .foldertree import AccountMixin
 from helper_sql import sqlExecute, sqlQuery
 from l10n import getTranslationLanguage
 from openclpow import openclEnabled
 from pyelliptic.openssl import OpenSSL
-from settings import getSOCKSProxyType
+from .settings import getSOCKSProxyType
 from version import softwareVersion
 from tr import _translate
 
@@ -67,12 +69,12 @@ Connected hosts: {}
 
 
 def checkAddressBook(myapp):
-    sqlExecute('DELETE from addressbook WHERE address=?', OLD_SUPPORT_ADDRESS)
+    sqlExecute('DELETE from addressbook WHERE address=?', dbstr(OLD_SUPPORT_ADDRESS))
-    queryreturn = sqlQuery('SELECT * FROM addressbook WHERE address=?', SUPPORT_ADDRESS)
+    queryreturn = sqlQuery('SELECT * FROM addressbook WHERE address=?', dbstr(SUPPORT_ADDRESS))
     if queryreturn == []:
         sqlExecute(
             'INSERT INTO addressbook VALUES (?,?)',
-            SUPPORT_LABEL.toUtf8(), SUPPORT_ADDRESS)
+            dbstr(SUPPORT_LABEL), dbstr(SUPPORT_ADDRESS))
         myapp.rerenderAddressBook()
 
 
@@ -88,7 +90,7 @@ def createAddressIfNeeded(myapp):
     if not checkHasNormalAddress():
         queues.addressGeneratorQueue.put((
             'createRandomAddress', 4, 1,
-            str(SUPPORT_MY_LABEL.toUtf8()),
+            ustr(SUPPORT_MY_LABEL),
             1, "", False,
             defaults.networkDefaultProofOfWorkNonceTrialsPerByte,
             defaults.networkDefaultPayloadLengthExtraBytes
@@ -122,7 +124,7 @@ def createSupportMessage(myapp):
     os = sys.platform
     if os == "win32":
         windowsversion = sys.getwindowsversion()
-        os = "Windows " + str(windowsversion[0]) + "." + str(windowsversion[1])
+        os = "Windows " + ustr(windowsversion[0]) + "." + ustr(windowsversion[1])
     else:
         try:
             from os import uname
@@ -141,7 +143,7 @@ def createSupportMessage(myapp):
         frozen = paths.frozen
     portablemode = "True" if state.appdata == paths.lookupExeFolder() else "False"
     cpow = "True" if proofofwork.bmpow else "False"
-    openclpow = str(
+    openclpow = ustr(
         config.safeGet('bitmessagesettings', 'opencl')
     ) if openclEnabled() else "None"
     locale = getTranslationLanguage()
@@ -149,9 +151,9 @@ def createSupportMessage(myapp):
     upnp = config.safeGet('bitmessagesettings', 'upnp', "N/A")
     connectedhosts = len(network.stats.connectedHostsList())
 
-    myapp.ui.textEditMessage.setText(unicode(SUPPORT_MESSAGE, 'utf-8').format(
+    myapp.ui.textEditMessage.setText(unic(ustr(SUPPORT_MESSAGE).format(
         version, os, architecture, pythonversion, opensslversion, frozen,
-        portablemode, cpow, openclpow, locale, socks, upnp, connectedhosts))
+        portablemode, cpow, openclpow, locale, socks, upnp, connectedhosts)))
 
     # single msg tab
     myapp.ui.tabWidgetSend.setCurrentIndex(

src/bitmessageqt/tests/addressbook.py

@@ -1,7 +1,7 @@
 import helper_addressbook
 from bitmessageqt.support import createAddressIfNeeded
 
-from main import TestBase
+from .main import TestBase
 
 
 class TestAddressbook(TestBase):

src/bitmessageqt/tests/main.py

@@ -1,5 +1,6 @@
 """Common definitions for bitmessageqt tests"""
 
+from six.moves import queue as Queue
 import sys
 import unittest
 
@@ -52,7 +53,7 @@ class TestMain(unittest.TestCase):
         """Check the results of _translate() with various args"""
         self.assertIsInstance(
             _translate("MainWindow", "Test"),
-            QtCore.QString
+            str
         )
 
 

src/bitmessageqt/tests/support.py

@@ -4,7 +4,9 @@ import sys
 
 from shared import isAddressInMyAddressBook
 
-from main import TestBase
+from .main import TestBase
+
+from unqstr import ustr
 
 
 class TestSupport(TestBase):
@@ -26,8 +28,8 @@ class TestSupport(TestBase):
         self.assertEqual(
             ui.tabWidget.currentIndex(), ui.tabWidget.indexOf(ui.send))
         self.assertEqual(
-            ui.lineEditTo.text(), self.SUPPORT_ADDRESS)
+            ustr(ui.lineEditTo.text()), ustr(self.SUPPORT_ADDRESS))
         self.assertEqual(
-            ui.lineEditSubject.text(), self.SUPPORT_SUBJECT)
+            ustr(ui.lineEditSubject.text()), ustr(self.SUPPORT_SUBJECT))
         self.assertIn(
             sys.version, ui.textEditMessage.toPlainText())

src/bitmessageqt/utils.py

@@ -38,7 +38,7 @@ def identiconize(address):
         # stripped from PIL and uses QT instead (by sendiulo, same license)
         import qidenticon
         icon_hash = hashlib.md5(
-            addBMIfNotPresent(address) + identiconsuffix).hexdigest()
+            (addBMIfNotPresent(address) + identiconsuffix).encode("utf-8", "replace")).hexdigest()
         use_two_colors = identicon_lib[:len('qidenticon_two')] == 'qidenticon_two'
         opacity = int(
             identicon_lib not in (
@@ -81,7 +81,7 @@ def avatarize(address):
     falls back to identiconize(address)
     """
     idcon = QtGui.QIcon()
-    icon_hash = hashlib.md5(addBMIfNotPresent(address)).hexdigest()
+    icon_hash = hashlib.md5(addBMIfNotPresent(address).encode("utf-8", "replace")).hexdigest()
     if address == str_broadcast_subscribers:
         # don't hash [Broadcast subscribers]
         icon_hash = address

src/bmconfigparser.py

@@ -15,13 +15,17 @@ try:
 except ImportError:
     from pybitmessage import state
 
-SafeConfigParser = configparser.SafeConfigParser
+try:
+    SafeConfigParser = configparser.SafeConfigParser
+except AttributeError:
+    # alpine linux, python3.12
+    SafeConfigParser = configparser.ConfigParser
 config_ready = Event()
 
 
 class BMConfigParser(SafeConfigParser):
     """
-    Singleton class inherited from :class:`ConfigParser.SafeConfigParser`
+    Singleton class inherited from :class:`configparser.SafeConfigParser`
     with additional methods specific to bitmessage config.
     """
     # pylint: disable=too-many-ancestors

src/class_addressGenerator.py

@@ -211,8 +211,8 @@ class addressGenerator(StoppableThread):
                         'updateStatusBar',
                         _translate(
                             "MainWindow",
-                            "Generating %1 new addresses."
+                            "Generating {0} new addresses."
-                        ).arg(str(numberOfAddressesToMake))
+                        ).format(str(numberOfAddressesToMake))
                     ))
                 signingKeyNonce = 0
                 encryptionKeyNonce = 1
@@ -302,9 +302,9 @@ class addressGenerator(StoppableThread):
                                 'updateStatusBar',
                                 _translate(
                                     "MainWindow",
-                                    "%1 is already in 'Your Identities'."
+                                    "{0} is already in 'Your Identities'."
                                     " Not adding it again."
-                                ).arg(address)
+                                ).format(address)
                             ))
                         else:
                             self.logger.debug('label: %s', label)

src/class_objectProcessor.py

@@ -12,6 +12,7 @@ import subprocess  # nosec B404
 import threading
 import time
 from binascii import hexlify
+import sqlite3
 
 import helper_bitcoin
 import helper_inbox
@@ -33,6 +34,7 @@ from helper_sql import (
 from network import knownnodes, invQueue
 from network.node import Peer
 from tr import _translate
+from dbcompat import dbstr
 
 logger = logging.getLogger('default')
 
@@ -121,7 +123,7 @@ class objectProcessor(threading.Thread):
                         objectType, data = queues.objectProcessorQueue.get()
                         sql.execute(
                             'INSERT INTO objectprocessorqueue VALUES (?,?)',
-                            objectType, data)
+                            objectType, sqlite3.Binary(data))
                         numberOfObjectsThatWereInTheObjectProcessorQueue += 1
                 logger.debug(
                     'Saved %s objects from the objectProcessorQueue to'
@@ -140,19 +142,24 @@ class objectProcessor(threading.Thread):
         # bypass nonce and time, retain object type/version/stream + body
         readPosition = 16
 
-        if data[readPosition:] in state.ackdataForWhichImWatching:
+        data_bytes = bytes(data[readPosition:])
+        if data_bytes in state.ackdataForWhichImWatching:
             logger.info('This object is an acknowledgement bound for me.')
-            del state.ackdataForWhichImWatching[data[readPosition:]]
+            del state.ackdataForWhichImWatching[data_bytes]
-            sqlExecute(
+            rowcount = sqlExecute(
                 "UPDATE sent SET status='ackreceived', lastactiontime=?"
-                " WHERE ackdata=?", int(time.time()), data[readPosition:])
+                " WHERE ackdata=?", int(time.time()), sqlite3.Binary(data_bytes))
+            if rowcount < 1:
+                rowcount = sqlExecute(
+                    "UPDATE sent SET status='ackreceived', lastactiontime=?"
+                    " WHERE ackdata=CAST(? AS TEXT)", int(time.time()), data_bytes)
             queues.UISignalQueue.put((
                 'updateSentItemStatusByAckdata', (
-                    data[readPosition:],
+                    data_bytes,
                     _translate(
                         "MainWindow",
-                        "Acknowledgement of the message received %1"
+                        "Acknowledgement of the message received {0}"
-                    ).arg(l10n.formatTimestamp()))
+                    ).format(l10n.formatTimestamp()))
             ))
         else:
             logger.debug('This object is not an acknowledgement bound for me.')
@@ -215,9 +222,10 @@ class objectProcessor(threading.Thread):
             logger.info(
                 'the hash requested in this getpubkey request is: %s',
                 hexlify(requestedHash))
+            requestedHash_bytes = bytes(requestedHash)
             # if this address hash is one of mine
-            if requestedHash in shared.myAddressesByHash:
+            if requestedHash_bytes in shared.myAddressesByHash:
-                myAddress = shared.myAddressesByHash[requestedHash]
+                myAddress = shared.myAddressesByHash[requestedHash_bytes]
         elif requestedAddressVersionNumber >= 4:
             requestedTag = data[readPosition:readPosition + 32]
             if len(requestedTag) != 32:
@@ -227,8 +235,9 @@ class objectProcessor(threading.Thread):
             logger.debug(
                 'the tag requested in this getpubkey request is: %s',
                 hexlify(requestedTag))
-            if requestedTag in shared.myAddressesByTag:
+            requestedTag_bytes = bytes(requestedTag)
-                myAddress = shared.myAddressesByTag[requestedTag]
+            if requestedTag_bytes in shared.myAddressesByTag:
+                myAddress = shared.myAddressesByTag[requestedTag_bytes]
 
         if myAddress == '':
             logger.info('This getpubkey request is not for any of my keys.')
@@ -299,12 +308,12 @@ class objectProcessor(threading.Thread):
                     '(within processpubkey) payloadLength less than 146.'
                     ' Sanity check failed.')
             readPosition += 4
-            pubSigningKey = '\x04' + data[readPosition:readPosition + 64]
+            pubSigningKey = b'\x04' + data[readPosition:readPosition + 64]
             # Is it possible for a public key to be invalid such that trying to
             # encrypt or sign with it will cause an error? If it is, it would
             # be easiest to test them here.
             readPosition += 64
-            pubEncryptionKey = '\x04' + data[readPosition:readPosition + 64]
+            pubEncryptionKey = b'\x04' + data[readPosition:readPosition + 64]
             if len(pubEncryptionKey) < 65:
                 return logger.debug(
                     'publicEncryptionKey length less than 64. Sanity check'
@@ -327,19 +336,19 @@ class objectProcessor(threading.Thread):
 
             queryreturn = sqlQuery(
                 "SELECT usedpersonally FROM pubkeys WHERE address=?"
-                " AND usedpersonally='yes'", address)
+                " AND usedpersonally='yes'", dbstr(address))
             # if this pubkey is already in our database and if we have
             # used it personally:
             if queryreturn != []:
                 logger.info(
                     'We HAVE used this pubkey personally. Updating time.')
-                t = (address, addressVersion, dataToStore,
+                t = (dbstr(address), addressVersion, sqlite3.Binary(dataToStore),
                      int(time.time()), 'yes')
             else:
                 logger.info(
                     'We have NOT used this pubkey personally. Inserting'
                     ' in database.')
-                t = (address, addressVersion, dataToStore,
+                t = (dbstr(address), addressVersion, sqlite3.Binary(dataToStore),
                      int(time.time()), 'no')
             sqlExecute('''INSERT INTO pubkeys VALUES (?,?,?,?,?)''', *t)
             self.possibleNewPubkey(address)
@@ -350,9 +359,9 @@ class objectProcessor(threading.Thread):
                     ' Sanity check failed.')
                 return
             readPosition += 4
-            pubSigningKey = '\x04' + data[readPosition:readPosition + 64]
+            pubSigningKey = b'\x04' + data[readPosition:readPosition + 64]
             readPosition += 64
-            pubEncryptionKey = '\x04' + data[readPosition:readPosition + 64]
+            pubEncryptionKey = b'\x04' + data[readPosition:readPosition + 64]
             readPosition += 64
             specifiedNonceTrialsPerByteLength = decodeVarint(
                 data[readPosition:readPosition + 10])[1]
@@ -389,20 +398,20 @@ class objectProcessor(threading.Thread):
             address = encodeAddress(addressVersion, streamNumber, ripe)
             queryreturn = sqlQuery(
                 "SELECT usedpersonally FROM pubkeys WHERE address=?"
-                " AND usedpersonally='yes'", address)
+                " AND usedpersonally='yes'", dbstr(address))
             # if this pubkey is already in our database and if we have
             # used it personally:
             if queryreturn != []:
                 logger.info(
                     'We HAVE used this pubkey personally. Updating time.')
-                t = (address, addressVersion, dataToStore,
+                t = (dbstr(address), addressVersion, sqlite3.Binary(dataToStore),
-                     int(time.time()), 'yes')
+                     int(time.time()), dbstr('yes'))
             else:
                 logger.info(
                     'We have NOT used this pubkey personally. Inserting'
                     ' in database.')
-                t = (address, addressVersion, dataToStore,
+                t = (dbstr(address), addressVersion, sqlite3.Binary(dataToStore),
-                     int(time.time()), 'no')
+                     int(time.time()), dbstr('no'))
             sqlExecute('''INSERT INTO pubkeys VALUES (?,?,?,?,?)''', *t)
             self.possibleNewPubkey(address)
 
@@ -413,12 +422,13 @@ class objectProcessor(threading.Thread):
                     ' Sanity check failed.')
 
             tag = data[readPosition:readPosition + 32]
-            if tag not in state.neededPubkeys:
+            tag_bytes = bytes(tag)
+            if tag_bytes not in state.neededPubkeys:
                 return logger.info(
                     'We don\'t need this v4 pubkey. We didn\'t ask for it.')
 
             # Let us try to decrypt the pubkey
-            toAddress = state.neededPubkeys[tag][0]
+            toAddress = state.neededPubkeys[tag_bytes][0]
             if protocol.decryptAndCheckPubkeyPayload(data, toAddress) == \
                     'successful':
                 # At this point we know that we have been waiting on this
@@ -483,7 +493,7 @@ class objectProcessor(threading.Thread):
 
         # This is a message bound for me.
         # Look up my address based on the RIPE hash.
-        toAddress = shared.myAddressesByHash[toRipe]
+        toAddress = shared.myAddressesByHash[bytes(toRipe)]
         readPosition = 0
         sendersAddressVersionNumber, sendersAddressVersionNumberLength = \
             decodeVarint(decryptedData[readPosition:readPosition + 10])
@@ -507,9 +517,9 @@ class objectProcessor(threading.Thread):
             return
         readPosition += sendersStreamNumberLength
         readPosition += 4
-        pubSigningKey = '\x04' + decryptedData[readPosition:readPosition + 64]
+        pubSigningKey = b'\x04' + decryptedData[readPosition:readPosition + 64]
         readPosition += 64
-        pubEncryptionKey = '\x04' + decryptedData[readPosition:readPosition + 64]
+        pubEncryptionKey = b'\x04' + decryptedData[readPosition:readPosition + 64]
         readPosition += 64
         if sendersAddressVersionNumber >= 3:
             requiredAverageProofOfWorkNonceTrialsPerByte, varintLength = \
@@ -558,7 +568,7 @@ class objectProcessor(threading.Thread):
         readPosition += signatureLengthLength
         signature = decryptedData[
             readPosition:readPosition + signatureLength]
-        signedData = data[8:20] + encodeVarint(1) + encodeVarint(
+        signedData = bytes(data[8:20]) + encodeVarint(1) + encodeVarint(
             streamNumberAsClaimedByMsg
         ) + decryptedData[:positionOfBottomOfAckData]
 
@@ -590,11 +600,11 @@ class objectProcessor(threading.Thread):
         # person.
         sqlExecute(
             '''INSERT INTO pubkeys VALUES (?,?,?,?,?)''',
-            fromAddress,
+            dbstr(fromAddress),
             sendersAddressVersionNumber,
-            decryptedData[:endOfThePublicKeyPosition],
+            sqlite3.Binary(decryptedData[:endOfThePublicKeyPosition]),
             int(time.time()),
-            'yes')
+            dbstr('yes'))
 
         # Check to see whether we happen to be awaiting this
         # pubkey in order to send a message. If we are, it will do the POW
@@ -631,7 +641,7 @@ class objectProcessor(threading.Thread):
                 'bitmessagesettings', 'blackwhitelist') == 'black':
             queryreturn = sqlQuery(
                 "SELECT label FROM blacklist where address=? and enabled='1'",
-                fromAddress)
+                dbstr(fromAddress))
             if queryreturn != []:
                 logger.info('Message ignored because address is in blacklist.')
 
@@ -639,7 +649,7 @@ class objectProcessor(threading.Thread):
         else:  # We're using a whitelist
             queryreturn = sqlQuery(
                 "SELECT label FROM whitelist where address=? and enabled='1'",
-                fromAddress)
+                dbstr(fromAddress))
             if queryreturn == []:
                 logger.info(
                     'Message ignored because address not in whitelist.')
@@ -808,13 +818,14 @@ class objectProcessor(threading.Thread):
         elif broadcastVersion == 5:
             embeddedTag = data[readPosition:readPosition + 32]
             readPosition += 32
-            if embeddedTag not in shared.MyECSubscriptionCryptorObjects:
+            embeddedTag_bytes = bytes(embeddedTag)
+            if embeddedTag_bytes not in shared.MyECSubscriptionCryptorObjects:
                 logger.debug('We\'re not interested in this broadcast.')
                 return
             # We are interested in this broadcast because of its tag.
             # We're going to add some more data which is signed further down.
-            signedData = data[8:readPosition]
+            signedData = bytes(data[8:readPosition])
-            cryptorObject = shared.MyECSubscriptionCryptorObjects[embeddedTag]
+            cryptorObject = shared.MyECSubscriptionCryptorObjects[embeddedTag_bytes]
             try:
                 decryptedData = cryptorObject.decrypt(data[readPosition:])
                 logger.debug('EC decryption successful')
@@ -854,10 +865,10 @@ class objectProcessor(threading.Thread):
             )
         readPosition += sendersStreamLength
         readPosition += 4
-        sendersPubSigningKey = '\x04' + \
+        sendersPubSigningKey = b'\x04' + \
             decryptedData[readPosition:readPosition + 64]
         readPosition += 64
-        sendersPubEncryptionKey = '\x04' + \
+        sendersPubEncryptionKey = b'\x04' + \
             decryptedData[readPosition:readPosition + 64]
         readPosition += 64
         if sendersAddressVersion >= 3:
@@ -927,11 +938,11 @@ class objectProcessor(threading.Thread):
 
         # Let's store the public key in case we want to reply to this person.
         sqlExecute('''INSERT INTO pubkeys VALUES (?,?,?,?,?)''',
-                   fromAddress,
+                   dbstr(fromAddress),
-                   sendersAddressVersion,
+                   dbstr(sendersAddressVersion),
-                   decryptedData[:endOfPubkeyPosition],
+                   sqlite3.Binary(decryptedData[:endOfPubkeyPosition]),
                    int(time.time()),
-                   'yes')
+                   dbstr('yes'))
 
         # Check to see whether we happen to be awaiting this
         # pubkey in order to send a message. If we are, it will do the POW
@@ -997,8 +1008,9 @@ class objectProcessor(threading.Thread):
                 encodeVarint(addressVersion) + encodeVarint(streamNumber)
                 + ripe
             )[32:]
-            if tag in state.neededPubkeys:
+            tag_bytes = bytes(tag)
-                del state.neededPubkeys[tag]
+            if tag_bytes in state.neededPubkeys:
+                del state.neededPubkeys[tag_bytes]
                 self.sendMessages(address)
 
     @staticmethod
@@ -1012,7 +1024,7 @@ class objectProcessor(threading.Thread):
             "UPDATE sent SET status='doingmsgpow', retrynumber=0"
             " WHERE toaddress=?"
             " AND (status='awaitingpubkey' OR status='doingpubkeypow')"
-            " AND folder='sent'", address)
+            " AND folder='sent'", dbstr(address))
         queues.workerQueue.put(('sendmessage', ''))
 
     @staticmethod
@@ -1047,8 +1059,8 @@ class objectProcessor(threading.Thread):
         if checksum != hashlib.sha512(payload).digest()[0:4]:
             logger.info('ackdata checksum wrong. Not sending ackdata.')
             return False
-        command = command.rstrip('\x00')
+        command = command.rstrip(b'\x00')
-        if command != 'object':
+        if command != b'object':
             return False
         return True
 

src/class_singleCleaner.py

@@ -22,6 +22,7 @@ It resends messages when there has been no response:
 import gc
 import os
 import time
+import sqlite3
 
 import queues
 import state
@@ -29,6 +30,7 @@ from bmconfigparser import config
 from helper_sql import sqlExecute, sqlQuery
 from network import connectionpool, knownnodes, StoppableThread
 from tr import _translate
+from dbcompat import dbstr
 
 
 #: Equals 4 weeks. You could make this longer if you want
@@ -99,6 +101,8 @@ class singleCleaner(StoppableThread):
                     tick - state.maximumLengthOfTimeToBotherResendingMessages
                 )
                 for toAddress, ackData, status in queryreturn:
+                    toAddress = toAddress.decode("utf-8", "replace")
+                    status = status.decode("utf-8", "replace")
                     if status == 'awaitingpubkey':
                         self.resendPubkeyRequest(toAddress)
                     elif status == 'msgsent':
@@ -168,7 +172,7 @@ class singleCleaner(StoppableThread):
         ))
         sqlExecute(
             "UPDATE sent SET status = 'msgqueued'"
-            " WHERE toaddress = ? AND folder = 'sent'", address)
+            " WHERE toaddress = ? AND folder = 'sent'", dbstr(address))
         queues.workerQueue.put(('sendmessage', ''))
 
     def resendMsg(self, ackdata):
@@ -177,9 +181,13 @@ class singleCleaner(StoppableThread):
             'It has been a long time and we haven\'t heard an acknowledgement'
             ' to our msg. Sending again.'
         )
+        rowcount = sqlExecute(
+            "UPDATE sent SET status = 'msgqueued'"
+            " WHERE ackdata = ? AND folder = 'sent'", sqlite3.Binary(ackdata))
+        if rowcount < 1:
             sqlExecute(
                 "UPDATE sent SET status = 'msgqueued'"
-            " WHERE ackdata = ? AND folder = 'sent'", ackdata)
+                " WHERE ackdata = CAST(? AS TEXT) AND folder = 'sent'", ackdata)
         queues.workerQueue.put(('sendmessage', ''))
         queues.UISignalQueue.put((
             'updateStatusBar',

src/class_singleWorker.py

@@ -11,6 +11,7 @@ import time
 from binascii import hexlify, unhexlify
 from struct import pack
 from subprocess import call  # nosec
+import sqlite3
 
 import defaults
 import helper_inbox
@@ -30,6 +31,9 @@ from bmconfigparser import config
 from helper_sql import sqlExecute, sqlQuery
 from network import knownnodes, StoppableThread, invQueue
 from six.moves import configparser, queue
+from six.moves.reprlib import repr
+import six
+from dbcompat import dbstr
 
 
 def sizeof_fmt(num, suffix='h/s'):
@@ -73,6 +77,7 @@ class singleWorker(StoppableThread):
             '''SELECT DISTINCT toaddress FROM sent'''
             ''' WHERE (status='awaitingpubkey' AND folder='sent')''')
         for toAddress, in queryreturn:
+            toAddress = toAddress.decode("utf-8", "replace")
             toAddressVersionNumber, toStreamNumber, toRipe = \
                 decodeAddress(toAddress)[1:]
             if toAddressVersionNumber <= 3:
@@ -87,7 +92,7 @@ class singleWorker(StoppableThread):
                 tag = doubleHashOfAddressData[32:]
                 # We'll need this for when we receive a pubkey reply:
                 # it will be encrypted and we'll need to decrypt it.
-                state.neededPubkeys[tag] = (
+                state.neededPubkeys[bytes(tag)] = (
                     toAddress,
                     highlevelcrypto.makeCryptor(
                         hexlify(privEncryptionKey))
@@ -99,17 +104,22 @@ class singleWorker(StoppableThread):
         for row in queryreturn:
             ackdata, = row
             self.logger.info('Watching for ackdata %s', hexlify(ackdata))
-            state.ackdataForWhichImWatching[ackdata] = 0
+            state.ackdataForWhichImWatching[bytes(ackdata)] = 0
 
         # Fix legacy (headerless) watched ackdata to include header
         for oldack in state.ackdataForWhichImWatching:
             if len(oldack) == 32:
                 # attach legacy header, always constant (msg/1/1)
-                newack = '\x00\x00\x00\x02\x01\x01' + oldack
+                newack = b'\x00\x00\x00\x02\x01\x01' + oldack
-                state.ackdataForWhichImWatching[newack] = 0
+                state.ackdataForWhichImWatching[bytes(newack)] = 0
-                sqlExecute(
+                rowcount = sqlExecute(
                     '''UPDATE sent SET ackdata=? WHERE ackdata=? AND folder = 'sent' ''',
-                    newack, oldack
+                    sqlite3.Binary(newack), sqlite3.Binary(oldack)
+                )
+                if rowcount < 1:
+                    sqlExecute(
+                        '''UPDATE sent SET ackdata=? WHERE ackdata=CAST(? AS TEXT) AND folder = 'sent' ''',
+                        sqlite3.Binary(newack), oldack
                     )
                 del state.ackdataForWhichImWatching[oldack]
 
@@ -261,7 +271,7 @@ class singleWorker(StoppableThread):
         TTL = int(28 * 24 * 60 * 60 + helper_random.randomrandrange(-300, 300))
         embeddedTime = int(time.time() + TTL)
         payload = pack('>Q', (embeddedTime))
-        payload += '\x00\x00\x00\x01'  # object type: pubkey
+        payload += b'\x00\x00\x00\x01'  # object type: pubkey
         payload += encodeVarint(addressVersionNumber)  # Address version number
         payload += encodeVarint(streamNumber)
         # bitfield of features supported by me (see the wiki).
@@ -338,7 +348,7 @@ class singleWorker(StoppableThread):
         # expiresTime time.
 
         payload = pack('>Q', (embeddedTime))
-        payload += '\x00\x00\x00\x01'  # object type: pubkey
+        payload += b'\x00\x00\x00\x01'  # object type: pubkey
         payload += encodeVarint(addressVersionNumber)  # Address version number
         payload += encodeVarint(streamNumber)
         # bitfield of features supported by me (see the wiki).
@@ -413,7 +423,7 @@ class singleWorker(StoppableThread):
         TTL = int(28 * 24 * 60 * 60 + helper_random.randomrandrange(-300, 300))
         embeddedTime = int(time.time() + TTL)
         payload = pack('>Q', (embeddedTime))
-        payload += '\x00\x00\x00\x01'  # object type: pubkey
+        payload += b'\x00\x00\x00\x01'  # object type: pubkey
         payload += encodeVarint(addressVersionNumber)  # Address version number
         payload += encodeVarint(streamNumber)
         dataToEncrypt = protocol.getBitfield(myAddress)
@@ -515,9 +525,15 @@ class singleWorker(StoppableThread):
             payload, TTL, log_prefix='(For onionpeer object)')
 
         inventoryHash = highlevelcrypto.calculateInventoryHash(payload)
+        if six.PY2:
+            payload_buffer = buffer(payload)
+            tag_buffer = buffer(tag)
+        else:  # assume six.PY3
+            payload_buffer = memoryview(payload)
+            tag_buffer = memoryview(tag)
         state.Inventory[inventoryHash] = (
-            objectType, streamNumber, buffer(payload),  # noqa: F821
+            objectType, streamNumber, payload_buffer,  # noqa: F821
-            embeddedTime, buffer(tag)  # noqa: F821
+            embeddedTime, tag_buffer  # noqa: F821
         )
         self.logger.info(
             'sending inv (within sendOnionPeerObj function) for object: %s',
@@ -534,10 +550,13 @@ class singleWorker(StoppableThread):
         queryreturn = sqlQuery(
             '''SELECT fromaddress, subject, message, '''
             ''' ackdata, ttl, encodingtype FROM sent '''
-            ''' WHERE status=? and folder='sent' ''', 'broadcastqueued')
+            ''' WHERE status=? and folder='sent' ''', dbstr('broadcastqueued'))
 
         for row in queryreturn:
             fromaddress, subject, body, ackdata, TTL, encoding = row
+            fromaddress = fromaddress.decode("utf-8", "replace")
+            subject = subject.decode("utf-8", "replace")
+            body = body.decode("utf-8", "replace")
             # status
             _, addressVersionNumber, streamNumber, ripe = \
                 decodeAddress(fromaddress)
@@ -578,11 +597,18 @@ class singleWorker(StoppableThread):
                 ))
                 continue
 
-            if not sqlExecute(
+            rowcount = sqlExecute(
                     '''UPDATE sent SET status='doingbroadcastpow' '''
                     ''' WHERE ackdata=? AND status='broadcastqueued' '''
                     ''' AND folder='sent' ''',
-                    ackdata):
+                    sqlite3.Binary(ackdata))
+            if rowcount < 1:
+                rowcount = sqlExecute(
+                        '''UPDATE sent SET status='doingbroadcastpow' '''
+                        ''' WHERE ackdata=CAST(? AS TEXT) AND status='broadcastqueued' '''
+                        ''' AND folder='sent' ''',
+                        ackdata)
+            if rowcount < 1:
                 continue
 
             # At this time these pubkeys are 65 bytes long
@@ -599,7 +625,7 @@ class singleWorker(StoppableThread):
             TTL = int(TTL + helper_random.randomrandrange(-300, 300))
             embeddedTime = int(time.time() + TTL)
             payload = pack('>Q', embeddedTime)
-            payload += '\x00\x00\x00\x03'  # object type: broadcast
+            payload += b'\x00\x00\x00\x03'  # object type: broadcast
 
             if addressVersionNumber <= 3:
                 payload += encodeVarint(4)  # broadcast version
@@ -615,7 +641,7 @@ class singleWorker(StoppableThread):
                 tag = doubleHashOfAddressData[32:]
                 payload += tag
             else:
-                tag = ''
+                tag = b''
 
             dataToEncrypt = encodeVarint(addressVersionNumber)
             dataToEncrypt += encodeVarint(streamNumber)
@@ -697,16 +723,22 @@ class singleWorker(StoppableThread):
                     ackdata,
                     tr._translate(
                         "MainWindow",
-                        "Broadcast sent on %1"
+                        "Broadcast sent on {0}"
-                    ).arg(l10n.formatTimestamp()))
+                    ).format(l10n.formatTimestamp()))
             ))
 
             # Update the status of the message in the 'sent' table to have
             # a 'broadcastsent' status
-            sqlExecute(
+            rowcount = sqlExecute(
                 '''UPDATE sent SET msgid=?, status=?, lastactiontime=? '''
                 ''' WHERE ackdata=? AND folder='sent' ''',
-                inventoryHash, 'broadcastsent', int(time.time()), ackdata
+                sqlite3.Binary(inventoryHash), dbstr('broadcastsent'), int(time.time()), sqlite3.Binary(ackdata)
+            )
+            if rowcount < 1:
+                sqlExecute(
+                    '''UPDATE sent SET msgid=?, status=?, lastactiontime=? '''
+                    ''' WHERE ackdata=CAST(? AS TEXT) AND folder='sent' ''',
+                    sqlite3.Binary(inventoryHash), 'broadcastsent', int(time.time()), ackdata
                 )
 
     def sendMsg(self):
@@ -726,6 +758,11 @@ class singleWorker(StoppableThread):
         for row in queryreturn:
             toaddress, fromaddress, subject, message, \
                 ackdata, status, TTL, retryNumber, encoding = row
+            toaddress = toaddress.decode("utf-8", "replace")
+            fromaddress = fromaddress.decode("utf-8", "replace")
+            subject = subject.decode("utf-8", "replace")
+            message = message.decode("utf-8", "replace")
+            status = status.decode("utf-8", "replace")
             # toStatus
             _, toAddressVersionNumber, toStreamNumber, toRipe = \
                 decodeAddress(toaddress)
@@ -755,7 +792,7 @@ class singleWorker(StoppableThread):
                 if not sqlExecute(
                     '''UPDATE sent SET status='doingmsgpow' '''
                     ''' WHERE toaddress=? AND status='msgqueued' AND folder='sent' ''',
-                    toaddress
+                    dbstr(toaddress)
                 ):
                     continue
                 status = 'doingmsgpow'
@@ -763,7 +800,7 @@ class singleWorker(StoppableThread):
                 # Let's see if we already have the pubkey in our pubkeys table
                 queryreturn = sqlQuery(
                     '''SELECT address FROM pubkeys WHERE address=?''',
-                    toaddress
+                    dbstr(toaddress)
                 )
                 # If we have the needed pubkey in the pubkey table already,
                 if queryreturn != []:
@@ -771,7 +808,7 @@ class singleWorker(StoppableThread):
                     if not sqlExecute(
                         '''UPDATE sent SET status='doingmsgpow' '''
                         ''' WHERE toaddress=? AND status='msgqueued' AND folder='sent' ''',
-                        toaddress
+                        dbstr(toaddress)
                     ):
                         continue
                     status = 'doingmsgpow'
@@ -783,26 +820,27 @@ class singleWorker(StoppableThread):
                     sqlExecute(
                         '''UPDATE pubkeys SET usedpersonally='yes' '''
                         ''' WHERE address=?''',
-                        toaddress
+                        dbstr(toaddress)
                     )
                 # We don't have the needed pubkey in the pubkeys table already.
                 else:
                     if toAddressVersionNumber <= 3:
-                        toTag = ''
+                        toTag = b''
                     else:
                         toTag = highlevelcrypto.double_sha512(
                             encodeVarint(toAddressVersionNumber)
                             + encodeVarint(toStreamNumber) + toRipe
                         )[32:]
+                    toTag_bytes = bytes(toTag)
                     if toaddress in state.neededPubkeys or \
-                            toTag in state.neededPubkeys:
+                            toTag_bytes in state.neededPubkeys:
                         # We already sent a request for the pubkey
                         sqlExecute(
                             '''UPDATE sent SET status='awaitingpubkey', '''
                             ''' sleeptill=? WHERE toaddress=? '''
                             ''' AND status='msgqueued' ''',
                             int(time.time()) + 2.5 * 24 * 60 * 60,
-                            toaddress
+                            dbstr(toaddress)
                         )
                         queues.UISignalQueue.put((
                             'updateSentItemStatusByToAddress', (
@@ -836,7 +874,8 @@ class singleWorker(StoppableThread):
                             privEncryptionKey = doubleHashOfToAddressData[:32]
                             # The second half of the sha512 hash.
                             tag = doubleHashOfToAddressData[32:]
-                            state.neededPubkeys[tag] = (
+                            tag_bytes = bytes(tag)
+                            state.neededPubkeys[tag_bytes] = (
                                 toaddress,
                                 highlevelcrypto.makeCryptor(
                                     hexlify(privEncryptionKey))
@@ -858,8 +897,8 @@ class singleWorker(StoppableThread):
                                         ''' status='awaitingpubkey' or '''
                                         ''' status='doingpubkeypow') AND '''
                                         ''' folder='sent' ''',
-                                        toaddress)
+                                        dbstr(toaddress))
-                                    del state.neededPubkeys[tag]
+                                    del state.neededPubkeys[tag_bytes]
                                     break
                                 # else:
                                 # There was something wrong with this
@@ -875,7 +914,7 @@ class singleWorker(StoppableThread):
                                 '''UPDATE sent SET '''
                                 ''' status='doingpubkeypow' WHERE '''
                                 ''' toaddress=? AND status='msgqueued' AND folder='sent' ''',
-                                toaddress
+                                dbstr(toaddress)
                             )
                             queues.UISignalQueue.put((
                                 'updateSentItemStatusByToAddress', (
@@ -901,7 +940,7 @@ class singleWorker(StoppableThread):
 
             # if we aren't sending this to ourselves or a chan
             if not config.has_section(toaddress):
-                state.ackdataForWhichImWatching[ackdata] = 0
+                state.ackdataForWhichImWatching[bytes(ackdata)] = 0
                 queues.UISignalQueue.put((
                     'updateSentItemStatusByAckdata', (
                         ackdata,
@@ -920,7 +959,7 @@ class singleWorker(StoppableThread):
                 # is too hard then we'll abort.
                 queryreturn = sqlQuery(
                     'SELECT transmitdata FROM pubkeys WHERE address=?',
-                    toaddress)
+                    dbstr(toaddress))
                 for row in queryreturn:  # pylint: disable=redefined-outer-name
                     pubkeyPayload, = row
 
@@ -969,8 +1008,8 @@ class singleWorker(StoppableThread):
                                     " device who requests that the"
                                     " destination be included in the"
                                     " message but this is disallowed in"
-                                    " your settings.  %1"
+                                    " your settings.  {0}"
-                                ).arg(l10n.formatTimestamp()))
+                                ).format(l10n.formatTimestamp()))
                         ))
                         # if the human changes their setting and then
                         # sends another message or restarts their client,
@@ -1035,14 +1074,13 @@ class singleWorker(StoppableThread):
                                 tr._translate(
                                     "MainWindow",
                                     "Doing work necessary to send message.\n"
-                                    "Receiver\'s required difficulty: %1"
+                                    "Receiver\'s required difficulty: {0}"
-                                    " and %2"
+                                    " and {1}"
-                                ).arg(
+                                ).format(
                                     str(
                                         float(requiredAverageProofOfWorkNonceTrialsPerByte)
                                         / defaults.networkDefaultProofOfWorkNonceTrialsPerByte
-                                    )
+                                    ),
-                                ).arg(
                                     str(
                                         float(requiredPayloadLengthExtraBytes)
                                         / defaults.networkDefaultPayloadLengthExtraBytes
@@ -1065,9 +1103,14 @@ class singleWorker(StoppableThread):
                         if cond1 or cond2:
                             # The demanded difficulty is more than
                             # we are willing to do.
-                            sqlExecute(
+                            rowcount = sqlExecute(
                                 '''UPDATE sent SET status='toodifficult' '''
                                 ''' WHERE ackdata=? AND folder='sent' ''',
+                                sqlite3.Binary(ackdata))
+                            if rowcount < 1:
+                                sqlExecute(
+                                    '''UPDATE sent SET status='toodifficult' '''
+                                    ''' WHERE ackdata=CAST(? AS TEXT) AND folder='sent' ''',
                                     ackdata)
                             queues.UISignalQueue.put((
                                 'updateSentItemStatusByAckdata', (
@@ -1075,14 +1118,14 @@ class singleWorker(StoppableThread):
                                     tr._translate(
                                         "MainWindow",
                                         "Problem: The work demanded by"
-                                        " the recipient (%1 and %2) is"
+                                        " the recipient ({0} and {1}) is"
                                         " more difficult than you are"
-                                        " willing to do. %3"
+                                        " willing to do. {2}"
-                                    ).arg(str(float(requiredAverageProofOfWorkNonceTrialsPerByte)
+                                    ).format(str(float(requiredAverageProofOfWorkNonceTrialsPerByte)
-                                          / defaults.networkDefaultProofOfWorkNonceTrialsPerByte)
+                                          / defaults.networkDefaultProofOfWorkNonceTrialsPerByte),
-                                          ).arg(str(float(requiredPayloadLengthExtraBytes)
+                                          str(float(requiredPayloadLengthExtraBytes)
-                                                / defaults.networkDefaultPayloadLengthExtraBytes)
+                                                / defaults.networkDefaultPayloadLengthExtraBytes),
-                                                ).arg(l10n.formatTimestamp()))))
+                                                l10n.formatTimestamp()))))
                             continue
             else:  # if we are sending a message to ourselves or a chan..
                 self.logger.info('Sending a message.')
@@ -1103,8 +1146,8 @@ class singleWorker(StoppableThread):
                                 " message to yourself or a chan but your"
                                 " encryption key could not be found in"
                                 " the keys.dat file. Could not encrypt"
-                                " message. %1"
+                                " message. {0}"
-                            ).arg(l10n.formatTimestamp()))
+                            ).format(l10n.formatTimestamp()))
                     ))
                     self.logger.error(
                         'Error within sendMsg. Could not read the keys'
@@ -1201,14 +1244,14 @@ class singleWorker(StoppableThread):
                     'Not bothering to include ackdata because we are'
                     ' sending to ourselves or a chan.'
                 )
-                fullAckPayload = ''
+                fullAckPayload = b''
             elif not protocol.checkBitfield(
                     behaviorBitfield, protocol.BITFIELD_DOESACK):
                 self.logger.info(
                     'Not bothering to include ackdata because'
                     ' the receiver said that they won\'t relay it anyway.'
                 )
-                fullAckPayload = ''
+                fullAckPayload = b''
             else:
                 # The fullAckPayload is a normal msg protocol message
                 # with the proof of work already completed that the
@@ -1217,7 +1260,7 @@ class singleWorker(StoppableThread):
                     ackdata, toStreamNumber, TTL)
             payload += encodeVarint(len(fullAckPayload))
             payload += fullAckPayload
-            dataToSign = pack('>Q', embeddedTime) + '\x00\x00\x00\x02' + \
+            dataToSign = pack('>Q', embeddedTime) + b'\x00\x00\x00\x02' + \
                 encodeVarint(1) + encodeVarint(toStreamNumber) + payload
             signature = highlevelcrypto.sign(
                 dataToSign, privSigningKeyHex, self.digestAlg)
@@ -1227,12 +1270,17 @@ class singleWorker(StoppableThread):
             # We have assembled the data that will be encrypted.
             try:
                 encrypted = highlevelcrypto.encrypt(
-                    payload, "04" + hexlify(pubEncryptionKeyBase256)
+                    payload, b"04" + hexlify(pubEncryptionKeyBase256)
                 )
             except:  # noqa:E722
                 self.logger.warning("highlevelcrypto.encrypt didn't work")
-                sqlExecute(
+                rowcount = sqlExecute(
                     '''UPDATE sent SET status='badkey' WHERE ackdata=? AND folder='sent' ''',
+                    sqlite3.Binary(ackdata)
+                )
+                if rowcount < 1:
+                    sqlExecute(
+                        '''UPDATE sent SET status='badkey' WHERE ackdata=CAST(? AS TEXT) AND folder='sent' ''',
                         ackdata
                     )
                 queues.UISignalQueue.put((
@@ -1241,13 +1289,13 @@ class singleWorker(StoppableThread):
                         tr._translate(
                             "MainWindow",
                             "Problem: The recipient\'s encryption key is"
-                            " no good. Could not encrypt message. %1"
+                            " no good. Could not encrypt message. {0}"
-                        ).arg(l10n.formatTimestamp()))
+                        ).format(l10n.formatTimestamp()))
                 ))
                 continue
 
             encryptedPayload = pack('>Q', embeddedTime)
-            encryptedPayload += '\x00\x00\x00\x02'  # object type: msg
+            encryptedPayload += b'\x00\x00\x00\x02'  # object type: msg
             encryptedPayload += encodeVarint(1)  # msg version
             encryptedPayload += encodeVarint(toStreamNumber) + encrypted
             target = 2 ** 64 / (
@@ -1309,8 +1357,8 @@ class singleWorker(StoppableThread):
                         ackdata,
                         tr._translate(
                             "MainWindow",
-                            "Message sent. Sent at %1"
+                            "Message sent. Sent at {0}"
-                        ).arg(l10n.formatTimestamp()))))
+                        ).format(l10n.formatTimestamp()))))
             else:
                 # not sending to a chan or one of my addresses
                 queues.UISignalQueue.put((
@@ -1319,8 +1367,8 @@ class singleWorker(StoppableThread):
                         tr._translate(
                             "MainWindow",
                             "Message sent. Waiting for acknowledgement."
-                            " Sent on %1"
+                            " Sent on {0}"
-                        ).arg(l10n.formatTimestamp()))
+                        ).format(l10n.formatTimestamp()))
                 ))
             self.logger.info(
                 'Broadcasting inv for my msg(within sendmsg function): %s',
@@ -1337,10 +1385,17 @@ class singleWorker(StoppableThread):
                 newStatus = 'msgsent'
             # wait 10% past expiration
             sleepTill = int(time.time() + TTL * 1.1)
-            sqlExecute(
+            rowcount = sqlExecute(
                 '''UPDATE sent SET msgid=?, status=?, retrynumber=?, '''
                 ''' sleeptill=?, lastactiontime=? WHERE ackdata=? AND folder='sent' ''',
-                inventoryHash, newStatus, retryNumber + 1,
+                sqlite3.Binary(inventoryHash), dbstr(newStatus), retryNumber + 1,
+                sleepTill, int(time.time()), sqlite3.Binary(ackdata)
+            )
+            if rowcount < 1:
+                sqlExecute(
+                    '''UPDATE sent SET msgid=?, status=?, retrynumber=?, '''
+                    ''' sleeptill=?, lastactiontime=? WHERE ackdata=CAST(? AS TEXT) AND folder='sent' ''',
+                    sqlite3.Binary(inventoryHash), newStatus, retryNumber + 1,
                     sleepTill, int(time.time()), ackdata
                 )
 
@@ -1386,7 +1441,7 @@ class singleWorker(StoppableThread):
             '''SELECT retrynumber FROM sent WHERE toaddress=? '''
             ''' AND (status='doingpubkeypow' OR status='awaitingpubkey') '''
             ''' AND folder='sent' LIMIT 1''',
-            toAddress
+            dbstr(toAddress)
         )
         if not queryReturn:
             self.logger.critical(
@@ -1412,10 +1467,11 @@ class singleWorker(StoppableThread):
             privEncryptionKey = doubleHashOfAddressData[:32]
             # Note that this is the second half of the sha512 hash.
             tag = doubleHashOfAddressData[32:]
-            if tag not in state.neededPubkeys:
+            tag_bytes = bytes(tag)
+            if tag_bytes not in state.neededPubkeys:
                 # We'll need this for when we receive a pubkey reply:
                 # it will be encrypted and we'll need to decrypt it.
-                state.neededPubkeys[tag] = (
+                state.neededPubkeys[tag_bytes] = (
                     toAddress,
                     highlevelcrypto.makeCryptor(hexlify(privEncryptionKey))
                 )
@@ -1429,7 +1485,7 @@ class singleWorker(StoppableThread):
         TTL = TTL + helper_random.randomrandrange(-300, 300)
         embeddedTime = int(time.time() + TTL)
         payload = pack('>Q', embeddedTime)
-        payload += '\x00\x00\x00\x00'  # object type: getpubkey
+        payload += b'\x00\x00\x00\x00'  # object type: getpubkey
         payload += encodeVarint(addressVersionNumber)
         payload += encodeVarint(streamNumber)
         if addressVersionNumber <= 3:
@@ -1468,7 +1524,7 @@ class singleWorker(StoppableThread):
             ''' status='awaitingpubkey', retrynumber=?, sleeptill=? '''
             ''' WHERE toaddress=? AND (status='doingpubkeypow' OR '''
             ''' status='awaitingpubkey') AND folder='sent' ''',
-            int(time.time()), retryNumber + 1, sleeptill, toAddress)
+            int(time.time()), retryNumber + 1, sleeptill, dbstr(toAddress))
 
         queues.UISignalQueue.put((
             'updateStatusBar',
@@ -1483,8 +1539,8 @@ class singleWorker(StoppableThread):
                 tr._translate(
                     "MainWindow",
                     "Sending public key request. Waiting for reply."
-                    " Requested at %1"
+                    " Requested at {0}"
-                ).arg(l10n.formatTimestamp()))
+                ).format(l10n.formatTimestamp()))
         ))
 
     def generateFullAckMessage(self, ackdata, _, TTL):
@@ -1511,4 +1567,4 @@ class singleWorker(StoppableThread):
         payload = self._doPOWDefaults(
             payload, TTL, log_prefix='(For ack message)', log_time=True)
 
-        return protocol.CreatePacket('object', payload)
+        return protocol.CreatePacket(b'object', payload)

src/class_smtpDeliver.py

@@ -4,9 +4,9 @@ SMTP client thread for delivering emails
 # pylint: disable=unused-variable
 
 import smtplib
-import urlparse
+from six.moves.urllib import parse as urlparse
 from email.header import Header
-from email.mime.text import MIMEText
+from six.moves import email_mime_text
 
 import queues
 import state
@@ -55,7 +55,7 @@ class smtpDeliver(StoppableThread):
                     u = urlparse.urlparse(dest)
                     to = urlparse.parse_qs(u.query)['to']
                     client = smtplib.SMTP(u.hostname, u.port)
-                    msg = MIMEText(body, 'plain', 'utf-8')
+                    msg = email_mime_text(body, 'plain', 'utf-8')
                     msg['Subject'] = Header(subject, 'utf-8')
                     msg['From'] = fromAddress + '@' + SMTPDOMAIN
                     toLabel = map(

src/class_smtpServer.py

@@ -12,6 +12,7 @@ import threading
 import time
 from email.header import decode_header
 from email.parser import Parser
+import sqlite3
 
 import queues
 from addresses import decodeAddress
@@ -20,6 +21,7 @@ from helper_ackPayload import genAckPayload
 from helper_sql import sqlExecute
 from network.threads import StoppableThread
 from version import softwareVersion
+from dbcompat import dbstr
 
 SMTPDOMAIN = "bmaddr.lan"
 LISTENPORT = 8425
@@ -88,19 +90,19 @@ class smtpServerPyBitmessage(smtpd.SMTPServer):
         ackdata = genAckPayload(streamNumber, stealthLevel)
         sqlExecute(
             '''INSERT INTO sent VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)''',
-            '',
+            sqlite3.Binary(b''),
-            toAddress,
+            dbstr(toAddress),
-            ripe,
+            sqlite3.Binary(ripe),
-            fromAddress,
+            dbstr(fromAddress),
-            subject,
+            dbstr(subject),
-            message,
+            dbstr(message),
-            ackdata,
+            sqlite3.Binary(ackdata),
             int(time.time()),  # sentTime (this will never change)
             int(time.time()),  # lastActionTime
             0,  # sleepTill time. This will get set when the POW gets done.
-            'msgqueued',
+            dbstr('msgqueued'),
             0,  # retryNumber
-            'sent',  # folder
+            dbstr('sent'),  # folder
             2,  # encodingtype
             # not necessary to have a TTL higher than 2 days
             min(config.getint('bitmessagesettings', 'ttl'), 86400 * 2)

src/class_sqlThread.py

@@ -8,6 +8,7 @@ import sqlite3
 import sys
 import threading
 import time
+from six.moves.reprlib import repr
 
 try:
     import helper_sql
@@ -38,7 +39,7 @@ class sqlThread(threading.Thread):
         helper_sql.sql_available = True
         config_ready.wait()
         self.conn = sqlite3.connect(state.appdata + 'messages.dat')
-        self.conn.text_factory = str
+        self.conn.text_factory = bytes
         self.cur = self.conn.cursor()
 
         self.cur.execute('PRAGMA secure_delete = true')
@@ -73,7 +74,7 @@ class sqlThread(threading.Thread):
                 '''INSERT INTO subscriptions VALUES'''
                 '''('Bitmessage new releases/announcements','BM-GtovgYdgs7qXPkoYaRgrLFuFKz1SFpsw',1)''')
             self.cur.execute(
-                '''CREATE TABLE settings (key blob, value blob, UNIQUE(key) ON CONFLICT REPLACE)''')
+                '''CREATE TABLE settings (key text, value blob, UNIQUE(key) ON CONFLICT REPLACE)''')
             self.cur.execute('''INSERT INTO settings VALUES('version','11')''')
             self.cur.execute('''INSERT INTO settings VALUES('lastvacuumtime',?)''', (
                 int(time.time()),))
@@ -542,7 +543,7 @@ class sqlThread(threading.Thread):
                 shutil.move(
                     paths.lookupAppdataFolder() + 'messages.dat', paths.lookupExeFolder() + 'messages.dat')
                 self.conn = sqlite3.connect(paths.lookupExeFolder() + 'messages.dat')
-                self.conn.text_factory = str
+                self.conn.text_factory = bytes
                 self.cur = self.conn.cursor()
             elif item == 'movemessagstoappdata':
                 logger.debug('the sqlThread is moving the messages.dat file to the Appdata folder.')
@@ -568,7 +569,7 @@ class sqlThread(threading.Thread):
                 shutil.move(
                     paths.lookupExeFolder() + 'messages.dat', paths.lookupAppdataFolder() + 'messages.dat')
                 self.conn = sqlite3.connect(paths.lookupAppdataFolder() + 'messages.dat')
-                self.conn.text_factory = str
+                self.conn.text_factory = bytes
                 self.cur = self.conn.cursor()
             elif item == 'deleteandvacuume':
                 self.cur.execute('''delete from inbox where folder='trash' ''')

src/dbcompat.py

@@ -0,0 +1,22 @@
+import logging
+import six
+
+logger = logging.getLogger("default")
+
+def dbstr(v):
+    if six.PY3:
+        if isinstance(v, str):
+            return v
+        elif isinstance(v, bytes):
+            return v.decode("utf-8", "replace")
+        logger.debug("unexpected type in dbstr(): {}".format(type(v)))
+        return v  # hope this never happens..
+    else:  # assume six.PY2
+        if isinstance(v, unicode):
+            return v.encode("utf-8", "replace")
+        elif isinstance(v, str):
+            return v
+        elif isinstance(v, bytes):
+            return str(v)
+        logger.debug("unexpected type in dbstr(): {}".format(type(v)))
+        return v  # hope this never happens..

src/depends.py

@@ -6,6 +6,7 @@ and suggest how it may be installed
 import os
 import re
 import sys
+import six
 
 # Only really old versions of Python don't have sys.hexversion. We don't
 # support them. The logging module was introduced in Python 2.3
@@ -383,6 +384,15 @@ def check_pyqt():
     Here we are checking for PyQt4 with its version, as for it require
     PyQt 4.8 or later.
     """
+    sip_found = False
+    try:
+        import sip
+        sip.setapi("QString", 2)
+        sip.setapi("QVariant", 2)
+        sip_found = True
+    except ImportError:
+        pass
+
     QtCore = try_import(
         'PyQt4.QtCore', 'PyBitmessage requires PyQt 4.8 or later and Qt 4.7 or later.')
 
@@ -402,6 +412,11 @@ def check_pyqt():
             'This version of Qt is too old. PyBitmessage requries'
             ' Qt 4.7 or later.')
         passed = False
+
+    if passed and not sip_found:
+        logger.info("sip is not found although PyQt is found")
+        return False
+
     return passed
 
 
@@ -435,14 +450,9 @@ def check_dependencies(verbose=False, optional=False):
     logger.info('Python version: %s', sys.version)
     if sys.hexversion < 0x20704F0:
         logger.error(
-            'PyBitmessage requires Python 2.7.4 or greater'
+            'PyBitmessage requires Python 2.7.4 or greater.'
-            ' (but not Python 3+)')
+            ' Python 2.7.18 is recommended.')
         has_all_dependencies = False
-    if sys.hexversion >= 0x3000000:
-        logger.error(
-            'PyBitmessage does not support Python 3+. Python 2.7.4'
-            ' or greater is required. Python 2.7.18 is recommended.')
-        sys.exit()
 
     # FIXME: This needs to be uncommented when more of the code is python3 compatible
     # if sys.hexversion >= 0x3000000 and sys.hexversion < 0x3060000:

src/fallback/umsgpack/umsgpack.py

@@ -49,10 +49,12 @@ License: MIT
 # pylint: disable=too-many-lines,too-many-branches,too-many-statements,global-statement,too-many-return-statements
 # pylint: disable=unused-argument
 
-import collections
+from collections import OrderedDict
-import io
+from six.moves.collections_abc import Hashable
+from six.moves import range as xrange
 import struct
 import sys
+import six
 
 __version__ = "2.4.1"
 "Module version string"
@@ -99,9 +101,9 @@ class Ext:  # pylint: disable=old-style-class
         if not isinstance(type, int) or not (type >= 0 and type <= 127):
             raise TypeError("ext type out of range")
         # Check data is type bytes
-        elif sys.version_info[0] == 3 and not isinstance(data, bytes):
+        elif six.PY3 and not isinstance(data, bytes):
             raise TypeError("ext data is not type \'bytes\'")
-        elif sys.version_info[0] == 2 and not isinstance(data, str):
+        elif six.PY2 and not isinstance(data, str):
             raise TypeError("ext data is not type \'str\'")
         self.type = type
         self.data = data
@@ -125,7 +127,7 @@ class Ext:  # pylint: disable=old-style-class
         String representation of this Ext object.
         """
         s = "Ext Object (Type: 0x%02x, Data: " % self.type
-        s += " ".join(["0x%02x" % ord(self.data[i:i + 1])
+        s += " ".join(["0x%02x" % six.byte2int(self.data[i:i + 1])
                        for i in xrange(min(len(self.data), 8))])
         if len(self.data) > 8:
             s += " ..."
@@ -549,7 +551,7 @@ def _packb2(obj, **options):
     '\x82\xa7compact\xc3\xa6schema\x00'
     >>>
     """
-    fp = io.BytesIO()
+    fp = six.BytesIO()
     _pack2(obj, fp, **options)
     return fp.getvalue()
 
@@ -582,7 +584,7 @@ def _packb3(obj, **options):
     b'\x82\xa7compact\xc3\xa6schema\x00'
     >>>
     """
-    fp = io.BytesIO()
+    fp = six.BytesIO()
     _pack3(obj, fp, **options)
     return fp.getvalue()
 
@@ -599,7 +601,7 @@ def _read_except(fp, n):
 
 
 def _unpack_integer(code, fp, options):
-    if (ord(code) & 0xe0) == 0xe0:
+    if (six.byte2int(code) & 0xe0) == 0xe0:
         return struct.unpack("b", code)[0]
     elif code == b'\xd0':
         return struct.unpack("b", _read_except(fp, 1))[0]
@@ -609,7 +611,7 @@ def _unpack_integer(code, fp, options):
         return struct.unpack(">i", _read_except(fp, 4))[0]
     elif code == b'\xd3':
         return struct.unpack(">q", _read_except(fp, 8))[0]
-    elif (ord(code) & 0x80) == 0x00:
+    elif (six.byte2int(code) & 0x80) == 0x00:
         return struct.unpack("B", code)[0]
     elif code == b'\xcc':
         return struct.unpack("B", _read_except(fp, 1))[0]
@@ -619,21 +621,21 @@ def _unpack_integer(code, fp, options):
         return struct.unpack(">I", _read_except(fp, 4))[0]
     elif code == b'\xcf':
         return struct.unpack(">Q", _read_except(fp, 8))[0]
-    raise Exception("logic error, not int: 0x%02x" % ord(code))
+    raise Exception("logic error, not int: 0x%02x" % six.byte2int(code))
 
 
 def _unpack_reserved(code, fp, options):
     if code == b'\xc1':
         raise ReservedCodeException(
-            "encountered reserved code: 0x%02x" % ord(code))
+            "encountered reserved code: 0x%02x" % six.byte2int(code))
     raise Exception(
-        "logic error, not reserved code: 0x%02x" % ord(code))
+        "logic error, not reserved code: 0x%02x" % six.byte2int(code))
 
 
 def _unpack_nil(code, fp, options):
     if code == b'\xc0':
         return None
-    raise Exception("logic error, not nil: 0x%02x" % ord(code))
+    raise Exception("logic error, not nil: 0x%02x" % six.byte2int(code))
 
 
 def _unpack_boolean(code, fp, options):
@@ -641,7 +643,7 @@ def _unpack_boolean(code, fp, options):
         return False
     elif code == b'\xc3':
         return True
-    raise Exception("logic error, not boolean: 0x%02x" % ord(code))
+    raise Exception("logic error, not boolean: 0x%02x" % six.byte2int(code))
 
 
 def _unpack_float(code, fp, options):
@@ -649,12 +651,12 @@ def _unpack_float(code, fp, options):
         return struct.unpack(">f", _read_except(fp, 4))[0]
     elif code == b'\xcb':
         return struct.unpack(">d", _read_except(fp, 8))[0]
-    raise Exception("logic error, not float: 0x%02x" % ord(code))
+    raise Exception("logic error, not float: 0x%02x" % six.byte2int(code))
 
 
 def _unpack_string(code, fp, options):
-    if (ord(code) & 0xe0) == 0xa0:
+    if (six.byte2int(code) & 0xe0) == 0xa0:
-        length = ord(code) & ~0xe0
+        length = six.byte2int(code) & ~0xe0
     elif code == b'\xd9':
         length = struct.unpack("B", _read_except(fp, 1))[0]
     elif code == b'\xda':
@@ -662,7 +664,7 @@ def _unpack_string(code, fp, options):
     elif code == b'\xdb':
         length = struct.unpack(">I", _read_except(fp, 4))[0]
     else:
-        raise Exception("logic error, not string: 0x%02x" % ord(code))
+        raise Exception("logic error, not string: 0x%02x" % six.byte2int(code))
 
     # Always return raw bytes in compatibility mode
     global compatibility
@@ -686,7 +688,7 @@ def _unpack_binary(code, fp, options):
     elif code == b'\xc6':
         length = struct.unpack(">I", _read_except(fp, 4))[0]
     else:
-        raise Exception("logic error, not binary: 0x%02x" % ord(code))
+        raise Exception("logic error, not binary: 0x%02x" % six.byte2int(code))
 
     return _read_except(fp, length)
 
@@ -709,9 +711,9 @@ def _unpack_ext(code, fp, options):
     elif code == b'\xc9':
         length = struct.unpack(">I", _read_except(fp, 4))[0]
     else:
-        raise Exception("logic error, not ext: 0x%02x" % ord(code))
+        raise Exception("logic error, not ext: 0x%02x" % six.byte2int(code))
 
-    ext = Ext(ord(_read_except(fp, 1)), _read_except(fp, length))
+    ext = Ext(six.byte2int(_read_except(fp, 1)), _read_except(fp, length))
 
     # Unpack with ext handler, if we have one
     ext_handlers = options.get("ext_handlers")
@@ -722,14 +724,14 @@ def _unpack_ext(code, fp, options):
 
 
 def _unpack_array(code, fp, options):
-    if (ord(code) & 0xf0) == 0x90:
+    if (six.byte2int(code) & 0xf0) == 0x90:
-        length = (ord(code) & ~0xf0)
+        length = (six.byte2int(code) & ~0xf0)
     elif code == b'\xdc':
         length = struct.unpack(">H", _read_except(fp, 2))[0]
     elif code == b'\xdd':
         length = struct.unpack(">I", _read_except(fp, 4))[0]
     else:
-        raise Exception("logic error, not array: 0x%02x" % ord(code))
+        raise Exception("logic error, not array: 0x%02x" % six.byte2int(code))
 
     return [_unpack(fp, options) for _ in xrange(length)]
 
@@ -741,17 +743,17 @@ def _deep_list_to_tuple(obj):
 
 
 def _unpack_map(code, fp, options):
-    if (ord(code) & 0xf0) == 0x80:
+    if (six.byte2int(code) & 0xf0) == 0x80:
-        length = (ord(code) & ~0xf0)
+        length = (six.byte2int(code) & ~0xf0)
     elif code == b'\xde':
         length = struct.unpack(">H", _read_except(fp, 2))[0]
     elif code == b'\xdf':
         length = struct.unpack(">I", _read_except(fp, 4))[0]
     else:
-        raise Exception("logic error, not map: 0x%02x" % ord(code))
+        raise Exception("logic error, not map: 0x%02x" % six.byte2int(code))
 
     d = {} if not options.get('use_ordered_dict') \
-        else collections.OrderedDict()
+        else OrderedDict()
     for _ in xrange(length):
         # Unpack key
         k = _unpack(fp, options)
@@ -759,7 +761,7 @@ def _unpack_map(code, fp, options):
         if isinstance(k, list):
             # Attempt to convert list into a hashable tuple
             k = _deep_list_to_tuple(k)
-        elif not isinstance(k, collections.Hashable):
+        elif not isinstance(k, Hashable):
             raise UnhashableKeyException(
                 "encountered unhashable key: %s, %s" % (str(k), str(type(k))))
         elif k in d:
@@ -911,7 +913,7 @@ def _unpackb2(s, **options):
     """
     if not isinstance(s, (str, bytearray)):
         raise TypeError("packed data must be type 'str' or 'bytearray'")
-    return _unpack(io.BytesIO(s), options)
+    return _unpack(six.BytesIO(s), options)
 
 
 # For Python 3, expects a bytes object
@@ -957,7 +959,7 @@ def _unpackb3(s, **options):
     """
     if not isinstance(s, (bytes, bytearray)):
         raise TypeError("packed data must be type 'bytes' or 'bytearray'")
-    return _unpack(io.BytesIO(s), options)
+    return _unpack(six.BytesIO(s), options)
 
 #############################################################################
 # Module Initialization
@@ -990,7 +992,7 @@ def __init():
         _float_precision = "single"
 
     # Map packb and unpackb to the appropriate version
-    if sys.version_info[0] == 3:
+    if six.PY3:
         pack = _pack3
         packb = _packb3
         dump = _pack3

src/helper_addressbook.py

@@ -4,11 +4,12 @@ Insert value into addressbook
 
 from bmconfigparser import config
 from helper_sql import sqlExecute
+from dbcompat import dbstr
 
 
 def insert(address, label):
     """perform insert into addressbook"""
 
     if address not in config.addresses():
-        return sqlExecute('''INSERT INTO addressbook VALUES (?,?)''', label, address) == 1
+        return sqlExecute('''INSERT INTO addressbook VALUES (?,?)''', dbstr(label), dbstr(address)) == 1
     return False

src/helper_bitcoin.py

@@ -19,17 +19,17 @@ def calculateBitcoinAddressFromPubkey(pubkey):
     sha = hashlib.new('sha256')
     sha.update(pubkey)
     ripe.update(sha.digest())
-    ripeWithProdnetPrefix = '\x00' + ripe.digest()
+    ripeWithProdnetPrefix = b'\x00' + ripe.digest()
 
     checksum = hashlib.sha256(hashlib.sha256(
         ripeWithProdnetPrefix).digest()).digest()[:4]
     binaryBitcoinAddress = ripeWithProdnetPrefix + checksum
     numberOfZeroBytesOnBinaryBitcoinAddress = 0
-    while binaryBitcoinAddress[0] == '\x00':
+    while binaryBitcoinAddress[0] == b'\x00':
         numberOfZeroBytesOnBinaryBitcoinAddress += 1
         binaryBitcoinAddress = binaryBitcoinAddress[1:]
     base58encoded = arithmetic.changebase(binaryBitcoinAddress, 256, 58)
-    return "1" * numberOfZeroBytesOnBinaryBitcoinAddress + base58encoded
+    return b"1" * numberOfZeroBytesOnBinaryBitcoinAddress + base58encoded
 
 
 def calculateTestnetAddressFromPubkey(pubkey):
@@ -43,14 +43,14 @@ def calculateTestnetAddressFromPubkey(pubkey):
     sha = hashlib.new('sha256')
     sha.update(pubkey)
     ripe.update(sha.digest())
-    ripeWithProdnetPrefix = '\x6F' + ripe.digest()
+    ripeWithProdnetPrefix = b'\x6F' + ripe.digest()
 
     checksum = hashlib.sha256(hashlib.sha256(
         ripeWithProdnetPrefix).digest()).digest()[:4]
     binaryBitcoinAddress = ripeWithProdnetPrefix + checksum
     numberOfZeroBytesOnBinaryBitcoinAddress = 0
-    while binaryBitcoinAddress[0] == '\x00':
+    while binaryBitcoinAddress[0] == b'\x00':
         numberOfZeroBytesOnBinaryBitcoinAddress += 1
         binaryBitcoinAddress = binaryBitcoinAddress[1:]
     base58encoded = arithmetic.changebase(binaryBitcoinAddress, 256, 58)
-    return "1" * numberOfZeroBytesOnBinaryBitcoinAddress + base58encoded
+    return b"1" * numberOfZeroBytesOnBinaryBitcoinAddress + base58encoded

src/helper_inbox.py

@@ -1,12 +1,16 @@
 """Helper Inbox performs inbox messages related operations"""
 
+import sqlite3
+
 import queues
 from helper_sql import sqlExecute, sqlQuery
+from dbcompat import dbstr
 
 
 def insert(t):
     """Perform an insert into the "inbox" table"""
-    sqlExecute('''INSERT INTO inbox VALUES (?,?,?,?,?,?,?,?,?,?)''', *t)
+    u = [sqlite3.Binary(t[0]), dbstr(t[1]), dbstr(t[2]), dbstr(t[3]), dbstr(t[4]), dbstr(t[5]), dbstr(t[6]), t[7], t[8], sqlite3.Binary(t[9])]
+    sqlExecute('''INSERT INTO inbox VALUES (?,?,?,?,?,?,?,?,?,?)''', *u)
     # shouldn't emit changedInboxUnread and displayNewInboxMessage
     # at the same time
     # queues.UISignalQueue.put(('changedInboxUnread', None))
@@ -14,22 +18,31 @@ def insert(t):
 
 def trash(msgid):
     """Mark a message in the `inbox` as `trash`"""
-    sqlExecute('''UPDATE inbox SET folder='trash' WHERE msgid=?''', msgid)
+    rowcount = sqlExecute('''UPDATE inbox SET folder='trash' WHERE msgid=?''', sqlite3.Binary(msgid))
+    if rowcount < 1:
+        sqlExecute('''UPDATE inbox SET folder='trash' WHERE msgid=CAST(? AS TEXT)''', msgid)
     queues.UISignalQueue.put(('removeInboxRowByMsgid', msgid))
 
 
 def delete(ack_data):
     """Permanent delete message from trash"""
-    sqlExecute("DELETE FROM inbox WHERE msgid = ?", ack_data)
+    rowcount = sqlExecute("DELETE FROM inbox WHERE msgid = ?", sqlite3.Binary(ack_data))
+    if rowcount < 1:
+        sqlExecute("DELETE FROM inbox WHERE msgid = CAST(? AS TEXT)", ack_data)
 
 
 def undeleteMessage(msgid):
     """Undelte the message"""
-    sqlExecute('''UPDATE inbox SET folder='inbox' WHERE msgid=?''', msgid)
+    rowcount = sqlExecute('''UPDATE inbox SET folder='inbox' WHERE msgid=?''', sqlite3.Binary(msgid))
+    if rowcount < 1:
+        sqlExecute('''UPDATE inbox SET folder='inbox' WHERE msgid=CAST(? AS TEXT)''', msgid)
 
 
 def isMessageAlreadyInInbox(sigHash):
     """Check for previous instances of this message"""
     queryReturn = sqlQuery(
-        '''SELECT COUNT(*) FROM inbox WHERE sighash=?''', sigHash)
+        '''SELECT COUNT(*) FROM inbox WHERE sighash=?''', sqlite3.Binary(sigHash))
+    if len(queryReturn) < 1:
+        queryReturn = sqlQuery(
+            '''SELECT COUNT(*) FROM inbox WHERE sighash=CAST(? AS TEXT)''', sigHash)
     return queryReturn[0][0] != 0

src/helper_msgcoding.py

@@ -2,7 +2,6 @@
 Message encoding end decoding functions
 """
 
-import string
 import zlib
 
 import messagetypes
@@ -71,7 +70,8 @@ class MsgEncode(object):
 
     def encodeSimple(self, message):
         """Handle simple encoding"""
-        self.data = 'Subject:%(subject)s\nBody:%(body)s' % message
+        data = 'Subject:%(subject)s\nBody:%(body)s' % message
+        self.data = data.encode("utf-8", "replace")
         self.length = len(self.data)
 
     def encodeTrivial(self, message):
@@ -99,14 +99,14 @@ class MsgDecode(object):
     def decodeExtended(self, data):
         """Handle extended encoding"""
         dc = zlib.decompressobj()
-        tmp = ""
+        tmp = b""
         while len(tmp) <= config.safeGetInt("zlib", "maxsize"):
             try:
                 got = dc.decompress(
                     data, config.safeGetInt("zlib", "maxsize")
                     + 1 - len(tmp))
                 # EOF
-                if got == "":
+                if got == b"":
                     break
                 tmp += got
                 data = dc.unconsumed_tail
@@ -142,7 +142,7 @@ class MsgDecode(object):
 
     def decodeSimple(self, data):
         """Handle simple encoding"""
-        bodyPositionIndex = string.find(data, '\nBody:')
+        bodyPositionIndex = data.find(b'\nBody:')
         if bodyPositionIndex > 1:
             subject = data[8:bodyPositionIndex]
             # Only save and show the first 500 characters of the subject.
@@ -150,10 +150,10 @@ class MsgDecode(object):
             subject = subject[:500]
             body = data[bodyPositionIndex + 6:]
         else:
-            subject = ''
+            subject = b''
             body = data
         # Throw away any extra lines (headers) after the subject.
         if subject:
             subject = subject.splitlines()[0]
-        self.subject = subject
+        self.subject = subject.decode("utf-8", "replace")
-        self.body = body
+        self.body = body.decode("utf-8", "replace")

src/helper_search.py

@@ -5,6 +5,7 @@ Used by :mod:`.bitmessageqt`.
 
 from helper_sql import sqlQuery
 from tr import _translate
+from dbcompat import dbstr
 
 
 def search_sql(
@@ -52,23 +53,23 @@ def search_sql(
     if account is not None:
         if xAddress == 'both':
             sqlStatementParts.append('(fromaddress = ? OR toaddress = ?)')
-            sqlArguments.append(account)
+            sqlArguments.append(dbstr(account))
-            sqlArguments.append(account)
+            sqlArguments.append(dbstr(account))
         else:
             sqlStatementParts.append(xAddress + ' = ? ')
-            sqlArguments.append(account)
+            sqlArguments.append(dbstr(account))
     if folder is not None:
         if folder == 'new':
             folder = 'inbox'
             unreadOnly = True
         sqlStatementParts.append('folder = ? ')
-        sqlArguments.append(folder)
+        sqlArguments.append(dbstr(folder))
     else:
         sqlStatementParts.append('folder != ?')
-        sqlArguments.append('trash')
+        sqlArguments.append(dbstr('trash'))
     if what:
         sqlStatementParts.append('%s LIKE ?' % (where))
-        sqlArguments.append(what)
+        sqlArguments.append(dbstr(what))
     if unreadOnly:
         sqlStatementParts.append('read = 0')
     if sqlStatementParts:

src/helper_sent.py

@@ -4,10 +4,12 @@ Insert values into sent table
 
 import time
 import uuid
+import sqlite3
 from addresses import decodeAddress
 from bmconfigparser import config
 from helper_ackPayload import genAckPayload
 from helper_sql import sqlExecute, sqlQuery
+from dbcompat import dbstr
 
 
 # pylint: disable=too-many-arguments
@@ -38,8 +40,8 @@ def insert(msgid=None, toAddress='[Broadcast subscribers]', fromAddress=None, su
 
         ttl = ttl if ttl else config.getint('bitmessagesettings', 'ttl')
 
-        t = (msgid, toAddress, ripe, fromAddress, subject, message, ackdata,
+        t = (sqlite3.Binary(msgid), dbstr(toAddress), sqlite3.Binary(ripe), dbstr(fromAddress), dbstr(subject), dbstr(message), sqlite3.Binary(ackdata),
-             sentTime, lastActionTime, sleeptill, status, retryNumber, folder,
+             sentTime, lastActionTime, sleeptill, dbstr(status), retryNumber, dbstr(folder),
              encoding, ttl)
 
         sqlExecute('''INSERT INTO sent VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)''', *t)
@@ -50,13 +52,19 @@ def insert(msgid=None, toAddress='[Broadcast subscribers]', fromAddress=None, su
 
 def delete(ack_data):
     """Perform Delete query"""
-    sqlExecute("DELETE FROM sent WHERE ackdata = ?", ack_data)
+    rowcount = sqlExecute("DELETE FROM sent WHERE ackdata = ?", sqlite3.Binary(ack_data))
+    if rowcount < 1:
+        sqlExecute("DELETE FROM sent WHERE ackdata = CAST(? AS TEXT)", ack_data)
 
 
 def retrieve_message_details(ack_data):
     """Retrieving Message details"""
     data = sqlQuery(
-        "select toaddress, fromaddress, subject, message, received from inbox where msgid = ?", ack_data
+        "select toaddress, fromaddress, subject, message, received from inbox where msgid = ?", sqlite3.Binary(ack_data)
+    )
+    if len(data) < 1:
+        data = sqlQuery(
+            "select toaddress, fromaddress, subject, message, received from inbox where msgid = CAST(? AS TEXT)", ack_data
         )
     return data
 
@@ -64,6 +72,10 @@ def retrieve_message_details(ack_data):
 def trash(ackdata):
     """Mark a message in the `sent` as `trash`"""
     rowcount = sqlExecute(
-        '''UPDATE sent SET folder='trash' WHERE ackdata=?''', ackdata
+        '''UPDATE sent SET folder='trash' WHERE ackdata=?''', sqlite3.Binary(ackdata)
+    )
+    if rowcount < 1:
+        rowcount = sqlExecute(
+            '''UPDATE sent SET folder='trash' WHERE ackdata=CAST(? AS TEXT)''', ackdata
         )
     return rowcount

src/helper_sql.py

@@ -61,7 +61,7 @@ def sqlQuery(sql_statement, *args):
     return queryreturn
 
 
-def sqlExecuteChunked(sql_statement, idCount, *args):
+def sqlExecuteChunked(sql_statement, as_text, idCount, *args):
     """Execute chunked SQL statement to avoid argument limit"""
     # SQLITE_MAX_VARIABLE_NUMBER,
     # unfortunately getting/setting isn't exposed to python
@@ -80,6 +80,15 @@ def sqlExecuteChunked(sql_statement, idCount, *args):
             chunk_slice = args[
                 i:i + sqlExecuteChunked.chunkSize - (len(args) - idCount)
             ]
+            if as_text:
+                q = ""
+                n = len(chunk_slice)
+                for i in range(n):
+                    q += "CAST(? AS TEXT)"
+                    if i != n - 1:
+                        q += ","
+                sqlSubmitQueue.put(sql_statement.format(q))
+            else:
                 sqlSubmitQueue.put(
                     sql_statement.format(','.join('?' * len(chunk_slice)))
                 )

src/highlevelcrypto.py

@@ -7,6 +7,8 @@ High level cryptographic functions based on `.pyelliptic` OpenSSL bindings.
   `More discussion. <https://github.com/yann2192/pyelliptic/issues/32>`_
 """
 
+from unqstr import unic
+
 import hashlib
 import os
 from binascii import hexlify
@@ -102,7 +104,7 @@ def random_keys():
 
 def deterministic_keys(passphrase, nonce):
     """Generate keys from *passphrase* and *nonce* (encoded as varint)"""
-    priv = hashlib.sha512(passphrase + nonce).digest()[:32]
+    priv = hashlib.sha512(unic(passphrase).encode("utf-8", "replace") + nonce).digest()[:32]
     pub = pointMult(priv)
     return priv, pub
 

src/inventory.py

@@ -28,8 +28,6 @@ class Inventory:
 
     # cheap inheritance copied from asyncore
     def __getattr__(self, attr):
-        if attr == "__contains__":
-            self.numberOfInventoryLookupsPerformed += 1
         try:
             realRet = getattr(self._realInventory, attr)
         except AttributeError:
@@ -40,6 +38,10 @@ class Inventory:
         else:
             return realRet
 
+    def __contains__(self, key):
+        self.numberOfInventoryLookupsPerformed += 1
+        return key in self._realInventory
+
     # hint for pylint: this is dictionary like object
     def __getitem__(self, key):
         return self._realInventory[key]

src/l10n.py

@@ -3,8 +3,8 @@
 import logging
 import os
 import re
-import sys
 import time
+import six
 
 from six.moves import range
 
@@ -61,7 +61,7 @@ if not re.search(r'\d', time.strftime(time_format)):
 
 # It seems some systems lie about the encoding they use
 # so we perform comprehensive decoding tests
-elif sys.version_info[0] == 2:
+elif six.PY2:
     try:
         # Check day names
         for i in range(7):
@@ -118,7 +118,7 @@ def formatTimestamp(timestamp=None):
         except ValueError:
             timestring = time.strftime(time_format)
 
-    if sys.version_info[0] == 2:
+    if six.PY2:
         return timestring.decode(encoding)
     return timestring
 

src/namecoin.py

@@ -4,7 +4,7 @@ Namecoin queries
 # pylint: disable=too-many-branches,protected-access
 
 import base64
-import httplib
+from six.moves import http_client as httplib
 import json
 import os
 import socket
@@ -96,8 +96,8 @@ class namecoinConnection(object):
                 res = res["reply"]
                 if not res:
                     return (_translate(
-                        "MainWindow", "The name %1 was not found."
+                        "MainWindow", "The name {0} was not found."
-                    ).arg(identity.decode("utf-8", "ignore")), None)
+                    ).format(identity.decode("utf-8", "ignore")), None)
             else:
                 assert False
         except RPCError as exc:
@@ -107,12 +107,12 @@ class namecoinConnection(object):
             else:
                 errmsg = exc.error
             return (_translate(
-                "MainWindow", "The namecoin query failed (%1)"
+                "MainWindow", "The namecoin query failed ({0})"
-            ).arg(errmsg.decode("utf-8", "ignore")), None)
+            ).format(errmsg.decode("utf-8", "ignore")), None)
         except AssertionError:
             return (_translate(
-                "MainWindow", "Unknown namecoin interface type: %1"
+                "MainWindow", "Unknown namecoin interface type: {0}"
-            ).arg(self.nmctype.decode("utf-8", "ignore")), None)
+            ).format(self.nmctype.decode("utf-8", "ignore")), None)
         except Exception:
             logger.exception("Namecoin query exception")
             return (_translate(
@@ -135,8 +135,8 @@ class namecoinConnection(object):
         ) if valid else (
             _translate(
                 "MainWindow",
-                "The name %1 has no associated Bitmessage address."
+                "The name {0} has no associated Bitmessage address."
-            ).arg(identity.decode("utf-8", "ignore")), None)
+            ).format(identity.decode("utf-8", "ignore")), None)
 
     def test(self):
         """
@@ -164,7 +164,7 @@ class namecoinConnection(object):
                     "success",
                     _translate(
                         "MainWindow",
-                        "Success!  Namecoind version %1 running.").arg(
+                        "Success!  Namecoind version {0} running.").format(
                             versStr.decode("utf-8", "ignore")))
 
             elif self.nmctype == "nmcontrol":

src/network/__init__.py

@@ -20,7 +20,7 @@ __all__ = ["StoppableThread"]
 def start(config, state):
     """Start network threads"""
     from .announcethread import AnnounceThread
-    import connectionpool  # pylint: disable=relative-import
+    from network import connectionpool
     from .addrthread import AddrThread
     from .downloadthread import DownloadThread
     from .invthread import InvThread

src/network/addrthread.py

@@ -5,11 +5,11 @@ import random
 from six.moves import queue
 
 # magic imports!
-import connectionpool
+from network import connectionpool
 from protocol import assembleAddrMessage
 from network import addrQueue  # FIXME: init with queue
 
-from threads import StoppableThread
+from .threads import StoppableThread
 
 
 class AddrThread(StoppableThread):

src/network/advanceddispatcher.py

@@ -7,7 +7,7 @@ import time
 
 import network.asyncore_pollchoose as asyncore
 import state
-from threads import BusyError, nonBlocking
+from .threads import BusyError, nonBlocking
 
 
 class ProcessingError(Exception):

src/network/announcethread.py

@@ -4,12 +4,12 @@ Announce myself (node address)
 import time
 
 # magic imports!
-import connectionpool
+from network import connectionpool
 from bmconfigparser import config
 from protocol import assembleAddrMessage
 
-from node import Peer
+from .node import Peer
-from threads import StoppableThread
+from .threads import StoppableThread
 
 
 class AnnounceThread(StoppableThread):

src/network/asyncore_pollchoose.py

@@ -19,6 +19,7 @@ from errno import (
     ENOTCONN, ENOTSOCK, EPIPE, ESHUTDOWN, ETIMEDOUT, EWOULDBLOCK, errorcode
 )
 from threading import current_thread
+from six.moves.reprlib import repr
 
 
 try:
@@ -723,21 +724,6 @@ class dispatcher(object):
             if why.args[0] not in (ENOTCONN, EBADF):
                 raise
 
-    # cheap inheritance, used to pass all other attribute
-    # references to the underlying socket object.
-    def __getattr__(self, attr):
-        try:
-            retattr = getattr(self.socket, attr)
-        except AttributeError:
-            raise AttributeError(
-                "%s instance has no attribute '%s'"
-                % (self.__class__.__name__, attr))
-        else:
-            msg = "%(me)s.%(attr)s is deprecated; use %(me)s.socket.%(attr)s"\
-                  " instead" % {'me': self.__class__.__name__, 'attr': attr}
-            warnings.warn(msg, DeprecationWarning, stacklevel=2)
-            return retattr
-
     # log and log_info may be overridden to provide more sophisticated
     # logging and warning methods. In general, log is for 'hit' logging
     # and 'log_info' is for informational, warning and error logging.

src/network/bmobject.py

@@ -6,7 +6,7 @@ import time
 
 import protocol
 import state
-import connectionpool
+import network.connectionpool  # use long name to address recursive import
 from network import dandelion_ins
 from highlevelcrypto import calculateInventoryHash
 
@@ -100,7 +100,7 @@ class BMObject(object):  # pylint: disable=too-many-instance-attributes
             logger.warning(
                 'The object has invalid stream: %s', self.streamNumber)
             raise BMObjectInvalidError()
-        if self.streamNumber not in connectionpool.pool.streams:
+        if self.streamNumber not in network.connectionpool.pool.streams:
             logger.debug(
                 'The streamNumber %i isn\'t one we are interested in.',
                 self.streamNumber)

src/network/bmproto.py

@@ -9,13 +9,14 @@ import re
 import socket
 import struct
 import time
+import six
 
 # magic imports!
 import addresses
-import knownnodes
+from network import knownnodes
 import protocol
 import state
-import connectionpool
+import network.connectionpool  # use long name to address recursive import
 from bmconfigparser import config
 from queues import objectProcessorQueue
 from randomtrackingdict import RandomTrackingDict
@@ -26,14 +27,27 @@ from network.bmobject import (
     BMObjectUnwantedStreamError
 )
 from network.proxy import ProxyError
+
 from network import dandelion_ins, invQueue, portCheckerQueue
-from node import Node, Peer
+from .node import Node, Peer
-from objectracker import ObjectTracker, missingObjects
+from .objectracker import ObjectTracker, missingObjects
 
 
 logger = logging.getLogger('default')
 
 
+def _hoststr(v):
+    if six.PY3:
+        return v
+    else:  # assume six.PY2
+        return str(v)
+
+def _restr(v):
+    if six.PY3:
+        return v.decode("utf-8", "replace")
+    else:  # assume six.PY2
+        return v
+
 class BMProtoError(ProxyError):
     """A Bitmessage Protocol Base Error"""
     errorCodes = ("Protocol error")
@@ -82,7 +96,7 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
         """Process incoming header"""
         self.magic, self.command, self.payloadLength, self.checksum = \
             protocol.Header.unpack(self.read_buf[:protocol.Header.size])
-        self.command = self.command.rstrip('\x00')
+        self.command = self.command.rstrip(b'\x00')
         if self.magic != protocol.magic:
             # skip 1 byte in order to sync
             self.set_state("bm_header", length=1)
@@ -107,7 +121,7 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
             self.invalid = True
         retval = True
         if not self.fullyEstablished and self.command not in (
-                "error", "version", "verack"):
+                b"error", b"version", b"verack"):
             logger.error(
                 'Received command %s before connection was fully'
                 ' established, ignoring', self.command)
@@ -115,7 +129,7 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
         if not self.invalid:
             try:
                 retval = getattr(
-                    self, "bm_command_" + str(self.command).lower())()
+                    self, "bm_command_" + self.command.decode("utf-8", "replace").lower())()
             except AttributeError:
                 # unimplemented command
                 logger.debug('unimplemented command %s', self.command)
@@ -168,17 +182,17 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
         """Decode node details from the payload"""
         # protocol.checkIPAddress()
         services, host, port = self.decode_payload_content("Q16sH")
-        if host[0:12] == '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xFF\xFF':
+        if host[0:12] == b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xFF\xFF':
-            host = socket.inet_ntop(socket.AF_INET, str(host[12:16]))
+            host = socket.inet_ntop(socket.AF_INET, _hoststr(host[12:16]))
-        elif host[0:6] == '\xfd\x87\xd8\x7e\xeb\x43':
+        elif host[0:6] == b'\xfd\x87\xd8\x7e\xeb\x43':
             # Onion, based on BMD/bitcoind
-            host = base64.b32encode(host[6:]).lower() + ".onion"
+            host = base64.b32encode(host[6:]).lower() + b".onion"
         else:
-            host = socket.inet_ntop(socket.AF_INET6, str(host))
+            host = socket.inet_ntop(socket.AF_INET6, _hoststr(host))
-        if host == "":
+        if host == b"":
             # This can happen on Windows systems which are not 64-bit
             # compatible so let us drop the IPv6 address.
-            host = socket.inet_ntop(socket.AF_INET, str(host[12:16]))
+            host = socket.inet_ntop(socket.AF_INET, _hoststr(host[12:16]))
 
         return Node(services, host, port)
 
@@ -334,7 +348,7 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
         if now < self.skipUntil:
             return True
         for i in items:
-            self.pendingUpload[str(i)] = now
+            self.pendingUpload[i] = now
         return True
 
     def _command_inv(self, extend_dandelion_stem=False):
@@ -353,7 +367,7 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
         if extend_dandelion_stem and not dandelion_ins.enabled:
             return True
 
-        for i in map(str, items):
+        for i in items:
             if i in state.Inventory and not dandelion_ins.hasHash(i):
                 continue
             if extend_dandelion_stem and not dandelion_ins.hasHash(i):
@@ -409,13 +423,17 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
 
         try:
             self.object.checkObjectByType()
+            if six.PY2:
+                data_buffer = buffer(self.object.data)
+            else:  # assume six.PY3
+                data_buffer = memoryview(self.object.data)
             objectProcessorQueue.put((
-                self.object.objectType, buffer(self.object.data)))  # noqa: F821
+                self.object.objectType, data_buffer))  # noqa: F821
         except BMObjectInvalidError:
             BMProto.stopDownloadingObject(self.object.inventoryHash, True)
         else:
             try:
-                del missingObjects[self.object.inventoryHash]
+                del missingObjects[bytes(self.object.inventoryHash)]
             except KeyError:
                 pass
 
@@ -424,10 +442,16 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
             dandelion_ins.removeHash(
                 self.object.inventoryHash, "cycle detection")
 
+        if six.PY2:
+            object_buffer = buffer(self.payload[objectOffset:])
+            tag_buffer = buffer(self.object.tag)
+        else:  # assume six.PY3
+            object_buffer = memoryview(self.payload[objectOffset:])
+            tag_buffer = memoryview(self.object.tag)
         state.Inventory[self.object.inventoryHash] = (
             self.object.objectType, self.object.streamNumber,
-            buffer(self.payload[objectOffset:]), self.object.expiresTime,  # noqa: F821
+            object_buffer, self.object.expiresTime,  # noqa: F821
-            buffer(self.object.tag)  # noqa: F821
+            tag_buffer  # noqa: F821
         )
         self.handleReceivedObject(
             self.object.streamNumber, self.object.inventoryHash)
@@ -443,11 +467,10 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
         """Incoming addresses, process them"""
         # not using services
         for seenTime, stream, _, ip, port in self._decode_addr():
-            ip = str(ip)
             if (
-                stream not in connectionpool.pool.streams
+                stream not in network.connectionpool.pool.streams
                 # FIXME: should check against complete list
-                or ip.startswith('bootstrap')
+                or ip.decode("utf-8", "replace").startswith('bootstrap')
             ):
                 continue
             decodedIP = protocol.checkIPAddress(ip)
@@ -477,7 +500,7 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
 
     def bm_command_ping(self):
         """Incoming ping, respond to it."""
-        self.append_write_buf(protocol.CreatePacket('pong'))
+        self.append_write_buf(protocol.CreatePacket(b'pong'))
         return True
 
     @staticmethod
@@ -526,21 +549,21 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
         logger.debug(
             'remote node incoming address: %s:%i',
             self.destination.host, self.peerNode.port)
-        logger.debug('user agent: %s', self.userAgent)
+        logger.debug('user agent: %s', self.userAgent.decode("utf-8", "replace"))
         logger.debug('streams: [%s]', ','.join(map(str, self.streams)))
         if not self.peerValidityChecks():
             # ABORT afterwards
             return True
-        self.append_write_buf(protocol.CreatePacket('verack'))
+        self.append_write_buf(protocol.CreatePacket(b'verack'))
         self.verackSent = True
         ua_valid = re.match(
-            r'^/[a-zA-Z]+:[0-9]+\.?[\w\s\(\)\./:;-]*/$', self.userAgent)
+            r'^/[a-zA-Z]+:[0-9]+\.?[\w\s\(\)\./:;-]*/$', _restr(self.userAgent))
         if not ua_valid:
-            self.userAgent = '/INVALID:0/'
+            self.userAgent = b'/INVALID:0/'
         if not self.isOutbound:
             self.append_write_buf(protocol.assembleVersionMessage(
                 self.destination.host, self.destination.port,
-                connectionpool.pool.streams, dandelion_ins.enabled, True,
+                network.connectionpool.pool.streams, dandelion_ins.enabled, True,
                 nodeid=self.nodeid))
             logger.debug(
                 '%(host)s:%(port)i sending version',
@@ -596,7 +619,7 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
                 'Closed connection to %s because there is no overlapping'
                 ' interest in streams.', self.destination)
             return False
-        if connectionpool.pool.inboundConnections.get(
+        if network.connectionpool.pool.inboundConnections.get(
                 self.destination):
             try:
                 if not protocol.checkSocksIP(self.destination.host):
@@ -614,8 +637,8 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
             # or server full report the same error to counter deanonymisation
             if (
                 Peer(self.destination.host, self.peerNode.port)
-                in connectionpool.pool.inboundConnections
+                in network.connectionpool.pool.inboundConnections
-                or len(connectionpool.pool)
+                or len(network.connectionpool.pool)
                 > config.safeGetInt(
                     'bitmessagesettings', 'maxtotalconnections')
                 + config.safeGetInt(
@@ -627,7 +650,7 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
                     'Closed connection to %s due to server full'
                     ' or duplicate inbound/outbound.', self.destination)
                 return False
-        if connectionpool.pool.isAlreadyConnected(self.nonce):
+        if network.connectionpool.pool.isAlreadyConnected(self.nonce):
             self.append_write_buf(protocol.assembleErrorMessage(
                 errorText="I'm connected to myself. Closing connection.",
                 fatal=2))
@@ -641,7 +664,7 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
     @staticmethod
     def stopDownloadingObject(hashId, forwardAnyway=False):
         """Stop downloading object *hashId*"""
-        for connection in connectionpool.pool.connections():
+        for connection in network.connectionpool.pool.connections():
             try:
                 del connection.objectsNewToMe[hashId]
             except KeyError:
@@ -653,7 +676,7 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
                 except KeyError:
                     pass
         try:
-            del missingObjects[hashId]
+            del missingObjects[bytes(hashId)]
         except KeyError:
             pass
 

src/network/connectionchooser.py

@@ -7,7 +7,7 @@ import random
 
 from six.moves import queue
 
-import knownnodes
+from network import knownnodes
 import protocol
 import state
 
@@ -17,10 +17,16 @@ from network import portCheckerQueue
 logger = logging.getLogger('default')
 
 
+def _ends_with(s, tail):
+    try:
+        return s.endswith(tail)
+    except:
+        return s.decode("utf-8", "replace").endswith(tail)
+
 def getDiscoveredPeer():
     """Get a peer from the local peer discovery list"""
     try:
-        peer = random.choice(state.discoveredPeers.keys())  # nosec B311
+        peer = random.choice(list(state.discoveredPeers.keys()))  # nosec B311
     except (IndexError, KeyError):
         raise ValueError
     try:
@@ -48,7 +54,7 @@ def chooseConnection(stream):
         return getDiscoveredPeer()
     for _ in range(50):
         peer = random.choice(  # nosec B311
-            knownnodes.knownNodes[stream].keys())
+            list(knownnodes.knownNodes[stream].keys()))
         try:
             peer_info = knownnodes.knownNodes[stream][peer]
             if peer_info.get('self'):
@@ -60,10 +66,10 @@ def chooseConnection(stream):
         if haveOnion:
             # do not connect to raw IP addresses
             # --keep all traffic within Tor overlay
-            if onionOnly and not peer.host.endswith('.onion'):
+            if onionOnly and not _ends_with(peer.host, '.onion'):
                 continue
             # onion addresses have a higher priority when SOCKS
-            if peer.host.endswith('.onion') and rating > 0:
+            if _ends_with(peer.host, '.onion') and rating > 0:
                 rating = 1
             # TODO: need better check
             elif not peer.host.startswith('bootstrap'):

src/network/connectionpool.py

@@ -9,22 +9,28 @@ import sys
 import time
 import random
 
-import asyncore_pollchoose as asyncore
+from network import asyncore_pollchoose as asyncore
-import knownnodes
+from network import knownnodes
 import protocol
 import state
 from bmconfigparser import config
-from connectionchooser import chooseConnection
+from .connectionchooser import chooseConnection
-from node import Peer
+from .node import Peer
-from proxy import Proxy
+from .proxy import Proxy
-from tcp import (
+from .tcp import (
     bootstrap, Socks4aBMConnection, Socks5BMConnection,
     TCPConnection, TCPServer)
-from udp import UDPSocket
+from .udp import UDPSocket
 
 logger = logging.getLogger('default')
 
 
+def _ends_with(s, tail):
+    try:
+        return s.endswith(tail)
+    except:
+        return s.decode("utf-8", "replace").endswith(tail)
+
 class BMConnectionPool(object):
     """Pool of all existing connections"""
     # pylint: disable=too-many-instance-attributes
@@ -78,7 +84,7 @@ class BMConnectionPool(object):
         Shortcut for combined list of connections from
         `inboundConnections` and `outboundConnections` dicts
         """
-        return self.inboundConnections.values() + self.outboundConnections.values()
+        return list(self.inboundConnections.values()) + list(self.outboundConnections.values())
 
     def establishedConnections(self):
         """Shortcut for list of connections having fullyEstablished == True"""
@@ -160,8 +166,8 @@ class BMConnectionPool(object):
     @staticmethod
     def getListeningIP():
         """What IP are we supposed to be listening on?"""
-        if config.safeGet(
+        if _ends_with(config.safeGet(
-                "bitmessagesettings", "onionhostname", "").endswith(".onion"):
+                "bitmessagesettings", "onionhostname", ""), ".onion"):
             host = config.safeGet(
                 "bitmessagesettings", "onionbindip")
         else:
@@ -314,7 +320,7 @@ class BMConnectionPool(object):
                         continue
 
                     try:
-                        if chosen.host.endswith(".onion") and Proxy.onion_proxy:
+                        if _ends_with(chosen.host, ".onion") and Proxy.onion_proxy:
                             if onionsocksproxytype == "SOCKS5":
                                 self.addConnection(Socks5BMConnection(chosen))
                             elif onionsocksproxytype == "SOCKS4a":
@@ -381,14 +387,14 @@ class BMConnectionPool(object):
                 minTx -= 300 - 20
             if i.lastTx < minTx:
                 if i.fullyEstablished:
-                    i.append_write_buf(protocol.CreatePacket('ping'))
+                    i.append_write_buf(protocol.CreatePacket(b'ping'))
                 else:
                     i.close_reason = "Timeout (%is)" % (
                         time.time() - i.lastTx)
                     i.set_state("close")
         for i in (
             self.connections()
-            + self.listeningSockets.values() + self.udpSockets.values()
+            + list(self.listeningSockets.values()) + list(self.udpSockets.values())
         ):
             if not (i.accepting or i.connecting or i.connected):
                 reaper.append(i)

src/network/dandelion.py

@@ -6,6 +6,8 @@ from collections import namedtuple
 from random import choice, expovariate, sample
 from threading import RLock
 from time import time
+import six
+from binascii import hexlify
 
 
 # randomise routes after 600 seconds
@@ -64,7 +66,7 @@ class Dandelion:  # pylint: disable=old-style-class
         """Add inventory vector to dandelion stem return status of dandelion enabled"""
         assert self.enabled is not None
         with self.lock:
-            self.hashMap[hashId] = Stem(
+            self.hashMap[bytes(hashId)] = Stem(
                 self.getNodeStem(source),
                 stream,
                 self.poissonTimeout())
@@ -75,9 +77,10 @@ class Dandelion:  # pylint: disable=old-style-class
         include streams, we only learn this after receiving the object)
         """
         with self.lock:
-            if hashId in self.hashMap:
+            hashId_bytes = bytes(hashId)
-                self.hashMap[hashId] = Stem(
+            if hashId_bytes in self.hashMap:
-                    self.hashMap[hashId].child,
+                self.hashMap[hashId_bytes] = Stem(
+                    self.hashMap[hashId_bytes].child,
                     stream,
                     self.poissonTimeout())
 
@@ -86,20 +89,20 @@ class Dandelion:  # pylint: disable=old-style-class
         if logger.isEnabledFor(logging.DEBUG):
             logger.debug(
                 '%s entering fluff mode due to %s.',
-                ''.join('%02x' % ord(i) for i in hashId), reason)
+                hexlify(hashId), reason)
         with self.lock:
             try:
-                del self.hashMap[hashId]
+                del self.hashMap[bytes(hashId)]
             except KeyError:
                 pass
 
     def hasHash(self, hashId):
         """Is inventory vector in stem mode?"""
-        return hashId in self.hashMap
+        return bytes(hashId) in self.hashMap
 
     def objectChildStem(self, hashId):
         """Child (i.e. next) node for an inventory vector during stem mode"""
-        return self.hashMap[hashId].child
+        return self.hashMap[bytes(hashId)].child
 
     def maybeAddStem(self, connection, invQueue):
         """
@@ -111,12 +114,12 @@ class Dandelion:  # pylint: disable=old-style-class
         with self.lock:
             if len(self.stem) < MAX_STEMS:
                 self.stem.append(connection)
-                for k in (k for k, v in self.nodeMap.iteritems() if v is None):
+                for k in (k for k, v in six.iteritems(self.nodeMap) if v is None):
                     self.nodeMap[k] = connection
-                for k, v in {
+                for k, v in six.iteritems({
-                        k: v for k, v in self.hashMap.iteritems()
+                        k: v for k, v in six.iteritems(self.hashMap)
                         if v.child is None
-                }.iteritems():
+                }):
                     self.hashMap[k] = Stem(
                         connection, v.stream, self.poissonTimeout())
                     invQueue.put((v.stream, k, v.child))
@@ -132,14 +135,14 @@ class Dandelion:  # pylint: disable=old-style-class
                 self.stem.remove(connection)
                 # active mappings to pointing to the removed node
                 for k in (
-                        k for k, v in self.nodeMap.iteritems()
+                        k for k, v in six.iteritems(self.nodeMap)
                         if v == connection
                 ):
                     self.nodeMap[k] = None
-                for k, v in {
+                for k, v in six.iteritems({
-                        k: v for k, v in self.hashMap.iteritems()
+                        k: v for k, v in six.iteritems(self.hashMap)
                         if v.child == connection
-                }.iteritems():
+                }):
                     self.hashMap[k] = Stem(
                         None, v.stream, self.poissonTimeout())
 
@@ -180,7 +183,7 @@ class Dandelion:  # pylint: disable=old-style-class
         with self.lock:
             deadline = time()
             toDelete = [
-                [v.stream, k, v.child] for k, v in self.hashMap.iteritems()
+                [v.stream, k, v.child] for k, v in six.iteritems(self.hashMap)
                 if v.timeout < deadline
             ]
 
@@ -199,10 +202,10 @@ class Dandelion:  # pylint: disable=old-style-class
             try:
                 # random two connections
                 self.stem = sample(
-                    self.pool.outboundConnections.values(), MAX_STEMS)
+                    sorted(self.pool.outboundConnections.values()), MAX_STEMS)
             # not enough stems available
             except ValueError:
-                self.stem = self.pool.outboundConnections.values()
+                self.stem = list(self.pool.outboundConnections.values())
             self.nodeMap = {}
             # hashMap stays to cater for pending stems
         self.refresh = time() + REASSIGN_INTERVAL

src/network/downloadthread.py

@@ -4,12 +4,13 @@
 import time
 import random
 import state
+import six
 import addresses
 import protocol
-import connectionpool
+from network import connectionpool
 from network import dandelion_ins
-from objectracker import missingObjects
+from .objectracker import missingObjects
-from threads import StoppableThread
+from .threads import StoppableThread
 
 
 class DownloadThread(StoppableThread):
@@ -29,7 +30,7 @@ class DownloadThread(StoppableThread):
         deadline = time.time() - self.requestExpires
         try:
             toDelete = [
-                k for k, v in missingObjects.iteritems()
+                k for k, v in six.iteritems(missingObjects)
                 if v < deadline]
         except RuntimeError:
             pass
@@ -68,11 +69,11 @@ class DownloadThread(StoppableThread):
                         continue
                     payload.extend(chunk)
                     chunkCount += 1
-                    missingObjects[chunk] = now
+                    missingObjects[bytes(chunk)] = now
                 if not chunkCount:
                     continue
                 payload[0:0] = addresses.encodeVarint(chunkCount)
-                i.append_write_buf(protocol.CreatePacket('getdata', payload))
+                i.append_write_buf(protocol.CreatePacket(b'getdata', payload))
                 self.logger.debug(
                     '%s:%i Requesting %i objects',
                     i.destination.host, i.destination.port, chunkCount)

src/network/http.py

@@ -1,10 +1,10 @@
 import socket
 
-from advanceddispatcher import AdvancedDispatcher
+from .advanceddispatcher import AdvancedDispatcher
-import asyncore_pollchoose as asyncore
+from network import asyncore_pollchoose as asyncore
-from proxy import ProxyError
+from .proxy import ProxyError
-from socks5 import Socks5Connection, Socks5Resolver
+from .socks5 import Socks5Connection, Socks5Resolver
-from socks4a import Socks4aConnection, Socks4aResolver
+from .socks4a import Socks4aConnection, Socks4aResolver
 
 
 class HttpError(ProxyError):

src/network/invthread.py

@@ -1,16 +1,16 @@
 """
 Thread to send inv annoucements
 """
-import Queue
+from six.moves import queue as Queue
 import random
 from time import time
 
 import addresses
 import protocol
 import state
-import connectionpool
+from network import connectionpool
 from network import dandelion_ins, invQueue
-from threads import StoppableThread
+from .threads import StoppableThread
 
 
 def handleExpiredDandelion(expired):
@@ -90,15 +90,15 @@ class InvThread(StoppableThread):
                     if fluffs:
                         random.shuffle(fluffs)
                         connection.append_write_buf(protocol.CreatePacket(
-                            'inv',
+                            b'inv',
                             addresses.encodeVarint(
-                                len(fluffs)) + ''.join(fluffs)))
+                                len(fluffs)) + b''.join(fluffs)))
                     if stems:
                         random.shuffle(stems)
                         connection.append_write_buf(protocol.CreatePacket(
-                            'dinv',
+                            b'dinv',
                             addresses.encodeVarint(
-                                len(stems)) + ''.join(stems)))
+                                len(stems)) + b''.join(stems)))
 
             invQueue.iterate()
             for _ in range(len(chunk)):

src/network/knownnodes.py

@@ -10,10 +10,8 @@ import os
 import pickle  # nosec B403
 import threading
 import time
-try:
+from six.moves.collections_abc import Iterable
-    from collections.abc import Iterable
+import six
-except ImportError:
-    from collections import Iterable
 
 import state
 from bmconfigparser import config
@@ -54,8 +52,8 @@ def json_serialize_knownnodes(output):
     Reorganize knownnodes dict and write it as JSON to output
     """
     _serialized = []
-    for stream, peers in knownNodes.iteritems():
+    for stream, peers in six.iteritems(knownNodes):
-        for peer, info in peers.iteritems():
+        for peer, info in six.iteritems(peers):
             info.update(rating=round(info.get('rating', 0), 2))
             _serialized.append({
                 'stream': stream, 'peer': peer._asdict(), 'info': info
@@ -87,7 +85,7 @@ def pickle_deserialize_old_knownnodes(source):
     global knownNodes
     knownNodes = pickle.load(source)  # nosec B301
     for stream in knownNodes.keys():
-        for node, params in knownNodes[stream].iteritems():
+        for node, params in six.iteritems(knownNodes[stream]):
             if isinstance(params, (float, int)):
                 addKnownNode(stream, node, params)
 
@@ -97,7 +95,7 @@ def saveKnownNodes(dirName=None):
     if dirName is None:
         dirName = state.appdata
     with knownNodesLock:
-        with open(os.path.join(dirName, 'knownnodes.dat'), 'wb') as output:
+        with open(os.path.join(dirName, 'knownnodes.dat'), 'w') as output:
             json_serialize_knownnodes(output)
 
 
@@ -108,6 +106,12 @@ def addKnownNode(stream, peer, lastseen=None, is_self=False):
     Returns True if added a new node.
     """
     # pylint: disable=too-many-branches
+    if not isinstance(peer.host, str):
+        try:
+            peer = Peer(peer.host.decode("ascii"), peer.port)
+        except UnicodeDecodeError as err:
+            logger.warning("Invalid host: {}".format(peer.host.decode("ascii", "backslashreplace")))
+            return
     if isinstance(stream, Iterable):
         with knownNodesLock:
             for s in stream:
@@ -153,7 +157,7 @@ def createDefaultKnownNodes():
 def readKnownNodes():
     """Load knownnodes from filesystem"""
     try:
-        with open(state.appdata + 'knownnodes.dat', 'rb') as source:
+        with open(state.appdata + 'knownnodes.dat', 'r') as source:
             with knownNodesLock:
                 try:
                     json_deserialize_knownnodes(source)

src/network/networkthread.py

@@ -2,9 +2,9 @@
 A thread to handle network concerns
 """
 import network.asyncore_pollchoose as asyncore
-import connectionpool
+from network import connectionpool
 from queues import excQueue
-from threads import StoppableThread
+from .threads import StoppableThread
 
 
 class BMNetworkThread(StoppableThread):

src/network/objectracker.py

@@ -3,8 +3,9 @@ Module for tracking objects
 """
 import time
 from threading import RLock
+import six
 
-import connectionpool
+import network.connectionpool  # use long name to address recursive import
 from network import dandelion_ins
 from randomtrackingdict import RandomTrackingDict
 
@@ -75,32 +76,35 @@ class ObjectTracker(object):
                 with self.objectsNewToThemLock:
                     self.objectsNewToThem = {
                         k: v
-                        for k, v in self.objectsNewToThem.iteritems()
+                        for k, v in six.iteritems(self.objectsNewToThem)
                         if v >= deadline}
             self.lastCleaned = time.time()
 
     def hasObj(self, hashid):
         """Do we already have object?"""
+        hashid_bytes = bytes(hashid)
         if haveBloom:
-            return hashid in self.invBloom
+            return hashid_bytes in self.invBloom
-        return hashid in self.objectsNewToMe
+        return hashid_bytes in self.objectsNewToMe
 
     def handleReceivedInventory(self, hashId):
         """Handling received inventory"""
+        hashId_bytes = bytes(hashId)
         if haveBloom:
-            self.invBloom.add(hashId)
+            self.invBloom.add(hashId_bytes)
         try:
             with self.objectsNewToThemLock:
-                del self.objectsNewToThem[hashId]
+                del self.objectsNewToThem[hashId_bytes]
         except KeyError:
             pass
-        if hashId not in missingObjects:
+        if hashId_bytes not in missingObjects:
-            missingObjects[hashId] = time.time()
+            missingObjects[hashId_bytes] = time.time()
         self.objectsNewToMe[hashId] = True
 
     def handleReceivedObject(self, streamNumber, hashid):
         """Handling received object"""
-        for i in connectionpool.pool.connections():
+        hashid_bytes = bytes(hashid)
+        for i in network.connectionpool.pool.connections():
             if not i.fullyEstablished:
                 continue
             try:
@@ -110,7 +114,7 @@ class ObjectTracker(object):
                         not dandelion_ins.hasHash(hashid)
                         or dandelion_ins.objectChildStem(hashid) == i):
                     with i.objectsNewToThemLock:
-                        i.objectsNewToThem[hashid] = time.time()
+                        i.objectsNewToThem[hashid_bytes] = time.time()
                     # update stream number,
                     # which we didn't have when we just received the dinv
                     # also resets expiration of the stem mode
@@ -119,7 +123,7 @@ class ObjectTracker(object):
             if i == self:
                 try:
                     with i.objectsNewToThemLock:
-                        del i.objectsNewToThem[hashid]
+                        del i.objectsNewToThem[hashid_bytes]
                 except KeyError:
                     pass
         self.objectsNewToMe.setLastObject()
@@ -133,4 +137,4 @@ class ObjectTracker(object):
     def addAddr(self, hashid):
         """WIP, should be moved to addrthread.py or removed"""
         if haveBloom:
-            self.addrBloom.add(hashid)
+            self.addrBloom.add(bytes(hashid))

src/network/proxy.py

@@ -6,14 +6,20 @@ import logging
 import socket
 import time
 
-import asyncore_pollchoose as asyncore
+from network import asyncore_pollchoose as asyncore
-from advanceddispatcher import AdvancedDispatcher
+from .advanceddispatcher import AdvancedDispatcher
 from bmconfigparser import config
-from node import Peer
+from .node import Peer
 
 logger = logging.getLogger('default')
 
 
+def _ends_with(s, tail):
+    try:
+        return s.endswith(tail)
+    except:
+        return s.decode("utf-8", "replace").endswith(tail)
+
 class ProxyError(Exception):
     """Base proxy exception class"""
     errorCodes = ("Unknown error",)
@@ -125,7 +131,7 @@ class Proxy(AdvancedDispatcher):
             self.auth = None
         self.connect(
             self.onion_proxy
-            if address.host.endswith(".onion") and self.onion_proxy else
+            if _ends_with(address.host, ".onion") and self.onion_proxy else
             self.proxy
         )
 

src/network/receivequeuethread.py

@@ -2,13 +2,13 @@
 Process data incoming from network
 """
 import errno
-import Queue
+from six.moves import queue as Queue
 import socket
 
-import connectionpool
+from network import connectionpool
 from network.advanceddispatcher import UnknownStateError
 from network import receiveDataQueue
-from threads import StoppableThread
+from .threads import StoppableThread
 
 
 class ReceiveQueueThread(StoppableThread):

src/network/socks4a.py

@@ -5,8 +5,9 @@ SOCKS4a proxy module
 import logging
 import socket
 import struct
+import six
 
-from proxy import GeneralProxyError, Proxy, ProxyError
+from .proxy import GeneralProxyError, Proxy, ProxyError
 
 logger = logging.getLogger('default')
 
@@ -39,16 +40,16 @@ class Socks4a(Proxy):
     def state_pre_connect(self):
         """Handle feedback from SOCKS4a while it is connecting on our behalf"""
         # Get the response
-        if self.read_buf[0:1] != chr(0x00).encode():
+        if self.read_buf[0:1] != six.int2byte(0x00):
             # bad data
             self.close()
             raise GeneralProxyError(1)
-        elif self.read_buf[1:2] != chr(0x5A).encode():
+        elif self.read_buf[1:2] != six.int2byte(0x5A):
             # Connection failed
             self.close()
-            if ord(self.read_buf[1:2]) in (91, 92, 93):
+            if six.byte2int(self.read_buf[1:2]) in (91, 92, 93):
                 # socks 4 error
-                raise Socks4aError(ord(self.read_buf[1:2]) - 90)
+                raise Socks4aError(six.byte2int(self.read_buf[1:2]) - 90)
             else:
                 raise Socks4aError(4)
         # Get the bound address/port
@@ -102,9 +103,9 @@ class Socks4aConnection(Socks4a):
                 self.append_write_buf(self.ipaddr)
         if self._auth:
             self.append_write_buf(self._auth[0])
-        self.append_write_buf(chr(0x00).encode())
+        self.append_write_buf(six.int2byte(0x00))
         if rmtrslv:
-            self.append_write_buf(self.destination[0] + chr(0x00).encode())
+            self.append_write_buf(self.destination[0].encode("utf-8", "replace") + six.int2byte(0x00))
         self.set_state("pre_connect", length=0, expectBytes=8)
         return True
 
@@ -132,8 +133,8 @@ class Socks4aResolver(Socks4a):
         self.append_write_buf(struct.pack("BBBB", 0x00, 0x00, 0x00, 0x01))
         if self._auth:
             self.append_write_buf(self._auth[0])
-        self.append_write_buf(chr(0x00).encode())
+        self.append_write_buf(six.int2byte(0x00))
-        self.append_write_buf(self.host + chr(0x00).encode())
+        self.append_write_buf(self.host + six.int2byte(0x00))
         self.set_state("pre_connect", length=0, expectBytes=8)
         return True
 

src/network/socks5.py

@@ -6,9 +6,10 @@ SOCKS5 proxy module
 import logging
 import socket
 import struct
+import six
 
-from node import Peer
+from .node import Peer
-from proxy import GeneralProxyError, Proxy, ProxyError
+from .proxy import GeneralProxyError, Proxy, ProxyError
 
 logger = logging.getLogger('default')
 
@@ -97,20 +98,20 @@ class Socks5(Proxy):
     def state_pre_connect(self):
         """Handle feedback from socks5 while it is connecting on our behalf."""
         # Get the response
-        if self.read_buf[0:1] != chr(0x05).encode():
+        if self.read_buf[0:1] != six.int2byte(0x05):
             self.close()
             raise GeneralProxyError(1)
-        elif self.read_buf[1:2] != chr(0x00).encode():
+        elif self.read_buf[1:2] != six.int2byte(0x00):
             # Connection failed
             self.close()
-            if ord(self.read_buf[1:2]) <= 8:
+            if six.byte2int(self.read_buf[1:2]) <= 8:
-                raise Socks5Error(ord(self.read_buf[1:2]))
+                raise Socks5Error(six.byte2int(self.read_buf[1:2]))
             else:
                 raise Socks5Error(9)
         # Get the bound address/port
-        elif self.read_buf[3:4] == chr(0x01).encode():
+        elif self.read_buf[3:4] == six.int2byte(0x01):
             self.set_state("proxy_addr_1", length=4, expectBytes=4)
-        elif self.read_buf[3:4] == chr(0x03).encode():
+        elif self.read_buf[3:4] == six.int2byte(0x03):
             self.set_state("proxy_addr_2_1", length=4, expectBytes=1)
         else:
             self.close()
@@ -129,7 +130,7 @@ class Socks5(Proxy):
         (e.g. IPv6, onion, ...). This is part 1 which retrieves the
         length of the data.
         """
-        self.address_length = ord(self.read_buf[0:1])
+        self.address_length = six.byte2int(self.read_buf[0:1])
         self.set_state(
             "proxy_addr_2_2", length=1, expectBytes=self.address_length)
         return True
@@ -171,19 +172,19 @@ class Socks5Connection(Socks5):
         # use the IPv4 address request even if remote resolving was specified.
         try:
             self.ipaddr = socket.inet_aton(self.destination[0])
-            self.append_write_buf(chr(0x01).encode() + self.ipaddr)
+            self.append_write_buf(six.int2byte(0x01) + self.ipaddr)
         except socket.error:  # may be IPv6!
             # Well it's not an IP number,  so it's probably a DNS name.
             if self._remote_dns:
                 # Resolve remotely
                 self.ipaddr = None
-                self.append_write_buf(chr(0x03).encode() + chr(
+                self.append_write_buf(six.int2byte(0x03) + six.int2byte(
-                    len(self.destination[0])).encode() + self.destination[0])
+                    len(self.destination[0])) + self.destination[0].encode("utf-8", "replace"))
             else:
                 # Resolve locally
                 self.ipaddr = socket.inet_aton(
                     socket.gethostbyname(self.destination[0]))
-                self.append_write_buf(chr(0x01).encode() + self.ipaddr)
+                self.append_write_buf(six.int2byte(0x01) + self.ipaddr)
         self.append_write_buf(struct.pack(">H", self.destination[1]))
         self.set_state("pre_connect", length=0, expectBytes=4)
         return True
@@ -208,8 +209,8 @@ class Socks5Resolver(Socks5):
         """Perform resolving"""
         # Now we can request the actual connection
         self.append_write_buf(struct.pack('BBB', 0x05, 0xF0, 0x00))
-        self.append_write_buf(chr(0x03).encode() + chr(
+        self.append_write_buf(six.int2byte(0x03) + six.int2byte(
-            len(self.host)).encode() + str(self.host))
+            len(self.host)) + bytes(self.host))
         self.append_write_buf(struct.pack(">H", self.port))
         self.set_state("pre_connect", length=0, expectBytes=4)
         return True

src/network/stats.py

@@ -3,9 +3,9 @@ Network statistics
 """
 import time
 
-import asyncore_pollchoose as asyncore
+from network import asyncore_pollchoose as asyncore
-import connectionpool
+from network import connectionpool
-from objectracker import missingObjects
+from .objectracker import missingObjects
 
 
 lastReceivedTimestamp = time.time()

src/network/tcp.py

@@ -8,28 +8,29 @@ import math
 import random
 import socket
 import time
+import six
 
 # magic imports!
 import addresses
 import l10n
 import protocol
 import state
-import connectionpool
+import network.connectionpool  # use long name to address recursive import
 from bmconfigparser import config
 from highlevelcrypto import randomBytes
 from network import dandelion_ins, invQueue, receiveDataQueue
 from queues import UISignalQueue
 from tr import _translate
 
-import asyncore_pollchoose as asyncore
+from network import asyncore_pollchoose as asyncore
-import knownnodes
+from network import knownnodes
 from network.advanceddispatcher import AdvancedDispatcher
 from network.bmproto import BMProto
 from network.objectracker import ObjectTracker
 from network.socks4a import Socks4aConnection
 from network.socks5 import Socks5Connection
 from network.tls import TLSDispatcher
-from node import Peer
+from .node import Peer
 
 
 logger = logging.getLogger('default')
@@ -39,6 +40,12 @@ maximumAgeOfNodesThatIAdvertiseToOthers = 10800  #: Equals three hours
 maximumTimeOffsetWrongCount = 3  #: Connections with wrong time offset
 
 
+def _ends_with(s, tail):
+    try:
+        return s.endswith(tail)
+    except:
+        return s.decode("utf-8", "replace").endswith(tail)
+
 class TCPConnection(BMProto, TLSDispatcher):
     # pylint: disable=too-many-instance-attributes
     """
@@ -138,9 +145,9 @@ class TCPConnection(BMProto, TLSDispatcher):
                 'updateStatusBar',
                 _translate(
                     "MainWindow",
-                    "The time on your computer, %1, may be wrong. "
+                    "The time on your computer, {0}, may be wrong. "
                     "Please verify your settings."
-                ).arg(l10n.formatTimestamp())))
+                ).format(l10n.formatTimestamp())))
 
     def state_connection_fully_established(self):
         """
@@ -191,10 +198,10 @@ class TCPConnection(BMProto, TLSDispatcher):
                     # only if more recent than 3 hours
                     # and having positive or neutral rating
                     filtered = [
-                        (k, v) for k, v in nodes.iteritems()
+                        (k, v) for k, v in six.iteritems(nodes)
                         if v["lastseen"] > int(time.time())
                         - maximumAgeOfNodesThatIAdvertiseToOthers
-                        and v["rating"] >= 0 and not k.host.endswith('.onion')
+                        and v["rating"] >= 0 and not _ends_with(k.host, '.onion')
                     ]
                     # sent 250 only if the remote isn't interested in it
                     elemCount = min(
@@ -220,7 +227,7 @@ class TCPConnection(BMProto, TLSDispatcher):
                 'Sending huge inv message with %i objects to just this'
                 ' one peer', objectCount)
             self.append_write_buf(protocol.CreatePacket(
-                'inv', addresses.encodeVarint(objectCount) + payload))
+                b'inv', addresses.encodeVarint(objectCount) + payload))
 
         # Select all hashes for objects in this stream.
         bigInvList = {}
@@ -267,7 +274,7 @@ class TCPConnection(BMProto, TLSDispatcher):
         self.append_write_buf(
             protocol.assembleVersionMessage(
                 self.destination.host, self.destination.port,
-                connectionpool.pool.streams, dandelion_ins.enabled,
+                network.connectionpool.pool.streams, dandelion_ins.enabled,
                 False, nodeid=self.nodeid))
         self.connectedAt = time.time()
         receiveDataQueue.put(self.destination)
@@ -318,7 +325,7 @@ class Socks5BMConnection(Socks5Connection, TCPConnection):
         self.append_write_buf(
             protocol.assembleVersionMessage(
                 self.destination.host, self.destination.port,
-                connectionpool.pool.streams, dandelion_ins.enabled,
+                network.connectionpool.pool.streams, dandelion_ins.enabled,
                 False, nodeid=self.nodeid))
         self.set_state("bm_header", expectBytes=protocol.Header.size)
         return True
@@ -342,7 +349,7 @@ class Socks4aBMConnection(Socks4aConnection, TCPConnection):
         self.append_write_buf(
             protocol.assembleVersionMessage(
                 self.destination.host, self.destination.port,
-                connectionpool.pool.streams, dandelion_ins.enabled,
+                network.connectionpool.pool.streams, dandelion_ins.enabled,
                 False, nodeid=self.nodeid))
         self.set_state("bm_header", expectBytes=protocol.Header.size)
         return True
@@ -430,7 +437,7 @@ class TCPServer(AdvancedDispatcher):
 
         state.ownAddresses[Peer(*sock.getsockname())] = True
         if (
-            len(connectionpool.pool)
+            len(network.connectionpool.pool)
             > config.safeGetInt(
                 'bitmessagesettings', 'maxtotalconnections')
                 + config.safeGetInt(
@@ -442,7 +449,7 @@ class TCPServer(AdvancedDispatcher):
             sock.close()
             return
         try:
-            connectionpool.pool.addConnection(
+            network.connectionpool.pool.addConnection(
                 TCPConnection(sock=sock))
         except socket.error:
             pass

src/network/tls.py

@@ -6,6 +6,7 @@ import os
 import socket
 import ssl
 import sys
+import six
 
 import network.asyncore_pollchoose as asyncore
 import paths
@@ -34,7 +35,7 @@ else:
 # ciphers
 if (
     ssl.OPENSSL_VERSION_NUMBER >= 0x10100000
-    and not ssl.OPENSSL_VERSION.startswith(b"LibreSSL")
+    and not ssl.OPENSSL_VERSION.startswith("LibreSSL")
 ):
     sslProtocolCiphers = "AECDH-AES256-SHA@SECLEVEL=0"
 else:
@@ -58,14 +59,29 @@ class TLSDispatcher(AdvancedDispatcher):
         self.tlsDone = False
         self.tlsVersion = "N/A"
         self.isSSL = False
+        if six.PY3 or ssl.OPENSSL_VERSION_NUMBER >= 0x30000000:
+            self.tlsPrepared = False
 
     def state_tls_init(self):
         """Prepare sockets for TLS handshake"""
         self.isSSL = True
         self.tlsStarted = True
+
+        if six.PY3 or ssl.OPENSSL_VERSION_NUMBER >= 0x30000000:
+            self.want_read = self.want_write = True
+            self.set_state("tls_handshake")
+            return False
+
+        return self.do_tls_init()
+
+    def do_tls_init(self):
         # Once the connection has been established,
         # it's safe to wrap the socket.
         if sys.version_info >= (2, 7, 9):
+            if ssl.OPENSSL_VERSION_NUMBER >= 0x30000000:
+                context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER
+                    if self.server_side else ssl.PROTOCOL_TLS_CLIENT)
+            else:
                 context = ssl.create_default_context(
                     purpose=ssl.Purpose.SERVER_AUTH
                     if self.server_side else ssl.Purpose.CLIENT_AUTH)
@@ -74,6 +90,11 @@ class TLSDispatcher(AdvancedDispatcher):
             context.check_hostname = False
             context.verify_mode = ssl.CERT_NONE
             # also exclude TLSv1 and TLSv1.1 in the future
+            if ssl.OPENSSL_VERSION_NUMBER >= 0x30000000:
+                context.options = ssl.OP_ALL | ssl.OP_NO_SSLv2 |\
+                    ssl.OP_NO_SSLv3 | ssl.OP_SINGLE_ECDH_USE |\
+                    ssl.OP_CIPHER_SERVER_PREFERENCE | ssl.OP_NO_TLSv1_3
+            else:
                 context.options = ssl.OP_ALL | ssl.OP_NO_SSLv2 |\
                     ssl.OP_NO_SSLv3 | ssl.OP_SINGLE_ECDH_USE |\
                     ssl.OP_CIPHER_SERVER_PREFERENCE
@@ -88,6 +109,9 @@ class TLSDispatcher(AdvancedDispatcher):
                 ciphers=self.ciphers, do_handshake_on_connect=False)
         self.sslSocket.setblocking(0)
         self.want_read = self.want_write = True
+        if six.PY3 or ssl.OPENSSL_VERSION_NUMBER >= 0x30000000:
+            self.tlsPrepared = True
+        else:
             self.set_state("tls_handshake")
         return False
 
@@ -114,6 +138,8 @@ class TLSDispatcher(AdvancedDispatcher):
             # during TLS handshake, and after flushing write buffer,
             # return status of last handshake attempt
             if self.tlsStarted and not self.tlsDone and not self.write_buf:
+                # with OpenSSL 3, excessive logs are spitted
+                if ssl.OPENSSL_VERSION_NUMBER < 0x30000000:
                     logger.debug('tls readable, %r', self.want_read)
                 return self.want_read
             # prior to TLS handshake,
@@ -134,6 +160,10 @@ class TLSDispatcher(AdvancedDispatcher):
         try:
             # wait for write buffer flush
             if self.tlsStarted and not self.tlsDone and not self.write_buf:
+                if six.PY3 or ssl.OPENSSL_VERSION_NUMBER >= 0x30000000:
+                    if not self.tlsPrepared:
+                        self.do_tls_init()
+                        return
                 self.tls_handshake()
             else:
                 AdvancedDispatcher.handle_read(self)
@@ -156,6 +186,10 @@ class TLSDispatcher(AdvancedDispatcher):
         try:
             # wait for write buffer flush
             if self.tlsStarted and not self.tlsDone and not self.write_buf:
+                if six.PY3 or ssl.OPENSSL_VERSION_NUMBER >= 0x30000000:
+                    if not self.tlsPrepared:
+                        self.do_tls_init()
+                        return
                 self.tls_handshake()
             else:
                 AdvancedDispatcher.handle_write(self)

src/network/udp.py

@@ -8,12 +8,12 @@ import time
 # magic imports!
 import protocol
 import state
-import connectionpool
+import network.connectionpool  # use long name to address recursive import
 
 from network import receiveDataQueue
-from bmproto import BMProto
+from .bmproto import BMProto
-from node import Peer
+from .node import Peer
-from objectracker import ObjectTracker
+from .objectracker import ObjectTracker
 
 
 logger = logging.getLogger('default')
@@ -81,8 +81,8 @@ class UDPSocket(BMProto):  # pylint: disable=too-many-instance-attributes
             return True
         remoteport = False
         for seenTime, stream, _, ip, port in addresses:
-            decodedIP = protocol.checkIPAddress(str(ip))
+            decodedIP = protocol.checkIPAddress(ip)
-            if stream not in connectionpool.pool.streams:
+            if stream not in network.connectionpool.pool.streams:
                 continue
             if (seenTime < time.time() - protocol.MAX_TIME_OFFSET
                     or seenTime > time.time() + protocol.MAX_TIME_OFFSET):

src/network/uploadthread.py

@@ -6,10 +6,10 @@ import time
 import random
 import protocol
 import state
-import connectionpool
+from network import connectionpool
 from randomtrackingdict import RandomTrackingDict
 from network import dandelion_ins
-from threads import StoppableThread
+from .threads import StoppableThread
 
 
 class UploadThread(StoppableThread):
@@ -50,7 +50,7 @@ class UploadThread(StoppableThread):
                         break
                     try:
                         payload.extend(protocol.CreatePacket(
-                            'object', state.Inventory[chunk].payload))
+                            b'object', state.Inventory[chunk].payload))
                         chunk_count += 1
                     except KeyError:
                         i.antiIntersectionDelay()

src/plugins/menu_qrcode.py

@@ -3,7 +3,7 @@
 A menu plugin showing QR-Code for bitmessage address in modal dialog.
 """
 
-import urllib
+from six.moves.urllib.parse import urlencode
 
 import qrcode
 from PyQt4 import QtCore, QtGui
@@ -93,7 +93,7 @@ def connect_plugin(form):
                 return
         dialog.render(
             'bitmessage:%s' % account.address + (
-                '?' + urllib.urlencode({'label': label.encode('utf-8')})
+                '?' + urlencode({'label': label.encode('utf-8')})
                 if label != account.address else '')
         )
         dialog.exec_()

src/protocol.py

@@ -12,6 +12,8 @@ import sys
 import time
 from binascii import hexlify
 from struct import Struct, pack, unpack
+import six
+import sqlite3
 
 import defaults
 import highlevelcrypto
@@ -23,6 +25,7 @@ from debug import logger
 from helper_sql import sqlExecute
 from network.node import Peer
 from version import softwareVersion
+from dbcompat import dbstr
 
 # Network constants
 magic = 0xE9BEB4D9
@@ -168,11 +171,11 @@ def checkIPAddress(host, private=False):
     otherwise returns False
     """
     if host[0:12] == b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xFF\xFF':
-        hostStandardFormat = socket.inet_ntop(socket.AF_INET, host[12:])
+        hostStandardFormat = socket.inet_ntop(socket.AF_INET, bytes(host[12:]))
         return checkIPv4Address(host[12:], hostStandardFormat, private)
     elif host[0:6] == b'\xfd\x87\xd8\x7e\xeb\x43':
         # Onion, based on BMD/bitcoind
-        hostStandardFormat = base64.b32encode(host[6:]).lower() + ".onion"
+        hostStandardFormat = base64.b32encode(host[6:]).lower() + b".onion"
         if private:
             return False
         return hostStandardFormat
@@ -227,7 +230,7 @@ def checkIPv6Address(host, hostStandardFormat, private=False):
             logger.debug('Ignoring loopback address: %s', hostStandardFormat)
         return False
     try:
-        host = [ord(c) for c in host[:2]]
+        host = [six.byte2int(c) for c in host[:2]]
     except TypeError:  # python3 has ints already
         pass
     if host[0] == 0xfe and host[1] & 0xc0 == 0x80:
@@ -293,7 +296,7 @@ def isProofOfWorkSufficient(
     if TTL < 300:
         TTL = 300
     POW, = unpack('>Q', highlevelcrypto.double_sha512(
-        data[:8] + hashlib.sha512(data[8:]).digest())[0:8])
+        bytes(data[:8]) + hashlib.sha512(data[8:]).digest())[0:8])
     return POW <= 2 ** 64 / (
         nonceTrialsPerByte * (
             len(data) + payloadLengthExtraBytes
@@ -417,7 +420,7 @@ def assembleVersionMessage(  # pylint: disable=too-many-arguments
     return CreatePacket(b'version', payload)
 
 
-def assembleErrorMessage(fatal=0, banTime=0, inventoryVector='', errorText=''):
+def assembleErrorMessage(fatal=0, banTime=0, inventoryVector=b'', errorText=''):
     """
     Construct the payload of an error message,
     return the resulting bytes of running `CreatePacket` on it
@@ -426,6 +429,8 @@ def assembleErrorMessage(fatal=0, banTime=0, inventoryVector='', errorText=''):
     payload += encodeVarint(banTime)
     payload += encodeVarint(len(inventoryVector))
     payload += inventoryVector
+    if isinstance(errorText, str):
+        errorText = errorText.encode("utf-8", "replace")
     payload += encodeVarint(len(errorText))
     payload += errorText
     return CreatePacket(b'error', payload)
@@ -465,7 +470,7 @@ def decryptAndCheckPubkeyPayload(data, address):
         readPosition += varintLength
         # We'll store the address version and stream number
         # (and some more) in the pubkeys table.
-        storedData = data[20:readPosition]
+        storedData = bytes(data[20:readPosition])
 
         if addressVersion != embeddedAddressVersion:
             logger.info(
@@ -482,11 +487,11 @@ def decryptAndCheckPubkeyPayload(data, address):
         readPosition += 32
         # the time through the tag. More data is appended onto
         # signedData below after the decryption.
-        signedData = data[8:readPosition]
+        signedData = bytes(data[8:readPosition])
         encryptedData = data[readPosition:]
 
         # Let us try to decrypt the pubkey
-        toAddress, cryptorObject = state.neededPubkeys[tag]
+        toAddress, cryptorObject = state.neededPubkeys[bytes(tag)]
         if toAddress != address:
             logger.critical(
                 'decryptAndCheckPubkeyPayload failed due to toAddress'
@@ -511,9 +516,9 @@ def decryptAndCheckPubkeyPayload(data, address):
         readPosition = 0
         # bitfieldBehaviors = decryptedData[readPosition:readPosition + 4]
         readPosition += 4
-        pubSigningKey = '\x04' + decryptedData[readPosition:readPosition + 64]
+        pubSigningKey = b'\x04' + decryptedData[readPosition:readPosition + 64]
         readPosition += 64
-        pubEncryptionKey = '\x04' + decryptedData[readPosition:readPosition + 64]
+        pubEncryptionKey = b'\x04' + decryptedData[readPosition:readPosition + 64]
         readPosition += 64
         specifiedNonceTrialsPerByteLength = decodeVarint(
             decryptedData[readPosition:readPosition + 10])[1]
@@ -558,7 +563,7 @@ def decryptAndCheckPubkeyPayload(data, address):
             hexlify(pubSigningKey), hexlify(pubEncryptionKey)
         )
 
-        t = (address, addressVersion, storedData, int(time.time()), 'yes')
+        t = (dbstr(address), addressVersion, sqlite3.Binary(storedData), int(time.time()), dbstr('yes'))
         sqlExecute('''INSERT INTO pubkeys VALUES (?,?,?,?,?)''', *t)
         return 'successful'
     except varintDecodeError:

src/py3bitmessage

@@ -0,0 +1,13 @@
+#!/usr/bin/python3
+
+import os
+import pkg_resources
+
+import pybitmessage
+
+dist = pkg_resources.get_distribution('pybitmessage')
+script_file = os.path.join(dist.location, dist.key, 'bitmessagemain.py')
+new_globals = globals()
+new_globals.update(__file__=script_file)
+
+execfile(script_file, new_globals)

src/pyelliptic/arithmetic.py

@@ -3,6 +3,7 @@ Arithmetic Expressions
 """
 import hashlib
 import re
+import six
 
 P = 2**256 - 2**32 - 2**9 - 2**8 - 2**7 - 2**6 - 2**4 - 1
 A = 0
@@ -34,7 +35,7 @@ def get_code_string(base):
         return b'123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
     if base == 256:
         try:
-            return b''.join([chr(x) for x in range(256)])
+            return b''.join([six.int2byte(x) for x in range(256)])
         except TypeError:
             return bytes([x for x in range(256)])
 

src/pyelliptic/cipher.py

@@ -30,7 +30,7 @@ class Cipher(object):
         self.ctx = OpenSSL.EVP_CIPHER_CTX_new()
         if do == 1 or do == 0:
             k = OpenSSL.malloc(key, len(key))
-            IV = OpenSSL.malloc(iv, len(iv))
+            IV = OpenSSL.malloc(bytes(iv), len(iv))
             OpenSSL.EVP_CipherInit_ex(
                 self.ctx, self.cipher.get_pointer(), 0, k, IV, do)
         else:
@@ -59,7 +59,7 @@ class Cipher(object):
         """Update result with more data"""
         i = OpenSSL.c_int(0)
         buffer = OpenSSL.malloc(b"", len(input) + self.cipher.get_blocksize())
-        inp = OpenSSL.malloc(input, len(input))
+        inp = OpenSSL.malloc(bytes(input), len(input))
         if OpenSSL.EVP_CipherUpdate(self.ctx, OpenSSL.byref(buffer),
                                     OpenSSL.byref(i), inp, len(input)) == 0:
             raise Exception("[OpenSSL] EVP_CipherUpdate FAIL ...")

src/pyelliptic/ecc.py

@@ -7,6 +7,7 @@ Asymmetric cryptography using elliptic curves
 
 from hashlib import sha512
 from struct import pack, unpack
+from ctypes import c_char_p
 
 from .cipher import Cipher
 from .hash import equals, hmac_sha256
@@ -218,8 +219,8 @@ class ECC(object):
             if other_key == 0:
                 raise Exception("[OpenSSL] EC_KEY_new_by_curve_name FAIL ...")
 
-            other_pub_key_x = OpenSSL.BN_bin2bn(pubkey_x, len(pubkey_x), None)
+            other_pub_key_x = OpenSSL.BN_bin2bn(c_char_p(bytes(pubkey_x)), len(pubkey_x), None)
-            other_pub_key_y = OpenSSL.BN_bin2bn(pubkey_y, len(pubkey_y), None)
+            other_pub_key_y = OpenSSL.BN_bin2bn(c_char_p(bytes(pubkey_y)), len(pubkey_y), None)
 
             other_group = OpenSSL.EC_KEY_get0_group(other_key)
             other_pub_key = OpenSSL.EC_POINT_new(other_group)

src/pyelliptic/hash.py

@@ -4,6 +4,8 @@ Wrappers for hash functions from OpenSSL.
 #  Copyright (C) 2011 Yann GUIBET <yannguibet@gmail.com>
 #  See LICENSE for details.
 
+import six
+
 from .openssl import OpenSSL
 
 
@@ -22,7 +24,7 @@ def _equals_str(a, b):
         return False
     result = 0
     for x, y in zip(a, b):
-        result |= ord(x) ^ ord(y)
+        result |= six.byte2int(x) ^ six.byte2int(y)
     return result == 0
 
 
@@ -38,7 +40,7 @@ def hmac_sha256(k, m):
     Compute the key and the message with HMAC SHA5256
     """
     key = OpenSSL.malloc(k, len(k))
-    d = OpenSSL.malloc(m, len(m))
+    d = OpenSSL.malloc(bytes(m), len(m))
     md = OpenSSL.malloc(0, 32)
     i = OpenSSL.pointer(OpenSSL.c_int(0))
     OpenSSL.HMAC(OpenSSL.EVP_sha256(), key, len(k), d, len(m), md, i)

src/pyelliptic/openssl.py

@@ -8,6 +8,7 @@ needed openssl functionality in class _OpenSSL.
 """
 import ctypes
 import sys
+import six
 
 # pylint: disable=protected-access
 
@@ -691,7 +692,7 @@ class _OpenSSL(object):
         length = self.BN_num_bytes(x)
         data = self.malloc(0, length)
         OpenSSL.BN_bn2bin(x, data)
-        return ord(data[length - 1]) & 1
+        return six.byte2int(data[length - 1]) & 1
 
     def get_cipher(self, name):
         """
@@ -745,7 +746,7 @@ class _OpenSSL(object):
         """
         buffer_ = None
         if data != 0:
-            if sys.version_info.major == 3 and isinstance(data, type('')):
+            if six.PY3 and isinstance(data, type('')):
                 data = data.encode()
             buffer_ = self.create_string_buffer(data, size)
         else:

src/randomtrackingdict.py

@@ -38,10 +38,10 @@ class RandomTrackingDict(object):
         return self.len
 
     def __contains__(self, key):
-        return key in self.dictionary
+        return bytes(key) in self.dictionary
 
     def __getitem__(self, key):
-        return self.dictionary[key][1]
+        return self.dictionary[bytes(key)][1]
 
     def _swap(self, i1, i2):
         with self.lock:
@@ -49,26 +49,28 @@ class RandomTrackingDict(object):
             key2 = self.indexDict[i2]
             self.indexDict[i1] = key2
             self.indexDict[i2] = key1
-            self.dictionary[key1][0] = i2
+            self.dictionary[bytes(key1)][0] = i2
-            self.dictionary[key2][0] = i1
+            self.dictionary[bytes(key2)][0] = i1
         # for quick reassignment
         return i2
 
     def __setitem__(self, key, value):
         with self.lock:
-            if key in self.dictionary:
+            key_bytes = bytes(key)
-                self.dictionary[key][1] = value
+            if key_bytes in self.dictionary:
+                self.dictionary[key_bytes][1] = value
             else:
                 self.indexDict.append(key)
-                self.dictionary[key] = [self.len, value]
+                self.dictionary[key_bytes] = [self.len, value]
                 self._swap(self.len, self.len - self.pendingLen)
                 self.len += 1
 
     def __delitem__(self, key):
-        if key not in self.dictionary:
+        key_bytes = bytes(key)
+        if key_bytes not in self.dictionary:
             raise KeyError
         with self.lock:
-            index = self.dictionary[key][0]
+            index = self.dictionary[key_bytes][0]
             # not pending
             if index < self.len - self.pendingLen:
                 # left of pending part
@@ -82,7 +84,7 @@ class RandomTrackingDict(object):
             # operation can improve 4x, but it's already very fast so we'll
             # ignore it for the time being
             del self.indexDict[-1]
-            del self.dictionary[key]
+            del self.dictionary[key_bytes]
             self.len -= 1
 
     def setMaxPending(self, maxPending):

src/revert_blob_to_text.py

@@ -0,0 +1,52 @@
+import helper_startup
+import state
+import shutil
+import sqlite3
+
+expected_ver = 11
+
+print("Looking up database file..")
+helper_startup.loadConfig()
+db_path = state.appdata + "messages.dat"
+print("Database path: {}".format(db_path))
+db_backup_path = db_path + ".blob-keys"
+print("Backup path: {}".format(db_backup_path))
+shutil.copyfile(db_path, db_backup_path)
+print("Copied to backup")
+
+print()
+
+print("Open the database")
+conn = sqlite3.connect(db_path)
+cur = conn.cursor()
+
+cur.execute("SELECT value FROM settings WHERE key='version';")
+ver = int(cur.fetchall()[0][0])
+print("PyBitmessage database version: {}".format(ver))
+if ver != expected_ver:
+    print("Error: version must be {}".format(expected_ver))
+    conn.close()
+    print("Quitting..")
+    quit()
+print("Version OK")
+
+print()
+
+print("Converting..")
+q = "UPDATE inbox SET msgid=CAST(msgid AS TEXT), sighash=CAST(sighash AS TEXT);"
+print(q)
+cur.execute(q)
+q = "UPDATE pubkeys SET transmitdata=CAST(transmitdata AS TEXT);"
+print(q)
+cur.execute(q)
+q = "UPDATE sent SET msgid=CAST(msgid AS TEXT), toripe=CAST(toripe AS TEXT), ackdata=CAST(ackdata AS TEXT);"
+print(q)
+cur.execute(q)
+
+print("Commiting..")
+conn.commit()
+print("Conversion done")
+
+print("Close the database")
+conn.close()
+print("Finished")

src/shared.py

@@ -14,6 +14,7 @@ import stat
 import subprocess  # nosec B404
 import sys
 from binascii import hexlify
+from six.moves.reprlib import repr
 
 # Project imports.
 import highlevelcrypto
@@ -22,6 +23,7 @@ from addresses import decodeAddress, encodeVarint
 from bmconfigparser import config
 from debug import logger
 from helper_sql import sqlQuery
+from dbcompat import dbstr
 
 
 myECCryptorObjects = {}
@@ -37,8 +39,8 @@ broadcastSendersForWhichImWatching = {}
 def isAddressInMyAddressBook(address):
     """Is address in my addressbook?"""
     queryreturn = sqlQuery(
-        '''select address from addressbook where address=?''',
+        '''select TRUE from addressbook where address=?''',
-        address)
+        dbstr(address))
     return queryreturn != []
 
 
@@ -46,8 +48,8 @@ def isAddressInMyAddressBook(address):
 def isAddressInMySubscriptionsList(address):
     """Am I subscribed to this address?"""
     queryreturn = sqlQuery(
-        '''select * from subscriptions where address=?''',
+        '''select TRUE from subscriptions where address=?''',
-        str(address))
+        dbstr(address))
     return queryreturn != []
 
 
@@ -61,14 +63,14 @@ def isAddressInMyAddressBookSubscriptionsListOrWhitelist(address):
     queryreturn = sqlQuery(
         '''SELECT address FROM whitelist where address=?'''
         ''' and enabled = '1' ''',
-        address)
+        dbstr(address))
     if queryreturn != []:
         return True
 
     queryreturn = sqlQuery(
         '''select address from subscriptions where address=?'''
         ''' and enabled = '1' ''',
-        address)
+        dbstr(address))
     if queryreturn != []:
         return True
     return False
@@ -114,11 +116,11 @@ def reloadMyAddressHashes():
         if len(privEncryptionKey) == 64:
             myECCryptorObjects[hashobj] = \
                 highlevelcrypto.makeCryptor(privEncryptionKey)
-            myAddressesByHash[hashobj] = addressInKeysFile
+            myAddressesByHash[bytes(hashobj)] = addressInKeysFile
             tag = highlevelcrypto.double_sha512(
                 encodeVarint(addressVersionNumber)
                 + encodeVarint(streamNumber) + hashobj)[32:]
-            myAddressesByTag[tag] = addressInKeysFile
+            myAddressesByTag[bytes(tag)] = addressInKeysFile
 
     if not keyfileSecure:
         fixSensitiveFilePermissions(os.path.join(
@@ -136,6 +138,7 @@ def reloadBroadcastSendersForWhichImWatching():
     logger.debug('reloading subscriptions...')
     for row in queryreturn:
         address, = row
+        address = address.decode("utf-8", "replace")
         # status
         addressVersionNumber, streamNumber, hashobj = decodeAddress(address)[1:]
         if addressVersionNumber == 2:
@@ -149,7 +152,7 @@ def reloadBroadcastSendersForWhichImWatching():
                 encodeVarint(addressVersionNumber)
                 + encodeVarint(streamNumber) + hashobj
             ).digest()[:32]
-            MyECSubscriptionCryptorObjects[hashobj] = \
+            MyECSubscriptionCryptorObjects[bytes(hashobj)] = \
                 highlevelcrypto.makeCryptor(hexlify(privEncryptionKey))
         else:
             doubleHashOfAddressData = highlevelcrypto.double_sha512(
@@ -158,7 +161,7 @@ def reloadBroadcastSendersForWhichImWatching():
             )
             tag = doubleHashOfAddressData[32:]
             privEncryptionKey = doubleHashOfAddressData[:32]
-            MyECSubscriptionCryptorObjects[tag] = \
+            MyECSubscriptionCryptorObjects[bytes(tag)] = \
                 highlevelcrypto.makeCryptor(hexlify(privEncryptionKey))
 
 
@@ -169,7 +172,7 @@ def fixPotentiallyInvalidUTF8Data(text):
         return text
     except UnicodeDecodeError:
         return 'Part of the message is corrupt. The message cannot be' \
-            ' displayed the normal way.\n\n' + repr(text)
+            ' displayed the normal way.\n\n' + text.decode("utf-8", "replace")
 
 
 def checkSensitiveFilePermissions(filename):
@@ -193,7 +196,7 @@ def checkSensitiveFilePermissions(filename):
             ['/usr/bin/stat', '-f', '-c', '%T', filename],
             stderr=subprocess.STDOUT
         )  # nosec B603
-        if 'fuseblk' in fstype:
+        if b'fuseblk' in fstype:
             logger.info(
                 'Skipping file permissions check for %s.'
                 ' Filesystem fuseblk detected.', filename)

src/shutdown.py

@@ -23,7 +23,11 @@ def doCleanShutdown():
 
     objectProcessorQueue.put(('checkShutdownVariable', 'no data'))
     for thread in threading.enumerate():
-        if thread.isAlive() and isinstance(thread, StoppableThread):
+        try:
+            alive = thread.isAlive()
+        except AttributeError:
+            alive = thread.is_alive()
+        if alive and isinstance(thread, StoppableThread):
             thread.stopThread()
 
     UISignalQueue.put((

src/storage/sqlite.py

@@ -1,6 +1,7 @@
 """
 Sqlite Inventory
 """
+import six
 import sqlite3
 import time
 from threading import RLock
@@ -29,20 +30,22 @@ class SqliteInventory(InventoryStorage):
 
     def __contains__(self, hash_):
         with self.lock:
-            if hash_ in self._objects:
+            hash_bytes = bytes(hash_)
+            if hash_bytes in self._objects:
                 return True
             rows = sqlQuery(
                 'SELECT streamnumber FROM inventory WHERE hash=?',
                 sqlite3.Binary(hash_))
             if not rows:
                 return False
-            self._objects[hash_] = rows[0][0]
+            self._objects[hash_bytes] = rows[0][0]
             return True
 
     def __getitem__(self, hash_):
         with self.lock:
-            if hash_ in self._inventory:
+            hash_bytes = bytes(hash_)
-                return self._inventory[hash_]
+            if hash_bytes in self._inventory:
+                return self._inventory[hash_bytes]
             rows = sqlQuery(
                 'SELECT objecttype, streamnumber, payload, expirestime, tag'
                 ' FROM inventory WHERE hash=?', sqlite3.Binary(hash_))
@@ -53,15 +56,16 @@ class SqliteInventory(InventoryStorage):
     def __setitem__(self, hash_, value):
         with self.lock:
             value = InventoryItem(*value)
-            self._inventory[hash_] = value
+            hash_bytes = bytes(hash_)
-            self._objects[hash_] = value.stream
+            self._inventory[hash_bytes] = value
+            self._objects[hash_bytes] = value.stream
 
     def __delitem__(self, hash_):
         raise NotImplementedError
 
     def __iter__(self):
         with self.lock:
-            hashes = self._inventory.keys()[:]
+            hashes = [] + self._inventory.keys()[:]
             hashes += (x for x, in sqlQuery('SELECT hash FROM inventory'))
             return hashes.__iter__()
 
@@ -95,7 +99,7 @@ class SqliteInventory(InventoryStorage):
             t = int(time.time())
             hashes = [x for x, value in self._inventory.items()
                       if value.stream == stream and value.expires > t]
-            hashes += (str(payload) for payload, in sqlQuery(
+            hashes += (bytes(payload) for payload, in sqlQuery(
                 'SELECT hash FROM inventory WHERE streamnumber=?'
                 ' AND expirestime>?', stream, t))
             return hashes
@@ -107,6 +111,10 @@ class SqliteInventory(InventoryStorage):
             # always use the inventoryLock OUTSIDE of the sqlLock.
             with SqlBulkExecute() as sql:
                 for objectHash, value in self._inventory.items():
+                    tag = value[4]
+                    if six.PY3 and isinstance(tag, str):
+                        tag = tag.encode("utf-8", "replace")
+                    value = [value[0], value[1], sqlite3.Binary(value[2]), value[3], sqlite3.Binary(tag)]
                     sql.execute(
                         'INSERT INTO inventory VALUES (?, ?, ?, ?, ?, ?)',
                         sqlite3.Binary(objectHash), *value)

src/storage/storage.py

@@ -4,10 +4,7 @@ Storing inventory items
 
 from abc import abstractmethod
 from collections import namedtuple
-try:
+from six.moves.collections_abc import MutableMapping  # pylint: disable=deprecated-class
-    from collections import MutableMapping  # pylint: disable=deprecated-class
-except ImportError:
-    from collections.abc import MutableMapping
 
 
 InventoryItem = namedtuple('InventoryItem', 'type stream payload expires tag')

src/tests/common.py

@@ -1,7 +1,7 @@
 import os
-import sys
 import time
 import unittest
+import six
 
 
 _files = (
@@ -33,7 +33,7 @@ def checkup():
 
 def skip_python3():
     """Raise unittest.SkipTest() if detected python3"""
-    if sys.hexversion >= 0x3000000:
+    if six.PY3:
         raise unittest.SkipTest('Module is not ported to python3')
 
 

src/tests/core.py

@@ -6,7 +6,7 @@ Tests for core and those that do not work outside
 import atexit
 import os
 import pickle  # nosec
-import Queue
+from six.moves import queue as Queue
 import random  # nosec
 import shutil
 import socket
@@ -15,6 +15,8 @@ import sys
 import threading
 import time
 import unittest
+import six
+import sqlite3
 
 import protocol
 import state
@@ -31,6 +33,7 @@ from network.node import Node, Peer
 from network.tcp import Socks4aBMConnection, Socks5BMConnection, TCPConnection
 from queues import excQueue
 from version import softwareVersion
+from dbcompat import dbstr
 
 from common import cleanup
 
@@ -137,8 +140,8 @@ class TestCore(unittest.TestCase):
     @staticmethod
     def _outdate_knownnodes():
         with knownnodes.knownNodesLock:
-            for nodes in knownnodes.knownNodes.itervalues():
+            for nodes in six.itervalues(knownnodes.knownNodes):
-                for node in nodes.itervalues():
+                for node in six.itervalues(nodes):
                     node['lastseen'] -= 2419205  # older than 28 days
 
     def test_knownnodes_pickle(self):
@@ -146,9 +149,9 @@ class TestCore(unittest.TestCase):
         pickle_knownnodes()
         self._wipe_knownnodes()
         knownnodes.readKnownNodes()
-        for nodes in knownnodes.knownNodes.itervalues():
+        for nodes in six.itervalues(knownnodes.knownNodes):
             self_count = n = 0
-            for n, node in enumerate(nodes.itervalues()):
+            for n, node in enumerate(six.itervalues(nodes)):
                 if node.get('self'):
                     self_count += 1
             self.assertEqual(n - self_count, 2)
@@ -202,7 +205,7 @@ class TestCore(unittest.TestCase):
         while c > 0:
             time.sleep(1)
             c -= 2
-            for peer, con in connectionpool.pool.outboundConnections.iteritems():
+            for peer, con in six.iteritems(connectionpool.pool.outboundConnections):
                 if (
                     peer.host.startswith('bootstrap')
                     or peer.host == 'quzwelsuziwqgpt2.onion'
@@ -223,7 +226,7 @@ class TestCore(unittest.TestCase):
             'Failed to connect during %.2f sec' % (time.time() - _started))
 
     def _check_knownnodes(self):
-        for stream in knownnodes.knownNodes.itervalues():
+        for stream in six.itervalues(knownnodes.knownNodes):
             for peer in stream:
                 if peer.host.startswith('bootstrap'):
                     self.fail(
@@ -346,12 +349,18 @@ class TestCore(unittest.TestCase):
             subject=subject, message=message
         )
         queryreturn = sqlQuery(
-            '''select msgid from sent where ackdata=?''', result)
+            '''select msgid from sent where ackdata=?''', sqlite3.Binary(result))
-        self.assertNotEqual(queryreturn[0][0] if queryreturn else '', '')
+        if len(queryreturn) < 1:
+            queryreturn = sqlQuery(
+                '''select msgid from sent where ackdata=CAST(? AS TEXT)''', result)
+        self.assertNotEqual(queryreturn[0][0] if queryreturn else b'', b'')
 
         column_type = sqlQuery(
-            '''select typeof(msgid) from sent where ackdata=?''', result)
+            '''select typeof(msgid) from sent where ackdata=?''', sqlite3.Binary(result))
-        self.assertEqual(column_type[0][0] if column_type else '', 'text')
+        if len(column_type) < 1:
+            column_type = sqlQuery(
+                '''select typeof(msgid) from sent where ackdata=CAST(? AS TEXT)''', result)
+        self.assertEqual(column_type[0][0] if column_type else '', 'blob')
 
     @unittest.skipIf(frozen, 'not packed test_pattern into the bundle')
     def test_old_knownnodes_pickle(self):
@@ -369,7 +378,7 @@ class TestCore(unittest.TestCase):
     @staticmethod
     def delete_address_from_addressbook(address):
         """Clean up addressbook"""
-        sqlQuery('''delete from addressbook where address=?''', address)
+        sqlQuery('''delete from addressbook where address=?''', dbstr(address))
 
     def test_add_same_address_twice_in_addressbook(self):
         """checking same address is added twice in addressbook"""
@@ -383,7 +392,7 @@ class TestCore(unittest.TestCase):
         """checking is address added in addressbook or not"""
         helper_addressbook.insert(label='test1', address=self.addr)
         queryreturn = sqlQuery(
-            'select count(*) from addressbook where address=?', self.addr)
+            'select count(*) from addressbook where address=?', dbstr(self.addr))
         self.assertEqual(queryreturn[0][0], 1)
         self.delete_address_from_addressbook(self.addr)