checking in python3 progress...

Dockerfile.test_py2

@@ -0,0 +1,27 @@
+# Dockerfile for testing PyBitmessage with Python 2
+# docker build -f Dockerfile.test_py2 .
+
+FROM ubuntu:xenial
+
+RUN apt-get update
+RUN apt-get install -yq --no-install-suggests --no-install-recommends \
+    build-essential libcap-dev python-all-dev python-setuptools virtualenv
+
+WORKDIR /home/bitmessage
+RUN ln -s src pybitmessage  # tests environment
+
+ENV VIRTUAL_ENV=/home/bitmessage/venv
+RUN virtualenv $VIRTUAL_ENV
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+
+RUN python --version
+RUN pip --version
+
+ADD requirements.txt .
+RUN pip install -r requirements.txt
+
+ADD . .
+
+RUN python setup.py install
+RUN coverage run --source=src setup.py test
+RUN coverage report

Dockerfile.test_py3

@@ -0,0 +1,28 @@
+# Dockerfile for testing PyBitmessage with Python 3
+# docker build -f Dockerfile.test_py3 .
+
+FROM ubuntu:xenial
+
+RUN apt-get update
+RUN apt-get install -yq --no-install-suggests --no-install-recommends \
+    build-essential libcap-dev libssl-dev python3-all-dev python3-setuptools \
+    virtualenv
+
+WORKDIR /home/bitmessage
+RUN ln -s src pybitmessage  # tests environment
+
+ENV VIRTUAL_ENV=/home/bitmessage/venv
+RUN virtualenv -p python3 $VIRTUAL_ENV
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+
+RUN python --version
+RUN pip --version
+
+ADD requirements.txt .
+RUN pip install -r requirements.txt
+
+ADD . .
+
+RUN python setup.py install
+RUN coverage run --source=src setup.py test
+RUN coverage report

requirements.txt

@@ -1,3 +1,6 @@
-python_prctl
+configparser
+coverage
+future
 psutil
 pycrypto
+python_prctl

src/bitmessagecurses/__init__.py

@@ -1,3 +1,7 @@
+from __future__ import print_function
+from __future__ import unicode_literals
+from __future__ import division
+from __future__ import absolute_import
 # Copyright (c) 2014 Luke Montalvo <lukemontalvo@gmail.com>
 # This file adds a alternative commandline interface, feel free to critique and fork
 # 
@@ -7,9 +11,17 @@
 #     * python2-pythondialog
 #  * dialog
 
+from future import standard_library
+standard_library.install_aliases()
+from builtins import chr
+from builtins import ascii
+from builtins import str
+from builtins import range
+from builtins import *
+from builtins import object
 import os
 import sys
-import StringIO
+import io
 from textwrap import *
 
 import time
@@ -23,7 +35,7 @@ from helper_sql import *
 from helper_ackPayload import genAckPayload
 
 from addresses import *
-import ConfigParser
+import configparser
 from bmconfigparser import BMConfigParser
 from inventory import Inventory
 import l10n
@@ -60,13 +72,13 @@ bwtype = "black"
 
 BROADCAST_STR = "[Broadcast subscribers]"
 
-class printLog:
+class printLog(object):
     def write(self, output):
         global log
         log += output
     def flush(self):
         pass
-class errLog:
+class errLog(object):
     def write(self, output):
         global log
         log += "!"+output
@@ -78,7 +90,7 @@ errlog = errLog()
 
 def cpair(a):
     r = curses.color_pair(a)
-    if r not in range(1, curses.COLOR_PAIRS-1):
+    if r not in list(range(1, curses.COLOR_PAIRS-1)):
         r = curses.color_pair(0)
     return r
 def ascii(s):
@@ -303,11 +315,11 @@ def handlech(c, stdscr):
                                 msg = ""
                                 for i, item in enumerate(data.split("\n")):
                                     msg += fill(item, replace_whitespace=False)+"\n"
-                                scrollbox(d, unicode(ascii(msg)), 30, 80)
+                                scrollbox(d, str(ascii(msg)), 30, 80)
                                 sqlExecute("UPDATE inbox SET read=1 WHERE msgid=?", inbox[inboxcur][0])
                                 inbox[inboxcur][7] = 1
                             else:
-                                scrollbox(d, unicode("Could not fetch message."))
+                                scrollbox(d, str("Could not fetch message."))
                         elif t == "2": # Mark unread
                             sqlExecute("UPDATE inbox SET read=0 WHERE msgid=?", inbox[inboxcur][0])
                             inbox[inboxcur][7] = 0
@@ -321,7 +333,7 @@ def handlech(c, stdscr):
                                     ischan = True
                                     break
                             if not addresses[i][1]:
-                                scrollbox(d, unicode("Sending address disabled, please either enable it or choose a different address."))
+                                scrollbox(d, str("Sending address disabled, please either enable it or choose a different address."))
                                 return
                             toaddr = m[2]
                             if ischan:
@@ -351,7 +363,7 @@ def handlech(c, stdscr):
                                     addrbook.append([label, addr])
                                     addrbook.reverse()
                             else:
-                                scrollbox(d, unicode("The selected address is already in the Address Book."))
+                                scrollbox(d, str("The selected address is already in the Address Book."))
                         elif t == "5": # Save message
                             set_background_title(d, "Save \""+inbox[inboxcur][5]+"\" as text file")
                             r, t = d.inputbox("Filename", init=inbox[inboxcur][5]+".txt")
@@ -365,11 +377,11 @@ def handlech(c, stdscr):
                                     fh.write(msg)
                                     fh.close()
                                 else:
-                                    scrollbox(d, unicode("Could not fetch message."))
+                                    scrollbox(d, str("Could not fetch message."))
                         elif t == "6": # Move to trash
                             sqlExecute("UPDATE inbox SET folder='trash' WHERE msgid=?", inbox[inboxcur][0])
                             del inbox[inboxcur]
-                            scrollbox(d, unicode("Message moved to trash. There is no interface to view your trash, \nbut the message is still on disk if you are desperate to recover it."))
+                            scrollbox(d, str("Message moved to trash. There is no interface to view your trash, \nbut the message is still on disk if you are desperate to recover it."))
                 elif menutab == 2:
                     a = ""
                     if addresses[addrcur][3] != 0: # if current address is a chan
@@ -392,13 +404,13 @@ def handlech(c, stdscr):
                                 msg = ""
                                 for i, item in enumerate(data.split("\n")):
                                     msg += fill(item, replace_whitespace=False)+"\n"
-                                scrollbox(d, unicode(ascii(msg)), 30, 80)
+                                scrollbox(d, str(ascii(msg)), 30, 80)
                             else:
-                                scrollbox(d, unicode("Could not fetch message."))
+                                scrollbox(d, str("Could not fetch message."))
                         elif t == "2": # Move to trash
                             sqlExecute("UPDATE sent SET folder='trash' WHERE subject=? AND ackdata=?", sentbox[sentcur][4], sentbox[sentcur][6])
                             del sentbox[sentcur]
-                            scrollbox(d, unicode("Message moved to trash. There is no interface to view your trash, \nbut the message is still on disk if you are desperate to recover it."))
+                            scrollbox(d, str("Message moved to trash. There is no interface to view your trash, \nbut the message is still on disk if you are desperate to recover it."))
                 elif menutab == 4:
                     set_background_title(d, "Your Identities Dialog Box")
                     if len(addresses) <= addrcur:
@@ -416,7 +428,7 @@ def handlech(c, stdscr):
                     if r == d.DIALOG_OK:
                         if t == "1": # Create new address
                             set_background_title(d, "Create new address")
-                            scrollbox(d, unicode("Here you may generate as many addresses as you like.\n"
+                            scrollbox(d, str("Here you may generate as many addresses as you like.\n"
                                 "Indeed, creating and abandoning addresses is encouraged.\n"
                                 "Deterministic addresses have several pros and cons:\n"
                                 "\nPros:\n"
@@ -474,12 +486,12 @@ def handlech(c, stdscr):
                                                     choices=[("1", "Spend time shortening the address", 1 if shorten else 0)])
                                                 if r == d.DIALOG_OK and "1" in t:
                                                     shorten = True
-                                                scrollbox(d, unicode("In addition to your passphrase, be sure to remember the following numbers:\n"
+                                                scrollbox(d, str("In addition to your passphrase, be sure to remember the following numbers:\n"
                                                     "\n  * Address version number: "+str(4)+"\n"
                                                     "  * Stream number: "+str(stream)))
                                                 queues.addressGeneratorQueue.put(('createDeterministicAddresses', 4, stream, "unused deterministic address", number, str(passphrase), shorten))
                                         else:
-                                            scrollbox(d, unicode("Passphrases do not match"))
+                                            scrollbox(d, str("Passphrases do not match"))
                         elif t == "2": # Send a message
                             a = ""
                             if addresses[addrcur][3] != 0: # if current address is a chan
@@ -527,7 +539,7 @@ def handlech(c, stdscr):
                             a = addresses[addrcur][2]
                             set_background_title(d, "Special address behavior")
                             if BMConfigParser().safeGetBoolean(a, "chan"):
-                                scrollbox(d, unicode("This is a chan address. You cannot use it as a pseudo-mailing list."))
+                                scrollbox(d, str("This is a chan address. You cannot use it as a pseudo-mailing list."))
                             else:
                                 m = BMConfigParser().safeGetBoolean(a, "mailinglist")
                                 r, t = d.radiolist("Select address behavior",
@@ -543,7 +555,7 @@ def handlech(c, stdscr):
                                     elif t == "2" and m == False:
                                         try:
                                             mn = BMConfigParser().get(a, "mailinglistname")
-                                        except ConfigParser.NoOptionError:
+                                        except configparser.NoOptionError:
                                            mn = ""
                                         r, t = d.inputbox("Mailing list name", init=mn)
                                         if r == d.DIALOG_OK:
@@ -632,7 +644,7 @@ def handlech(c, stdscr):
                                         addrbook.append([t, addr])
                                         addrbook.reverse()
                                 else:
-                                    scrollbox(d, unicode("The selected address is already in the Address Book."))
+                                    scrollbox(d, str("The selected address is already in the Address Book."))
                         elif t == "4":
                             r, t = d.inputbox("Type in \"I want to delete this Address Book entry\"")
                             if r == d.DIALOG_OK and t == "I want to delete this Address Book entry":
@@ -771,20 +783,20 @@ def sendMessage(sender="", recv="", broadcast=None, subject="", body="", reply=F
                         err += "Some data encoded in the address is malformed. There might be something wrong with the software of your acquaintance."
                     else:
                         err += "It is unknown what is wrong with the address."
-                    scrollbox(d, unicode(err))
+                    scrollbox(d, str(err))
                 else:
                     addr = addBMIfNotPresent(addr)
                     if version > 4 or version <= 1:
                         set_background_title(d, "Recipient address error")
-                        scrollbox(d, unicode("Could not understand version number " + version + "of address" + addr + "."))
+                        scrollbox(d, str("Could not understand version number " + version + "of address" + addr + "."))
                         continue
                     if stream > 1 or stream == 0:
                         set_background_title(d, "Recipient address error")
-                        scrollbox(d, unicode("Bitmessage currently only supports stream numbers of 1, unlike as requested for address " + addr + "."))
+                        scrollbox(d, str("Bitmessage currently only supports stream numbers of 1, unlike as requested for address " + addr + "."))
                         continue
                     if not network.stats.connectedHostsList():
                         set_background_title(d, "Not connected warning")
-                        scrollbox(d, unicode("Because you are not currently connected to the network, "))
+                        scrollbox(d, str("Because you are not currently connected to the network, "))
                     stealthLevel = BMConfigParser().safeGetInt('bitmessagesettings', 'ackstealthlevel')
                     ackdata = genAckPayload(streamNumber, stealthLevel)
                     sqlExecute(
@@ -808,7 +820,7 @@ def sendMessage(sender="", recv="", broadcast=None, subject="", body="", reply=F
     else: # Broadcast
         if recv == "":
             set_background_title(d, "Empty sender error")
-            scrollbox(d, unicode("You must specify an address to send the message from."))
+            scrollbox(d, str("You must specify an address to send the message from."))
         else:
             # dummy ackdata, no need for stealth
             ackdata = genAckPayload(streamNumber, 0)

src/bitmessagekivy/kivy_helper_search.py

@@ -1,3 +1,10 @@
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
 from helper_sql import *
 
 

src/bitmessagekivy/mpybit.py

@@ -1,4 +1,11 @@
-import kivy_helper_search
+from __future__ import print_function
+from __future__ import absolute_import
+from __future__ import unicode_literals
+from __future__ import division
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
+from . import kivy_helper_search
 import os
 import queues
 import shutdown
@@ -308,7 +315,7 @@ class Create(Screen):
         message = self.ids.message.text
         subject = self.ids.subject.text
         encoding = 3
-        print("message: ", self.ids.message.text)
+        print(("message: ", self.ids.message.text))
         sendMessageToPeople = True
         if sendMessageToPeople:
             if toAddress != '':

src/bitmessagemain.py

@@ -53,8 +53,7 @@ from bmconfigparser import BMConfigParser
 
 from inventory import Inventory
 
-from network.connectionpool import BMConnectionPool
-from network.dandelion import Dandelion
+from network.fix_circular_imports import BMConnectionPool, Dandelion
 from network.networkthread import BMNetworkThread
 from network.receivequeuethread import ReceiveQueueThread
 from network.announcethread import AnnounceThread
@@ -472,8 +471,8 @@ class Main:
         # signal.signal(signal.SIGINT, signal.SIG_DFL)
 
     def usage(self):
-        print 'Usage: ' + sys.argv[0] + ' [OPTIONS]'
-        print '''
+        print('Usage: ' + sys.argv[0] + ' [OPTIONS]')
+        print('''
 Options:
   -h, --help            show this help message and exit
   -c, --curses          use curses (text mode) interface
@@ -482,6 +481,7 @@ Options:
 
 All parameters are optional.
 '''
+        )
 
     def stop(self):
         with shared.printLock:

src/bitmessageqt/__init__.py

@@ -1,7 +1,17 @@
 """
 PyQt based UI for bitmessage, the main module
 """
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+from __future__ import print_function
 
+from future import standard_library
+standard_library.install_aliases()
+from builtins import str
+from builtins import range
+from builtins import *
+from past.utils import old_div
 import hashlib
 import locale
 import os
@@ -21,41 +31,41 @@ from debug import logger
 from tr import _translate
 from addresses import decodeAddress, addBMIfNotPresent
 import shared
-from bitmessageui import Ui_MainWindow
+from .bitmessageui import Ui_MainWindow
 from bmconfigparser import BMConfigParser
 import defaults
 import namecoin
-from messageview import MessageView
-from migrationwizard import Ui_MigrationWizard
-from foldertree import (
+from .messageview import MessageView
+from .migrationwizard import Ui_MigrationWizard
+from .foldertree import (
     AccountMixin, Ui_FolderWidget, Ui_AddressWidget, Ui_SubscriptionWidget,
     MessageList_AddressWidget, MessageList_SubjectWidget,
     Ui_AddressBookWidgetItemLabel, Ui_AddressBookWidgetItemAddress)
-from settings import Ui_settingsDialog
-import settingsmixin
-import support
+from .settings import Ui_settingsDialog
+from . import settingsmixin
+from . import support
 import debug
 from helper_ackPayload import genAckPayload
 from helper_sql import sqlQuery, sqlExecute, sqlExecuteChunked, sqlStoredProcedure
 import helper_search
 import l10n
 import openclpow
-from utils import str_broadcast_subscribers, avatarize
-from account import (
+from .utils import str_broadcast_subscribers, avatarize
+from .account import (
     getSortedAccounts, getSortedSubscriptions, accountClass, BMAccount,
     GatewayAccount, MailchuckAccount, AccountColor)
-import dialogs
+from . import dialogs
 from network.stats import pendingDownload, pendingUpload
-from uisignaler import UISignaler
+from .uisignaler import UISignaler
 import knownnodes
 import paths
 from proofofwork import getPowType
 import queues
 import shutdown
 import state
-from statusbar import BMStatusBar
+from .statusbar import BMStatusBar
 from network.asyncore_pollchoose import set_rates
-import sound
+from . import sound
 
 
 try:
@@ -404,7 +414,7 @@ class MyForm(settingsmixin.SMainWindow):
 
     def rerenderTabTreeSubscriptions(self):
         treeWidget = self.ui.treeWidgetSubscriptions
-        folders = Ui_FolderWidget.folderWeight.keys()
+        folders = list(Ui_FolderWidget.folderWeight.keys())
         folders.remove("new")
 
         # sort ascending when creating
@@ -452,7 +462,7 @@ class MyForm(settingsmixin.SMainWindow):
             # add missing folders
             if len(db[toAddress]) > 0:
                 j = 0
-                for f, c in db[toAddress].iteritems():
+                for f, c in db[toAddress].items():
                     try:
                         subwidget = Ui_FolderWidget(widget, j, toAddress, f, c['count'])
                     except KeyError:
@@ -491,7 +501,7 @@ class MyForm(settingsmixin.SMainWindow):
             treeWidget = self.ui.treeWidgetYourIdentities
         elif tab == 'chan':
             treeWidget = self.ui.treeWidgetChans
-        folders = Ui_FolderWidget.folderWeight.keys()
+        folders = list(Ui_FolderWidget.folderWeight.keys())
         
         # sort ascending when creating
         if treeWidget.topLevelItemCount() == 0:
@@ -572,7 +582,7 @@ class MyForm(settingsmixin.SMainWindow):
             # add missing folders
             if len(db[toAddress]) > 0:
                 j = 0
-                for f, c in db[toAddress].iteritems():
+                for f, c in db[toAddress].items():
                     if toAddress is not None and tab == 'messages' and folder == "new":
                         continue
                     subwidget = Ui_FolderWidget(widget, j, toAddress, f, c)
@@ -835,7 +845,7 @@ class MyForm(settingsmixin.SMainWindow):
         BMConfigParser().save()
 
     def updateHumanFriendlyTTLDescription(self, TTL):
-        numberOfHours = int(round(TTL / (60*60)))
+        numberOfHours = int(round(old_div(TTL, (60*60))))
         font = QtGui.QFont()
         stylesheet = ""
 
@@ -848,7 +858,7 @@ class MyForm(settingsmixin.SMainWindow):
             stylesheet = "QLabel { color : red; }"
             font.setBold(True)
         else:
-            numberOfDays = int(round(TTL / (24*60*60)))
+            numberOfDays = int(round(old_div(TTL, (24*60*60))))
             self.ui.labelHumanFriendlyTTLDescription.setText(_translate("MainWindow", "%n day(s)", None, QtCore.QCoreApplication.CodecForTr, numberOfDays))
             font.setBold(False)
         self.ui.labelHumanFriendlyTTLDescription.setStyleSheet(stylesheet)
@@ -955,7 +965,7 @@ class MyForm(settingsmixin.SMainWindow):
             # rrow = related.row(msgid), msgid should be QTableWidgetItem
             # related = related.findItems(msgid, QtCore.Qt.MatchExactly),
             # returns an empty list
-            for rrow in xrange(related.rowCount()):
+            for rrow in range(related.rowCount()):
                 if msgid == str(related.item(rrow, 3).data(
                         QtCore.Qt.UserRole).toPyObject()):
                     break
@@ -1022,7 +1032,7 @@ class MyForm(settingsmixin.SMainWindow):
             for i in range(root.childCount()):
                 addressItem = root.child(i)
                 if addressItem.type == AccountMixin.ALL:
-                    newCount = sum(totalUnread.itervalues())
+                    newCount = sum(totalUnread.values())
                     self.drawTrayIcon(self.currentTrayIconFileName, newCount)
                 else:
                     try:
@@ -1030,7 +1040,7 @@ class MyForm(settingsmixin.SMainWindow):
                             broadcastsUnread
                             if addressItem.type == AccountMixin.SUBSCRIPTION
                             else normalUnread
-                        )[addressItem.address].itervalues())
+                        )[addressItem.address].values())
                     except KeyError:
                         newCount = 0
                 if newCount != addressItem.unreadCount:
@@ -1073,9 +1083,9 @@ class MyForm(settingsmixin.SMainWindow):
         acct.parseMessage(toAddress, fromAddress, subject, "")
 
         items = []
-        MessageList_AddressWidget(items, str(toAddress), unicode(acct.toLabel, 'utf-8'))
-        MessageList_AddressWidget(items, str(fromAddress), unicode(acct.fromLabel, 'utf-8'))
-        MessageList_SubjectWidget(items, str(subject), unicode(acct.subject, 'utf-8', 'replace'))
+        MessageList_AddressWidget(items, str(toAddress), str(acct.toLabel, 'utf-8'))
+        MessageList_AddressWidget(items, str(fromAddress), str(acct.fromLabel, 'utf-8'))
+        MessageList_SubjectWidget(items, str(subject), str(acct.subject, 'utf-8', 'replace'))
 
         if status == 'awaitingpubkey':
             statusText = _translate(
@@ -1144,11 +1154,11 @@ class MyForm(settingsmixin.SMainWindow):
             
         items = []
         #to
-        MessageList_AddressWidget(items, toAddress, unicode(acct.toLabel, 'utf-8'), not read)
+        MessageList_AddressWidget(items, toAddress, str(acct.toLabel, 'utf-8'), not read)
         # from
-        MessageList_AddressWidget(items, fromAddress, unicode(acct.fromLabel, 'utf-8'), not read)
+        MessageList_AddressWidget(items, fromAddress, str(acct.fromLabel, 'utf-8'), not read)
         # subject
-        MessageList_SubjectWidget(items, str(subject), unicode(acct.subject, 'utf-8', 'replace'), not read)
+        MessageList_SubjectWidget(items, str(subject), str(acct.subject, 'utf-8', 'replace'), not read)
         # time received
         time_item = myTableWidgetItem(l10n.formatTimestamp(received))
         time_item.setToolTip(l10n.formatTimestamp(received))
@@ -1427,7 +1437,7 @@ class MyForm(settingsmixin.SMainWindow):
             self, title, subtitle, category, label=None, icon=None):
         self.playSound(category, label)
         self._notifier(
-            unicode(title), unicode(subtitle), category, label, icon)
+            str(title), str(subtitle), category, label, icon)
 
     # tree
     def treeWidgetKeyPressEvent(self, event):
@@ -1856,9 +1866,9 @@ class MyForm(settingsmixin.SMainWindow):
     def rerenderAddressBook(self):
         def addRow (address, label, type):
             self.ui.tableWidgetAddressBook.insertRow(0)
-            newItem = Ui_AddressBookWidgetItemLabel(address, unicode(label, 'utf-8'), type)
+            newItem = Ui_AddressBookWidgetItemLabel(address, str(label, 'utf-8'), type)
             self.ui.tableWidgetAddressBook.setItem(0, 0, newItem)
-            newItem = Ui_AddressBookWidgetItemAddress(address, unicode(label, 'utf-8'), type)
+            newItem = Ui_AddressBookWidgetItemAddress(address, str(label, 'utf-8'), type)
             self.ui.tableWidgetAddressBook.setItem(0, 1, newItem)
 
         oldRows = {}
@@ -1892,13 +1902,13 @@ class MyForm(settingsmixin.SMainWindow):
         completerList = []
         for address in sorted(oldRows, key = lambda x: oldRows[x][2], reverse = True):
             if address in newRows:
-                completerList.append(unicode(newRows[address][0], encoding="UTF-8") + " <" + address + ">")
+                completerList.append(str(newRows[address][0], encoding="UTF-8") + " <" + address + ">")
                 newRows.pop(address)
             else:
                 self.ui.tableWidgetAddressBook.removeRow(oldRows[address][2])
         for address in newRows:
             addRow(address, newRows[address][0], newRows[address][1])
-            completerList.append(unicode(newRows[address][0], encoding="UTF-8") + " <" + address + ">")
+            completerList.append(str(newRows[address][0], encoding="UTF-8") + " <" + address + ">")
 
         # sort
         self.ui.tableWidgetAddressBook.sortByColumn(
@@ -2010,7 +2020,7 @@ class MyForm(settingsmixin.SMainWindow):
                         toAddress)
                     if status != 'success':
                         try:
-                            toAddress = unicode(toAddress, 'utf-8', 'ignore')
+                            toAddress = str(toAddress, 'utf-8', 'ignore')
                         except:
                             pass
                         logger.error('Error: Could not decode recipient address ' + toAddress + ':' + status)
@@ -2235,7 +2245,7 @@ class MyForm(settingsmixin.SMainWindow):
                 addressInKeysFile, 'enabled')  # I realize that this is poor programming practice but I don't care. It's easier for others to read.
             isMaillinglist = BMConfigParser().safeGetBoolean(addressInKeysFile, 'mailinglist')
             if isEnabled and not isMaillinglist:
-                label = unicode(BMConfigParser().get(addressInKeysFile, 'label'), 'utf-8', 'ignore').strip()
+                label = str(BMConfigParser().get(addressInKeysFile, 'label'), 'utf-8', 'ignore').strip()
                 if label == "":
                     label = addressInKeysFile
                 self.ui.comboBoxSendFrom.addItem(avatarize(addressInKeysFile), label, addressInKeysFile)
@@ -2259,7 +2269,7 @@ class MyForm(settingsmixin.SMainWindow):
                 addressInKeysFile, 'enabled')  # I realize that this is poor programming practice but I don't care. It's easier for others to read.
             isChan = BMConfigParser().safeGetBoolean(addressInKeysFile, 'chan')
             if isEnabled and not isChan:
-                label = unicode(BMConfigParser().get(addressInKeysFile, 'label'), 'utf-8', 'ignore').strip()
+                label = str(BMConfigParser().get(addressInKeysFile, 'label'), 'utf-8', 'ignore').strip()
                 if label == "":
                     label = addressInKeysFile
                 self.ui.comboBoxSendFromBroadcast.addItem(avatarize(addressInKeysFile), label, addressInKeysFile)
@@ -2298,7 +2308,7 @@ class MyForm(settingsmixin.SMainWindow):
                 continue
             
             self.addMessageListItemSent(sent, toAddress, fromAddress, subject, "msgqueued", ackdata, time.time())
-            self.getAccountTextedit(acct).setPlainText(unicode(message, 'utf-8', 'replace'))
+            self.getAccountTextedit(acct).setPlainText(str(message, 'utf-8', 'replace'))
             sent.setCurrentCell(0, 0)
 
     def displayNewInboxMessage(self, inventoryHash, toAddress, fromAddress, subject, message):
@@ -2331,7 +2341,7 @@ class MyForm(settingsmixin.SMainWindow):
             self.notifierShow(
                 _translate("MainWindow", "New Message"),
                 _translate("MainWindow", "From %1").arg(
-                    unicode(acct.fromLabel, 'utf-8')),
+                    str(acct.fromLabel, 'utf-8')),
                 sound.SOUND_UNKNOWN
             )
         if self.getCurrentAccount() is not None and ((self.getCurrentFolder(treeWidget) != "inbox" and self.getCurrentFolder(treeWidget) is not None) or self.getCurrentAccount(treeWidget) != acct.address):
@@ -2879,8 +2889,8 @@ class MyForm(settingsmixin.SMainWindow):
                 if curWorkerQueue > 0:
                     self.updateStatusBar(_translate(
                         "MainWindow", "Waiting for PoW to finish... %1%"
-                    ).arg(50 * (maxWorkerQueue - curWorkerQueue) /
-                          maxWorkerQueue))
+                    ).arg(old_div(50 * (maxWorkerQueue - curWorkerQueue),
+                          maxWorkerQueue)))
                     time.sleep(0.5)
                     QtCore.QCoreApplication.processEvents(
                         QtCore.QEventLoop.AllEvents, 1000
@@ -2908,7 +2918,7 @@ class MyForm(settingsmixin.SMainWindow):
                 self.updateStatusBar(_translate(
                     "MainWindow",
                     "Waiting for objects to be sent... %1%"
-                ).arg(int(50 + 20 * (pendingUpload() / maxPendingUpload))))
+                ).arg(int(50 + 20 * (old_div(pendingUpload(), maxPendingUpload)))))
                 time.sleep(0.5)
                 QtCore.QCoreApplication.processEvents(
                     QtCore.QEventLoop.AllEvents, 1000
@@ -2928,7 +2938,7 @@ class MyForm(settingsmixin.SMainWindow):
             QtCore.QEventLoop.AllEvents, 1000
         )
         self.saveSettings()
-        for attr, obj in self.ui.__dict__.iteritems():
+        for attr, obj in self.ui.__dict__.items():
             if hasattr(obj, "__class__") \
                     and isinstance(obj, settingsmixin.SettingsMixin):
                 saveMethod = getattr(obj, "saveSettings", None)
@@ -2978,7 +2988,7 @@ class MyForm(settingsmixin.SMainWindow):
 
         lines = messageText.split('\n')
         totalLines = len(lines)
-        for i in xrange(totalLines):
+        for i in range(totalLines):
             if 'Message ostensibly from ' in lines[i]:
                 lines[i] = '<p style="font-size: 12px; color: grey;">%s</span></p>' % (
                     lines[i])
@@ -2989,7 +2999,7 @@ class MyForm(settingsmixin.SMainWindow):
                 lines[i] = '<br><br>'
         content = ' '.join(lines) # To keep the whitespace between lines
         content = shared.fixPotentiallyInvalidUTF8Data(content)
-        content = unicode(content, 'utf-8)')
+        content = str(content, 'utf-8)')
         textEdit.setHtml(QtCore.QString(content))
 
     def on_action_InboxMarkUnread(self):
@@ -3172,7 +3182,7 @@ class MyForm(settingsmixin.SMainWindow):
         self.setSendFromComboBox(toAddressAtCurrentInboxRow)
 
         quotedText = self.quoted_text(
-            unicode(messageAtCurrentInboxRow, 'utf-8', 'replace'))
+            str(messageAtCurrentInboxRow, 'utf-8', 'replace'))
         widget['message'].setPlainText(quotedText)
         if acct.subject[0:3] in ['Re:', 'RE:']:
             widget['subject'].setText(
@@ -3416,7 +3426,7 @@ class MyForm(settingsmixin.SMainWindow):
             return self.updateStatusBar(_translate(
                 "MainWindow", "No addresses selected."))
 
-        addresses_string = unicode(
+        addresses_string = str(
             self.ui.lineEditTo.text().toUtf8(), 'utf-8')
         for item in selected_items:
             address_string = item.accountString()
@@ -3804,7 +3814,7 @@ class MyForm(settingsmixin.SMainWindow):
             text = str(tableWidget.item(currentRow, currentColumn).label)
         else:
             text = tableWidget.item(currentRow, currentColumn).data(QtCore.Qt.UserRole)
-        text = unicode(str(text), 'utf-8', 'ignore')
+        text = str(str(text), 'utf-8', 'ignore')
         clipboard = QtGui.QApplication.clipboard()
         clipboard.setText(text)
 
@@ -3900,11 +3910,11 @@ class MyForm(settingsmixin.SMainWindow):
         self.setAddressSound(widget.item(widget.currentRow(), 0).text())
 
     def setAddressSound(self, addr):
-        filters = [unicode(_translate(
+        filters = [str(_translate(
             "MainWindow", "Sound files (%s)" %
             ' '.join(['*%s%s' % (os.extsep, ext) for ext in sound.extensions])
         ))]
-        sourcefile = unicode(QtGui.QFileDialog.getOpenFileName(
+        sourcefile = str(QtGui.QFileDialog.getOpenFileName(
             self, _translate("MainWindow", "Set notification sound..."),
             filter=';;'.join(filters)
         ))
@@ -3913,7 +3923,7 @@ class MyForm(settingsmixin.SMainWindow):
             return
 
         destdir = os.path.join(state.appdata, 'sounds')
-        destfile = unicode(addr) + os.path.splitext(sourcefile)[-1]
+        destfile = str(addr) + os.path.splitext(sourcefile)[-1]
         destination = os.path.join(destdir, destfile)
 
         if sourcefile == destination:
@@ -4120,7 +4130,7 @@ class MyForm(settingsmixin.SMainWindow):
         if item.type == AccountMixin.ALL:
             return
         
-        newLabel = unicode(item.text(0), 'utf-8', 'ignore')
+        newLabel = str(item.text(0), 'utf-8', 'ignore')
         oldLabel = item.defaultLabel()
 
         # unchanged, do not do anything either
@@ -4191,7 +4201,7 @@ class MyForm(settingsmixin.SMainWindow):
         self.rerenderMessagelistToLabels()
         completerList = self.ui.lineEditTo.completer().model().stringList()
         for i in range(len(completerList)):
-            if unicode(completerList[i]).endswith(" <" + item.address + ">"):
+            if str(completerList[i]).endswith(" <" + item.address + ">"):
                 completerList[i] = item.label + " <" + item.address + ">"
         self.ui.lineEditTo.completer().model().setStringList(completerList)
 
@@ -4245,7 +4255,7 @@ class MyForm(settingsmixin.SMainWindow):
         QtCore.QCoreApplication.setOrganizationDomain("bitmessage.org")
         QtCore.QCoreApplication.setApplicationName("pybitmessageqt")
         self.loadSettings()
-        for attr, obj in self.ui.__dict__.iteritems():
+        for attr, obj in self.ui.__dict__.items():
             if hasattr(obj, "__class__") and \
                     isinstance(obj, settingsmixin.SettingsMixin):
                 loadMethod = getattr(obj, "loadSettings", None)

src/bitmessageqt/account.py

@@ -8,7 +8,16 @@ Account related functions.
 """
 
 from __future__ import absolute_import
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
 
+from past.builtins import cmp
+from future import standard_library
+standard_library.install_aliases()
+from builtins import str
+from builtins import *
+from builtins import object
 import inspect
 import re
 import sys
@@ -31,12 +40,12 @@ def getSortedAccounts():
     configSections = BMConfigParser().addresses()
     configSections.sort(
         cmp=lambda x, y: cmp(
-            unicode(
+            str(
                 BMConfigParser().get(
                     x,
                     'label'),
                 'utf-8').lower(),
-            unicode(
+            str(
                 BMConfigParser().get(
                     y,
                     'label'),
@@ -169,7 +178,7 @@ class BMAccount(object):
 
         self.toAddress = toAddress
         self.fromAddress = fromAddress
-        if isinstance(subject, unicode):
+        if isinstance(subject, str):
             self.subject = str(subject)
         else:
             self.subject = subject

src/bitmessageqt/address_dialogs.py

@@ -3,18 +3,27 @@ src/bitmessageqt/address_dialogs.py
 ===================================
 
 """
+from __future__ import absolute_import
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
 # pylint: disable=attribute-defined-outside-init
 
+from future import standard_library
+standard_library.install_aliases()
+from builtins import str
+from builtins import *
+from builtins import object
 import hashlib
 
 from PyQt4 import QtCore, QtGui
 
 import queues
-import widgets
-from account import AccountMixin, GatewayAccount, MailchuckAccount, accountClass, getSortedAccounts
+from . import widgets
+from .account import AccountMixin, GatewayAccount, MailchuckAccount, accountClass, getSortedAccounts
 from addresses import addBMIfNotPresent, decodeAddress, encodeVarint
 from inventory import Inventory
-from retranslateui import RetranslateMixin
+from .retranslateui import RetranslateMixin
 from tr import _translate
 
 
@@ -262,7 +271,7 @@ class SpecialAddressBehaviorDialog(QtGui.QDialog, RetranslateMixin):
                 except:
                     mailingListName = ''
                 self.lineEditMailingListName.setText(
-                    unicode(mailingListName, 'utf-8')
+                    str(mailingListName, 'utf-8')
                 )
 
         QtGui.QWidget.resize(self, QtGui.QWidget.sizeHint(self))

src/bitmessageqt/addressvalidator.py

@@ -1,13 +1,22 @@
+from __future__ import absolute_import
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from future import standard_library
+standard_library.install_aliases()
+from builtins import str
+from builtins import *
+from builtins import object
 from PyQt4 import QtGui
-from Queue import Empty
+from queue import Empty
 
 from addresses import decodeAddress, addBMIfNotPresent
-from account import getSortedAccounts
+from .account import getSortedAccounts
 from queues import apiAddressGeneratorReturnQueue, addressGeneratorQueue
 from tr import _translate
-from utils import str_chan
+from .utils import str_chan
 
-class AddressPassPhraseValidatorMixin():
+class AddressPassPhraseValidatorMixin(object):
     def setParams(self, passPhraseObject=None, addressObject=None, feedBackObject=None, buttonBox=None, addressMandatory=True):
         self.addressObject = addressObject
         self.passPhraseObject = passPhraseObject

src/bitmessageqt/bitmessage_icons_rc.py

@@ -7,9 +7,16 @@
 #
 # WARNING! All changes made in this file will be lost!
 
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
 from PyQt4 import QtCore
 
-qt_resource_data = "\
+qt_resource_data = b"\
 \x00\x00\x03\x66\
 \x89\
 \x50\x4e\x47\x0d\x0a\x1a\x0a\x00\x00\x00\x0d\x49\x48\x44\x52\x00\
@@ -1534,7 +1541,7 @@ qt_resource_data = "\
 \x82\
 "
 
-qt_resource_name = "\
+qt_resource_name = b"\
 \x00\x09\
 \x0c\x78\x54\x88\
 \x00\x6e\
@@ -1639,7 +1646,7 @@ qt_resource_name = "\
 \x00\x70\x00\x6e\x00\x67\
 "
 
-qt_resource_struct = "\
+qt_resource_struct = b"\
 \x00\x00\x00\x00\x00\x02\x00\x00\x00\x01\x00\x00\x00\x01\
 \x00\x00\x00\x00\x00\x02\x00\x00\x00\x01\x00\x00\x00\x02\
 \x00\x00\x00\x18\x00\x02\x00\x00\x00\x15\x00\x00\x00\x03\

src/bitmessageqt/bitmessageui.py

@@ -7,14 +7,23 @@
 #
 # WARNING! All changes made in this file will be lost!
 
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+from __future__ import print_function
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
+from builtins import object
+from past.utils import old_div
 from PyQt4 import QtCore, QtGui
 from bmconfigparser import BMConfigParser
-from foldertree import AddressBookCompleter
-from messageview import MessageView
-from messagecompose import MessageCompose
-import settingsmixin
-from networkstatus import NetworkStatus
-from blacklist import Blacklist
+from .foldertree import AddressBookCompleter
+from .messageview import MessageView
+from .messagecompose import MessageCompose
+from . import settingsmixin
+from .networkstatus import NetworkStatus
+from .blacklist import Blacklist
 
 try:
     _fromUtf8 = QtCore.QString.fromUtf8
@@ -708,7 +717,7 @@ class Ui_MainWindow(object):
         self.pushButtonTTL.setText(_translate("MainWindow", "TTL:", None))
         hours = 48
         try:
-            hours = int(BMConfigParser().getint('bitmessagesettings', 'ttl')/60/60)
+            hours = int(old_div(BMConfigParser().getint('bitmessagesettings', 'ttl'),60/60))
         except:
             pass
         self.labelHumanFriendlyTTLDescription.setText(_translate("MainWindow", "%n hour(s)", None, QtCore.QCoreApplication.CodecForTr, hours))
@@ -771,7 +780,7 @@ class Ui_MainWindow(object):
         self.actionDeleteAllTrashedMessages.setText(_translate("MainWindow", "Delete all trashed messages", None))
         self.actionJoinChan.setText(_translate("MainWindow", "Join / Create chan", None))
 
-import bitmessage_icons_rc
+from . import bitmessage_icons_rc
 
 if __name__ == "__main__":
     import sys

src/bitmessageqt/blacklist.py

@@ -1,15 +1,23 @@
+from __future__ import absolute_import
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from future import standard_library
+standard_library.install_aliases()
+from builtins import str
+from builtins import *
 from PyQt4 import QtCore, QtGui
 
-import widgets
+from . import widgets
 from addresses import addBMIfNotPresent
 from bmconfigparser import BMConfigParser
-from dialogs import AddAddressDialog
+from .dialogs import AddAddressDialog
 from helper_sql import sqlExecute, sqlQuery
 from queues import UISignalQueue
-from retranslateui import RetranslateMixin
+from .retranslateui import RetranslateMixin
 from tr import _translate
-from uisignaler import UISignaler
-from utils import avatarize
+from .uisignaler import UISignaler
+from .utils import avatarize
 
 
 class Blacklist(QtGui.QWidget, RetranslateMixin):
@@ -73,7 +81,7 @@ class Blacklist(QtGui.QWidget, RetranslateMixin):
                 if queryreturn == []:
                     self.tableWidgetBlacklist.setSortingEnabled(False)
                     self.tableWidgetBlacklist.insertRow(0)
-                    newItem = QtGui.QTableWidgetItem(unicode(
+                    newItem = QtGui.QTableWidgetItem(str(
                         self.NewBlacklistDialogInstance.lineEditLabel.text().toUtf8(), 'utf-8'))
                     newItem.setIcon(avatarize(address))
                     self.tableWidgetBlacklist.setItem(0, 0, newItem)
@@ -172,7 +180,7 @@ class Blacklist(QtGui.QWidget, RetranslateMixin):
         for row in queryreturn:
             label, address, enabled = row
             self.tableWidgetBlacklist.insertRow(0)
-            newItem = QtGui.QTableWidgetItem(unicode(label, 'utf-8'))
+            newItem = QtGui.QTableWidgetItem(str(label, 'utf-8'))
             if not enabled:
                 newItem.setTextColor(QtGui.QColor(128, 128, 128))
             newItem.setIcon(avatarize(address))

src/bitmessageqt/dialogs.py

@@ -2,19 +2,27 @@
 src/bitmessageqt/dialogs.py
 ===========================
 """
+from __future__ import absolute_import
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
 
+from future import standard_library
+standard_library.install_aliases()
+from builtins import str
+from builtins import *
 from PyQt4 import QtGui
 
 from version import softwareVersion
 
 import paths
-import widgets
-from address_dialogs import (
+from . import widgets
+from .address_dialogs import (
     AddAddressDialog, EmailGatewayDialog, NewAddressDialog, NewSubscriptionDialog, RegenerateAddressesDialog,
     SpecialAddressBehaviorDialog
 )
-from newchandialog import NewChanDialog
-from retranslateui import RetranslateMixin
+from .newchandialog import NewChanDialog
+from .retranslateui import RetranslateMixin
 from tr import _translate
 
 __all__ = [

src/bitmessageqt/foldertree.py

@@ -2,17 +2,27 @@
 src/bitmessageqt/foldertree.py
 ==============================
 """
+from __future__ import absolute_import
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
 # pylint: disable=too-many-arguments,bad-super-call,attribute-defined-outside-init
 
+from future import standard_library
+standard_library.install_aliases()
+from builtins import str
+from builtins import range
+from builtins import *
+from builtins import object
 from cgi import escape
 
 from PyQt4 import QtCore, QtGui
 
 from bmconfigparser import BMConfigParser
 from helper_sql import sqlExecute, sqlQuery
-from settingsmixin import SettingsMixin
+from .settingsmixin import SettingsMixin
 from tr import _translate
-from utils import avatarize
+from .utils import avatarize
 
 # for pylupdate
 _translate("MainWindow", "inbox")
@@ -122,7 +132,7 @@ class AccountMixin(object):
                 AccountMixin.NORMAL,
                 AccountMixin.CHAN, AccountMixin.MAILINGLIST):
             try:
-                retval = unicode(
+                retval = str(
                     BMConfigParser().get(self.address, 'label'), 'utf-8')
             except Exception:
                 queryreturn = sqlQuery(
@@ -134,12 +144,12 @@ class AccountMixin(object):
             if queryreturn != []:
                 for row in queryreturn:
                     retval, = row
-                    retval = unicode(retval, 'utf-8')
+                    retval = str(retval, 'utf-8')
         elif self.address is None or self.type == AccountMixin.ALL:
-            return unicode(
+            return str(
                 str(_translate("MainWindow", "All accounts")), 'utf-8')
 
-        return retval or unicode(self.address, 'utf-8')
+        return retval or str(self.address, 'utf-8')
 
 
 class BMTreeWidgetItem(QtGui.QTreeWidgetItem, AccountMixin):
@@ -230,15 +240,15 @@ class Ui_AddressWidget(BMTreeWidgetItem, SettingsMixin):
 
     def _getLabel(self):
         if self.address is None:
-            return unicode(_translate(
+            return str(_translate(
                 "MainWindow", "All accounts").toUtf8(), 'utf-8', 'ignore')
         else:
             try:
-                return unicode(
+                return str(
                     BMConfigParser().get(self.address, 'label'),
                     'utf-8', 'ignore')
             except:
-                return unicode(self.address, 'utf-8')
+                return str(self.address, 'utf-8')
 
     def _getAddressBracket(self, unreadCount=False):
         ret = "" if self.isExpanded() \
@@ -309,8 +319,8 @@ class Ui_SubscriptionWidget(Ui_AddressWidget):
         if queryreturn != []:
             for row in queryreturn:
                 retval, = row
-            return unicode(retval, 'utf-8', 'ignore')
-        return unicode(self.address, 'utf-8')
+            return str(retval, 'utf-8', 'ignore')
+        return str(self.address, 'utf-8')
 
     def setType(self):
         """Set account type"""
@@ -324,7 +334,7 @@ class Ui_SubscriptionWidget(Ui_AddressWidget):
                 label = str(
                     value.toString().toUtf8()).decode('utf-8', 'ignore')
             else:
-                label = unicode(value, 'utf-8', 'ignore')
+                label = str(value, 'utf-8', 'ignore')
             sqlExecute(
                 '''UPDATE subscriptions SET label=? WHERE address=?''',
                 label, self.address)
@@ -407,7 +417,7 @@ class MessageList_AddressWidget(BMAddressWidget):
                 AccountMixin.NORMAL,
                 AccountMixin.CHAN, AccountMixin.MAILINGLIST):
             try:
-                newLabel = unicode(
+                newLabel = str(
                     BMConfigParser().get(self.address, 'label'),
                     'utf-8', 'ignore')
             except:
@@ -418,7 +428,7 @@ class MessageList_AddressWidget(BMAddressWidget):
                 '''select label from subscriptions where address=?''', self.address)
         if queryreturn:
             for row in queryreturn:
-                newLabel = unicode(row[0], 'utf-8', 'ignore')
+                newLabel = str(row[0], 'utf-8', 'ignore')
 
         self.label = newLabel
 
@@ -459,7 +469,7 @@ class MessageList_SubjectWidget(BMTableWidgetItem):
         if role == QtCore.Qt.UserRole:
             return self.subject
         if role == QtCore.Qt.ToolTipRole:
-            return escape(unicode(self.subject, 'utf-8'))
+            return escape(str(self.subject, 'utf-8'))
         return super(MessageList_SubjectWidget, self).data(role)
 
     # label (or address) alphabetically, disabled at the end
@@ -557,14 +567,14 @@ class AddressBookCompleter(QtGui.QCompleter):
 
     def splitPath(self, path):
         """Split on semicolon"""
-        text = unicode(path.toUtf8(), 'utf-8')
+        text = str(path.toUtf8(), 'utf-8')
         return [text[:self.widget().cursorPosition()].split(';')[-1].strip()]
 
     def pathFromIndex(self, index):
         """Perform autocompletion (reimplemented QCompleter method)"""
-        autoString = unicode(
+        autoString = str(
             index.data(QtCore.Qt.EditRole).toString().toUtf8(), 'utf-8')
-        text = unicode(self.widget().text().toUtf8(), 'utf-8')
+        text = str(self.widget().text().toUtf8(), 'utf-8')
 
         # If cursor position was saved, restore it, else save it
         if self.cursorPos != -1:

src/bitmessageqt/languagebox.py

@@ -1,3 +1,11 @@
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import range
+from builtins import *
 import glob
 import os
 from PyQt4 import QtCore, QtGui

src/bitmessageqt/messagecompose.py

@@ -1,3 +1,12 @@
+from __future__ import division
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import str
+from builtins import *
+from past.utils import old_div
 from PyQt4 import QtCore, QtGui
 
 class MessageCompose(QtGui.QTextEdit):
@@ -13,7 +22,7 @@ class MessageCompose(QtGui.QTextEdit):
                 self.zoomIn(1)
             else:
                 self.zoomOut(1)
-            zoom = self.currentFont().pointSize() * 100 / self.defaultFontPointSize
+            zoom = old_div(self.currentFont().pointSize() * 100, self.defaultFontPointSize)
             QtGui.QApplication.activeWindow().statusBar().showMessage(QtGui.QApplication.translate("MainWindow", "Zoom level %1%").arg(str(zoom)))
         else:
             # in QTextEdit, super does not zoom, only scroll

src/bitmessageqt/messageview.py

@@ -3,10 +3,19 @@ src/bitmessageqt/messageview.py
 ===============================
 
 """
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+from __future__ import print_function
 
+from future import standard_library
+standard_library.install_aliases()
+from builtins import str
+from builtins import *
+from past.utils import old_div
 from PyQt4 import QtCore, QtGui
 
-from safehtmlparser import SafeHTMLParser
+from .safehtmlparser import SafeHTMLParser
 
 
 class MessageView(QtGui.QTextBrowser):
@@ -51,7 +60,7 @@ class MessageView(QtGui.QTextBrowser):
         super(MessageView, self).wheelEvent(event)
         if (QtGui.QApplication.queryKeyboardModifiers() &
                 QtCore.Qt.ControlModifier) == QtCore.Qt.ControlModifier and event.orientation() == QtCore.Qt.Vertical:
-            zoom = self.currentFont().pointSize() * 100 / self.defaultFontPointSize
+            zoom = old_div(self.currentFont().pointSize() * 100, self.defaultFontPointSize)
             QtGui.QApplication.activeWindow().statusBar().showMessage(
                 QtGui.QApplication.translate("MainWindow", "Zoom level %1%").arg(str(zoom)))
 
@@ -88,7 +97,7 @@ class MessageView(QtGui.QTextBrowser):
             QtGui.QApplication.translate(
                 "MessageView",
                 "The link \"%1\" will open in a browser. It may be a security risk, it could de-anonymise you"
-                " or download malicious data. Are you sure?").arg(unicode(link.toString())),
+                " or download malicious data. Are you sure?").arg(str(link.toString())),
             QtGui.QMessageBox.Yes,
             QtGui.QMessageBox.No)
         if reply == QtGui.QMessageBox.Yes:
@@ -130,7 +139,7 @@ class MessageView(QtGui.QTextBrowser):
         self.mode = MessageView.MODE_PLAIN
         out = self.html.raw
         if self.html.has_html:
-            out = "<div align=\"center\" style=\"text-decoration: underline;\"><b>" + unicode(
+            out = "<div align=\"center\" style=\"text-decoration: underline;\"><b>" + str(
                 QtGui.QApplication.translate(
                     "MessageView", "HTML detected, click here to display")) + "</b></div><br/>" + out
         self.out = out
@@ -142,7 +151,7 @@ class MessageView(QtGui.QTextBrowser):
         """Render message as HTML"""
         self.mode = MessageView.MODE_HTML
         out = self.html.sanitised
-        out = "<div align=\"center\" style=\"text-decoration: underline;\"><b>" + unicode(
+        out = "<div align=\"center\" style=\"text-decoration: underline;\"><b>" + str(
             QtGui.QApplication.translate("MessageView", "Click here to disable HTML")) + "</b></div><br/>" + out
         self.out = out
         self.outpos = 0

src/bitmessageqt/migrationwizard.py

@@ -1,4 +1,11 @@
 #!/usr/bin/env python2.7
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
 from PyQt4 import QtCore, QtGui
 
 class MigrationWizardIntroPage(QtGui.QWizardPage):

src/bitmessageqt/networkstatus.py

@@ -3,7 +3,18 @@ src/bitmessageqt/networkstatus.py
 =================================
 
 """
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+from __future__ import print_function
 
+from future import standard_library
+standard_library.install_aliases()
+from builtins import str
+from builtins import map
+from builtins import range
+from builtins import *
+from past.utils import old_div
 import time
 
 from PyQt4 import QtCore, QtGui
@@ -12,12 +23,12 @@ import knownnodes
 import l10n
 import network.stats
 import shared
-import widgets
+from . import widgets
 from inventory import Inventory
-from network.connectionpool import BMConnectionPool
-from retranslateui import RetranslateMixin
+from network.fix_circular_imports import BMConnectionPool
+from .retranslateui import RetranslateMixin
 from tr import _translate
-from uisignaler import UISignaler
+from .uisignaler import UISignaler
 
 
 class NetworkStatus(QtGui.QWidget, RetranslateMixin):
@@ -234,7 +245,7 @@ class NetworkStatus(QtGui.QWidget, RetranslateMixin):
     def runEveryTwoSeconds(self):
         """Updates counters, runs every 2 seconds if the timer is running"""
         self.labelLookupsPerSecond.setText(_translate("networkstatus", "Inventory lookups per second: %1").arg(
-            str(Inventory().numberOfInventoryLookupsPerformed / 2)))
+            str(old_div(Inventory().numberOfInventoryLookupsPerformed, 2))))
         Inventory().numberOfInventoryLookupsPerformed = 0
         self.updateNumberOfBytes()
         self.updateNumberOfObjectsToBeSynced()

src/bitmessageqt/newaddresswizard.py

@@ -1,4 +1,12 @@
 #!/usr/bin/env python2.7
+from __future__ import print_function
+from __future__ import unicode_literals
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import range
+from builtins import *
 from PyQt4 import QtCore, QtGui
 
 class NewAddressWizardIntroPage(QtGui.QWizardPage):
@@ -234,7 +242,7 @@ class NewAddressWizardWaitPage(QtGui.QWizardPage):
             self.wizard().button(QtGui.QWizard.NextButton).click()
             return
         elif i == 101:
-            print "haha"
+            print("haha")
             return
         self.progressBar.setValue(i)
         if i == 50:
@@ -347,8 +355,8 @@ if __name__ == '__main__':
 
     wizard = Ui_NewAddressWizard(["a", "b", "c", "d"])
     if (wizard.exec_()):
-        print "Email: " + ("yes" if wizard.field("emailAsWell").toBool() else "no")
-        print "BM: " + ("yes" if wizard.field("onlyBM").toBool() else "no")
+        print("Email: " + ("yes" if wizard.field("emailAsWell").toBool() else "no"))
+        print("BM: " + ("yes" if wizard.field("onlyBM").toBool() else "no"))
     else:
-        print "Wizard cancelled"
+        print("Wizard cancelled")
     sys.exit()

src/bitmessageqt/newchandialog.py

@@ -3,16 +3,24 @@ src/bitmessageqt/newchandialog.py
 =================================
 
 """
+from __future__ import absolute_import
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
 
+from future import standard_library
+standard_library.install_aliases()
+from builtins import str
+from builtins import *
 from PyQt4 import QtCore, QtGui
 
-import widgets
+from . import widgets
 from addresses import addBMIfNotPresent
-from addressvalidator import AddressValidator, PassPhraseValidator
+from .addressvalidator import AddressValidator, PassPhraseValidator
 from queues import UISignalQueue, addressGeneratorQueue, apiAddressGeneratorReturnQueue
-from retranslateui import RetranslateMixin
+from .retranslateui import RetranslateMixin
 from tr import _translate
-from utils import str_chan
+from .utils import str_chan
 
 
 class NewChanDialog(QtGui.QDialog, RetranslateMixin):
@@ -66,7 +74,7 @@ class NewChanDialog(QtGui.QDialog, RetranslateMixin):
         addressGeneratorReturnValue = apiAddressGeneratorReturnQueue.get(True)
         if addressGeneratorReturnValue and addressGeneratorReturnValue[0] != 'chan name does not match address':
             UISignalQueue.put(('updateStatusBar', _translate(
-                "newchandialog", "Successfully created / joined chan %1").arg(unicode(self.chanPassPhrase.text()))))
+                "newchandialog", "Successfully created / joined chan %1").arg(str(self.chanPassPhrase.text()))))
             self.parent.ui.tabWidget.setCurrentIndex(
                 self.parent.ui.tabWidget.indexOf(self.parent.ui.chans)
             )

src/bitmessageqt/retranslateui.py

@@ -1,13 +1,22 @@
+from __future__ import absolute_import
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from future import standard_library
+standard_library.install_aliases()
+from builtins import range
+from builtins import *
+from builtins import object
 from os import path
 from PyQt4 import QtGui
 from debug import logger
-import widgets
+from . import widgets
 
 class RetranslateMixin(object):
     def retranslateUi(self):
         defaults = QtGui.QWidget()
         widgets.load(self.__class__.__name__.lower() + '.ui', defaults)
-        for attr, value in defaults.__dict__.iteritems():
+        for attr, value in defaults.__dict__.items():
             setTextMethod = getattr(value, "setText", None)
             if callable(setTextMethod):
                 getattr(self, attr).setText(getattr(defaults, attr).text())

src/bitmessageqt/safehtmlparser.py

@@ -1,8 +1,16 @@
-from HTMLParser import HTMLParser
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import str
+from builtins import *
+from html.parser import HTMLParser
 import inspect
 import re
-from urllib import quote, quote_plus
-from urlparse import urlparse
+from urllib.parse import quote, quote_plus
+from urllib.parse import urlparse
 
 class SafeHTMLParser(HTMLParser):
     # from html5lib.sanitiser
@@ -99,9 +107,9 @@ class SafeHTMLParser(HTMLParser):
 
     def feed(self, data):
         try:
-            data = unicode(data, 'utf-8')
+            data = str(data, 'utf-8')
         except UnicodeDecodeError:
-            data = unicode(data, 'utf-8', errors='replace')
+            data = str(data, 'utf-8', errors='replace')
         HTMLParser.feed(self, data)
         tmp = SafeHTMLParser.replace_pre(data)
         tmp = SafeHTMLParser.uriregex1.sub(

src/bitmessageqt/settings.py

@@ -11,7 +11,15 @@ Created: Thu Dec 25 23:21:20 2014
 
 WARNING! All changes made in this file will be lost!
 """
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
 
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
+from builtins import object
 from sys import platform
 
 from PyQt4 import QtCore, QtGui

src/bitmessageqt/settingsmixin.py

@@ -4,7 +4,16 @@ src/settingsmixin.py
 ====================
 
 """
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
 
+from future import standard_library
+standard_library.install_aliases()
+from builtins import str
+from builtins import *
+from builtins import object
 from PyQt4 import QtCore, QtGui
 
 

src/bitmessageqt/sound.py

@@ -1,6 +1,13 @@
 # -*- coding: utf-8 -*-
 
 # sound type constants
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
 SOUND_NONE = 0
 SOUND_KNOWN = 1
 SOUND_UNKNOWN = 2

src/bitmessageqt/statusbar.py

@@ -1,5 +1,12 @@
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
 from PyQt4 import QtCore, QtGui
-from Queue import Queue
+from queue import Queue
 from time import time
 
 class BMStatusBar(QtGui.QStatusBar):

src/bitmessageqt/support.py

@@ -1,14 +1,22 @@
+from __future__ import absolute_import
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from future import standard_library
+standard_library.install_aliases()
+from builtins import str
+from builtins import *
 import ctypes
 from PyQt4 import QtCore, QtGui
 import ssl
 import sys
 import time
 
-import account
+from . import account
 from bmconfigparser import BMConfigParser
 from debug import logger
 import defaults
-from foldertree import AccountMixin
+from .foldertree import AccountMixin
 from helper_sql import *
 from l10n import getTranslationLanguage
 from openclpow import openclAvailable, openclEnabled

src/bitmessageqt/uisignaler.py

@@ -1,4 +1,12 @@
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
 
+from future import standard_library
+standard_library.install_aliases()
+from builtins import str
+from builtins import *
 from PyQt4.QtCore import QThread, SIGNAL
 import sys
 

src/bitmessageqt/utils.py

@@ -1,3 +1,10 @@
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
 from PyQt4 import QtGui
 import hashlib
 import os

src/bitmessageqt/widgets.py

@@ -1,3 +1,10 @@
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
 from PyQt4 import uic
 import os.path
 import paths

src/bmconfigparser.py

@@ -2,13 +2,20 @@
 BMConfigParser class definition and default configuration settings
 """
 
-import ConfigParser
-import shutil
-import os
+from configparser import (
+    ConfigParser,
+    InterpolationError,
+    NoOptionError,
+    NoSectionError,
+)
 from datetime import datetime
+import os
+from past.builtins import basestring
+import shutil
+from singleton import Singleton
 
 import state
-from singleton import Singleton
+
 
 BMConfigDefaults = {
     "bitmessagesettings": {
@@ -42,9 +49,9 @@ BMConfigDefaults = {
 
 
 @Singleton
-class BMConfigParser(ConfigParser.SafeConfigParser):
-    """Singleton class inherited from ConfigParser.SafeConfigParser
-    with additional methods specific to bitmessage config."""
+class BMConfigParser(ConfigParser):
+    """Singleton class inherited from ConfigParser with additional methods
+    specific to bitmessage config."""
 
     def set(self, section, option, value=None):
         if self._optcre is self.OPTCRE or value:
@@ -52,19 +59,16 @@ class BMConfigParser(ConfigParser.SafeConfigParser):
                 raise TypeError("option values must be strings")
         if not self.validate(section, option, value):
             raise ValueError("Invalid value %s" % value)
-        return ConfigParser.ConfigParser.set(self, section, option, value)
+        return ConfigParser.set(self, section, option, value)
 
-    def get(self, section, option, raw=False, variables=None):
+    def get(self, section, option, *args, raw=False, vars=None, **kwargs):
         try:
             if section == "bitmessagesettings" and option == "timeformat":
-                return ConfigParser.ConfigParser.get(
-                    self, section, option, raw, variables)
-            return ConfigParser.ConfigParser.get(
-                self, section, option, True, variables)
-        except ConfigParser.InterpolationError:
-            return ConfigParser.ConfigParser.get(
-                self, section, option, True, variables)
-        except (ConfigParser.NoSectionError, ConfigParser.NoOptionError) as e:
+                return ConfigParser.get(self, section, option, raw=raw, vars=vars)
+            return ConfigParser.get(self, section, option, raw=True, vars=vars)
+        except InterpolationError:
+            return ConfigParser.get(self, section, option, raw=True, vars=vars)
+        except (NoSectionError, NoOptionError) as e:
             try:
                 return BMConfigDefaults[section][option]
             except (KeyError, ValueError, AttributeError):
@@ -73,47 +77,45 @@ class BMConfigParser(ConfigParser.SafeConfigParser):
     def safeGetBoolean(self, section, field):
         try:
             return self.getboolean(section, field)
-        except (ConfigParser.NoSectionError, ConfigParser.NoOptionError,
-                ValueError, AttributeError):
+        except (NoSectionError, NoOptionError, ValueError, AttributeError):
             return False
 
     def safeGetInt(self, section, field, default=0):
         try:
             return self.getint(section, field)
-        except (ConfigParser.NoSectionError, ConfigParser.NoOptionError,
+        except (NoSectionError, NoOptionError,
                 ValueError, AttributeError):
             return default
 
     def safeGet(self, section, option, default=None):
         try:
             return self.get(section, option)
-        except (ConfigParser.NoSectionError, ConfigParser.NoOptionError,
+        except (NoSectionError, NoOptionError,
                 ValueError, AttributeError):
             return default
 
-    def items(self, section, raw=False, variables=None):
-        return ConfigParser.ConfigParser.items(self, section, True, variables)
+    def items(self, section, vars=None, **kwargs):
+        return ConfigParser.items(self, section, raw=True, vars=vars)
 
     def addresses(self):
         return filter(
             lambda x: x.startswith('BM-'), BMConfigParser().sections())
 
     def read(self, filenames):
-        ConfigParser.ConfigParser.read(self, filenames)
+        ConfigParser.read(self, filenames)
         for section in self.sections():
             for option in self.options(section):
                 try:
                     if not self.validate(
                         section, option,
-                        ConfigParser.ConfigParser.get(self, section, option)
+                        self.get(section, option)
                     ):
                         try:
                             newVal = BMConfigDefaults[section][option]
                         except KeyError:
                             continue
-                        ConfigParser.ConfigParser.set(
-                            self, section, option, newVal)
-                except ConfigParser.InterpolationError:
+                        self.set(section, option, newVal)
+                except InterpolationError:
                     continue
 
     def save(self):
@@ -131,7 +133,7 @@ class BMConfigParser(ConfigParser.SafeConfigParser):
             # didn't exist before.
             fileNameExisted = False
         # write the file
-        with open(fileName, 'wb') as configfile:
+        with open(fileName, 'w') as configfile:
             self.write(configfile)
         # delete the backup
         if fileNameExisted:

src/class_objectProcessor.py

@@ -19,7 +19,7 @@ import helper_msgcoding
 import helper_sent
 from helper_sql import SqlBulkExecute, sqlExecute, sqlQuery
 from helper_ackPayload import genAckPayload
-from network import bmproto
+from network.fix_circular_imports import BMStringParser
 import protocol
 import queues
 import state
@@ -50,7 +50,7 @@ class objectProcessor(threading.Thread):
         logger.debug(
             'Loaded %s objects from disk into the objectProcessorQueue.',
             len(queryreturn))
-        self._ack_obj = bmproto.BMStringParser()
+        self._ack_obj = BMStringParser()
         self.successfullyDecryptMessageTimings = []
 
     def run(self):
@@ -151,7 +151,7 @@ class objectProcessor(threading.Thread):
         readPosition += length
         stream, length = decodeVarint(data[readPosition:readPosition + 10])
         readPosition += length
-        # it seems that stream is checked in network.bmproto
+        # it seems that stream is checked in network.fix_circular_imports
         port, length = decodeVarint(data[readPosition:readPosition + 10])
         host = protocol.checkIPAddress(data[readPosition + length:])
 

src/class_objectProcessorQueue.py

@@ -1,12 +1,12 @@
-import Queue
+from queue import Queue
 import threading
 import time
 
-class ObjectProcessorQueue(Queue.Queue):
+class ObjectProcessorQueue(Queue):
     maxSize = 32000000
 
     def __init__(self):
-        Queue.Queue.__init__(self)
+        Queue.__init__(self)
         self.sizeLock = threading.Lock()
         self.curSize = 0 # in Bytes. We maintain this to prevent nodes from flooing us with objects which take up too much memory. If this gets too big we'll sleep before asking for further objects.
 
@@ -15,10 +15,10 @@ class ObjectProcessorQueue(Queue.Queue):
             time.sleep(1)
         with self.sizeLock:
             self.curSize += len(item[1])
-        Queue.Queue.put(self, item, block, timeout)
+        Queue.put(self, item, block, timeout)
 
     def get(self, block = True, timeout = None):
-        item = Queue.Queue.get(self, block, timeout)
+        item = Queue.get(self, block, timeout)
         with self.sizeLock:
             self.curSize -= len(item[1])
         return item

src/class_singleCleaner.py

@@ -29,7 +29,7 @@ from bmconfigparser import BMConfigParser
 from helper_sql import sqlQuery, sqlExecute
 from helper_threading import StoppableThread
 from inventory import Inventory
-from network.connectionpool import BMConnectionPool
+from network.fix_circular_imports import BMConnectionPool
 from debug import logger
 import knownnodes
 import queues

src/debug.py

@@ -22,8 +22,7 @@ Use: `from debug import logger` to import this facility into whatever module you
     Logging is thread-safe so you don't have to worry about locks, just import and log.
 
 """
-
-import ConfigParser
+import configparser
 import logging
 import logging.config
 import os
@@ -53,7 +52,7 @@ def configureLogging():
             False,
             'Loaded logger configuration from %s' % logging_config
         )
-    except (OSError, ConfigParser.NoSectionError):
+    except (KeyError, OSError, configparser.NoSectionError):
         if os.path.isfile(logging_config):
             fail_msg = \
                 'Failed to load logger configuration from %s, using default' \

src/depends.py

@@ -299,7 +299,7 @@ def check_openssl():
                 ' OpenSSL 0.9.8b or later with AES, Elliptic Curves (EC),'
                 ' ECDH, and ECDSA enabled.')
             return False
-        matches = cflags_regex.findall(openssl_cflags)
+        matches = cflags_regex.findall(openssl_cflags.decode())
         if len(matches) > 0:
             logger.error(
                 'This OpenSSL library is missing the following required'
@@ -408,19 +408,12 @@ def check_dependencies(verbose=False, optional=False):
 
     # Python 2.7.4 is the required minimum.
     # (https://bitmessage.org/forum/index.php?topic=4081.0)
-    # Python 3+ is not supported, but it is still useful to provide
-    # information about our other requirements.
     logger.info('Python version: %s', sys.version)
     if sys.hexversion < 0x20704F0:
         logger.error(
             'PyBitmessage requires Python 2.7.4 or greater'
             ' (but not Python 3+)')
         has_all_dependencies = False
-    if sys.hexversion >= 0x3000000:
-        logger.error(
-            'PyBitmessage does not support Python 3+. Python 2.7.4'
-            ' or greater is required.')
-        has_all_dependencies = False
 
     check_functions = [check_ripemd160, check_sqlite, check_openssl]
     if optional:

src/fallback/__init__.py

@@ -1,7 +1,14 @@
 """
 .. todo:: hello world
 """
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
 
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
 import hashlib
 
 # We need to check hashlib for RIPEMD-160, as it won't be available

src/helper_bitcoin.py

@@ -4,7 +4,7 @@ from pyelliptic import arithmetic
 # This function expects that pubkey begin with \x04
 def calculateBitcoinAddressFromPubkey(pubkey):
     if len(pubkey) != 65:
-        print 'Could not calculate Bitcoin address from pubkey because function was passed a pubkey that was', len(pubkey), 'bytes long rather than 65.'
+        print('Could not calculate Bitcoin address from pubkey because function was passed a pubkey that was', len(pubkey), 'bytes long rather than 65.')
         return "error"
     ripe = hashlib.new('ripemd160')
     sha = hashlib.new('sha256')
@@ -25,7 +25,7 @@ def calculateBitcoinAddressFromPubkey(pubkey):
 
 def calculateTestnetAddressFromPubkey(pubkey):
     if len(pubkey) != 65:
-        print 'Could not calculate Bitcoin address from pubkey because function was passed a pubkey that was', len(pubkey), 'bytes long rather than 65.'
+        print('Could not calculate Bitcoin address from pubkey because function was passed a pubkey that was', len(pubkey), 'bytes long rather than 65.')
         return "error"
     ripe = hashlib.new('ripemd160')
     sha = hashlib.new('sha256')

src/helper_sql.py

@@ -1,12 +1,12 @@
 """Helper Sql performs sql operations."""
 
 import threading
-import Queue
+from queue import Queue
 
-sqlSubmitQueue = Queue.Queue()
+sqlSubmitQueue = Queue()
 # SQLITE3 is so thread-unsafe that they won't even let you call it from different threads using your own locks.
 # SQL objects #can only be called from one thread.
-sqlReturnQueue = Queue.Queue()
+sqlReturnQueue = Queue()
 sqlLock = threading.Lock()
 
 

src/helper_startup.py

@@ -7,7 +7,7 @@ Helper Start performs all the startup operations.
 # pylint: disable=too-many-branches,too-many-statements
 from __future__ import print_function
 
-import ConfigParser
+import configparser
 import os
 import platform
 import sys
@@ -27,7 +27,7 @@ StoreConfigFilesInSameDirectoryAsProgramByDefault = False
 def _loadTrustedPeer():
     try:
         trustedPeer = BMConfigParser().get('bitmessagesettings', 'trustedpeer')
-    except ConfigParser.Error:
+    except configparser.Error:
         # This probably means the trusted peer wasn't specified so we
         # can just leave it as None
         return

src/helper_threading.py

@@ -3,23 +3,10 @@
 from contextlib import contextmanager
 import threading
 
-try:
-    import prctl
-except ImportError:
+
 def set_thread_name(name):
     """Set the thread name for external use (visible from the OS)."""
     threading.current_thread().name = name
-else:
-    def set_thread_name(name):
-        """Set a name for the thread for python internal use."""
-        prctl.set_name(name)
-
-    def _thread_name_hack(self):
-        set_thread_name(self.name)
-        threading.Thread.__bootstrap_original__(self)
-
-    threading.Thread.__bootstrap_original__ = threading.Thread._Thread__bootstrap
-    threading.Thread._Thread__bootstrap = _thread_name_hack
 
 
 class StoppableThread(object):

src/kivymd/__init__.py

@@ -1,4 +1,11 @@
 # -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
 import os
 
 path = os.path.dirname(__file__)

src/kivymd/accordion.py

@@ -1,5 +1,12 @@
 # -*- coding: utf-8 -*-
 
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
 from kivy.lang import Builder
 from kivy.properties import StringProperty, ListProperty, OptionProperty
 from kivy.utils import get_color_from_hex

src/kivymd/backgroundcolorbehavior.py

@@ -1,4 +1,11 @@
 # -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
 from kivy.lang import Builder
 from kivy.properties import BoundedNumericProperty, ReferenceListProperty
 from kivy.uix.widget import Widget

src/kivymd/bottomsheet.py

@@ -39,6 +39,13 @@ For :class:`MDListBottomSheet`:
 API
 ---
 '''
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
 from kivy.clock import Clock
 from kivy.lang import Builder
 from kivy.metrics import dp

src/kivymd/button.py

@@ -9,6 +9,13 @@ Buttons
 
 TO-DO: DOCUMENT MODULE
 '''
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
 from kivy.clock import Clock
 from kivy.lang import Builder
 from kivy.metrics import dp

src/kivymd/card.py

@@ -1,4 +1,11 @@
 # -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
 from kivy.lang import Builder
 from kivy.properties import BoundedNumericProperty, ReferenceListProperty, ListProperty,BooleanProperty
 from kivy.uix.boxlayout import BoxLayout

src/kivymd/color_definitions.py

@@ -1,3 +1,10 @@
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
 colors = {
     'Pink': {
         '50': 'fce4ec',

src/kivymd/date_picker.py

@@ -1,4 +1,13 @@
 # -*- coding: utf-8 -*-
+from __future__ import print_function
+from __future__ import unicode_literals
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import str
+from builtins import range
+from builtins import *
 from kivy.lang import Builder
 from kivy.uix.modalview import ModalView
 from kivymd.label import MDLabel

src/kivymd/dialog.py

@@ -1,5 +1,12 @@
 # -*- coding: utf-8 -*-
 
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
 from kivy.lang import Builder
 from kivy.properties import StringProperty, ObjectProperty, ListProperty
 from kivy.metrics import dp

src/kivymd/elevationbehavior.py

@@ -1,5 +1,15 @@
 # -*- coding: utf-8 -*-
 
+from __future__ import division
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import str
+from builtins import *
+from builtins import object
+from past.utils import old_div
 from kivy.app import App
 from kivy.lang import Builder
 from kivy.properties import (ListProperty, ObjectProperty, NumericProperty)
@@ -75,7 +85,7 @@ class ElevationBehavior(object):
 
     def _update_shadow(self, *args):
         if self.elevation > 0:
-            ratio = self.width / (self.height if self.height != 0 else 1)
+            ratio = old_div(self.width, (self.height if self.height != 0 else 1))
             if ratio > -2 and ratio < 2:
                 self._shadow = App.get_running_app().theme_cls.quad_shadow
                 width = soft_width = self.width * 1.9
@@ -106,19 +116,19 @@ class ElevationBehavior(object):
                 #                 soft_width = self.width + dp(ratio) + dp(self.elevation) * .9
                 #                 height = soft_height = self.height * 1.9
 
-            x = self.center_x - width / 2
-            soft_x = self.center_x - soft_width / 2
+            x = self.center_x - old_div(width, 2)
+            soft_x = self.center_x - old_div(soft_width, 2)
             self._soft_shadow_size = (soft_width, soft_height)
             self._hard_shadow_size = (width, height)
 
-            y = self.center_y - soft_height / 2 - dp(
+            y = self.center_y - old_div(soft_height, 2) - dp(
                 .1 * 1.5 ** self.elevation)
             self._soft_shadow_pos = (soft_x, y)
             self._soft_shadow_a = 0.1 * 1.1 ** self.elevation
             self._soft_shadow_texture = self._shadow.textures[
                 str(int(round(self.elevation - 1)))]
 
-            y = self.center_y - height / 2 - dp(.5 * 1.18 ** self.elevation)
+            y = self.center_y - old_div(height, 2) - dp(.5 * 1.18 ** self.elevation)
             self._hard_shadow_pos = (x, y)
             self._hard_shadow_a = .4 * .9 ** self.elevation
             self._hard_shadow_texture = self._shadow.textures[
@@ -165,18 +175,18 @@ class RoundElevationBehavior(object):
             width = self.width * 2
             height = self.height * 2
 
-            x = self.center_x - width / 2
+            x = self.center_x - old_div(width, 2)
             self._soft_shadow_size = (width, height)
 
             self._hard_shadow_size = (width, height)
 
-            y = self.center_y - height / 2 - dp(.1 * 1.5 ** self.elevation)
+            y = self.center_y - old_div(height, 2) - dp(.1 * 1.5 ** self.elevation)
             self._soft_shadow_pos = (x, y)
             self._soft_shadow_a = 0.1 * 1.1 ** self.elevation
             self._soft_shadow_texture = self._shadow.textures[
                 str(int(round(self.elevation)))]
 
-            y = self.center_y - height / 2 - dp(.5 * 1.18 ** self.elevation)
+            y = self.center_y - old_div(height, 2) - dp(.5 * 1.18 ** self.elevation)
             self._hard_shadow_pos = (x, y)
             self._hard_shadow_a = .4 * .9 ** self.elevation
             self._hard_shadow_texture = self._shadow.textures[

src/kivymd/grid.py

@@ -1,4 +1,12 @@
 # coding=utf-8
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
+from builtins import object
 from kivy.lang import Builder
 from kivy.properties import StringProperty, BooleanProperty, ObjectProperty, \
     NumericProperty, ListProperty, OptionProperty
@@ -154,14 +162,14 @@ class SmartTileWithLabel(SmartTile):
     """Determines the text for the box footer/header"""
 
 
-class IBoxOverlay():
+class IBoxOverlay(object):
     """An interface to specify widgets that belong to to the image overlay
     in the :class:`SmartTile` widget when added as a child.
     """
     pass
 
 
-class IOverlay():
+class IOverlay(object):
     """An interface to specify widgets that belong to to the image overlay
     in the :class:`SmartTile` widget when added as a child.
     """

src/kivymd/icon_definitions.py

@@ -8,6 +8,13 @@
 
 # LAST UPDATED: version 2.2.0 of Material Design Iconic Font
 
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
 md_icons = {
     '3d-rotation': u'',
 

src/kivymd/label.py

@@ -1,4 +1,11 @@
 # -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
 from kivy.lang import Builder
 from kivy.metrics import sp
 from kivy.properties import OptionProperty, DictProperty, ListProperty

src/kivymd/list.py

@@ -139,7 +139,15 @@ Python example:
 API
 ---
 '''
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
 
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
+from builtins import object
 from kivy.lang import Builder
 from kivy.metrics import dp
 from kivy.properties import ObjectProperty, StringProperty, NumericProperty, \
@@ -338,7 +346,7 @@ class BaseListItem(ThemableBehavior, RectangularRippleBehavior,
     _num_lines = 2
 
 
-class ILeftBody:
+class ILeftBody(object):
     '''Pseudo-interface for widgets that go in the left container for
     ListItems that support it.
 
@@ -347,14 +355,14 @@ class ILeftBody:
     pass
 
 
-class ILeftBodyTouch:
+class ILeftBodyTouch(object):
     '''Same as :class:`~ILeftBody`, but allows the widget to receive touch
     events instead of triggering the ListItem's ripple effect
     '''
     pass
 
 
-class IRightBody:
+class IRightBody(object):
     '''Pseudo-interface for widgets that go in the right container for
     ListItems that support it.
 
@@ -363,14 +371,14 @@ class IRightBody:
     pass
 
 
-class IRightBodyTouch:
+class IRightBodyTouch(object):
     '''Same as :class:`~IRightBody`, but allows the widget to receive touch
     events instead of triggering the ListItem's ripple effect
     '''
     pass
 
 
-class ContainerSupport:
+class ContainerSupport(object):
     '''Overrides add_widget in a ListItem to include support for I*Body
     widgets when the appropiate containers are present.
     '''

src/kivymd/material_resources.py

@@ -1,4 +1,11 @@
 # -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
 from kivy import platform
 from kivy.core.window import Window
 from kivy.metrics import dp

src/kivymd/menu.py

@@ -1,4 +1,12 @@
 # -*- coding: utf-8 -*-
+from __future__ import division
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
+from past.utils import old_div
 from kivy.animation import Animation
 from kivy.clock import Clock
 from kivy.core.window import Window
@@ -109,7 +117,7 @@ class MDDropdownMenu(ThemableBehavior, BoxLayout):
         if target_width > Window.width:
             # ...reduce our multiplier to max allowed.
             target_width = int(
-                Window.width / m_res.STANDARD_INCREMENT) * m_res.STANDARD_INCREMENT
+                old_div(Window.width, m_res.STANDARD_INCREMENT)) * m_res.STANDARD_INCREMENT
 
         target_height = sum([dp(48) for i in self.items])
         # If we're over max_height...

src/kivymd/navigationdrawer.py

@@ -1,4 +1,11 @@
 # -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
 from kivy.animation import Animation
 from kivy.lang import Builder
 from kivy.properties import StringProperty, ObjectProperty

src/kivymd/progressbar.py

@@ -1,5 +1,12 @@
 # -*- coding: utf-8 -*-
 
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
 from kivy.lang import Builder
 from kivy.properties import ListProperty, OptionProperty, BooleanProperty
 from kivy.utils import get_color_from_hex

src/kivymd/ripplebehavior.py

@@ -1,4 +1,13 @@
 # -*- coding: utf-8 -*-
+from __future__ import division
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
+from past.utils import old_div
+from builtins import object
 from kivy.properties import ListProperty, NumericProperty, StringProperty, \
     BooleanProperty
 from kivy.animation import Animation
@@ -146,10 +155,10 @@ class CircularRippleBehavior(CommonRipple):
             StencilPush()
             self.stencil = Ellipse(size=(self.width * self.ripple_scale,
                                          self.height * self.ripple_scale),
-                                   pos=(self.center_x - (
-                                       self.width * self.ripple_scale) / 2,
-                                        self.center_y - (
-                                            self.height * self.ripple_scale) / 2))
+                                   pos=(self.center_x - old_div((
+                                       self.width * self.ripple_scale), 2),
+                                        self.center_y - old_div((
+                                            self.height * self.ripple_scale), 2)))
             StencilUse()
             self.col_instruction = Color(rgba=self.ripple_color)
             self.ellipse = Ellipse(size=(self.ripple_rad, self.ripple_rad),

src/kivymd/selectioncontrols.py

@@ -1,5 +1,13 @@
 # -*- coding: utf-8 -*-
 
+from __future__ import division
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
+from past.utils import old_div
 from kivy.lang import Builder
 from kivy.properties import StringProperty, ListProperty, NumericProperty
 from kivy.uix.behaviors import ToggleButtonBehavior
@@ -124,8 +132,8 @@ class Thumb(RoundElevationBehavior, CircularRippleBehavior, ButtonBehavior,
         self.ellipse.pos = (self.center_x - self.ripple_rad / 2.,
                             self.center_y - self.ripple_rad / 2.)
         self.stencil.pos = (
-            self.center_x - (self.width * self.ripple_scale) / 2,
-            self.center_y - (self.height * self.ripple_scale) / 2)
+            self.center_x - old_div((self.width * self.ripple_scale), 2),
+            self.center_y - old_div((self.height * self.ripple_scale), 2))
 
 
 class MDSwitch(ThemableBehavior, ButtonBehavior, FloatLayout):

src/kivymd/slider.py

@@ -1,5 +1,12 @@
 # -*- coding: utf-8 -*-
 
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
 from kivy.lang import Builder
 from kivy.properties import StringProperty, ListProperty, NumericProperty,AliasProperty, BooleanProperty
 from kivy.utils import get_color_from_hex

src/kivymd/slidingpanel.py

@@ -1,4 +1,11 @@
 # -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
 from kivy.animation import Animation
 from kivy.clock import Clock
 from kivy.core.window import Window

src/kivymd/snackbar.py

@@ -1,4 +1,11 @@
 # -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
 from collections import deque
 from kivy.animation import Animation
 from kivy.clock import Clock

src/kivymd/spinner.py

@@ -1,5 +1,12 @@
 # -*- coding: utf-8 -*-
 
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
 from kivy.lang import Builder
 from kivy.uix.widget import Widget
 from kivy.properties import NumericProperty, ListProperty, BooleanProperty

src/kivymd/tabs.py

@@ -1,3 +1,7 @@
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
 # Created on Jul 8, 2016
 #
 # The default kivy tab implementation seems like a stupid design to me. The
@@ -5,6 +9,10 @@
 #
 # @author: jrm
 
+from future import standard_library
+standard_library.install_aliases()
+from builtins import str
+from builtins import *
 from kivy.properties import StringProperty, DictProperty, ListProperty, \
     ObjectProperty, OptionProperty, BoundedNumericProperty
 from kivy.uix.screenmanager import Screen

src/kivymd/textfields.py

@@ -1,5 +1,12 @@
 # -*- coding: utf-8 -*-
 
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
 from kivy.lang import Builder
 from kivy.uix.textinput import TextInput
 from kivy.properties import ObjectProperty, NumericProperty, StringProperty, \

src/kivymd/theme_picker.py

@@ -1,5 +1,12 @@
 # -*- coding: utf-8 -*-
 
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
 from kivy.lang import Builder
 from kivy.uix.modalview import ModalView
 from kivy.uix.floatlayout import FloatLayout

src/kivymd/theming.py

@@ -1,4 +1,12 @@
 # -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
+from builtins import object
 from kivy.app import App
 from kivy.core.text import LabelBase
 from kivy.core.window import Window

src/kivymd/time_picker.py

@@ -1,5 +1,12 @@
 # -*- coding: utf-8 -*-
 
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
 from kivy.lang import Builder
 from kivy.uix.modalview import ModalView
 from kivy.uix.floatlayout import FloatLayout

src/kivymd/toolbar.py

@@ -1,4 +1,11 @@
 # -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
 from kivy.clock import Clock
 from kivy.lang import Builder
 from kivy.metrics import dp

src/messagetypes/__init__.py

@@ -1,6 +1,13 @@
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
+from builtins import object
 from importlib import import_module
 from os import path, listdir
-from string import lower
 
 from debug import logger
 import messagetypes
@@ -8,7 +15,7 @@ import paths
 
 class MsgBase(object):
     def encode(self):
-        self.data = {"": lower(type(self).__name__)}
+        self.data = {"": type(self).__name__.lower()}
 
 
 def constructObject(data):

src/messagetypes/message.py

@@ -1,3 +1,11 @@
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import str
+from builtins import *
 from debug import logger
 from messagetypes import MsgBase
 
@@ -9,13 +17,13 @@ class Message(MsgBase):
     def decode(self, data):
         # UTF-8 and variable type validator
         if type(data["subject"]) is str:
-            self.subject = unicode(data["subject"], 'utf-8', 'replace')
+            self.subject = str(data["subject"], 'utf-8', 'replace')
         else:
-            self.subject = unicode(str(data["subject"]), 'utf-8', 'replace')
+            self.subject = str(str(data["subject"]), 'utf-8', 'replace')
         if type(data["body"]) is str:
-            self.body = unicode(data["body"], 'utf-8', 'replace')
+            self.body = str(data["body"], 'utf-8', 'replace')
         else:
-            self.body = unicode(str(data["body"]), 'utf-8', 'replace')
+            self.body = str(str(data["body"]), 'utf-8', 'replace')
 
     def encode(self, data):
         super(Message, self).encode()

src/messagetypes/vote.py

@@ -1,3 +1,10 @@
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
 from debug import logger
 from messagetypes import MsgBase
 

src/multiqueue.py

@@ -3,13 +3,13 @@ src/multiqueue.py
 =================
 """
 
-import Queue
+from queue import Queue
 from collections import deque
 
 import helper_random
 
 
-class MultiQueue(Queue.Queue):
+class MultiQueue(Queue):
     """A base queue class"""
     # pylint: disable=redefined-builtin,attribute-defined-outside-init
     defaultQueueCount = 10
@@ -19,7 +19,7 @@ class MultiQueue(Queue.Queue):
             self.queueCount = MultiQueue.defaultQueueCount
         else:
             self.queueCount = count
-        Queue.Queue.__init__(self, maxsize)
+        Queue.__init__(self, maxsize)
 
     # Initialize the queue representation
     def _init(self, maxsize):

src/navigationdrawer/__init__.py

@@ -1,4 +1,11 @@
 # -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
 from kivy.animation import Animation
 from kivy.lang import Builder
 from kivy.properties import StringProperty, ObjectProperty

src/network/addrthread.py

@@ -1,9 +1,17 @@
-import Queue
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import range
+from builtins import *
+import queue
 import threading
 
 import addresses
 from helper_threading import StoppableThread
-from network.connectionpool import BMConnectionPool
+from .fix_circular_imports import BMConnectionPool
 from queues import addrQueue
 import protocol
 import state
@@ -23,7 +31,7 @@ class AddrThread(threading.Thread, StoppableThread):
                     chunk.append((data[0], data[1]))
                     if len(data) > 2:
                         source = BMConnectionPool().getConnectionByAddr(data[2])
-                except Queue.Empty:
+                except queue.Empty:
                     break
                 except KeyError:
                     continue

src/network/advanceddispatcher.py

@@ -2,8 +2,16 @@
 src/network/advanceddispatcher.py
 =================================
 """
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
 # pylint: disable=attribute-defined-outside-init
 
+from future import standard_library
+standard_library.install_aliases()
+from builtins import str
+from builtins import *
 import socket
 import threading
 import time

src/network/announcethread.py

@@ -1,11 +1,17 @@
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
 import threading
 import time
 
 from bmconfigparser import BMConfigParser
 from debug import logger
 from helper_threading import StoppableThread
-from network.bmproto import BMProto
-from network.connectionpool import BMConnectionPool
+from .fix_circular_imports import BMConnectionPool, BMProto
 from network.udp import UDPSocket
 import state
 
@@ -27,7 +33,7 @@ class AnnounceThread(threading.Thread, StoppableThread):
                 self.stop.wait(10)
 
     def announceSelf(self):
-        for connection in BMConnectionPool().udpSockets.values():
+        for connection in list(BMConnectionPool().udpSockets.values()):
             if not connection.announcing:
                 continue
             for stream in state.streamsInWhichIAmParticipating:

src/network/asyncore_pollchoose.py

@@ -50,7 +50,16 @@ control than multi-threaded programming. The module documented here solves
 many of the difficult problems for you, making the task of building
 sophisticated high-performance network servers and clients a snap.
 """
+from __future__ import print_function
+from __future__ import unicode_literals
+from __future__ import division
+from __future__ import absolute_import
 
+from future import standard_library
+standard_library.install_aliases()
+from builtins import str
+from builtins import *
+from builtins import object
 import os
 import select
 import socket
@@ -365,7 +374,7 @@ def epoll_poller(timeout=0.0, map=None):
     except AttributeError:
         epoll_poller.pollster = select.epoll()
     if map:
-        for fd, obj in map.items():
+        for fd, obj in list(map.items()):
             flags = newflags = 0
             if obj.readable():
                 flags |= select.POLLIN | select.POLLPRI
@@ -423,7 +432,7 @@ def kqueue_poller(timeout=0.0, map=None):
     if map:
         updates = []
         selectables = 0
-        for fd, obj in map.items():
+        for fd, obj in list(map.items()):
             kq_filter = 0
             if obj.readable():
                 kq_filter |= 1
@@ -520,7 +529,7 @@ def loop(timeout=30.0, use_poll=False, map=None, count=None, poller=None):
             count = count - 1
 
 
-class dispatcher:
+class dispatcher(object):
     """Dispatcher for socket objects"""
     # pylint: disable=too-many-public-methods,too-many-instance-attributes,old-style-class
 
@@ -788,7 +797,7 @@ class dispatcher:
     def log_info(self, message, log_type='info'):
         """Conditionally print a message"""
         if log_type not in self.ignore_log_types:
-            print '%s: %s' % (log_type, message)
+            print('%s: %s' % (log_type, message))
 
     def handle_read_event(self):
         """Handle a read event"""
@@ -998,7 +1007,7 @@ def close_all(map=None, ignore_all=False):
 if os.name == 'posix':
     import fcntl
 
-    class file_wrapper:
+    class file_wrapper(object):
         """
         Here we override just enough to make a file look like a socket for the purposes of asyncore.
 

src/network/bmobject.py

@@ -1,155 +0,0 @@
-"""
-src/network/bmobject.py
-======================
-
-"""
-
-import time
-
-import protocol
-import state
-from addresses import calculateInventoryHash
-from debug import logger
-from inventory import Inventory
-from network.dandelion import Dandelion
-
-
-class BMObjectInsufficientPOWError(Exception):
-    """Exception indicating the object doesn't have sufficient proof of work."""
-    errorCodes = ("Insufficient proof of work")
-
-
-class BMObjectInvalidDataError(Exception):
-    """Exception indicating the data being parsed does not match the specification."""
-    errorCodes = ("Data invalid")
-
-
-class BMObjectExpiredError(Exception):
-    """Exception indicating the object's lifetime has expired."""
-    errorCodes = ("Object expired")
-
-
-class BMObjectUnwantedStreamError(Exception):
-    """Exception indicating the object is in a stream we didn't advertise as being interested in."""
-    errorCodes = ("Object in unwanted stream")
-
-
-class BMObjectInvalidError(Exception):
-    """The object's data does not match object specification."""
-    errorCodes = ("Invalid object")
-
-
-class BMObjectAlreadyHaveError(Exception):
-    """We received a duplicate object (one we already have)"""
-    errorCodes = ("Already have this object")
-
-
-class BMObject(object):
-    """Bitmessage Object as a class."""
-    # pylint: disable=too-many-instance-attributes
-
-    # max TTL, 28 days and 3 hours
-    maxTTL = 28 * 24 * 60 * 60 + 10800
-    # min TTL, 3 hour (in the past
-    minTTL = -3600
-
-    def __init__(
-            self,
-            nonce,
-            expiresTime,
-            objectType,
-            version,
-            streamNumber,
-            data,
-            payloadOffset
-    ):  # pylint: disable=too-many-arguments
-        self.nonce = nonce
-        self.expiresTime = expiresTime
-        self.objectType = objectType
-        self.version = version
-        self.streamNumber = streamNumber
-        self.inventoryHash = calculateInventoryHash(data)
-        # copy to avoid memory issues
-        self.data = bytearray(data)
-        self.tag = self.data[payloadOffset:payloadOffset + 32]
-
-    def checkProofOfWorkSufficient(self):
-        """Perform a proof of work check for sufficiency."""
-        # Let us check to make sure that the proof of work is sufficient.
-        if not protocol.isProofOfWorkSufficient(self.data):
-            logger.info('Proof of work is insufficient.')
-            raise BMObjectInsufficientPOWError()
-
-    def checkEOLSanity(self):
-        """Check if object's lifetime isn't ridiculously far in the past or future."""
-        # EOL sanity check
-        if self.expiresTime - int(time.time()) > BMObject.maxTTL:
-            logger.info(
-                'This object\'s End of Life time is too far in the future. Ignoring it. Time is %i',
-                self.expiresTime)
-            # .. todo::  remove from download queue
-            raise BMObjectExpiredError()
-
-        if self.expiresTime - int(time.time()) < BMObject.minTTL:
-            logger.info(
-                'This object\'s End of Life time was too long ago. Ignoring the object. Time is %i',
-                self.expiresTime)
-            # .. todo::  remove from download queue
-            raise BMObjectExpiredError()
-
-    def checkStream(self):
-        """Check if object's stream matches streams we are interested in"""
-        if self.streamNumber not in state.streamsInWhichIAmParticipating:
-            logger.debug('The streamNumber %i isn\'t one we are interested in.', self.streamNumber)
-            raise BMObjectUnwantedStreamError()
-
-    def checkAlreadyHave(self):
-        """
-        Check if we already have the object (so that we don't duplicate it in inventory or advertise it unnecessarily)
-        """
-        # if it's a stem duplicate, pretend we don't have it
-        if Dandelion().hasHash(self.inventoryHash):
-            return
-        if self.inventoryHash in Inventory():
-            raise BMObjectAlreadyHaveError()
-
-    def checkObjectByType(self):
-        """Call a object type specific check (objects can have additional checks based on their types)"""
-        if self.objectType == protocol.OBJECT_GETPUBKEY:
-            self.checkGetpubkey()
-        elif self.objectType == protocol.OBJECT_PUBKEY:
-            self.checkPubkey()
-        elif self.objectType == protocol.OBJECT_MSG:
-            self.checkMessage()
-        elif self.objectType == protocol.OBJECT_BROADCAST:
-            self.checkBroadcast()
-        # other objects don't require other types of tests
-
-    def checkMessage(self):
-        """"Message" object type checks."""
-        # pylint: disable=no-self-use
-        return
-
-    def checkGetpubkey(self):
-        """"Getpubkey" object type checks."""
-        if len(self.data) < 42:
-            logger.info('getpubkey message doesn\'t contain enough data. Ignoring.')
-            raise BMObjectInvalidError()
-
-    def checkPubkey(self):
-        """"Pubkey" object type checks."""
-        if len(self.data) < 146 or len(self.data) > 440:  # sanity check
-            logger.info('pubkey object too short or too long. Ignoring.')
-            raise BMObjectInvalidError()
-
-    def checkBroadcast(self):
-        """"Broadcast" object type checks."""
-        if len(self.data) < 180:
-            logger.debug(
-                'The payload length of this broadcast packet is unreasonably low.'
-                ' Someone is probably trying funny business. Ignoring message.')
-            raise BMObjectInvalidError()
-
-        # this isn't supported anymore
-        if self.version < 2:
-            raise BMObjectInvalidError()

src/network/bmproto.py

@@ -1,702 +0,0 @@
-import base64
-import hashlib
-import socket
-import struct
-import time
-from binascii import hexlify
-
-import addresses
-import connectionpool
-import knownnodes
-import protocol
-import state
-from bmconfigparser import BMConfigParser
-from debug import logger
-from inventory import Inventory
-from network.advanceddispatcher import AdvancedDispatcher
-from network.dandelion import Dandelion
-from network.bmobject import (
-    BMObject, BMObjectInsufficientPOWError, BMObjectInvalidDataError,
-    BMObjectExpiredError, BMObjectUnwantedStreamError,
-    BMObjectInvalidError, BMObjectAlreadyHaveError)
-from network.node import Node
-from network.proxy import ProxyError
-from objectracker import missingObjects, ObjectTracker
-from queues import objectProcessorQueue, portCheckerQueue, invQueue, addrQueue
-from randomtrackingdict import RandomTrackingDict
-
-
-class BMProtoError(ProxyError):
-    """A Bitmessage Protocol Base Error"""
-    errorCodes = ("Protocol error")
-
-
-class BMProtoInsufficientDataError(BMProtoError):
-    """A Bitmessage Protocol Insufficient Data Error"""
-    errorCodes = ("Insufficient data")
-
-
-class BMProtoExcessiveDataError(BMProtoError):
-    """A Bitmessage Protocol Excessive Data Error"""
-    errorCodes = ("Too much data")
-
-
-class BMProto(AdvancedDispatcher, ObjectTracker):
-    """A parser for the Bitmessage Protocol"""
-    # ~1.6 MB which is the maximum possible size of an inv message.
-    maxMessageSize = 1600100
-    # 2**18 = 256kB is the maximum size of an object payload
-    maxObjectPayloadSize = 2**18
-    # protocol specification says max 1000 addresses in one addr command
-    maxAddrCount = 1000
-    # protocol specification says max 50000 objects in one inv command
-    maxObjectCount = 50000
-    # address is online if online less than this many seconds ago
-    addressAlive = 10800
-    # maximum time offset
-    maxTimeOffset = 3600
-    timeOffsetWrongCount = 0
-
-    def __init__(self, address=None, sock=None):
-        AdvancedDispatcher.__init__(self, sock)
-        self.isOutbound = False
-        # packet/connection from a local IP
-        self.local = False
-        self.pendingUpload = RandomTrackingDict()
-
-    def bm_proto_reset(self):
-        """Reset the bitmessage object parser"""
-        self.magic = None
-        self.command = None
-        self.payloadLength = 0
-        self.checksum = None
-        self.payload = None
-        self.invalid = False
-        self.payloadOffset = 0
-        self.expectBytes = protocol.Header.size
-        self.object = None
-
-    def state_bm_header(self):
-        """Process incoming header"""
-        self.magic, self.command, self.payloadLength, self.checksum = \
-            protocol.Header.unpack(self.read_buf[:protocol.Header.size])
-        self.command = self.command.rstrip('\x00')
-        if self.magic != 0xE9BEB4D9:
-            # skip 1 byte in order to sync
-            self.set_state("bm_header", length=1)
-            self.bm_proto_reset()
-            logger.debug('Bad magic')
-            if self.socket.type == socket.SOCK_STREAM:
-                self.close_reason = "Bad magic"
-                self.set_state("close")
-            return False
-        if self.payloadLength > BMProto.maxMessageSize:
-            self.invalid = True
-        self.set_state(
-            "bm_command",
-            length=protocol.Header.size, expectBytes=self.payloadLength)
-        return True
-
-    def state_bm_command(self):
-        """Process incoming command"""
-        self.payload = self.read_buf[:self.payloadLength]
-        if self.checksum != hashlib.sha512(self.payload).digest()[0:4]:
-            logger.debug('Bad checksum, ignoring')
-            self.invalid = True
-        retval = True
-        if not self.fullyEstablished and self.command not in (
-                "error", "version", "verack"):
-            logger.error(
-                'Received command %s before connection was fully'
-                ' established, ignoring', self.command)
-            self.invalid = True
-        if not self.invalid:
-            try:
-                retval = getattr(
-                    self, "bm_command_" + str(self.command).lower())()
-            except AttributeError:
-                # unimplemented command
-                logger.debug('unimplemented command %s', self.command)
-            except BMProtoInsufficientDataError:
-                logger.debug('packet length too short, skipping')
-            except BMProtoExcessiveDataError:
-                logger.debug('too much data, skipping')
-            except BMObjectInsufficientPOWError:
-                logger.debug('insufficient PoW, skipping')
-            except BMObjectInvalidDataError:
-                logger.debug('object invalid data, skipping')
-            except BMObjectExpiredError:
-                logger.debug('object expired, skipping')
-            except BMObjectUnwantedStreamError:
-                logger.debug('object not in wanted stream, skipping')
-            except BMObjectInvalidError:
-                logger.debug('object invalid, skipping')
-            except BMObjectAlreadyHaveError:
-                logger.debug(
-                    '%(host)s:%(port)i already got object, skipping',
-                    self.destination._asdict())
-            except struct.error:
-                logger.debug('decoding error, skipping')
-        elif self.socket.type == socket.SOCK_DGRAM:
-            # broken read, ignore
-            pass
-        else:
-            logger.debug('Closing due to invalid command %s', self.command)
-            self.close_reason = "Invalid command %s" % self.command
-            self.set_state("close")
-            return False
-        if retval:
-            self.set_state("bm_header", length=self.payloadLength)
-            self.bm_proto_reset()
-        # else assume the command requires a different state to follow
-        return True
-
-    def decode_payload_string(self, length):
-        """Read and return `length` bytes from payload"""
-        value = self.payload[self.payloadOffset:self.payloadOffset + length]
-        self.payloadOffset += length
-        return value
-
-    def decode_payload_varint(self):
-        """Decode a varint from the payload"""
-        value, offset = addresses.decodeVarint(self.payload[self.payloadOffset:])
-        self.payloadOffset += offset
-        return value
-
-    def decode_payload_node(self):
-        """Decode node details from the payload"""
-        # protocol.checkIPAddress()
-        services, host, port = self.decode_payload_content("Q16sH")
-        if host[0:12] == '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xFF\xFF':
-            host = socket.inet_ntop(socket.AF_INET, str(host[12:16]))
-        elif host[0:6] == '\xfd\x87\xd8\x7e\xeb\x43':
-            # Onion, based on BMD/bitcoind
-            host = base64.b32encode(host[6:]).lower() + ".onion"
-        else:
-            host = socket.inet_ntop(socket.AF_INET6, str(host))
-        if host == "":
-            # This can happen on Windows systems which are not 64-bit
-            # compatible so let us drop the IPv6 address.
-            host = socket.inet_ntop(socket.AF_INET, str(host[12:16]))
-
-        return Node(services, host, port)
-
-    def decode_payload_content(self, pattern="v"):
-        """
-        Decode the payload depending on pattern:
-
-        L = varint indicating the length of the next array
-        l = varint indicating the length of the next item
-        v = varint (or array)
-        H = uint16
-        I = uint32
-        Q = uint64
-        i = net_addr (without time and stream number)
-        s = string
-        0-9 = length of the next item
-        , = end of array
-        """
-
-        def decode_simple(self, char="v"):
-            """Decode the payload using one char pattern"""
-            if char == "v":
-                return self.decode_payload_varint()
-            if char == "i":
-                return self.decode_payload_node()
-            if char == "H":
-                self.payloadOffset += 2
-                return struct.unpack(">H", self.payload[
-                    self.payloadOffset - 2:self.payloadOffset])[0]
-            if char == "I":
-                self.payloadOffset += 4
-                return struct.unpack(">I", self.payload[
-                    self.payloadOffset - 4:self.payloadOffset])[0]
-            if char == "Q":
-                self.payloadOffset += 8
-                return struct.unpack(">Q", self.payload[
-                    self.payloadOffset - 8:self.payloadOffset])[0]
-
-        size = None
-        isArray = False
-
-        # size
-        # iterator starting from size counting to 0
-        # isArray?
-        # subpattern
-        # position of parser in subpattern
-        # retval (array)
-        parserStack = [[1, 1, False, pattern, 0, []]]
-
-        while True:
-            i = parserStack[-1][3][parserStack[-1][4]]
-            if i in "0123456789" and (
-                size is None or parserStack[-1][3][parserStack[-1][4] - 1]
-                    not in "lL"):
-                try:
-                    size = size * 10 + int(i)
-                except TypeError:
-                    size = int(i)
-                isArray = False
-            elif i in "Ll" and size is None:
-                size = self.decode_payload_varint()
-                isArray = i == "L"
-            elif size is not None:
-                if isArray:
-                    parserStack.append([
-                        size, size, isArray,
-                        parserStack[-1][3][parserStack[-1][4]:], 0, []
-                    ])
-                    parserStack[-2][4] = len(parserStack[-2][3])
-                else:
-                    for j in range(parserStack[-1][4], len(parserStack[-1][3])):
-                        if parserStack[-1][3][j] not in "lL0123456789":
-                            break
-                    parserStack.append([
-                        size, size, isArray,
-                        parserStack[-1][3][parserStack[-1][4]:j + 1], 0, []
-                    ])
-                    parserStack[-2][4] += len(parserStack[-1][3]) - 1
-                size = None
-                continue
-            elif i == "s":
-                # if parserStack[-2][2]:
-                #    parserStack[-1][5].append(self.payload[
-                #        self.payloadOffset:self.payloadOffset + parserStack[-1][0]])
-                # else:
-                parserStack[-1][5] = self.payload[
-                    self.payloadOffset:self.payloadOffset + parserStack[-1][0]]
-                self.payloadOffset += parserStack[-1][0]
-                parserStack[-1][1] = 0
-                parserStack[-1][2] = True
-                # del parserStack[-1]
-                size = None
-            elif i in "viHIQ":
-                parserStack[-1][5].append(decode_simple(
-                    self, parserStack[-1][3][parserStack[-1][4]]))
-                size = None
-            else:
-                size = None
-            for depth in range(len(parserStack) - 1, -1, -1):
-                parserStack[depth][4] += 1
-                if parserStack[depth][4] >= len(parserStack[depth][3]):
-                    parserStack[depth][1] -= 1
-                    parserStack[depth][4] = 0
-                    if depth > 0:
-                        if parserStack[depth][2]:
-                            parserStack[depth - 1][5].append(
-                                parserStack[depth][5])
-                        else:
-                            parserStack[depth - 1][5].extend(
-                                parserStack[depth][5])
-                        parserStack[depth][5] = []
-                    if parserStack[depth][1] <= 0:
-                        if depth == 0:
-                            # we're done, at depth 0 counter is at 0
-                            # and pattern is done parsing
-                            return parserStack[depth][5]
-                        del parserStack[-1]
-                        continue
-                    break
-                break
-        if self.payloadOffset > self.payloadLength:
-            logger.debug(
-                'Insufficient data %i/%i',
-                self.payloadOffset, self.payloadLength)
-            raise BMProtoInsufficientDataError()
-
-    def bm_command_error(self):
-        """Decode an error message and log it"""
-        fatalStatus, banTime, inventoryVector, errorText = \
-            self.decode_payload_content("vvlsls")
-        logger.error(
-            '%s:%i error: %i, %s', self.destination.host,
-            self.destination.port, fatalStatus, errorText)
-        return True
-
-    def bm_command_getdata(self):
-        """
-        Incoming request for object(s).
-        If we have them and some other conditions are fulfilled,
-        append them to the write queue.
-        """
-        items = self.decode_payload_content("l32s")
-        # skip?
-        now = time.time()
-        if now < self.skipUntil:
-            return True
-        for i in items:
-            self.pendingUpload[str(i)] = now
-        return True
-
-    def _command_inv(self, dandelion=False):
-        items = self.decode_payload_content("l32s")
-
-        if len(items) > BMProto.maxObjectCount:
-            logger.error(
-                'Too many items in %sinv message!', 'd' if dandelion else '')
-            raise BMProtoExcessiveDataError()
-
-        # ignore dinv if dandelion turned off
-        if dandelion and not state.dandelion:
-            return True
-
-        for i in map(str, items):
-            if i in Inventory() and not Dandelion().hasHash(i):
-                continue
-            if dandelion and not Dandelion().hasHash(i):
-                Dandelion().addHash(i, self)
-            self.handleReceivedInventory(i)
-
-        return True
-
-    def bm_command_inv(self):
-        """Non-dandelion announce"""
-        return self._command_inv(False)
-
-    def bm_command_dinv(self):
-        """Dandelion stem announce"""
-        return self._command_inv(True)
-
-    def bm_command_object(self):
-        """Incoming object, process it"""
-        objectOffset = self.payloadOffset
-        nonce, expiresTime, objectType, version, streamNumber = \
-            self.decode_payload_content("QQIvv")
-        self.object = BMObject(
-            nonce, expiresTime, objectType, version, streamNumber,
-            self.payload, self.payloadOffset)
-
-        if len(self.payload) - self.payloadOffset > BMProto.maxObjectPayloadSize:
-            logger.info(
-                'The payload length of this object is too large (%d bytes).'
-                ' Ignoring it.', len(self.payload) - self.payloadOffset)
-            raise BMProtoExcessiveDataError()
-
-        try:
-            self.object.checkProofOfWorkSufficient()
-            self.object.checkEOLSanity()
-            self.object.checkAlreadyHave()
-        except (BMObjectExpiredError, BMObjectAlreadyHaveError,
-                BMObjectInsufficientPOWError):
-            BMProto.stopDownloadingObject(self.object.inventoryHash)
-            raise
-        try:
-            self.object.checkStream()
-        except BMObjectUnwantedStreamError:
-            acceptmismatch = BMConfigParser().get(
-                "inventory", "acceptmismatch")
-            BMProto.stopDownloadingObject(
-                self.object.inventoryHash, acceptmismatch)
-            if not acceptmismatch:
-                raise
-
-        try:
-            self.object.checkObjectByType()
-            objectProcessorQueue.put((
-                self.object.objectType, buffer(self.object.data)))
-        except BMObjectInvalidError:
-            BMProto.stopDownloadingObject(self.object.inventoryHash, True)
-        else:
-            try:
-                del missingObjects[self.object.inventoryHash]
-            except KeyError:
-                pass
-
-        if self.object.inventoryHash in Inventory() and Dandelion().hasHash(self.object.inventoryHash):
-            Dandelion().removeHash(self.object.inventoryHash, "cycle detection")
-
-        Inventory()[self.object.inventoryHash] = (
-            self.object.objectType, self.object.streamNumber,
-            buffer(self.payload[objectOffset:]), self.object.expiresTime,
-            buffer(self.object.tag)
-        )
-        self.handleReceivedObject(
-            self.object.streamNumber, self.object.inventoryHash)
-        invQueue.put((
-            self.object.streamNumber, self.object.inventoryHash,
-            self.destination))
-        return True
-
-    def _decode_addr(self):
-        return self.decode_payload_content("LQIQ16sH")
-
-    def bm_command_addr(self):
-        """Incoming addresses, process them"""
-        addresses = self._decode_addr()
-        for i in addresses:
-            seenTime, stream, services, ip, port = i
-            decodedIP = protocol.checkIPAddress(str(ip))
-            if stream not in state.streamsInWhichIAmParticipating:
-                continue
-            if (
-                decodedIP and time.time() - seenTime > 0 and
-                seenTime > time.time() - BMProto.addressAlive and
-                port > 0
-            ):
-                peer = state.Peer(decodedIP, port)
-                try:
-                    if knownnodes.knownNodes[stream][peer]["lastseen"] > seenTime:
-                        continue
-                except KeyError:
-                    pass
-                if len(knownnodes.knownNodes[stream]) < BMConfigParser().safeGetInt("knownnodes", "maxnodes"):
-                    with knownnodes.knownNodesLock:
-                        try:
-                            knownnodes.knownNodes[stream][peer]["lastseen"] = seenTime
-                        except (TypeError, KeyError):
-                            knownnodes.knownNodes[stream][peer] = {
-                                "lastseen": seenTime,
-                                "rating": 0,
-                                "self": False,
-                            }
-                addrQueue.put((stream, peer, self.destination))
-        return True
-
-    def bm_command_portcheck(self):
-        """Incoming port check request, queue it."""
-        portCheckerQueue.put(state.Peer(self.destination, self.peerNode.port))
-        return True
-
-    def bm_command_ping(self):
-        """Incoming ping, respond to it."""
-        self.append_write_buf(protocol.CreatePacket('pong'))
-        return True
-
-    def bm_command_pong(self):
-        """
-        Incoming pong.
-        Ignore it. PyBitmessage pings connections after about 5 minutes
-        of inactivity, and leaves it to the TCP stack to handle actual
-        timeouts. So there is no need to do anything when a pong arrives.
-        """
-        # nothing really
-        return True
-
-    def bm_command_verack(self):
-        """
-        Incoming verack.
-        If already sent my own verack, handshake is complete (except
-        potentially waiting for buffers to flush), so we can continue
-        to the main connection phase. If not sent verack yet,
-        continue processing.
-        """
-        self.verackReceived = True
-        if not self.verackSent:
-            return True
-        self.set_state(
-            "tls_init" if self.isSSL else "connection_fully_established",
-            length=self.payloadLength, expectBytes=0)
-        return False
-
-    def bm_command_version(self):
-        """
-        Incoming version.
-        Parse and log, remember important things, like streams, bitfields, etc.
-        """
-        (self.remoteProtocolVersion, self.services, self.timestamp,
-         self.sockNode, self.peerNode, self.nonce, self.userAgent,
-         self.streams) = self.decode_payload_content("IQQiiQlsLv")
-        self.nonce = struct.pack('>Q', self.nonce)
-        self.timeOffset = self.timestamp - int(time.time())
-        logger.debug('remoteProtocolVersion: %i', self.remoteProtocolVersion)
-        logger.debug('services: 0x%08X', self.services)
-        logger.debug('time offset: %i', self.timestamp - int(time.time()))
-        logger.debug('my external IP: %s', self.sockNode.host)
-        logger.debug(
-            'remote node incoming address: %s:%i',
-            self.destination.host, self.peerNode.port)
-        logger.debug('user agent: %s', self.userAgent)
-        logger.debug('streams: [%s]', ','.join(map(str, self.streams)))
-        if not self.peerValidityChecks():
-            # ABORT afterwards
-            return True
-        self.append_write_buf(protocol.CreatePacket('verack'))
-        self.verackSent = True
-        if not self.isOutbound:
-            self.append_write_buf(protocol.assembleVersionMessage(
-                self.destination.host, self.destination.port,
-                connectionpool.BMConnectionPool().streams, True,
-                nodeid=self.nodeid))
-            logger.debug(
-                '%(host)s:%(port)i sending version',
-                self.destination._asdict())
-        if ((self.services & protocol.NODE_SSL == protocol.NODE_SSL) and
-                protocol.haveSSL(not self.isOutbound)):
-            self.isSSL = True
-        if not self.verackReceived:
-            return True
-        self.set_state(
-            "tls_init" if self.isSSL else "connection_fully_established",
-            length=self.payloadLength, expectBytes=0)
-        return False
-
-    def peerValidityChecks(self):
-        """Check the validity of the peer"""
-        if self.remoteProtocolVersion < 3:
-            self.append_write_buf(protocol.assembleErrorMessage(
-                errorText="Your is using an old protocol. Closing connection.",
-                fatal=2))
-            logger.debug(
-                'Closing connection to old protocol version %s, node: %s',
-                self.remoteProtocolVersion, self.destination)
-            return False
-        if self.timeOffset > BMProto.maxTimeOffset:
-            self.append_write_buf(protocol.assembleErrorMessage(
-                errorText="Your time is too far in the future compared to mine."
-                " Closing connection.", fatal=2))
-            logger.info(
-                "%s's time is too far in the future (%s seconds)."
-                " Closing connection to it.", self.destination, self.timeOffset)
-            BMProto.timeOffsetWrongCount += 1
-            return False
-        elif self.timeOffset < -BMProto.maxTimeOffset:
-            self.append_write_buf(protocol.assembleErrorMessage(
-                errorText="Your time is too far in the past compared to mine."
-                " Closing connection.", fatal=2))
-            logger.info(
-                "%s's time is too far in the past (timeOffset %s seconds)."
-                " Closing connection to it.", self.destination, self.timeOffset)
-            BMProto.timeOffsetWrongCount += 1
-            return False
-        else:
-            BMProto.timeOffsetWrongCount = 0
-        if not self.streams:
-            self.append_write_buf(protocol.assembleErrorMessage(
-                errorText="We don't have shared stream interests."
-                " Closing connection.", fatal=2))
-            logger.debug(
-                'Closed connection to %s because there is no overlapping interest'
-                ' in streams.', self.destination)
-            return False
-        if self.destination in connectionpool.BMConnectionPool().inboundConnections:
-            try:
-                if not protocol.checkSocksIP(self.destination.host):
-                    self.append_write_buf(protocol.assembleErrorMessage(
-                        errorText="Too many connections from your IP."
-                        " Closing connection.", fatal=2))
-                    logger.debug(
-                        'Closed connection to %s because we are already connected'
-                        ' to that IP.', self.destination)
-                    return False
-            except:
-                pass
-        if not self.isOutbound:
-            # incoming from a peer we're connected to as outbound,
-            # or server full report the same error to counter deanonymisation
-            if (
-                state.Peer(self.destination.host, self.peerNode.port) in
-                connectionpool.BMConnectionPool().inboundConnections or
-                len(connectionpool.BMConnectionPool().inboundConnections) +
-                len(connectionpool.BMConnectionPool().outboundConnections) >
-                BMConfigParser().safeGetInt("bitmessagesettings", "maxtotalconnections") +
-                BMConfigParser().safeGetInt("bitmessagesettings", "maxbootstrapconnections")
-            ):
-                self.append_write_buf(protocol.assembleErrorMessage(
-                    errorText="Server full, please try again later.", fatal=2))
-                logger.debug(
-                    'Closed connection to %s due to server full'
-                    ' or duplicate inbound/outbound.', self.destination)
-                return False
-        if connectionpool.BMConnectionPool().isAlreadyConnected(
-                self.nonce):
-            self.append_write_buf(protocol.assembleErrorMessage(
-                errorText="I'm connected to myself. Closing connection.",
-                fatal=2))
-            logger.debug(
-                "Closed connection to %s because I'm connected to myself.",
-                self.destination)
-            return False
-
-        return True
-
-    @staticmethod
-    def assembleAddr(peerList):
-        """Build up a packed address"""
-        if isinstance(peerList, state.Peer):
-            peerList = (peerList)
-        if not peerList:
-            return b''
-        retval = b''
-        for i in range(0, len(peerList), BMProto.maxAddrCount):
-            payload = addresses.encodeVarint(
-                len(peerList[i:i + BMProto.maxAddrCount]))
-            for address in peerList[i:i + BMProto.maxAddrCount]:
-                stream, peer, timestamp = address
-                payload += struct.pack(
-                    '>Q', timestamp)  # 64-bit time
-                payload += struct.pack('>I', stream)
-                payload += struct.pack(
-                    '>q', 1)  # service bit flags offered by this node
-                payload += protocol.encodeHost(peer.host)
-                payload += struct.pack('>H', peer.port)  # remote port
-            retval += protocol.CreatePacket('addr', payload)
-        return retval
-
-    @staticmethod
-    def stopDownloadingObject(hashId, forwardAnyway=False):
-        """Stop downloading an object"""
-        for connection in (
-            connectionpool.BMConnectionPool().inboundConnections.values() +
-            connectionpool.BMConnectionPool().outboundConnections.values()
-        ):
-            try:
-                del connection.objectsNewToMe[hashId]
-            except KeyError:
-                pass
-            if not forwardAnyway:
-                try:
-                    with connection.objectsNewToThemLock:
-                        del connection.objectsNewToThem[hashId]
-                except KeyError:
-                    pass
-        try:
-            del missingObjects[hashId]
-        except KeyError:
-            pass
-
-    def handle_close(self):
-        """Handle close"""
-        self.set_state("close")
-        if not (self.accepting or self.connecting or self.connected):
-            # already disconnected
-            return
-        try:
-            logger.debug(
-                '%s:%i: closing, %s', self.destination.host,
-                self.destination.port, self.close_reason)
-        except AttributeError:
-            try:
-                logger.debug(
-                    '%(host)s:%(port)i: closing', self.destination._asdict())
-            except AttributeError:
-                logger.debug('Disconnected socket closing')
-        AdvancedDispatcher.handle_close(self)
-
-
-class BMStringParser(BMProto):
-    """
-    A special case of BMProto used by objectProcessor to send ACK
-    """
-    def __init__(self):
-        super(BMStringParser, self).__init__()
-        self.destination = state.Peer('127.0.0.1', 8444)
-        self.payload = None
-        ObjectTracker.__init__(self)
-
-    def send_data(self, data):
-        """Send object given by the data string"""
-        # This class is introduced specially for ACK sending, please
-        # change log strings if you are going to use it for something else
-        self.bm_proto_reset()
-        self.payload = data
-        try:
-            self.bm_command_object()
-        except BMObjectAlreadyHaveError:
-            pass  # maybe the same msg received on different nodes
-        except BMObjectExpiredError:
-            logger.debug(
-                'Sending ACK failure (expired): %s', hexlify(data))
-        except Exception as e:
-            logger.debug(
-                'Exception of type %s while sending ACK',
-                type(e), exc_info=True)

src/network/connectionchooser.py

@@ -1,3 +1,11 @@
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import range
+from builtins import *
 import random  # nosec
 
 import knownnodes
@@ -10,7 +18,7 @@ from queues import Queue, portCheckerQueue
 
 def getDiscoveredPeer():
     try:
-        peer = random.choice(state.discoveredPeers.keys())
+        peer = random.choice(list(state.discoveredPeers.keys()))
     except (IndexError, KeyError):
         raise ValueError
     try:
@@ -36,7 +44,7 @@ def chooseConnection(stream):
         # discovered peers are already filtered by allowed streams
         return getDiscoveredPeer()
     for _ in range(50):
-        peer = random.choice(knownnodes.knownNodes[stream].keys())
+        peer = random.choice(list(knownnodes.knownNodes[stream].keys()))
         try:
             rating = knownnodes.knownNodes[stream][peer]['rating']
         except TypeError:

src/network/connectionpool.py

@@ -1,319 +0,0 @@
-import errno
-import re
-import socket
-import time
-
-import asyncore_pollchoose as asyncore
-import helper_bootstrap
-import helper_random
-import knownnodes
-import protocol
-import state
-from bmconfigparser import BMConfigParser
-from connectionchooser import chooseConnection
-from debug import logger
-from proxy import Proxy
-from singleton import Singleton
-from tcp import (
-    TCPServer, Socks5BMConnection, Socks4aBMConnection, TCPConnection)
-from udp import UDPSocket
-
-
-@Singleton
-class BMConnectionPool(object):
-    """Pool of all existing connections"""
-    def __init__(self):
-        asyncore.set_rates(
-            BMConfigParser().safeGetInt(
-                "bitmessagesettings", "maxdownloadrate"),
-            BMConfigParser().safeGetInt(
-                "bitmessagesettings", "maxuploadrate")
-        )
-        self.outboundConnections = {}
-        self.inboundConnections = {}
-        self.listeningSockets = {}
-        self.udpSockets = {}
-        self.streams = []
-        self.lastSpawned = 0
-        self.spawnWait = 2
-        self.bootstrapped = False
-
-    def connectToStream(self, streamNumber):
-        """Connect to a bitmessage stream"""
-        self.streams.append(streamNumber)
-
-    def getConnectionByAddr(self, addr):
-        """
-        Return an (existing) connection object based on a `Peer` object
-        (IP and port)
-        """
-        try:
-            return self.inboundConnections[addr]
-        except KeyError:
-            pass
-        try:
-            return self.inboundConnections[addr.host]
-        except (KeyError, AttributeError):
-            pass
-        try:
-            return self.outboundConnections[addr]
-        except KeyError:
-            pass
-        try:
-            return self.udpSockets[addr.host]
-        except (KeyError, AttributeError):
-            pass
-        raise KeyError
-
-    def isAlreadyConnected(self, nodeid):
-        """Check if we're already connected to this peer"""
-        for i in (
-            self.inboundConnections.values() +
-            self.outboundConnections.values()
-        ):
-            try:
-                if nodeid == i.nodeid:
-                    return True
-            except AttributeError:
-                pass
-        return False
-
-    def addConnection(self, connection):
-        """Add a connection object to our internal dict"""
-        if isinstance(connection, UDPSocket):
-            return
-        if connection.isOutbound:
-            self.outboundConnections[connection.destination] = connection
-        else:
-            if connection.destination.host in self.inboundConnections:
-                self.inboundConnections[connection.destination] = connection
-            else:
-                self.inboundConnections[connection.destination.host] = \
-                    connection
-
-    def removeConnection(self, connection):
-        """Remove a connection from our internal dict"""
-        if isinstance(connection, UDPSocket):
-            del self.udpSockets[connection.listening.host]
-        elif isinstance(connection, TCPServer):
-            del self.listeningSockets[state.Peer(
-                connection.destination.host, connection.destination.port)]
-        elif connection.isOutbound:
-            try:
-                del self.outboundConnections[connection.destination]
-            except KeyError:
-                pass
-        else:
-            try:
-                del self.inboundConnections[connection.destination]
-            except KeyError:
-                try:
-                    del self.inboundConnections[connection.destination.host]
-                except KeyError:
-                    pass
-        connection.handle_close()
-
-    def getListeningIP(self):
-        """What IP are we supposed to be listening on?"""
-        if BMConfigParser().safeGet(
-                "bitmessagesettings", "onionhostname").endswith(".onion"):
-            host = BMConfigParser().safeGet(
-                "bitmessagesettings", "onionbindip")
-        else:
-            host = '127.0.0.1'
-        if (BMConfigParser().safeGetBoolean(
-                "bitmessagesettings", "sockslisten") or
-            BMConfigParser().safeGet(
-                "bitmessagesettings", "socksproxytype") == "none"):
-            # python doesn't like bind + INADDR_ANY?
-            # host = socket.INADDR_ANY
-            host = BMConfigParser().get("network", "bind")
-        return host
-
-    def startListening(self, bind=None):
-        """Open a listening socket and start accepting connections on it"""
-        if bind is None:
-            bind = self.getListeningIP()
-        port = BMConfigParser().safeGetInt("bitmessagesettings", "port")
-        # correct port even if it changed
-        ls = TCPServer(host=bind, port=port)
-        self.listeningSockets[ls.destination] = ls
-
-    def startUDPSocket(self, bind=None):
-        """
-        Open an UDP socket. Depending on settings, it can either only
-        accept incoming UDP packets, or also be able to send them.
-        """
-        if bind is None:
-            host = self.getListeningIP()
-            udpSocket = UDPSocket(host=host, announcing=True)
-        else:
-            if bind is False:
-                udpSocket = UDPSocket(announcing=False)
-            else:
-                udpSocket = UDPSocket(host=bind, announcing=True)
-        self.udpSockets[udpSocket.listening.host] = udpSocket
-
-    def loop(self):
-        """Main Connectionpool's loop"""
-        # defaults to empty loop if outbound connections are maxed
-        spawnConnections = False
-        acceptConnections = True
-        if BMConfigParser().safeGetBoolean(
-                'bitmessagesettings', 'dontconnect'):
-            acceptConnections = False
-        elif BMConfigParser().safeGetBoolean(
-                'bitmessagesettings', 'sendoutgoingconnections'):
-            spawnConnections = True
-        socksproxytype = BMConfigParser().safeGet(
-            'bitmessagesettings', 'socksproxytype', '')
-        onionsocksproxytype = BMConfigParser().safeGet(
-            'bitmessagesettings', 'onionsocksproxytype', '')
-        if (socksproxytype[:5] == 'SOCKS' and
-            not BMConfigParser().safeGetBoolean(
-                'bitmessagesettings', 'sockslisten') and
-            '.onion' not in BMConfigParser().safeGet(
-                'bitmessagesettings', 'onionhostname', '')):
-            acceptConnections = False
-
-        if spawnConnections:
-            if not knownnodes.knownNodesActual:
-                helper_bootstrap.dns()
-            if not self.bootstrapped:
-                self.bootstrapped = True
-                Proxy.proxy = (
-                    BMConfigParser().safeGet(
-                        'bitmessagesettings', 'sockshostname'),
-                    BMConfigParser().safeGetInt(
-                        'bitmessagesettings', 'socksport')
-                )
-                # TODO AUTH
-                # TODO reset based on GUI settings changes
-                try:
-                    if not onionsocksproxytype.startswith("SOCKS"):
-                        raise ValueError
-                    Proxy.onion_proxy = (
-                        BMConfigParser().safeGet(
-                            'network', 'onionsockshostname', None),
-                        BMConfigParser().safeGet(
-                            'network', 'onionsocksport', None)
-                    )
-                except ValueError:
-                    Proxy.onion_proxy = None
-            established = sum(
-                1 for c in self.outboundConnections.values()
-                if (c.connected and c.fullyEstablished))
-            pending = len(self.outboundConnections) - established
-            if established < BMConfigParser().safeGetInt(
-                    'bitmessagesettings', 'maxoutboundconnections'):
-                for i in range(
-                        state.maximumNumberOfHalfOpenConnections - pending):
-                    try:
-                        chosen = chooseConnection(
-                            helper_random.randomchoice(self.streams))
-                    except ValueError:
-                        continue
-                    if chosen in self.outboundConnections:
-                        continue
-                    if chosen.host in self.inboundConnections:
-                        continue
-                    # don't connect to self
-                    if chosen in state.ownAddresses:
-                        continue
-
-                    try:
-                        if (chosen.host.endswith(".onion") and
-                                Proxy.onion_proxy is not None):
-                            if onionsocksproxytype == "SOCKS5":
-                                self.addConnection(Socks5BMConnection(chosen))
-                            elif onionsocksproxytype == "SOCKS4a":
-                                self.addConnection(Socks4aBMConnection(chosen))
-                        elif socksproxytype == "SOCKS5":
-                            self.addConnection(Socks5BMConnection(chosen))
-                        elif socksproxytype == "SOCKS4a":
-                            self.addConnection(Socks4aBMConnection(chosen))
-                        else:
-                            self.addConnection(TCPConnection(chosen))
-                    except socket.error as e:
-                        if e.errno == errno.ENETUNREACH:
-                            continue
-
-                    self.lastSpawned = time.time()
-        else:
-            for i in (
-                self.inboundConnections.values() +
-                self.outboundConnections.values()
-            ):
-                # FIXME: rating will be increased after next connection
-                i.handle_close()
-
-        if acceptConnections:
-            if not self.listeningSockets:
-                if BMConfigParser().safeGet('network', 'bind') == '':
-                    self.startListening()
-                else:
-                    for bind in re.sub(
-                        "[^\w.]+", " ",
-                        BMConfigParser().safeGet('network', 'bind')
-                    ).split():
-                        self.startListening(bind)
-                logger.info('Listening for incoming connections.')
-            if not self.udpSockets:
-                if BMConfigParser().safeGet('network', 'bind') == '':
-                    self.startUDPSocket()
-                else:
-                    for bind in re.sub(
-                        "[^\w.]+", " ",
-                        BMConfigParser().safeGet('network', 'bind')
-                    ).split():
-                        self.startUDPSocket(bind)
-                    self.startUDPSocket(False)
-                logger.info('Starting UDP socket(s).')
-        else:
-            if self.listeningSockets:
-                for i in self.listeningSockets.values():
-                    i.close_reason = "Stopping listening"
-                    i.accepting = i.connecting = i.connected = False
-                logger.info('Stopped listening for incoming connections.')
-            if self.udpSockets:
-                for i in self.udpSockets.values():
-                    i.close_reason = "Stopping UDP socket"
-                    i.accepting = i.connecting = i.connected = False
-                logger.info('Stopped udp sockets.')
-
-        loopTime = float(self.spawnWait)
-        if self.lastSpawned < time.time() - self.spawnWait:
-            loopTime = 2.0
-        asyncore.loop(timeout=loopTime, count=1000)
-
-        reaper = []
-        for i in (
-            self.inboundConnections.values() +
-            self.outboundConnections.values()
-        ):
-            minTx = time.time() - 20
-            if i.fullyEstablished:
-                minTx -= 300 - 20
-            if i.lastTx < minTx:
-                if i.fullyEstablished:
-                    i.append_write_buf(protocol.CreatePacket('ping'))
-                else:
-                    i.close_reason = "Timeout (%is)" % (
-                        time.time() - i.lastTx)
-                    i.set_state("close")
-        for i in (
-            self.inboundConnections.values() +
-            self.outboundConnections.values() +
-            self.listeningSockets.values() +
-            self.udpSockets.values()
-        ):
-            if not (i.accepting or i.connecting or i.connected):
-                reaper.append(i)
-            else:
-                try:
-                    if i.state == "close":
-                        reaper.append(i)
-                except AttributeError:
-                    pass
-        for i in reaper:
-            self.removeConnection(i)

src/network/dandelion.py

@@ -1,188 +0,0 @@
-from collections import namedtuple
-from random import choice, sample, expovariate
-from threading import RLock
-from time import time
-
-import connectionpool
-import state
-from debug import logging
-from queues import invQueue
-from singleton import Singleton
-
-# randomise routes after 600 seconds
-REASSIGN_INTERVAL = 600
-
-# trigger fluff due to expiration
-FLUFF_TRIGGER_FIXED_DELAY = 10
-FLUFF_TRIGGER_MEAN_DELAY = 30
-
-MAX_STEMS = 2
-
-Stem = namedtuple('Stem', ['child', 'stream', 'timeout'])
-
-
-@Singleton
-class Dandelion():
-    """Dandelion class for tracking stem/fluff stages."""
-    def __init__(self):
-        # currently assignable child stems
-        self.stem = []
-        # currently assigned parent <-> child mappings
-        self.nodeMap = {}
-        # currently existing objects in stem mode
-        self.hashMap = {}
-        # when to rerandomise routes
-        self.refresh = time() + REASSIGN_INTERVAL
-        self.lock = RLock()
-
-    def poissonTimeout(self, start=None, average=0):
-        """Generate deadline using Poisson distribution"""
-        if start is None:
-            start = time()
-        if average == 0:
-            average = FLUFF_TRIGGER_MEAN_DELAY
-        return start + expovariate(1.0 / average) + FLUFF_TRIGGER_FIXED_DELAY
-
-    def addHash(self, hashId, source=None, stream=1):
-        """Add inventory vector to dandelion stem"""
-        if not state.dandelion:
-            return
-        with self.lock:
-            self.hashMap[hashId] = Stem(
-                self.getNodeStem(source),
-                stream,
-                self.poissonTimeout())
-
-    def setHashStream(self, hashId, stream=1):
-        """
-        Update stream for inventory vector (as inv/dinv commands don't
-        include streams, we only learn this after receiving the object)
-        """
-        with self.lock:
-            if hashId in self.hashMap:
-                self.hashMap[hashId] = Stem(
-                    self.hashMap[hashId].child,
-                    stream,
-                    self.poissonTimeout())
-
-    def removeHash(self, hashId, reason="no reason specified"):
-        """Switch inventory vector from stem to fluff mode"""
-        logging.debug(
-            "%s entering fluff mode due to %s.",
-            ''.join('%02x' % ord(i) for i in hashId), reason)
-        with self.lock:
-            try:
-                del self.hashMap[hashId]
-            except KeyError:
-                pass
-
-    def hasHash(self, hashId):
-        """Is inventory vector in stem mode?"""
-        return hashId in self.hashMap
-
-    def objectChildStem(self, hashId):
-        """Child (i.e. next) node for an inventory vector during stem mode"""
-        return self.hashMap[hashId].child
-
-    def maybeAddStem(self, connection):
-        """
-        If we had too few outbound connections, add the current one to the
-        current stem list. Dandelion as designed by the authors should
-        always have two active stem child connections.
-        """
-        # fewer than MAX_STEMS outbound connections at last reshuffle?
-        with self.lock:
-            if len(self.stem) < MAX_STEMS:
-                self.stem.append(connection)
-                for k in (k for k, v in self.nodeMap.iteritems() if v is None):
-                    self.nodeMap[k] = connection
-                for k, v in {
-                    k: v for k, v in self.hashMap.iteritems()
-                    if v.child is None
-                }.iteritems():
-                    self.hashMap[k] = Stem(
-                        connection, v.stream, self.poissonTimeout())
-                    invQueue.put((v.stream, k, v.child))
-
-    def maybeRemoveStem(self, connection):
-        """
-        Remove current connection from the stem list (called e.g. when
-        a connection is closed).
-        """
-        # is the stem active?
-        with self.lock:
-            if connection in self.stem:
-                self.stem.remove(connection)
-                # active mappings to pointing to the removed node
-                for k in (
-                    k for k, v in self.nodeMap.iteritems() if v == connection
-                ):
-                    self.nodeMap[k] = None
-                for k, v in {
-                    k: v for k, v in self.hashMap.iteritems()
-                    if v.child == connection
-                }.iteritems():
-                    self.hashMap[k] = Stem(
-                        None, v.stream, self.poissonTimeout())
-
-    def pickStem(self, parent=None):
-        """
-        Pick a random active stem, but not the parent one
-        (the one where an object came from)
-        """
-        try:
-            # pick a random from available stems
-            stem = choice(range(len(self.stem)))
-            if self.stem[stem] == parent:
-                # one stem available and it's the parent
-                if len(self.stem) == 1:
-                    return None
-                # else, pick the other one
-                return self.stem[1 - stem]
-            # all ok
-            return self.stem[stem]
-        except IndexError:
-            # no stems available
-            return None
-
-    def getNodeStem(self, node=None):
-        """
-        Return child stem node for a given parent stem node
-        (the mapping is static for about 10 minutes, then it reshuffles)
-        """
-        with self.lock:
-            try:
-                return self.nodeMap[node]
-            except KeyError:
-                self.nodeMap[node] = self.pickStem(node)
-                return self.nodeMap[node]
-
-    def expire(self):
-        """Switch expired objects from stem to fluff mode"""
-        with self.lock:
-            deadline = time()
-            toDelete = [
-                [v.stream, k, v.child] for k, v in self.hashMap.iteritems()
-                if v.timeout < deadline
-            ]
-
-            for row in toDelete:
-                self.removeHash(row[1], 'expiration')
-                invQueue.put(row)
-        return toDelete
-
-    def reRandomiseStems(self):
-        """Re-shuffle stem mapping (parent <-> child pairs)"""
-        with self.lock:
-            try:
-                # random two connections
-                self.stem = sample(
-                    connectionpool.BMConnectionPool(
-                    ).outboundConnections.values(), MAX_STEMS)
-            # not enough stems available
-            except ValueError:
-                self.stem = connectionpool.BMConnectionPool(
-                ).outboundConnections.values()
-            self.nodeMap = {}
-            # hashMap stays to cater for pending stems
-        self.refresh = time() + REASSIGN_INTERVAL

src/network/downloadthread.py

@@ -1,15 +1,21 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+from __future__ import print_function
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
+from past.utils import old_div
 import threading
 import time
 
 import addresses
 import helper_random
 import protocol
-from dandelion import Dandelion
+from .fix_circular_imports import BMConnectionPool, Dandelion, missingObjects
 from debug import logger
 from helper_threading import StoppableThread
 from inventory import Inventory
-from network.connectionpool import BMConnectionPool
-from objectracker import missingObjects
 
 
 class DownloadThread(threading.Thread, StoppableThread):
@@ -29,7 +35,7 @@ class DownloadThread(threading.Thread, StoppableThread):
     def cleanPending(self):
         deadline = time.time() - DownloadThread.requestExpires
         try:
-            toDelete = [k for k, v in missingObjects.iteritems() if v < deadline]
+            toDelete = [k for k, v in missingObjects.items() if v < deadline]
         except RuntimeError:
             pass
         else:
@@ -41,10 +47,10 @@ class DownloadThread(threading.Thread, StoppableThread):
         while not self._stopped:
             requested = 0
             # Choose downloading peers randomly
-            connections = [x for x in BMConnectionPool().inboundConnections.values() + BMConnectionPool().outboundConnections.values() if x.fullyEstablished]
+            connections = [x for x in list(BMConnectionPool().inboundConnections.values()) + list(BMConnectionPool().outboundConnections.values()) if x.fullyEstablished]
             helper_random.randomshuffle(connections)
             try:
-                requestChunk =  max(int(min(DownloadThread.maxRequestChunk, len(missingObjects)) / len(connections)), 1)
+                requestChunk =  max(int(old_div(min(DownloadThread.maxRequestChunk, len(missingObjects)), len(connections))), 1)
             except ZeroDivisionError:
                 requestChunk = 1
             for i in connections:

src/network/fix_circular_imports.py

@@ -0,0 +1,1927 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+from __future__ import print_function
+
+
+
+
+##########################
+# src/network/bmproto.py #
+##########################
+
+
+
+
+from future import standard_library
+standard_library.install_aliases()
+from builtins import map
+from builtins import str
+from builtins import range
+from builtins import *
+from past.utils import old_div
+from builtins import object
+import time
+
+import protocol
+import state
+from addresses import calculateInventoryHash
+from debug import logger
+from inventory import Inventory
+
+
+class BMObjectInsufficientPOWError(Exception):
+    """Exception indicating the object doesn't have sufficient proof of work."""
+    errorCodes = ("Insufficient proof of work")
+
+
+class BMObjectInvalidDataError(Exception):
+    """Exception indicating the data being parsed does not match the specification."""
+    errorCodes = ("Data invalid")
+
+
+class BMObjectExpiredError(Exception):
+    """Exception indicating the object's lifetime has expired."""
+    errorCodes = ("Object expired")
+
+
+class BMObjectUnwantedStreamError(Exception):
+    """Exception indicating the object is in a stream we didn't advertise as being interested in."""
+    errorCodes = ("Object in unwanted stream")
+
+
+class BMObjectInvalidError(Exception):
+    """The object's data does not match object specification."""
+    errorCodes = ("Invalid object")
+
+
+class BMObjectAlreadyHaveError(Exception):
+    """We received a duplicate object (one we already have)"""
+    errorCodes = ("Already have this object")
+
+
+class BMObject(object):
+    """Bitmessage Object as a class."""
+    # pylint: disable=too-many-instance-attributes
+
+    # max TTL, 28 days and 3 hours
+    maxTTL = 28 * 24 * 60 * 60 + 10800
+    # min TTL, 3 hour (in the past
+    minTTL = -3600
+
+    def __init__(
+            self,
+            nonce,
+            expiresTime,
+            objectType,
+            version,
+            streamNumber,
+            data,
+            payloadOffset
+    ):  # pylint: disable=too-many-arguments
+        self.nonce = nonce
+        self.expiresTime = expiresTime
+        self.objectType = objectType
+        self.version = version
+        self.streamNumber = streamNumber
+        self.inventoryHash = calculateInventoryHash(data)
+        # copy to avoid memory issues
+        self.data = bytearray(data)
+        self.tag = self.data[payloadOffset:payloadOffset + 32]
+
+    def checkProofOfWorkSufficient(self):
+        """Perform a proof of work check for sufficiency."""
+        # Let us check to make sure that the proof of work is sufficient.
+        if not protocol.isProofOfWorkSufficient(self.data):
+            logger.info('Proof of work is insufficient.')
+            raise BMObjectInsufficientPOWError()
+
+    def checkEOLSanity(self):
+        """Check if object's lifetime isn't ridiculously far in the past or future."""
+        # EOL sanity check
+        if self.expiresTime - int(time.time()) > BMObject.maxTTL:
+            logger.info(
+                'This object\'s End of Life time is too far in the future. Ignoring it. Time is %i',
+                self.expiresTime)
+            # .. todo::  remove from download queue
+            raise BMObjectExpiredError()
+
+        if self.expiresTime - int(time.time()) < BMObject.minTTL:
+            logger.info(
+                'This object\'s End of Life time was too long ago. Ignoring the object. Time is %i',
+                self.expiresTime)
+            # .. todo::  remove from download queue
+            raise BMObjectExpiredError()
+
+    def checkStream(self):
+        """Check if object's stream matches streams we are interested in"""
+        if self.streamNumber not in state.streamsInWhichIAmParticipating:
+            logger.debug('The streamNumber %i isn\'t one we are interested in.', self.streamNumber)
+            raise BMObjectUnwantedStreamError()
+
+    def checkAlreadyHave(self):
+        """
+        Check if we already have the object (so that we don't duplicate it in inventory or advertise it unnecessarily)
+        """
+        # if it's a stem duplicate, pretend we don't have it
+        if Dandelion().hasHash(self.inventoryHash):
+            return
+        if self.inventoryHash in Inventory():
+            raise BMObjectAlreadyHaveError()
+
+    def checkObjectByType(self):
+        """Call a object type specific check (objects can have additional checks based on their types)"""
+        if self.objectType == protocol.OBJECT_GETPUBKEY:
+            self.checkGetpubkey()
+        elif self.objectType == protocol.OBJECT_PUBKEY:
+            self.checkPubkey()
+        elif self.objectType == protocol.OBJECT_MSG:
+            self.checkMessage()
+        elif self.objectType == protocol.OBJECT_BROADCAST:
+            self.checkBroadcast()
+        # other objects don't require other types of tests
+
+    def checkMessage(self):
+        """"Message" object type checks."""
+        # pylint: disable=no-self-use
+        return
+
+    def checkGetpubkey(self):
+        """"Getpubkey" object type checks."""
+        if len(self.data) < 42:
+            logger.info('getpubkey message doesn\'t contain enough data. Ignoring.')
+            raise BMObjectInvalidError()
+
+    def checkPubkey(self):
+        """"Pubkey" object type checks."""
+        if len(self.data) < 146 or len(self.data) > 440:  # sanity check
+            logger.info('pubkey object too short or too long. Ignoring.')
+            raise BMObjectInvalidError()
+
+    def checkBroadcast(self):
+        """"Broadcast" object type checks."""
+        if len(self.data) < 180:
+            logger.debug(
+                'The payload length of this broadcast packet is unreasonably low.'
+                ' Someone is probably trying funny business. Ignoring message.')
+            raise BMObjectInvalidError()
+
+        # this isn't supported anymore
+        if self.version < 2:
+            raise BMObjectInvalidError()
+
+
+
+
+###############################
+# src/network/objectracker.py #
+###############################
+
+
+
+
+import time
+from threading import RLock
+
+from randomtrackingdict import RandomTrackingDict
+
+haveBloom = False
+
+try:
+    # pybloomfiltermmap
+    from pybloomfilter import BloomFilter
+    haveBloom = True
+except ImportError:
+    try:
+        # pybloom
+        from pybloom import BloomFilter
+        haveBloom = True
+    except ImportError:
+        pass
+
+# it isn't actually implemented yet so no point in turning it on
+haveBloom = False
+
+# tracking pending downloads globally, for stats
+missingObjects = {}
+
+
+class ObjectTracker(object):
+    invCleanPeriod = 300
+    invInitialCapacity = 50000
+    invErrorRate = 0.03
+    trackingExpires = 3600
+    initialTimeOffset = 60
+
+    def __init__(self):
+        self.objectsNewToMe = RandomTrackingDict()
+        self.objectsNewToThem = {}
+        self.objectsNewToThemLock = RLock()
+        self.initInvBloom()
+        self.initAddrBloom()
+        self.lastCleaned = time.time()
+
+    def initInvBloom(self):
+        if haveBloom:
+            # lock?
+            self.invBloom = BloomFilter(capacity=ObjectTracker.invInitialCapacity,
+                                        error_rate=ObjectTracker.invErrorRate)
+
+    def initAddrBloom(self):
+        if haveBloom:
+            # lock?
+            self.addrBloom = BloomFilter(capacity=ObjectTracker.invInitialCapacity,
+                                         error_rate=ObjectTracker.invErrorRate)
+
+    def clean(self):
+        if self.lastCleaned < time.time() - ObjectTracker.invCleanPeriod:
+            if haveBloom:
+                if len(missingObjects) == 0:
+                    self.initInvBloom()
+                self.initAddrBloom()
+            else:
+                # release memory
+                deadline = time.time() - ObjectTracker.trackingExpires
+                with self.objectsNewToThemLock:
+                    self.objectsNewToThem = {k: v for k, v in self.objectsNewToThem.items() if v >= deadline}
+            self.lastCleaned = time.time()
+
+    def hasObj(self, hashid):
+        if haveBloom:
+            return hashid in self.invBloom
+        else:
+            return hashid in self.objectsNewToMe
+
+    def handleReceivedInventory(self, hashId):
+        if haveBloom:
+            self.invBloom.add(hashId)
+        try:
+            with self.objectsNewToThemLock:
+                del self.objectsNewToThem[hashId]
+        except KeyError:
+            pass
+        if hashId not in missingObjects:
+            missingObjects[hashId] = time.time()
+        self.objectsNewToMe[hashId] = True
+
+    def handleReceivedObject(self, streamNumber, hashid):
+        for i in list(BMConnectionPool().inboundConnections.values()) + list(BMConnectionPool().outboundConnections.values()):
+            if not i.fullyEstablished:
+                continue
+            try:
+                del i.objectsNewToMe[hashid]
+            except KeyError:
+                if streamNumber in i.streams and \
+                    (not Dandelion().hasHash(hashid) or \
+                    Dandelion().objectChildStem(hashid) == i):
+                    with i.objectsNewToThemLock:
+                        i.objectsNewToThem[hashid] = time.time()
+                    # update stream number, which we didn't have when we just received the dinv
+                    # also resets expiration of the stem mode
+                    Dandelion().setHashStream(hashid, streamNumber)
+
+            if i == self:
+                try:
+                    with i.objectsNewToThemLock:
+                        del i.objectsNewToThem[hashid]
+                except KeyError:
+                    pass
+        self.objectsNewToMe.setLastObject()
+
+    def hasAddr(self, addr):
+        if haveBloom:
+            return addr in self.invBloom
+
+    def addAddr(self, hashid):
+        if haveBloom:
+            self.addrBloom.add(hashid)
+
+# addr sending -> per node upload queue, and flush every minute or so
+# inv sending -> if not in bloom, inv immediately, otherwise put into a per node upload queue and flush every minute or so
+# data sending -> a simple queue
+
+# no bloom
+# - if inv arrives
+#   - if we don't have it, add tracking and download queue
+#   - if we do have it, remove from tracking
+# tracking downloads
+# - per node hash of items the node has but we don't
+# tracking inv
+# - per node hash of items that neither the remote node nor we have
+#
+
+
+
+
+##########################
+# src/network/bmproto.py #
+##########################
+
+
+
+
+import base64
+import hashlib
+import socket
+import struct
+import time
+from binascii import hexlify
+
+import addresses
+import knownnodes
+import protocol
+import state
+from bmconfigparser import BMConfigParser
+from debug import logger
+from inventory import Inventory
+from network.advanceddispatcher import AdvancedDispatcher
+from network.node import Node
+from network.proxy import ProxyError
+from queues import objectProcessorQueue, portCheckerQueue, invQueue, addrQueue
+from randomtrackingdict import RandomTrackingDict
+
+
+class BMProtoError(ProxyError):
+    """A Bitmessage Protocol Base Error"""
+    errorCodes = ("Protocol error")
+
+
+class BMProtoInsufficientDataError(BMProtoError):
+    """A Bitmessage Protocol Insufficient Data Error"""
+    errorCodes = ("Insufficient data")
+
+
+class BMProtoExcessiveDataError(BMProtoError):
+    """A Bitmessage Protocol Excessive Data Error"""
+    errorCodes = ("Too much data")
+
+
+class BMProto(AdvancedDispatcher, ObjectTracker):
+    """A parser for the Bitmessage Protocol"""
+    # ~1.6 MB which is the maximum possible size of an inv message.
+    maxMessageSize = 1600100
+    # 2**18 = 256kB is the maximum size of an object payload
+    maxObjectPayloadSize = 2**18
+    # protocol specification says max 1000 addresses in one addr command
+    maxAddrCount = 1000
+    # protocol specification says max 50000 objects in one inv command
+    maxObjectCount = 50000
+    # address is online if online less than this many seconds ago
+    addressAlive = 10800
+    # maximum time offset
+    maxTimeOffset = 3600
+    timeOffsetWrongCount = 0
+
+    def __init__(self, address=None, sock=None):
+        AdvancedDispatcher.__init__(self, sock)
+        self.isOutbound = False
+        # packet/connection from a local IP
+        self.local = False
+        self.pendingUpload = RandomTrackingDict()
+
+    def bm_proto_reset(self):
+        """Reset the bitmessage object parser"""
+        self.magic = None
+        self.command = None
+        self.payloadLength = 0
+        self.checksum = None
+        self.payload = None
+        self.invalid = False
+        self.payloadOffset = 0
+        self.expectBytes = protocol.Header.size
+        self.object = None
+
+    def state_bm_header(self):
+        """Process incoming header"""
+        self.magic, self.command, self.payloadLength, self.checksum = \
+            protocol.Header.unpack(self.read_buf[:protocol.Header.size])
+        self.command = self.command.rstrip('\x00')
+        if self.magic != 0xE9BEB4D9:
+            # skip 1 byte in order to sync
+            self.set_state("bm_header", length=1)
+            self.bm_proto_reset()
+            logger.debug('Bad magic')
+            if self.socket.type == socket.SOCK_STREAM:
+                self.close_reason = "Bad magic"
+                self.set_state("close")
+            return False
+        if self.payloadLength > BMProto.maxMessageSize:
+            self.invalid = True
+        self.set_state(
+            "bm_command",
+            length=protocol.Header.size, expectBytes=self.payloadLength)
+        return True
+
+    def state_bm_command(self):
+        """Process incoming command"""
+        self.payload = self.read_buf[:self.payloadLength]
+        if self.checksum != hashlib.sha512(self.payload).digest()[0:4]:
+            logger.debug('Bad checksum, ignoring')
+            self.invalid = True
+        retval = True
+        if not self.fullyEstablished and self.command not in (
+                "error", "version", "verack"):
+            logger.error(
+                'Received command %s before connection was fully'
+                ' established, ignoring', self.command)
+            self.invalid = True
+        if not self.invalid:
+            try:
+                retval = getattr(
+                    self, "bm_command_" + str(self.command).lower())()
+            except AttributeError:
+                # unimplemented command
+                logger.debug('unimplemented command %s', self.command)
+            except BMProtoInsufficientDataError:
+                logger.debug('packet length too short, skipping')
+            except BMProtoExcessiveDataError:
+                logger.debug('too much data, skipping')
+            except BMObjectInsufficientPOWError:
+                logger.debug('insufficient PoW, skipping')
+            except BMObjectInvalidDataError:
+                logger.debug('object invalid data, skipping')
+            except BMObjectExpiredError:
+                logger.debug('object expired, skipping')
+            except BMObjectUnwantedStreamError:
+                logger.debug('object not in wanted stream, skipping')
+            except BMObjectInvalidError:
+                logger.debug('object invalid, skipping')
+            except BMObjectAlreadyHaveError:
+                logger.debug(
+                    '%(host)s:%(port)i already got object, skipping',
+                    self.destination._asdict())
+            except struct.error:
+                logger.debug('decoding error, skipping')
+        elif self.socket.type == socket.SOCK_DGRAM:
+            # broken read, ignore
+            pass
+        else:
+            logger.debug('Closing due to invalid command %s', self.command)
+            self.close_reason = "Invalid command %s" % self.command
+            self.set_state("close")
+            return False
+        if retval:
+            self.set_state("bm_header", length=self.payloadLength)
+            self.bm_proto_reset()
+        # else assume the command requires a different state to follow
+        return True
+
+    def decode_payload_string(self, length):
+        """Read and return `length` bytes from payload"""
+        value = self.payload[self.payloadOffset:self.payloadOffset + length]
+        self.payloadOffset += length
+        return value
+
+    def decode_payload_varint(self):
+        """Decode a varint from the payload"""
+        value, offset = addresses.decodeVarint(self.payload[self.payloadOffset:])
+        self.payloadOffset += offset
+        return value
+
+    def decode_payload_node(self):
+        """Decode node details from the payload"""
+        # protocol.checkIPAddress()
+        services, host, port = self.decode_payload_content("Q16sH")
+        if host[0:12] == '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xFF\xFF':
+            host = socket.inet_ntop(socket.AF_INET, str(host[12:16]))
+        elif host[0:6] == '\xfd\x87\xd8\x7e\xeb\x43':
+            # Onion, based on BMD/bitcoind
+            host = base64.b32encode(host[6:]).lower() + ".onion"
+        else:
+            host = socket.inet_ntop(socket.AF_INET6, str(host))
+        if host == "":
+            # This can happen on Windows systems which are not 64-bit
+            # compatible so let us drop the IPv6 address.
+            host = socket.inet_ntop(socket.AF_INET, str(host[12:16]))
+
+        return Node(services, host, port)
+
+    def decode_payload_content(self, pattern="v"):
+        """
+        Decode the payload depending on pattern:
+
+        L = varint indicating the length of the next array
+        l = varint indicating the length of the next item
+        v = varint (or array)
+        H = uint16
+        I = uint32
+        Q = uint64
+        i = net_addr (without time and stream number)
+        s = string
+        0-9 = length of the next item
+        , = end of array
+        """
+
+        def decode_simple(self, char="v"):
+            """Decode the payload using one char pattern"""
+            if char == "v":
+                return self.decode_payload_varint()
+            if char == "i":
+                return self.decode_payload_node()
+            if char == "H":
+                self.payloadOffset += 2
+                return struct.unpack(">H", self.payload[
+                    self.payloadOffset - 2:self.payloadOffset])[0]
+            if char == "I":
+                self.payloadOffset += 4
+                return struct.unpack(">I", self.payload[
+                    self.payloadOffset - 4:self.payloadOffset])[0]
+            if char == "Q":
+                self.payloadOffset += 8
+                return struct.unpack(">Q", self.payload[
+                    self.payloadOffset - 8:self.payloadOffset])[0]
+
+        size = None
+        isArray = False
+
+        # size
+        # iterator starting from size counting to 0
+        # isArray?
+        # subpattern
+        # position of parser in subpattern
+        # retval (array)
+        parserStack = [[1, 1, False, pattern, 0, []]]
+
+        while True:
+            i = parserStack[-1][3][parserStack[-1][4]]
+            if i in "0123456789" and (
+                size is None or parserStack[-1][3][parserStack[-1][4] - 1]
+                    not in "lL"):
+                try:
+                    size = size * 10 + int(i)
+                except TypeError:
+                    size = int(i)
+                isArray = False
+            elif i in "Ll" and size is None:
+                size = self.decode_payload_varint()
+                isArray = i == "L"
+            elif size is not None:
+                if isArray:
+                    parserStack.append([
+                        size, size, isArray,
+                        parserStack[-1][3][parserStack[-1][4]:], 0, []
+                    ])
+                    parserStack[-2][4] = len(parserStack[-2][3])
+                else:
+                    for j in range(parserStack[-1][4], len(parserStack[-1][3])):
+                        if parserStack[-1][3][j] not in "lL0123456789":
+                            break
+                    parserStack.append([
+                        size, size, isArray,
+                        parserStack[-1][3][parserStack[-1][4]:j + 1], 0, []
+                    ])
+                    parserStack[-2][4] += len(parserStack[-1][3]) - 1
+                size = None
+                continue
+            elif i == "s":
+                # if parserStack[-2][2]:
+                #    parserStack[-1][5].append(self.payload[
+                #        self.payloadOffset:self.payloadOffset + parserStack[-1][0]])
+                # else:
+                parserStack[-1][5] = self.payload[
+                    self.payloadOffset:self.payloadOffset + parserStack[-1][0]]
+                self.payloadOffset += parserStack[-1][0]
+                parserStack[-1][1] = 0
+                parserStack[-1][2] = True
+                # del parserStack[-1]
+                size = None
+            elif i in "viHIQ":
+                parserStack[-1][5].append(decode_simple(
+                    self, parserStack[-1][3][parserStack[-1][4]]))
+                size = None
+            else:
+                size = None
+            for depth in range(len(parserStack) - 1, -1, -1):
+                parserStack[depth][4] += 1
+                if parserStack[depth][4] >= len(parserStack[depth][3]):
+                    parserStack[depth][1] -= 1
+                    parserStack[depth][4] = 0
+                    if depth > 0:
+                        if parserStack[depth][2]:
+                            parserStack[depth - 1][5].append(
+                                parserStack[depth][5])
+                        else:
+                            parserStack[depth - 1][5].extend(
+                                parserStack[depth][5])
+                        parserStack[depth][5] = []
+                    if parserStack[depth][1] <= 0:
+                        if depth == 0:
+                            # we're done, at depth 0 counter is at 0
+                            # and pattern is done parsing
+                            return parserStack[depth][5]
+                        del parserStack[-1]
+                        continue
+                    break
+                break
+        if self.payloadOffset > self.payloadLength:
+            logger.debug(
+                'Insufficient data %i/%i',
+                self.payloadOffset, self.payloadLength)
+            raise BMProtoInsufficientDataError()
+
+    def bm_command_error(self):
+        """Decode an error message and log it"""
+        fatalStatus, banTime, inventoryVector, errorText = \
+            self.decode_payload_content("vvlsls")
+        logger.error(
+            '%s:%i error: %i, %s', self.destination.host,
+            self.destination.port, fatalStatus, errorText)
+        return True
+
+    def bm_command_getdata(self):
+        """
+        Incoming request for object(s).
+        If we have them and some other conditions are fulfilled,
+        append them to the write queue.
+        """
+        items = self.decode_payload_content("l32s")
+        # skip?
+        now = time.time()
+        if now < self.skipUntil:
+            return True
+        for i in items:
+            self.pendingUpload[str(i)] = now
+        return True
+
+    def _command_inv(self, dandelion=False):
+        items = self.decode_payload_content("l32s")
+
+        if len(items) > BMProto.maxObjectCount:
+            logger.error(
+                'Too many items in %sinv message!', 'd' if dandelion else '')
+            raise BMProtoExcessiveDataError()
+
+        # ignore dinv if dandelion turned off
+        if dandelion and not state.dandelion:
+            return True
+
+        for i in map(str, items):
+            if i in Inventory() and not Dandelion().hasHash(i):
+                continue
+            if dandelion and not Dandelion().hasHash(i):
+                Dandelion().addHash(i, self)
+            self.handleReceivedInventory(i)
+
+        return True
+
+    def bm_command_inv(self):
+        """Non-dandelion announce"""
+        return self._command_inv(False)
+
+    def bm_command_dinv(self):
+        """Dandelion stem announce"""
+        return self._command_inv(True)
+
+    def bm_command_object(self):
+        """Incoming object, process it"""
+        objectOffset = self.payloadOffset
+        nonce, expiresTime, objectType, version, streamNumber = \
+            self.decode_payload_content("QQIvv")
+        self.object = BMObject(
+            nonce, expiresTime, objectType, version, streamNumber,
+            self.payload, self.payloadOffset)
+
+        if len(self.payload) - self.payloadOffset > BMProto.maxObjectPayloadSize:
+            logger.info(
+                'The payload length of this object is too large (%d bytes).'
+                ' Ignoring it.', len(self.payload) - self.payloadOffset)
+            raise BMProtoExcessiveDataError()
+
+        try:
+            self.object.checkProofOfWorkSufficient()
+            self.object.checkEOLSanity()
+            self.object.checkAlreadyHave()
+        except (BMObjectExpiredError, BMObjectAlreadyHaveError,
+                BMObjectInsufficientPOWError):
+            BMProto.stopDownloadingObject(self.object.inventoryHash)
+            raise
+        try:
+            self.object.checkStream()
+        except BMObjectUnwantedStreamError:
+            acceptmismatch = BMConfigParser().get(
+                "inventory", "acceptmismatch")
+            BMProto.stopDownloadingObject(
+                self.object.inventoryHash, acceptmismatch)
+            if not acceptmismatch:
+                raise
+
+        try:
+            self.object.checkObjectByType()
+            objectProcessorQueue.put((
+                self.object.objectType, buffer(self.object.data)))
+        except BMObjectInvalidError:
+            BMProto.stopDownloadingObject(self.object.inventoryHash, True)
+        else:
+            try:
+                del missingObjects[self.object.inventoryHash]
+            except KeyError:
+                pass
+
+        if self.object.inventoryHash in Inventory() and Dandelion().hasHash(self.object.inventoryHash):
+            Dandelion().removeHash(self.object.inventoryHash, "cycle detection")
+
+        Inventory()[self.object.inventoryHash] = (
+            self.object.objectType, self.object.streamNumber,
+            buffer(self.payload[objectOffset:]), self.object.expiresTime,
+            buffer(self.object.tag)
+        )
+        self.handleReceivedObject(
+            self.object.streamNumber, self.object.inventoryHash)
+        invQueue.put((
+            self.object.streamNumber, self.object.inventoryHash,
+            self.destination))
+        return True
+
+    def _decode_addr(self):
+        return self.decode_payload_content("LQIQ16sH")
+
+    def bm_command_addr(self):
+        """Incoming addresses, process them"""
+        addresses = self._decode_addr()
+        for i in addresses:
+            seenTime, stream, services, ip, port = i
+            decodedIP = protocol.checkIPAddress(str(ip))
+            if stream not in state.streamsInWhichIAmParticipating:
+                continue
+            if (
+                decodedIP and time.time() - seenTime > 0 and
+                seenTime > time.time() - BMProto.addressAlive and
+                port > 0
+            ):
+                peer = state.Peer(decodedIP, port)
+                try:
+                    if knownnodes.knownNodes[stream][peer]["lastseen"] > seenTime:
+                        continue
+                except KeyError:
+                    pass
+                if len(knownnodes.knownNodes[stream]) < BMConfigParser().safeGetInt("knownnodes", "maxnodes"):
+                    with knownnodes.knownNodesLock:
+                        try:
+                            knownnodes.knownNodes[stream][peer]["lastseen"] = seenTime
+                        except (TypeError, KeyError):
+                            knownnodes.knownNodes[stream][peer] = {
+                                "lastseen": seenTime,
+                                "rating": 0,
+                                "self": False,
+                            }
+                addrQueue.put((stream, peer, self.destination))
+        return True
+
+    def bm_command_portcheck(self):
+        """Incoming port check request, queue it."""
+        portCheckerQueue.put(state.Peer(self.destination, self.peerNode.port))
+        return True
+
+    def bm_command_ping(self):
+        """Incoming ping, respond to it."""
+        self.append_write_buf(protocol.CreatePacket('pong'))
+        return True
+
+    def bm_command_pong(self):
+        """
+        Incoming pong.
+        Ignore it. PyBitmessage pings connections after about 5 minutes
+        of inactivity, and leaves it to the TCP stack to handle actual
+        timeouts. So there is no need to do anything when a pong arrives.
+        """
+        # nothing really
+        return True
+
+    def bm_command_verack(self):
+        """
+        Incoming verack.
+        If already sent my own verack, handshake is complete (except
+        potentially waiting for buffers to flush), so we can continue
+        to the main connection phase. If not sent verack yet,
+        continue processing.
+        """
+        self.verackReceived = True
+        if not self.verackSent:
+            return True
+        self.set_state(
+            "tls_init" if self.isSSL else "connection_fully_established",
+            length=self.payloadLength, expectBytes=0)
+        return False
+
+    def bm_command_version(self):
+        """
+        Incoming version.
+        Parse and log, remember important things, like streams, bitfields, etc.
+        """
+        (self.remoteProtocolVersion, self.services, self.timestamp,
+         self.sockNode, self.peerNode, self.nonce, self.userAgent,
+         self.streams) = self.decode_payload_content("IQQiiQlsLv")
+        self.nonce = struct.pack('>Q', self.nonce)
+        self.timeOffset = self.timestamp - int(time.time())
+        logger.debug('remoteProtocolVersion: %i', self.remoteProtocolVersion)
+        logger.debug('services: 0x%08X', self.services)
+        logger.debug('time offset: %i', self.timestamp - int(time.time()))
+        logger.debug('my external IP: %s', self.sockNode.host)
+        logger.debug(
+            'remote node incoming address: %s:%i',
+            self.destination.host, self.peerNode.port)
+        logger.debug('user agent: %s', self.userAgent)
+        logger.debug('streams: [%s]', ','.join(map(str, self.streams)))
+        if not self.peerValidityChecks():
+            # ABORT afterwards
+            return True
+        self.append_write_buf(protocol.CreatePacket('verack'))
+        self.verackSent = True
+        if not self.isOutbound:
+            self.append_write_buf(protocol.assembleVersionMessage(
+                self.destination.host, self.destination.port,
+                BMConnectionPool().streams, True,
+                nodeid=self.nodeid))
+            logger.debug(
+                '%(host)s:%(port)i sending version',
+                self.destination._asdict())
+        if ((self.services & protocol.NODE_SSL == protocol.NODE_SSL) and
+                protocol.haveSSL(not self.isOutbound)):
+            self.isSSL = True
+        if not self.verackReceived:
+            return True
+        self.set_state(
+            "tls_init" if self.isSSL else "connection_fully_established",
+            length=self.payloadLength, expectBytes=0)
+        return False
+
+    def peerValidityChecks(self):
+        """Check the validity of the peer"""
+        if self.remoteProtocolVersion < 3:
+            self.append_write_buf(protocol.assembleErrorMessage(
+                errorText="Your is using an old protocol. Closing connection.",
+                fatal=2))
+            logger.debug(
+                'Closing connection to old protocol version %s, node: %s',
+                self.remoteProtocolVersion, self.destination)
+            return False
+        if self.timeOffset > BMProto.maxTimeOffset:
+            self.append_write_buf(protocol.assembleErrorMessage(
+                errorText="Your time is too far in the future compared to mine."
+                " Closing connection.", fatal=2))
+            logger.info(
+                "%s's time is too far in the future (%s seconds)."
+                " Closing connection to it.", self.destination, self.timeOffset)
+            BMProto.timeOffsetWrongCount += 1
+            return False
+        elif self.timeOffset < -BMProto.maxTimeOffset:
+            self.append_write_buf(protocol.assembleErrorMessage(
+                errorText="Your time is too far in the past compared to mine."
+                " Closing connection.", fatal=2))
+            logger.info(
+                "%s's time is too far in the past (timeOffset %s seconds)."
+                " Closing connection to it.", self.destination, self.timeOffset)
+            BMProto.timeOffsetWrongCount += 1
+            return False
+        else:
+            BMProto.timeOffsetWrongCount = 0
+        if not self.streams:
+            self.append_write_buf(protocol.assembleErrorMessage(
+                errorText="We don't have shared stream interests."
+                " Closing connection.", fatal=2))
+            logger.debug(
+                'Closed connection to %s because there is no overlapping interest'
+                ' in streams.', self.destination)
+            return False
+        if self.destination in BMConnectionPool().inboundConnections:
+            try:
+                if not protocol.checkSocksIP(self.destination.host):
+                    self.append_write_buf(protocol.assembleErrorMessage(
+                        errorText="Too many connections from your IP."
+                        " Closing connection.", fatal=2))
+                    logger.debug(
+                        'Closed connection to %s because we are already connected'
+                        ' to that IP.', self.destination)
+                    return False
+            except:
+                pass
+        if not self.isOutbound:
+            # incoming from a peer we're connected to as outbound,
+            # or server full report the same error to counter deanonymisation
+            if (
+                state.Peer(self.destination.host, self.peerNode.port) in
+                BMConnectionPool().inboundConnections or
+                len(BMConnectionPool().inboundConnections) +
+                len(BMConnectionPool().outboundConnections) >
+                BMConfigParser().safeGetInt("bitmessagesettings", "maxtotalconnections") +
+                BMConfigParser().safeGetInt("bitmessagesettings", "maxbootstrapconnections")
+            ):
+                self.append_write_buf(protocol.assembleErrorMessage(
+                    errorText="Server full, please try again later.", fatal=2))
+                logger.debug(
+                    'Closed connection to %s due to server full'
+                    ' or duplicate inbound/outbound.', self.destination)
+                return False
+        if BMConnectionPool().isAlreadyConnected(
+                self.nonce):
+            self.append_write_buf(protocol.assembleErrorMessage(
+                errorText="I'm connected to myself. Closing connection.",
+                fatal=2))
+            logger.debug(
+                "Closed connection to %s because I'm connected to myself.",
+                self.destination)
+            return False
+
+        return True
+
+    @staticmethod
+    def assembleAddr(peerList):
+        """Build up a packed address"""
+        if isinstance(peerList, state.Peer):
+            peerList = (peerList)
+        if not peerList:
+            return b''
+        retval = b''
+        for i in range(0, len(peerList), BMProto.maxAddrCount):
+            payload = addresses.encodeVarint(
+                len(peerList[i:i + BMProto.maxAddrCount]))
+            for address in peerList[i:i + BMProto.maxAddrCount]:
+                stream, peer, timestamp = address
+                payload += struct.pack(
+                    '>Q', timestamp)  # 64-bit time
+                payload += struct.pack('>I', stream)
+                payload += struct.pack(
+                    '>q', 1)  # service bit flags offered by this node
+                payload += protocol.encodeHost(peer.host)
+                payload += struct.pack('>H', peer.port)  # remote port
+            retval += protocol.CreatePacket('addr', payload)
+        return retval
+
+    @staticmethod
+    def stopDownloadingObject(hashId, forwardAnyway=False):
+        """Stop downloading an object"""
+        for connection in (
+            list(BMConnectionPool().inboundConnections.values()) +
+            list(BMConnectionPool().outboundConnections.values())
+        ):
+            try:
+                del connection.objectsNewToMe[hashId]
+            except KeyError:
+                pass
+            if not forwardAnyway:
+                try:
+                    with connection.objectsNewToThemLock:
+                        del connection.objectsNewToThem[hashId]
+                except KeyError:
+                    pass
+        try:
+            del missingObjects[hashId]
+        except KeyError:
+            pass
+
+    def handle_close(self):
+        """Handle close"""
+        self.set_state("close")
+        if not (self.accepting or self.connecting or self.connected):
+            # already disconnected
+            return
+        try:
+            logger.debug(
+                '%s:%i: closing, %s', self.destination.host,
+                self.destination.port, self.close_reason)
+        except AttributeError:
+            try:
+                logger.debug(
+                    '%(host)s:%(port)i: closing', self.destination._asdict())
+            except AttributeError:
+                logger.debug('Disconnected socket closing')
+        AdvancedDispatcher.handle_close(self)
+
+
+class BMStringParser(BMProto):
+    """
+    A special case of BMProto used by objectProcessor to send ACK
+    """
+    def __init__(self):
+        super(BMStringParser, self).__init__()
+        self.destination = state.Peer('127.0.0.1', 8444)
+        self.payload = None
+        ObjectTracker.__init__(self)
+
+    def send_data(self, data):
+        """Send object given by the data string"""
+        # This class is introduced specially for ACK sending, please
+        # change log strings if you are going to use it for something else
+        self.bm_proto_reset()
+        self.payload = data
+        try:
+            self.bm_command_object()
+        except BMObjectAlreadyHaveError:
+            pass  # maybe the same msg received on different nodes
+        except BMObjectExpiredError:
+            logger.debug(
+                'Sending ACK failure (expired): %s', hexlify(data))
+        except Exception as e:
+            logger.debug(
+                'Exception of type %s while sending ACK',
+                type(e), exc_info=True)
+
+
+
+
+############################
+# src/network/dandelion.py #
+############################
+
+
+
+
+from collections import namedtuple
+from random import choice, sample, expovariate
+from threading import RLock
+import time
+
+import state
+from debug import logging
+from queues import invQueue
+from singleton import Singleton
+
+# randomise routes after 600 seconds
+REASSIGN_INTERVAL = 600
+
+# trigger fluff due to expiration
+FLUFF_TRIGGER_FIXED_DELAY = 10
+FLUFF_TRIGGER_MEAN_DELAY = 30
+
+MAX_STEMS = 2
+
+Stem = namedtuple('Stem', ['child', 'stream', 'timeout'])
+
+
+@Singleton
+class Dandelion(object):
+    """Dandelion class for tracking stem/fluff stages."""
+    def __init__(self):
+        # currently assignable child stems
+        self.stem = []
+        # currently assigned parent <-> child mappings
+        self.nodeMap = {}
+        # currently existing objects in stem mode
+        self.hashMap = {}
+        # when to rerandomise routes
+        self.refresh = time.time() + REASSIGN_INTERVAL
+        self.lock = RLock()
+
+    def poissonTimeout(self, start=None, average=0):
+        """Generate deadline using Poisson distribution"""
+        if start is None:
+            start = time.time()
+        if average == 0:
+            average = FLUFF_TRIGGER_MEAN_DELAY
+        return start + expovariate(1.0 / average) + FLUFF_TRIGGER_FIXED_DELAY
+
+    def addHash(self, hashId, source=None, stream=1):
+        """Add inventory vector to dandelion stem"""
+        if not state.dandelion:
+            return
+        with self.lock:
+            self.hashMap[hashId] = Stem(
+                self.getNodeStem(source),
+                stream,
+                self.poissonTimeout())
+
+    def setHashStream(self, hashId, stream=1):
+        """
+        Update stream for inventory vector (as inv/dinv commands don't
+        include streams, we only learn this after receiving the object)
+        """
+        with self.lock:
+            if hashId in self.hashMap:
+                self.hashMap[hashId] = Stem(
+                    self.hashMap[hashId].child,
+                    stream,
+                    self.poissonTimeout())
+
+    def removeHash(self, hashId, reason="no reason specified"):
+        """Switch inventory vector from stem to fluff mode"""
+        logging.debug(
+            "%s entering fluff mode due to %s.",
+            ''.join('%02x' % ord(i) for i in hashId), reason)
+        with self.lock:
+            try:
+                del self.hashMap[hashId]
+            except KeyError:
+                pass
+
+    def hasHash(self, hashId):
+        """Is inventory vector in stem mode?"""
+        return hashId in self.hashMap
+
+    def objectChildStem(self, hashId):
+        """Child (i.e. next) node for an inventory vector during stem mode"""
+        return self.hashMap[hashId].child
+
+    def maybeAddStem(self, connection):
+        """
+        If we had too few outbound connections, add the current one to the
+        current stem list. Dandelion as designed by the authors should
+        always have two active stem child connections.
+        """
+        # fewer than MAX_STEMS outbound connections at last reshuffle?
+        with self.lock:
+            if len(self.stem) < MAX_STEMS:
+                self.stem.append(connection)
+                for k in (k for k, v in self.nodeMap.items() if v is None):
+                    self.nodeMap[k] = connection
+                for k, v in {
+                    k: v for k, v in self.hashMap.items()
+                    if v.child is None
+                }.items():
+                    self.hashMap[k] = Stem(
+                        connection, v.stream, self.poissonTimeout())
+                    invQueue.put((v.stream, k, v.child))
+
+    def maybeRemoveStem(self, connection):
+        """
+        Remove current connection from the stem list (called e.g. when
+        a connection is closed).
+        """
+        # is the stem active?
+        with self.lock:
+            if connection in self.stem:
+                self.stem.remove(connection)
+                # active mappings to pointing to the removed node
+                for k in (
+                    k for k, v in self.nodeMap.items() if v == connection
+                ):
+                    self.nodeMap[k] = None
+                for k, v in {
+                    k: v for k, v in self.hashMap.items()
+                    if v.child == connection
+                }.items():
+                    self.hashMap[k] = Stem(
+                        None, v.stream, self.poissonTimeout())
+
+    def pickStem(self, parent=None):
+        """
+        Pick a random active stem, but not the parent one
+        (the one where an object came from)
+        """
+        try:
+            # pick a random from available stems
+            stem = choice(list(range(len(self.stem))))
+            if self.stem[stem] == parent:
+                # one stem available and it's the parent
+                if len(self.stem) == 1:
+                    return None
+                # else, pick the other one
+                return self.stem[1 - stem]
+            # all ok
+            return self.stem[stem]
+        except IndexError:
+            # no stems available
+            return None
+
+    def getNodeStem(self, node=None):
+        """
+        Return child stem node for a given parent stem node
+        (the mapping is static for about 10 minutes, then it reshuffles)
+        """
+        with self.lock:
+            try:
+                return self.nodeMap[node]
+            except KeyError:
+                self.nodeMap[node] = self.pickStem(node)
+                return self.nodeMap[node]
+
+    def expire(self):
+        """Switch expired objects from stem to fluff mode"""
+        with self.lock:
+            deadline = time.time()
+            toDelete = [
+                [v.stream, k, v.child] for k, v in self.hashMap.items()
+                if v.timeout < deadline
+            ]
+
+            for row in toDelete:
+                self.removeHash(row[1], 'expiration')
+                invQueue.put(row)
+        return toDelete
+
+    def reRandomiseStems(self):
+        """Re-shuffle stem mapping (parent <-> child pairs)"""
+        with self.lock:
+            try:
+                # random two connections
+                self.stem = sample(
+                    list(BMConnectionPool(
+                    ).outboundConnections.values()), MAX_STEMS)
+            # not enough stems available
+            except ValueError:
+                self.stem = list(BMConnectionPool(
+                ).outboundConnections.values())
+            self.nodeMap = {}
+            # hashMap stays to cater for pending stems
+        self.refresh = time.time() + REASSIGN_INTERVAL
+
+
+
+
+#################################
+# src/network/connectionpool.py #
+#################################
+
+
+
+
+import errno
+import re
+import socket
+import time
+
+from . import asyncore_pollchoose as asyncore
+import helper_bootstrap
+import helper_random
+import knownnodes
+import protocol
+import state
+from bmconfigparser import BMConfigParser
+from .connectionchooser import chooseConnection
+from debug import logger
+from .proxy import Proxy
+from singleton import Singleton
+from .udp import UDPSocket
+
+
+@Singleton
+class BMConnectionPool(object):
+    """Pool of all existing connections"""
+    def __init__(self):
+        asyncore.set_rates(
+            BMConfigParser().safeGetInt(
+                "bitmessagesettings", "maxdownloadrate"),
+            BMConfigParser().safeGetInt(
+                "bitmessagesettings", "maxuploadrate")
+        )
+        self.outboundConnections = {}
+        self.inboundConnections = {}
+        self.listeningSockets = {}
+        self.udpSockets = {}
+        self.streams = []
+        self.lastSpawned = 0
+        self.spawnWait = 2
+        self.bootstrapped = False
+
+    def connectToStream(self, streamNumber):
+        """Connect to a bitmessage stream"""
+        self.streams.append(streamNumber)
+
+    def getConnectionByAddr(self, addr):
+        """
+        Return an (existing) connection object based on a `Peer` object
+        (IP and port)
+        """
+        try:
+            return self.inboundConnections[addr]
+        except KeyError:
+            pass
+        try:
+            return self.inboundConnections[addr.host]
+        except (KeyError, AttributeError):
+            pass
+        try:
+            return self.outboundConnections[addr]
+        except KeyError:
+            pass
+        try:
+            return self.udpSockets[addr.host]
+        except (KeyError, AttributeError):
+            pass
+        raise KeyError
+
+    def isAlreadyConnected(self, nodeid):
+        """Check if we're already connected to this peer"""
+        for i in (
+            list(self.inboundConnections.values()) +
+            list(self.outboundConnections.values())
+        ):
+            try:
+                if nodeid == i.nodeid:
+                    return True
+            except AttributeError:
+                pass
+        return False
+
+    def addConnection(self, connection):
+        """Add a connection object to our internal dict"""
+        if isinstance(connection, UDPSocket):
+            return
+        if connection.isOutbound:
+            self.outboundConnections[connection.destination] = connection
+        else:
+            if connection.destination.host in self.inboundConnections:
+                self.inboundConnections[connection.destination] = connection
+            else:
+                self.inboundConnections[connection.destination.host] = \
+                    connection
+
+    def removeConnection(self, connection):
+        """Remove a connection from our internal dict"""
+        if isinstance(connection, UDPSocket):
+            del self.udpSockets[connection.listening.host]
+        elif isinstance(connection, TCPServer):
+            del self.listeningSockets[state.Peer(
+                connection.destination.host, connection.destination.port)]
+        elif connection.isOutbound:
+            try:
+                del self.outboundConnections[connection.destination]
+            except KeyError:
+                pass
+        else:
+            try:
+                del self.inboundConnections[connection.destination]
+            except KeyError:
+                try:
+                    del self.inboundConnections[connection.destination.host]
+                except KeyError:
+                    pass
+        connection.handle_close()
+
+    def getListeningIP(self):
+        """What IP are we supposed to be listening on?"""
+        if BMConfigParser().safeGet(
+                "bitmessagesettings", "onionhostname").endswith(".onion"):
+            host = BMConfigParser().safeGet(
+                "bitmessagesettings", "onionbindip")
+        else:
+            host = '127.0.0.1'
+        if (BMConfigParser().safeGetBoolean(
+                "bitmessagesettings", "sockslisten") or
+            BMConfigParser().safeGet(
+                "bitmessagesettings", "socksproxytype") == "none"):
+            # python doesn't like bind + INADDR_ANY?
+            # host = socket.INADDR_ANY
+            host = BMConfigParser().get("network", "bind")
+        return host
+
+    def startListening(self, bind=None):
+        """Open a listening socket and start accepting connections on it"""
+        if bind is None:
+            bind = self.getListeningIP()
+        port = BMConfigParser().safeGetInt("bitmessagesettings", "port")
+        # correct port even if it changed
+        ls = TCPServer(host=bind, port=port)
+        self.listeningSockets[ls.destination] = ls
+
+    def startUDPSocket(self, bind=None):
+        """
+        Open an UDP socket. Depending on settings, it can either only
+        accept incoming UDP packets, or also be able to send them.
+        """
+        if bind is None:
+            host = self.getListeningIP()
+            udpSocket = UDPSocket(host=host, announcing=True)
+        else:
+            if bind is False:
+                udpSocket = UDPSocket(announcing=False)
+            else:
+                udpSocket = UDPSocket(host=bind, announcing=True)
+        self.udpSockets[udpSocket.listening.host] = udpSocket
+
+    def loop(self):
+        """Main Connectionpool's loop"""
+        # defaults to empty loop if outbound connections are maxed
+        spawnConnections = False
+        acceptConnections = True
+        if BMConfigParser().safeGetBoolean(
+                'bitmessagesettings', 'dontconnect'):
+            acceptConnections = False
+        elif BMConfigParser().safeGetBoolean(
+                'bitmessagesettings', 'sendoutgoingconnections'):
+            spawnConnections = True
+        socksproxytype = BMConfigParser().safeGet(
+            'bitmessagesettings', 'socksproxytype', '')
+        onionsocksproxytype = BMConfigParser().safeGet(
+            'bitmessagesettings', 'onionsocksproxytype', '')
+        if (socksproxytype[:5] == 'SOCKS' and
+            not BMConfigParser().safeGetBoolean(
+                'bitmessagesettings', 'sockslisten') and
+            '.onion' not in BMConfigParser().safeGet(
+                'bitmessagesettings', 'onionhostname', '')):
+            acceptConnections = False
+
+        if spawnConnections:
+            if not knownnodes.knownNodesActual:
+                helper_bootstrap.dns()
+            if not self.bootstrapped:
+                self.bootstrapped = True
+                Proxy.proxy = (
+                    BMConfigParser().safeGet(
+                        'bitmessagesettings', 'sockshostname'),
+                    BMConfigParser().safeGetInt(
+                        'bitmessagesettings', 'socksport')
+                )
+                # TODO AUTH
+                # TODO reset based on GUI settings changes
+                try:
+                    if not onionsocksproxytype.startswith("SOCKS"):
+                        raise ValueError
+                    Proxy.onion_proxy = (
+                        BMConfigParser().safeGet(
+                            'network', 'onionsockshostname', None),
+                        BMConfigParser().safeGet(
+                            'network', 'onionsocksport', None)
+                    )
+                except ValueError:
+                    Proxy.onion_proxy = None
+            established = sum(
+                1 for c in list(self.outboundConnections.values())
+                if (c.connected and c.fullyEstablished))
+            pending = len(self.outboundConnections) - established
+            if established < BMConfigParser().safeGetInt(
+                    'bitmessagesettings', 'maxoutboundconnections'):
+                for i in range(
+                        state.maximumNumberOfHalfOpenConnections - pending):
+                    try:
+                        chosen = chooseConnection(
+                            helper_random.randomchoice(self.streams))
+                    except ValueError:
+                        continue
+                    if chosen in self.outboundConnections:
+                        continue
+                    if chosen.host in self.inboundConnections:
+                        continue
+                    # don't connect to self
+                    if chosen in state.ownAddresses:
+                        continue
+
+                    try:
+                        if (chosen.host.endswith(".onion") and
+                                Proxy.onion_proxy is not None):
+                            if onionsocksproxytype == "SOCKS5":
+                                self.addConnection(Socks5BMConnection(chosen))
+                            elif onionsocksproxytype == "SOCKS4a":
+                                self.addConnection(Socks4aBMConnection(chosen))
+                        elif socksproxytype == "SOCKS5":
+                            self.addConnection(Socks5BMConnection(chosen))
+                        elif socksproxytype == "SOCKS4a":
+                            self.addConnection(Socks4aBMConnection(chosen))
+                        else:
+                            self.addConnection(TCPConnection(chosen))
+                    except socket.error as e:
+                        if e.errno == errno.ENETUNREACH:
+                            continue
+
+                    self.lastSpawned = time.time()
+        else:
+            for i in (
+                list(self.inboundConnections.values()) +
+                list(self.outboundConnections.values())
+            ):
+                # FIXME: rating will be increased after next connection
+                i.handle_close()
+
+        if acceptConnections:
+            if not self.listeningSockets:
+                if BMConfigParser().safeGet('network', 'bind') == '':
+                    self.startListening()
+                else:
+                    for bind in re.sub(
+                        "[^\w.]+", " ",
+                        BMConfigParser().safeGet('network', 'bind')
+                    ).split():
+                        self.startListening(bind)
+                logger.info('Listening for incoming connections.')
+            if not self.udpSockets:
+                if BMConfigParser().safeGet('network', 'bind') == '':
+                    self.startUDPSocket()
+                else:
+                    for bind in re.sub(
+                        "[^\w.]+", " ",
+                        BMConfigParser().safeGet('network', 'bind')
+                    ).split():
+                        self.startUDPSocket(bind)
+                    self.startUDPSocket(False)
+                logger.info('Starting UDP socket(s).')
+        else:
+            if self.listeningSockets:
+                for i in list(self.listeningSockets.values()):
+                    i.close_reason = "Stopping listening"
+                    i.accepting = i.connecting = i.connected = False
+                logger.info('Stopped listening for incoming connections.')
+            if self.udpSockets:
+                for i in list(self.udpSockets.values()):
+                    i.close_reason = "Stopping UDP socket"
+                    i.accepting = i.connecting = i.connected = False
+                logger.info('Stopped udp sockets.')
+
+        loopTime = float(self.spawnWait)
+        if self.lastSpawned < time.time() - self.spawnWait:
+            loopTime = 2.0
+        asyncore.loop(timeout=loopTime, count=1000)
+
+        reaper = []
+        for i in (
+            list(self.inboundConnections.values()) +
+            list(self.outboundConnections.values())
+        ):
+            minTx = time.time() - 20
+            if i.fullyEstablished:
+                minTx -= 300 - 20
+            if i.lastTx < minTx:
+                if i.fullyEstablished:
+                    i.append_write_buf(protocol.CreatePacket('ping'))
+                else:
+                    i.close_reason = "Timeout (%is)" % (
+                        time.time() - i.lastTx)
+                    i.set_state("close")
+        for i in (
+            list(self.inboundConnections.values()) +
+            list(self.outboundConnections.values()) +
+            list(self.listeningSockets.values()) +
+            list(self.udpSockets.values())
+        ):
+            if not (i.accepting or i.connecting or i.connected):
+                reaper.append(i)
+            else:
+                try:
+                    if i.state == "close":
+                        reaper.append(i)
+                except AttributeError:
+                    pass
+        for i in reaper:
+            self.removeConnection(i)
+
+
+
+
+######################
+# src/network/tcp.py #
+######################
+
+
+
+
+import math
+import random
+import socket
+import time
+
+import addresses
+from . import asyncore_pollchoose as asyncore
+import helper_random
+import knownnodes
+import protocol
+import shared
+import state
+from bmconfigparser import BMConfigParser
+from debug import logger
+from helper_random import randomBytes
+from inventory import Inventory
+from network.advanceddispatcher import AdvancedDispatcher
+from network.socks4a import Socks4aConnection
+from network.socks5 import Socks5Connection
+from network.tls import TLSDispatcher
+from queues import UISignalQueue, invQueue, receiveDataQueue
+
+
+class TCPConnection(BMProto, TLSDispatcher):
+    # pylint: disable=too-many-instance-attributes
+    """
+
+    .. todo:: Look to understand and/or fix the non-parent-init-called
+    """
+
+    def __init__(self, address=None, sock=None):
+        BMProto.__init__(self, address=address, sock=sock)
+        self.verackReceived = False
+        self.verackSent = False
+        self.streams = [0]
+        self.fullyEstablished = False
+        self.connectedAt = 0
+        self.skipUntil = 0
+        if address is None and sock is not None:
+            self.destination = state.Peer(*sock.getpeername())
+            self.isOutbound = False
+            TLSDispatcher.__init__(self, sock, server_side=True)
+            self.connectedAt = time.time()
+            logger.debug(
+                'Received connection from %s:%i',
+                self.destination.host, self.destination.port)
+            self.nodeid = randomBytes(8)
+        elif address is not None and sock is not None:
+            TLSDispatcher.__init__(self, sock, server_side=False)
+            self.isOutbound = True
+            logger.debug(
+                'Outbound proxy connection to %s:%i',
+                self.destination.host, self.destination.port)
+        else:
+            self.destination = address
+            self.isOutbound = True
+            self.create_socket(
+                socket.AF_INET6 if ":" in address.host else socket.AF_INET,
+                socket.SOCK_STREAM)
+            self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+            TLSDispatcher.__init__(self, sock, server_side=False)
+            self.connect(self.destination)
+            logger.debug(
+                'Connecting to %s:%i',
+                self.destination.host, self.destination.port)
+        encodedAddr = protocol.encodeHost(self.destination.host)
+        self.local = all([
+            protocol.checkIPAddress(encodedAddr, True),
+            not protocol.checkSocksIP(self.destination.host)
+        ])
+        ObjectTracker.__init__(self)  # pylint: disable=non-parent-init-called
+        self.bm_proto_reset()
+        self.set_state("bm_header", expectBytes=protocol.Header.size)
+
+    def antiIntersectionDelay(self, initial=False):
+        """
+        This is a defense against the so called intersection attacks.
+
+        It is called when you notice peer is requesting non-existing
+        objects, or right after the connection is established. It will
+        estimate how long an object will take to propagate across the
+        network, and skip processing "getdata" requests until then. This
+        means an attacker only has one shot per IP to perform the attack.
+        """
+        # estimated time for a small object to propagate across the
+        # whole network
+        max_known_nodes = max(
+            len(knownnodes.knownNodes[x]) for x in knownnodes.knownNodes)
+        delay = math.ceil(math.log(max_known_nodes + 2, 20)) * (
+            0.2 + invQueue.queueCount / 2.0)
+        # take the stream with maximum amount of nodes
+        # +2 is to avoid problems with log(0) and log(1)
+        # 20 is avg connected nodes count
+        # 0.2 is avg message transmission time
+        if delay > 0:
+            if initial:
+                self.skipUntil = self.connectedAt + delay
+                if self.skipUntil > time.time():
+                    logger.debug(
+                        'Initial skipping processing getdata for %.2fs',
+                        self.skipUntil - time.time())
+            else:
+                logger.debug(
+                    'Skipping processing getdata due to missing object'
+                    ' for %.2fs', delay)
+                self.skipUntil = time.time() + delay
+
+    def state_connection_fully_established(self):
+        """
+        State after the bitmessage protocol handshake is completed
+        (version/verack exchange, and if both side support TLS,
+        the TLS handshake as well).
+        """
+        self.set_connection_fully_established()
+        self.set_state("bm_header")
+        self.bm_proto_reset()
+        return True
+
+    def set_connection_fully_established(self):
+        """Initiate inventory synchronisation."""
+        if not self.isOutbound and not self.local:
+            shared.clientHasReceivedIncomingConnections = True
+            UISignalQueue.put(('setStatusIcon', 'green'))
+        UISignalQueue.put((
+            'updateNetworkStatusTab',
+            (self.isOutbound, True, self.destination)
+        ))
+        self.antiIntersectionDelay(True)
+        self.fullyEstablished = True
+        if self.isOutbound:
+            knownnodes.increaseRating(self.destination)
+            Dandelion().maybeAddStem(self)
+        self.sendAddr()
+        self.sendBigInv()
+
+    def sendAddr(self):
+        """Send a partial list of known addresses to peer."""
+        # We are going to share a maximum number of 1000 addrs (per overlapping
+        # stream) with our peer. 500 from overlapping streams, 250 from the
+        # left child stream, and 250 from the right child stream.
+        maxAddrCount = BMConfigParser().safeGetInt(
+            "bitmessagesettings", "maxaddrperstreamsend", 500)
+
+        templist = []
+        addrs = {}
+        for stream in self.streams:
+            with knownnodes.knownNodesLock:
+                for n, s in enumerate((stream, stream * 2, stream * 2 + 1)):
+                    nodes = knownnodes.knownNodes.get(s)
+                    if not nodes:
+                        continue
+                    # only if more recent than 3 hours
+                    # and having positive or neutral rating
+                    filtered = [
+                        (k, v) for k, v in nodes.items()
+                        if v["lastseen"] > int(time.time()) -
+                        shared.maximumAgeOfNodesThatIAdvertiseToOthers and
+                        v["rating"] >= 0 and len(k.host) <= 22
+                    ]
+                    # sent 250 only if the remote isn't interested in it
+                    elemCount = min(
+                        len(filtered),
+                        old_div(maxAddrCount, 2) if n else maxAddrCount)
+                    addrs[s] = helper_random.randomsample(filtered, elemCount)
+        for substream in addrs:
+            for peer, params in addrs[substream]:
+                templist.append((substream, peer, params["lastseen"]))
+        if templist:
+            self.append_write_buf(BMProto.assembleAddr(templist))
+
+    def sendBigInv(self):
+        """
+        Send hashes of all inventory objects, chunked as the protocol has
+        a per-command limit.
+        """
+        def sendChunk():
+            """Send one chunk of inv entries in one command"""
+            if objectCount == 0:
+                return
+            logger.debug(
+                'Sending huge inv message with %i objects to just this'
+                ' one peer', objectCount)
+            self.append_write_buf(protocol.CreatePacket(
+                'inv', addresses.encodeVarint(objectCount) + payload))
+
+        # Select all hashes for objects in this stream.
+        bigInvList = {}
+        for stream in self.streams:
+            # may lock for a long time, but I think it's better than
+            # thousands of small locks
+            with self.objectsNewToThemLock:
+                for objHash in Inventory().unexpired_hashes_by_stream(stream):
+                    # don't advertise stem objects on bigInv
+                    if Dandelion().hasHash(objHash):
+                        continue
+                    bigInvList[objHash] = 0
+        objectCount = 0
+        payload = b''
+        # Now let us start appending all of these hashes together. They will be
+        # sent out in a big inv message to our new peer.
+        for obj_hash, _ in list(bigInvList.items()):
+            payload += obj_hash
+            objectCount += 1
+
+            # Remove -1 below when sufficient time has passed for users to
+            # upgrade to versions of PyBitmessage that accept inv with 50,000
+            # items
+            if objectCount >= BMProto.maxObjectCount - 1:
+                sendChunk()
+                payload = b''
+                objectCount = 0
+
+        # flush
+        sendChunk()
+
+    def handle_connect(self):
+        """Callback for TCP connection being established."""
+        try:
+            AdvancedDispatcher.handle_connect(self)
+        except socket.error as e:
+            # pylint: disable=protected-access
+            if e.errno in asyncore._DISCONNECTED:
+                logger.debug(
+                    '%s:%i: Connection failed: %s',
+                    self.destination.host, self.destination.port, e)
+                return
+        self.nodeid = randomBytes(8)
+        self.append_write_buf(
+            protocol.assembleVersionMessage(
+                self.destination.host, self.destination.port,
+                BMConnectionPool().streams,
+                False, nodeid=self.nodeid))
+        self.connectedAt = time.time()
+        receiveDataQueue.put(self.destination)
+
+    def handle_read(self):
+        """Callback for reading from a socket"""
+        TLSDispatcher.handle_read(self)
+        if self.isOutbound and self.fullyEstablished:
+            for s in self.streams:
+                try:
+                    with knownnodes.knownNodesLock:
+                        knownnodes.knownNodes[s][self.destination][
+                            "lastseen"] = time.time()
+                except KeyError:
+                    pass
+        receiveDataQueue.put(self.destination)
+
+    def handle_write(self):
+        """Callback for writing to a socket"""
+        TLSDispatcher.handle_write(self)
+
+    def handle_close(self):
+        """Callback for connection being closed."""
+        if self.isOutbound and not self.fullyEstablished:
+            knownnodes.decreaseRating(self.destination)
+        if self.fullyEstablished:
+            UISignalQueue.put((
+                'updateNetworkStatusTab',
+                (self.isOutbound, False, self.destination)
+            ))
+            if self.isOutbound:
+                Dandelion().maybeRemoveStem(self)
+        BMProto.handle_close(self)
+
+
+class Socks5BMConnection(Socks5Connection, TCPConnection):
+    """SOCKS5 wrapper for TCP connections"""
+
+    def __init__(self, address):
+        Socks5Connection.__init__(self, address=address)
+        TCPConnection.__init__(self, address=address, sock=self.socket)
+        self.set_state("init")
+
+    def state_proxy_handshake_done(self):
+        """
+        State when SOCKS5 connection succeeds, we need to send a
+        Bitmessage handshake to peer.
+        """
+        Socks5Connection.state_proxy_handshake_done(self)
+        self.nodeid = randomBytes(8)
+        self.append_write_buf(
+            protocol.assembleVersionMessage(
+                self.destination.host, self.destination.port,
+                BMConnectionPool().streams,
+                False, nodeid=self.nodeid))
+        self.set_state("bm_header", expectBytes=protocol.Header.size)
+        return True
+
+
+class Socks4aBMConnection(Socks4aConnection, TCPConnection):
+    """SOCKS4a wrapper for TCP connections"""
+
+    def __init__(self, address):
+        Socks4aConnection.__init__(self, address=address)
+        TCPConnection.__init__(self, address=address, sock=self.socket)
+        self.set_state("init")
+
+    def state_proxy_handshake_done(self):
+        """
+        State when SOCKS4a connection succeeds, we need to send a
+        Bitmessage handshake to peer.
+        """
+        Socks4aConnection.state_proxy_handshake_done(self)
+        self.nodeid = randomBytes(8)
+        self.append_write_buf(
+            protocol.assembleVersionMessage(
+                self.destination.host, self.destination.port,
+                BMConnectionPool().streams,
+                False, nodeid=self.nodeid))
+        self.set_state("bm_header", expectBytes=protocol.Header.size)
+        return True
+
+
+class TCPServer(AdvancedDispatcher):
+    """TCP connection server for Bitmessage protocol"""
+
+    def __init__(self, host='127.0.0.1', port=8444):
+        if not hasattr(self, '_map'):
+            AdvancedDispatcher.__init__(self)
+        self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
+        self.set_reuse_addr()
+        for attempt in range(50):
+            try:
+                if attempt > 0:
+                    port = random.randint(32767, 65535)
+                self.bind((host, port))
+            except socket.error as e:
+                if e.errno in (asyncore.EADDRINUSE, asyncore.WSAEADDRINUSE):
+                    continue
+            else:
+                if attempt > 0:
+                    BMConfigParser().set(
+                        'bitmessagesettings', 'port', str(port))
+                    BMConfigParser().save()
+                break
+        self.destination = state.Peer(host, port)
+        self.bound = True
+        self.listen(5)
+
+    def is_bound(self):
+        """Is the socket bound?"""
+        try:
+            return self.bound
+        except AttributeError:
+            return False
+
+    def handle_accept(self):
+        """Incoming connection callback"""
+        try:
+            sock = self.accept()[0]
+        except (TypeError, IndexError):
+            return
+
+        state.ownAddresses[state.Peer(*sock.getsockname())] = True
+        if (
+            len(BMConnectionPool().inboundConnections) +
+            len(BMConnectionPool().outboundConnections) >
+            BMConfigParser().safeGetInt(
+                'bitmessagesettings', 'maxtotalconnections') +
+            BMConfigParser().safeGetInt(
+                'bitmessagesettings', 'maxbootstrapconnections') + 10
+        ):
+            # 10 is a sort of buffer, in between it will go through
+            # the version handshake and return an error to the peer
+            logger.warning("Server full, dropping connection")
+            sock.close()
+            return
+        try:
+            BMConnectionPool().addConnection(
+                TCPConnection(sock=sock))
+        except socket.error:
+            pass

src/network/http-old.py

@@ -1,3 +1,12 @@
+from __future__ import print_function
+from __future__ import division
+from __future__ import unicode_literals
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import range
+from builtins import *
+from past.utils import old_div
 import asyncore
 import socket
 import time
@@ -43,7 +52,7 @@ if __name__ == "__main__":
         if (len(asyncore.socket_map) < parallel):
             for i in range(parallel - len(asyncore.socket_map)):
                 HTTPClient('127.0.0.1', '/')
-        print "Active connections: %i" % (len(asyncore.socket_map))
-        asyncore.loop(count=len(asyncore.socket_map)/2)
+        print("Active connections: %i" % (len(asyncore.socket_map)))
+        asyncore.loop(count=old_div(len(asyncore.socket_map),2))
         if requestCount % 100 == 0:
-            print "Processed %i total messages" % (requestCount)
+            print("Processed %i total messages" % (requestCount))

src/network/http.py

@@ -1,10 +1,17 @@
+from __future__ import print_function
+from __future__ import absolute_import
+from __future__ import unicode_literals
+from __future__ import division
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
 import socket
 
-from advanceddispatcher import AdvancedDispatcher
-import asyncore_pollchoose as asyncore
-from proxy import Proxy, ProxyError, GeneralProxyError
-from socks5 import Socks5Connection, Socks5Resolver, Socks5AuthError, Socks5Error
-from socks4a import Socks4aConnection, Socks4aResolver, Socks4aError
+from .advanceddispatcher import AdvancedDispatcher
+from . import asyncore_pollchoose as asyncore
+from .proxy import Proxy, ProxyError, GeneralProxyError
+from .socks5 import Socks5Connection, Socks5Resolver, Socks5AuthError, Socks5Error
+from .socks4a import Socks4aConnection, Socks4aResolver, Socks4aError
 
 class HttpError(ProxyError): pass
 
@@ -16,17 +23,17 @@ class HttpConnection(AdvancedDispatcher):
         self.destination = (host, 80)
         self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
         self.connect(self.destination)
-        print "connecting in background to %s:%i" % (self.destination[0], self.destination[1])
+        print("connecting in background to %s:%i" % (self.destination[0], self.destination[1]))
 
     def state_init(self):
         self.append_write_buf("GET %s HTTP/1.1\r\nHost: %s\r\nConnection: close\r\n\r\n" % (self.path, self.destination[0]))
-        print "Sending %ib"  % (len(self.write_buf))
+        print("Sending %ib"  % (len(self.write_buf)))
         self.set_state("http_request_sent", 0)
         return False
 
     def state_http_request_sent(self):
         if len(self.read_buf) > 0:
-            print "Received %ib" % (len(self.read_buf))
+            print("Received %ib" % (len(self.read_buf)))
             self.read_buf = b""
         if not self.connected:
             self.set_state("close", 0)
@@ -59,13 +66,13 @@ if __name__ == "__main__":
     for host in ("bootstrap8080.bitmessage.org", "bootstrap8444.bitmessage.org"):
         proxy = Socks5Resolver(host=host)
         while len(asyncore.socket_map) > 0:
-            print "loop %s, len %i" % (proxy.state, len(asyncore.socket_map))
+            print("loop %s, len %i" % (proxy.state, len(asyncore.socket_map)))
             asyncore.loop(timeout=1, count=1)
         proxy.resolved()
 
         proxy = Socks4aResolver(host=host)
         while len(asyncore.socket_map) > 0:
-            print "loop %s, len %i" % (proxy.state, len(asyncore.socket_map))
+            print("loop %s, len %i" % (proxy.state, len(asyncore.socket_map)))
             asyncore.loop(timeout=1, count=1)
         proxy.resolved()
 

src/network/httpd.py

@@ -1,7 +1,14 @@
+from __future__ import absolute_import
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
 import asyncore
 import socket
 
-from tls import TLSHandshake
+from .tls import TLSHandshake
 
 class HTTPRequestHandler(asyncore.dispatcher):
     response = """HTTP/1.0 200 OK\r

src/network/https.py

@@ -1,8 +1,15 @@
+from __future__ import absolute_import
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
 import asyncore
 
-from http import HTTPClient
+from .http import HTTPClient
 import paths
-from tls import TLSHandshake
+from .tls import TLSHandshake
 
 # self.sslSock = ssl.wrap_socket(self.sock, keyfile = os.path.join(paths.codePath(), 'sslkeys', 'key.pem'), certfile = os.path.join(paths.codePath(), 'sslkeys', 'cert.pem'), server_side = not self.initiatedConnection, ssl_version=ssl.PROTOCOL_TLSv1, do_handshake_on_connect=False, ciphers='AECDH-AES256-SHA')
 

src/network/invthread.py

@@ -1,4 +1,12 @@
-import Queue
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import range
+from builtins import *
+import queue
 from random import randint, shuffle
 import threading
 from time import time
@@ -6,8 +14,7 @@ from time import time
 import addresses
 from bmconfigparser import BMConfigParser
 from helper_threading import StoppableThread
-from network.connectionpool import BMConnectionPool
-from network.dandelion import Dandelion
+from .fix_circular_imports import BMConnectionPool, Dandelion
 from queues import invQueue
 import protocol
 import state
@@ -19,8 +26,8 @@ def handleExpiredDandelion(expired):
     if not expired:
         return
     for i in \
-        BMConnectionPool().inboundConnections.values() + \
-            BMConnectionPool().outboundConnections.values():
+        list(BMConnectionPool().inboundConnections.values()) + \
+            list(BMConnectionPool().outboundConnections.values()):
         if not i.fullyEstablished:
             continue
         for x in expired:
@@ -42,8 +49,8 @@ class InvThread(threading.Thread, StoppableThread):
     def handleLocallyGenerated(self, stream, hashId):
         Dandelion().addHash(hashId, stream=stream)
         for connection in \
-                BMConnectionPool().inboundConnections.values() + \
-                BMConnectionPool().outboundConnections.values():
+                list(BMConnectionPool().inboundConnections.values()) + \
+                list(BMConnectionPool().outboundConnections.values()):
                 if state.dandelion and connection != Dandelion().objectChildStem(hashId):
                     continue
                 connection.objectsNewToThem[hashId] = time()
@@ -60,12 +67,12 @@ class InvThread(threading.Thread, StoppableThread):
                     # locally generated
                     if len(data) == 2 or data[2] is None:
                         self.handleLocallyGenerated(data[0], data[1])
-                except Queue.Empty:
+                except queue.Empty:
                     break
 
             if chunk:
-                for connection in BMConnectionPool().inboundConnections.values() + \
-                        BMConnectionPool().outboundConnections.values():
+                for connection in list(BMConnectionPool().inboundConnections.values()) + \
+                        list(BMConnectionPool().outboundConnections.values()):
                     fluffs = []
                     stems = []
                     for inv in chunk:

src/network/networkthread.py

@@ -1,10 +1,17 @@
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
 import threading
 
 import network.asyncore_pollchoose as asyncore
 import state
 from debug import logger
 from helper_threading import StoppableThread
-from network.connectionpool import BMConnectionPool
+from .fix_circular_imports import BMConnectionPool
 from queues import excQueue
 
 
@@ -25,17 +32,17 @@ class BMNetworkThread(threading.Thread, StoppableThread):
 
     def stopThread(self):
         super(BMNetworkThread, self).stopThread()
-        for i in BMConnectionPool().listeningSockets.values():
+        for i in list(BMConnectionPool().listeningSockets.values()):
             try:
                 i.close()
             except:
                 pass
-        for i in BMConnectionPool().outboundConnections.values():
+        for i in list(BMConnectionPool().outboundConnections.values()):
             try:
                 i.close()
             except:
                 pass
-        for i in BMConnectionPool().inboundConnections.values():
+        for i in list(BMConnectionPool().inboundConnections.values()):
             try:
                 i.close()
             except:

src/network/node.py

@@ -1,3 +1,10 @@
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
 import collections
 
 Node = collections.namedtuple('Node', ['services', 'host', 'port'])

src/network/objectracker.py

@@ -1,131 +0,0 @@
-import time
-from threading import RLock
-
-import network.connectionpool
-from network.dandelion import Dandelion
-from randomtrackingdict import RandomTrackingDict
-
-haveBloom = False
-
-try:
-    # pybloomfiltermmap
-    from pybloomfilter import BloomFilter
-    haveBloom = True
-except ImportError:
-    try:
-        # pybloom
-        from pybloom import BloomFilter
-        haveBloom = True
-    except ImportError:
-        pass
-
-# it isn't actually implemented yet so no point in turning it on
-haveBloom = False
-
-# tracking pending downloads globally, for stats
-missingObjects = {}
-
-
-class ObjectTracker(object):
-    invCleanPeriod = 300
-    invInitialCapacity = 50000
-    invErrorRate = 0.03
-    trackingExpires = 3600
-    initialTimeOffset = 60
-
-    def __init__(self):
-        self.objectsNewToMe = RandomTrackingDict()
-        self.objectsNewToThem = {}
-        self.objectsNewToThemLock = RLock()
-        self.initInvBloom()
-        self.initAddrBloom()
-        self.lastCleaned = time.time()
-
-    def initInvBloom(self):
-        if haveBloom:
-            # lock?
-            self.invBloom = BloomFilter(capacity=ObjectTracker.invInitialCapacity,
-                                        error_rate=ObjectTracker.invErrorRate)
-
-    def initAddrBloom(self):
-        if haveBloom:
-            # lock?
-            self.addrBloom = BloomFilter(capacity=ObjectTracker.invInitialCapacity,
-                                         error_rate=ObjectTracker.invErrorRate)
-
-    def clean(self):
-        if self.lastCleaned < time.time() - ObjectTracker.invCleanPeriod:
-            if haveBloom:
-                if len(missingObjects) == 0:
-                    self.initInvBloom()
-                self.initAddrBloom()
-            else:
-                # release memory
-                deadline = time.time() - ObjectTracker.trackingExpires
-                with self.objectsNewToThemLock:
-                    self.objectsNewToThem = {k: v for k, v in self.objectsNewToThem.iteritems() if v >= deadline}
-            self.lastCleaned = time.time()
-
-    def hasObj(self, hashid):
-        if haveBloom:
-            return hashid in self.invBloom
-        else:
-            return hashid in self.objectsNewToMe
-
-    def handleReceivedInventory(self, hashId):
-        if haveBloom:
-            self.invBloom.add(hashId)
-        try:
-            with self.objectsNewToThemLock:
-                del self.objectsNewToThem[hashId]
-        except KeyError:
-            pass
-        if hashId not in missingObjects:
-            missingObjects[hashId] = time.time()
-        self.objectsNewToMe[hashId] = True
-
-    def handleReceivedObject(self, streamNumber, hashid):
-        for i in network.connectionpool.BMConnectionPool().inboundConnections.values() + network.connectionpool.BMConnectionPool().outboundConnections.values():
-            if not i.fullyEstablished:
-                continue
-            try:
-                del i.objectsNewToMe[hashid]
-            except KeyError:
-                if streamNumber in i.streams and \
-                    (not Dandelion().hasHash(hashid) or \
-                    Dandelion().objectChildStem(hashid) == i):
-                    with i.objectsNewToThemLock:
-                        i.objectsNewToThem[hashid] = time.time()
-                    # update stream number, which we didn't have when we just received the dinv
-                    # also resets expiration of the stem mode
-                    Dandelion().setHashStream(hashid, streamNumber)
-
-            if i == self:
-                try:
-                    with i.objectsNewToThemLock:
-                        del i.objectsNewToThem[hashid]
-                except KeyError:
-                    pass
-        self.objectsNewToMe.setLastObject()
-
-    def hasAddr(self, addr):
-        if haveBloom:
-            return addr in self.invBloom
-
-    def addAddr(self, hashid):
-        if haveBloom:
-            self.addrBloom.add(hashid)
-
-# addr sending -> per node upload queue, and flush every minute or so
-# inv sending -> if not in bloom, inv immediately, otherwise put into a per node upload queue and flush every minute or so
-# data sending -> a simple queue
-
-# no bloom
-# - if inv arrives
-#   - if we don't have it, add tracking and download queue
-#   - if we do have it, remove from tracking
-# tracking downloads
-# - per node hash of items the node has but we don't
-# tracking inv
-# - per node hash of items that neither the remote node nor we have
-#

src/network/proxy.py

@@ -1,9 +1,16 @@
+from __future__ import absolute_import
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
 import socket
 import time
 
-import asyncore_pollchoose as asyncore
+from . import asyncore_pollchoose as asyncore
 import state
-from advanceddispatcher import AdvancedDispatcher
+from .advanceddispatcher import AdvancedDispatcher
 from bmconfigparser import BMConfigParser
 from debug import logger
 

src/network/receivequeuethread.py

@@ -1,5 +1,13 @@
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import str
+from builtins import *
 import errno
-import Queue
+import queue
 import socket
 import sys
 import threading
@@ -10,8 +18,7 @@ from bmconfigparser import BMConfigParser
 from debug import logger
 from helper_threading import StoppableThread
 from inventory import Inventory
-from network.connectionpool import BMConnectionPool
-from network.bmproto import BMProto
+from .fix_circular_imports import BMConnectionPool, BMProto
 from network.advanceddispatcher import UnknownStateError
 from queues import receiveDataQueue
 import protocol
@@ -28,7 +35,7 @@ class ReceiveQueueThread(threading.Thread, StoppableThread):
         while not self._stopped and state.shutdown == 0:
             try:
                 dest = receiveDataQueue.get(block=True, timeout=1)
-            except Queue.Empty:
+            except queue.Empty:
                 continue
 
             if self._stopped or state.shutdown: