diff --git a/.buildbot/appimage/build.sh b/.buildbot/appimage/build.sh index c8dc4f56..10f5ad75 100755 --- a/.buildbot/appimage/build.sh +++ b/.buildbot/appimage/build.sh @@ -4,7 +4,11 @@ export APPIMAGE_EXTRACT_AND_RUN=1 BUILDER=appimage-builder-x86_64.AppImage RECIPE=packages/AppImage/AppImageBuilder.yml +git remote add -f upstream https://github.com/Bitmessage/PyBitmessage.git +HEAD="$(git rev-parse HEAD)" +UPSTREAM="$(git merge-base --fork-point upstream/v0.6)" export APP_VERSION=$(git describe --tags | cut -d- -f1,3 | tr -d v) +[ $HEAD != $UPSTREAM ] && APP_VERSION="${APP_VERSION}-alpha" function set_sourceline { if [ ${ARCH} == amd64 ]; then diff --git a/INSTALL.md b/INSTALL.md index 7942a957..4f11b199 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -1,12 +1,43 @@ # PyBitmessage Installation Instructions - Binary (64bit, no separate installation of dependencies required) - - Windows: https://download.bitmessage.org/snapshots/ + - Windows: https://artifacts.bitmessage.at/winebuild/ - Linux AppImages: https://artifacts.bitmessage.at/appimage/ - - Linux snaps: https://artifacts.bitmessage.at/snap/ + - Linux snaps: https://artifacts.bitmessage.at/snap/ - Mac (not up to date): https://github.com/Bitmessage/PyBitmessage/releases/tag/v0.6.1 - Source `git clone git://github.com/Bitmessage/PyBitmessage.git` +## Notes on the AppImages + +The [AppImage](https://docs.appimage.org/introduction/index.html) +is a bundle, built by the +[appimage-builder](https://github.com/AppImageCrafters/appimage-builder) from +the Ubuntu Bionic deb files, the sources and `bitmsghash.so`, precompiled for +3 architectures, using the `packages/AppImage/AppImageBuilder.yml` recipe. + +When you run the appimage the bundle is loop mounted to a location like +`/tmp/.mount_PyBitm97wj4K` with `squashfs-tools`. + +The appimage name has several informational filds: +``` +PyBitmessage--g[-alpha]-.AppImage +``` + +E.g. `PyBitmessage-0.6.3.2-ge571ba8a-x86_64.AppImage` is an appimage, built from +the `v0.6` for x86_64 and `PyBitmessage-0.6.3.2-g9de2aaf1-alpha-aarch64.AppImage` +is one, built from some development branch for arm64. + +You can also build the appimage with local code. For that you need installed +docker: + +``` +$ docker build -t bm-appimage -f .buildbot/appimage/Dockerfile . +$ docker run -t --rm -v "$(pwd)"/dist:/out bm-appimage .buildbot/appimage/build.sh +``` + +The appimages should be in the dist dir. + + ## Helper Script for building from source Go to the directory with PyBitmessage source code and run: ``` diff --git a/src/bitmessagemain.py b/src/bitmessagemain.py index 9acd1278..ab131a4c 100755 --- a/src/bitmessagemain.py +++ b/src/bitmessagemain.py @@ -12,6 +12,7 @@ The PyBitmessage startup script import os import sys + try: import pathmagic except ImportError: @@ -156,13 +157,6 @@ class Main(object): set_thread_name("PyBitmessage") - state.dandelion_enabled = config.safeGetInt('network', 'dandelion') - # dandelion requires outbound connections, without them, - # stem objects will get stuck forever - if state.dandelion_enabled and not config.safeGetBoolean( - 'bitmessagesettings', 'sendoutgoingconnections'): - state.dandelion_enabled = 0 - if state.testmode or config.safeGetBoolean( 'bitmessagesettings', 'extralowdifficulty'): defaults.networkDefaultProofOfWorkNonceTrialsPerByte = int( diff --git a/src/bitmessageqt/__init__.py b/src/bitmessageqt/__init__.py index 1acedd96..de1e974e 100644 --- a/src/bitmessageqt/__init__.py +++ b/src/bitmessageqt/__init__.py @@ -70,6 +70,9 @@ except ImportError: logger = logging.getLogger('default') +is_windows = sys.platform.startswith('win') + + # TODO: rewrite def powQueueSize(): """Returns the size of queues.workerQueue including current unfinished work""" @@ -88,7 +91,7 @@ def openKeysFile(): keysfile = os.path.join(state.appdata, 'keys.dat') if 'linux' in sys.platform: subprocess.call(["xdg-open", keysfile]) - elif sys.platform.startswith('win'): + elif is_windows: os.startfile(keysfile) # pylint: disable=no-member @@ -848,7 +851,7 @@ class MyForm(settingsmixin.SMainWindow): """ startonlogon = config.safeGetBoolean( 'bitmessagesettings', 'startonlogon') - if sys.platform.startswith('win'): # Auto-startup for Windows + if is_windows: # Auto-startup for Windows RUN_PATH = "HKEY_CURRENT_USER\\Software\\Microsoft\\Windows\\CurrentVersion\\Run" settings = QtCore.QSettings( RUN_PATH, QtCore.QSettings.NativeFormat) @@ -1919,9 +1922,10 @@ class MyForm(settingsmixin.SMainWindow): continue for i in range(sent.rowCount()): # toAddress = sent.item(i, 0).data(QtCore.Qt.UserRole) - # decodeAddress(toAddress) - - if as_msgid(sent.item(i, 3).data()) == ackdata: + tableAckdata = as_msgid(sent.item(i, 3).data()) + # status, addressVersionNumber, streamNumber, ripe = decodeAddress( + # toAddress) + if ackdata == tableAckdata: sent.item(i, 3).setToolTip(textToDisplay) try: newlinePosition = textToDisplay.find('\n') @@ -4327,6 +4331,14 @@ class BitmessageQtApplication(QtWidgets.QApplication): # Unique identifier for this application uuid = '6ec0149b-96e1-4be1-93ab-1465fb3ebf7c' + @staticmethod + def get_windowstyle(): + """Get window style set in config or default""" + return config.safeGet( + 'bitmessagesettings', 'windowstyle', + 'Windows' if is_windows else 'GTK+' + ) + def __init__(self, *argv): super(BitmessageQtApplication, self).__init__(*argv) id = BitmessageQtApplication.uuid @@ -4335,6 +4347,14 @@ class BitmessageQtApplication(QtWidgets.QApplication): QtCore.QCoreApplication.setOrganizationDomain("bitmessage.org") QtCore.QCoreApplication.setApplicationName("pybitmessageqt") + self.setStyle(self.get_windowstyle()) + + font = config.safeGet('bitmessagesettings', 'font') + if font: + # family, size, weight = font.split(',') + family, size = font.split(',') + self.setFont(QtGui.QFont(family, int(size))) + self.server = None self.is_running = False diff --git a/src/bitmessageqt/bitmessageui.py b/src/bitmessageqt/bitmessageui.py index 58e01e5c..1a98d4d4 100644 --- a/src/bitmessageqt/bitmessageui.py +++ b/src/bitmessageqt/bitmessageui.py @@ -34,7 +34,8 @@ class Ui_MainWindow(object): self.tabWidget.setMinimumSize(QtCore.QSize(0, 0)) self.tabWidget.setBaseSize(QtCore.QSize(0, 0)) font = QtGui.QFont() - font.setPointSize(9) + base_size = QtWidgets.QApplication.instance().font().pointSize() + font.setPointSize(int(base_size * 0.75)) self.tabWidget.setFont(font) self.tabWidget.setTabPosition(QtWidgets.QTabWidget.North) self.tabWidget.setTabShape(QtWidgets.QTabWidget.Rounded) diff --git a/src/bitmessageqt/settings.py b/src/bitmessageqt/settings.py index af441afa..d54a0f68 100644 --- a/src/bitmessageqt/settings.py +++ b/src/bitmessageqt/settings.py @@ -48,14 +48,17 @@ def getSOCKSProxyType(config): class SettingsDialog(QtWidgets.QDialog): """The "Settings" dialog""" + # pylint: disable=too-many-instance-attributes def __init__(self, parent=None, firstrun=False): super(SettingsDialog, self).__init__(parent) widgets.load('settings.ui', self) + self.app = QtWidgets.QApplication.instance() self.parent = parent self.firstrun = firstrun self.config = config_obj self.net_restart_needed = False + self.font_setting = None self.timer = QtCore.QTimer() if self.config.safeGetBoolean('bitmessagesettings', 'dontconnect'): @@ -92,6 +95,15 @@ class SettingsDialog(QtWidgets.QDialog): def adjust_from_config(self, config): """Adjust all widgets state according to config settings""" # pylint: disable=too-many-branches,too-many-statements + + current_style = self.app.get_windowstyle() + for i, sk in enumerate(QtWidgets.QStyleFactory.keys()): + self.comboBoxStyle.addItem(sk) + if sk == current_style: + self.comboBoxStyle.setCurrentIndex(i) + + self.save_font_setting(self.app.font()) + if not self.parent.tray.isSystemTrayAvailable(): self.groupBoxTray.setEnabled(False) self.groupBoxTray.setTitle(_translate( @@ -144,7 +156,7 @@ class SettingsDialog(QtWidgets.QDialog): "MainWindow", "Tray notifications not yet supported on your OS.")) - if 'win' not in sys.platform and not self.parent.desktop: + if not sys.platform.startswith('win') and not self.parent.desktop: self.checkBoxStartOnLogon.setDisabled(True) self.checkBoxStartOnLogon.setText(_translate( "MainWindow", "Start-on-login not yet supported on your OS.")) @@ -329,6 +341,18 @@ class SettingsDialog(QtWidgets.QDialog): if status == 'success': self.parent.namecoin = nc + def save_font_setting(self, font): + """Save user font setting and set the buttonFont text""" + font_setting = (font.family(), font.pointSize()) + self.buttonFont.setText('{} {}'.format(*font_setting)) + self.font_setting = '{},{}'.format(*font_setting) + + def choose_font(self): + """Show the font selection dialog""" + font, valid = QtWidgets.QFontDialog.getFont() + if valid: + self.save_font_setting(font) + def accept(self): """A callback for accepted event of buttonBox (OK button pressed)""" # pylint: disable=too-many-branches,too-many-statements @@ -355,6 +379,20 @@ class SettingsDialog(QtWidgets.QDialog): self.config.set('bitmessagesettings', 'replybelow', str( self.checkBoxReplyBelow.isChecked())) + window_style = ustr(self.comboBoxStyle.currentText()) + if self.app.get_windowstyle() != window_style or self.config.safeGet( + 'bitmessagesettings', 'font' + ) != self.font_setting: + self.config.set('bitmessagesettings', 'windowstyle', window_style) + self.config.set('bitmessagesettings', 'font', self.font_setting) + queues.UISignalQueue.put(( + 'updateStatusBar', ( + _translate( + "MainWindow", + "You need to restart the application to apply" + " the window style or default font."), 1) + )) + lang = ustr(self.languageComboBox.itemData( self.languageComboBox.currentIndex())) self.config.set('bitmessagesettings', 'userlocale', lang) diff --git a/src/bitmessageqt/settings.ui b/src/bitmessageqt/settings.ui index 7ce1e389..cdad7ba5 100644 --- a/src/bitmessageqt/settings.ui +++ b/src/bitmessageqt/settings.ui @@ -147,6 +147,32 @@ + + + + Custom Style + + + + + + + 100 + 0 + + + + + + + + Font + + + + + + @@ -1199,5 +1225,11 @@ + + buttonFont + clicked() + settingsDialog + choose_font + diff --git a/src/bitmessageqt/tests/main.py b/src/bitmessageqt/tests/main.py index 7f0d204f..32e488c0 100644 --- a/src/bitmessageqt/tests/main.py +++ b/src/bitmessageqt/tests/main.py @@ -1,20 +1,24 @@ """Common definitions for bitmessageqt tests""" -from six.moves import queue as Queue import sys import unittest from qtpy import QtCore, QtWidgets from six import string_types +from six.moves import queue import bitmessageqt -import queues -from tr import _translate +from bitmessageqt import _translate, config, queues class TestBase(unittest.TestCase): """Base class for bitmessageqt test case""" + @classmethod + def setUpClass(cls): + """Provide the UI test cases with common settings""" + cls.config = config + def setUp(self): self.app = ( QtWidgets.QApplication.instance() @@ -25,14 +29,21 @@ class TestBase(unittest.TestCase): self.window.appIndicatorInit(self.app) def tearDown(self): + """Search for exceptions in closures called by timer and fail if any""" # self.app.deleteLater() + concerning = [] while True: try: thread, exc = queues.excQueue.get(block=False) - except Queue.Empty: - return + except queue.Empty: + break if thread == 'tests': - self.fail('Exception in the main thread: %s' % exc) + concerning.append(exc) + if concerning: + self.fail( + 'Exceptions found in the main thread:\n%s' % '\n'.join(( + str(e) for e in concerning + ))) class TestMain(unittest.TestCase): diff --git a/src/bitmessageqt/tests/settings.py b/src/bitmessageqt/tests/settings.py index e7927ea0..9c2e6452 100644 --- a/src/bitmessageqt/tests/settings.py +++ b/src/bitmessageqt/tests/settings.py @@ -1,10 +1,14 @@ +"""Tests for PyBitmessage settings""" import threading import time -from .main import TestBase +from qtpy import QtCore, QtWidgets, QtTest + from bmconfigparser import config from bitmessageqt import settings +from .main import TestBase + class TestSettings(TestBase): """A test case for the "Settings" dialog""" @@ -14,8 +18,7 @@ class TestSettings(TestBase): def test_udp(self): """Test the effect of checkBoxUDP""" - udp_setting = config.safeGetBoolean( - 'bitmessagesettings', 'udp') + udp_setting = config.safeGetBoolean('bitmessagesettings', 'udp') self.assertEqual(udp_setting, self.dialog.checkBoxUDP.isChecked()) self.dialog.checkBoxUDP.setChecked(not udp_setting) self.dialog.accept() @@ -32,3 +35,44 @@ class TestSettings(TestBase): else: if not udp_setting: self.fail('No Announcer thread found while udp set to True') + + def test_styling(self): + """Test custom windows style and font""" + style_setting = config.safeGet('bitmessagesettings', 'windowstyle') + font_setting = config.safeGet('bitmessagesettings', 'font') + self.assertIs(style_setting, None) + self.assertIs(font_setting, None) + style_control = self.dialog.comboBoxStyle + self.assertEqual( + style_control.currentText(), self.app.get_windowstyle()) + + def call_font_dialog(): + """A function to get the open font dialog and accept it""" + font_dialog = QtWidgets.QApplication.activeModalWidget() + self.assertTrue(isinstance(font_dialog, QtWidgets.QFontDialog)) + selected_font = font_dialog.currentFont() + self.assertEqual( + config.safeGet('bitmessagesettings', 'font'), '{},{}'.format( + selected_font.family(), selected_font.pointSize())) + + font_dialog.accept() + self.dialog.accept() + self.assertEqual( + config.safeGet('bitmessagesettings', 'windowstyle'), + style_control.currentText()) + + def click_font_button(): + """Use QtTest to click the button""" + QtTest.QTest.mouseClick( + self.dialog.buttonFont, QtCore.Qt.MouseButton.LeftButton) + + style_count = style_control.count() + self.assertGreater(style_count, 1) + for i in range(style_count): + if i != style_control.currentIndex(): + style_control.setCurrentIndex(i) + break + + QtCore.QTimer.singleShot(30, click_font_button) + QtCore.QTimer.singleShot(60, call_font_dialog) + time.sleep(2) diff --git a/src/network/__init__.py b/src/network/__init__.py index 58fca104..e950df55 100644 --- a/src/network/__init__.py +++ b/src/network/__init__.py @@ -1,9 +1,10 @@ """ Network subsystem package """ - +from .dandelion import Dandelion from .threads import StoppableThread +dandelion_ins = Dandelion() __all__ = ["StoppableThread"] @@ -21,6 +22,11 @@ def start(config, state): from .receivequeuethread import ReceiveQueueThread from .uploadthread import UploadThread + # check and set dandelion enabled value at network startup + dandelion_ins.init_dandelion_enabled(config) + # pass pool instance into dandelion class instance + dandelion_ins.init_pool(connectionpool.pool) + readKnownNodes() connectionpool.pool.connectToStream(1) for thread in ( diff --git a/src/network/addrthread.py b/src/network/addrthread.py index 452598e8..6b998883 100644 --- a/src/network/addrthread.py +++ b/src/network/addrthread.py @@ -1,11 +1,11 @@ """ Announce addresses as they are received from other hosts """ +import random from six.moves import queue # magic imports! from network import connectionpool -from helper_random import randomshuffle from protocol import assembleAddrMessage from queues import addrQueue # FIXME: init with queue @@ -29,9 +29,9 @@ class AddrThread(StoppableThread): if chunk: # Choose peers randomly connections = connectionpool.pool.establishedConnections() - randomshuffle(connections) + random.shuffle(connections) for i in connections: - randomshuffle(chunk) + random.shuffle(chunk) filtered = [] for stream, peer, seen, destination in chunk: # peer's own address or address received from peer diff --git a/src/network/asyncore_pollchoose.py b/src/network/asyncore_pollchoose.py index 5de3a18f..a1d34878 100644 --- a/src/network/asyncore_pollchoose.py +++ b/src/network/asyncore_pollchoose.py @@ -9,6 +9,7 @@ Basic infrastructure for asynchronous socket service clients and servers. import os import select import socket +import random import sys import time import warnings @@ -20,7 +21,6 @@ from errno import ( from threading import current_thread from six.moves.reprlib import repr -import helper_random try: from errno import WSAEWOULDBLOCK @@ -234,13 +234,13 @@ def select_poller(timeout=0.0, map=None): if err.args[0] in (WSAENOTSOCK, ): return - for fd in helper_random.randomsample(r, len(r)): + for fd in random.sample(r, len(r)): obj = map.get(fd) if obj is None: continue read(obj) - for fd in helper_random.randomsample(w, len(w)): + for fd in random.sample(w, len(w)): obj = map.get(fd) if obj is None: continue @@ -298,7 +298,7 @@ def poll_poller(timeout=0.0, map=None): except socket.error as err: if err.args[0] in (EBADF, WSAENOTSOCK, EINTR): return - for fd, flags in helper_random.randomsample(r, len(r)): + for fd, flags in random.sample(r, len(r)): obj = map.get(fd) if obj is None: continue @@ -358,7 +358,7 @@ def epoll_poller(timeout=0.0, map=None): if err.args[0] != EINTR: raise r = [] - for fd, flags in helper_random.randomsample(r, len(r)): + for fd, flags in random.sample(r, len(r)): obj = map.get(fd) if obj is None: continue @@ -421,7 +421,7 @@ def kqueue_poller(timeout=0.0, map=None): events = kqueue_poller.pollster.control(updates, selectables, timeout) if len(events) > 1: - events = helper_random.randomsample(events, len(events)) + events = random.sample(events, len(events)) for event in events: fd = event.ident diff --git a/src/network/bmobject.py b/src/network/bmobject.py index 67652001..cc34231e 100644 --- a/src/network/bmobject.py +++ b/src/network/bmobject.py @@ -7,7 +7,7 @@ import time import protocol import state import network.connectionpool # use long name to address recursive import -from network import dandelion +from network import dandelion_ins from highlevelcrypto import calculateInventoryHash logger = logging.getLogger('default') @@ -113,7 +113,7 @@ class BMObject(object): # pylint: disable=too-many-instance-attributes or advertise it unnecessarily) """ # if it's a stem duplicate, pretend we don't have it - if dandelion.instance.hasHash(self.inventoryHash): + if dandelion_ins.hasHash(self.inventoryHash): return if self.inventoryHash in state.Inventory: raise BMObjectAlreadyHaveError() diff --git a/src/network/bmproto.py b/src/network/bmproto.py index 9fae2305..445d7c01 100644 --- a/src/network/bmproto.py +++ b/src/network/bmproto.py @@ -17,7 +17,6 @@ from network import knownnodes import protocol import state import network.connectionpool # use long name to address recursive import -from network import dandelion from bmconfigparser import config from queues import invQueue, objectProcessorQueue, portCheckerQueue from randomtrackingdict import RandomTrackingDict @@ -29,6 +28,7 @@ from network.bmobject import ( ) from network.proxy import ProxyError +from network import dandelion_ins from .node import Node, Peer from .objectracker import ObjectTracker, missingObjects @@ -364,14 +364,14 @@ class BMProto(AdvancedDispatcher, ObjectTracker): raise BMProtoExcessiveDataError() # ignore dinv if dandelion turned off - if extend_dandelion_stem and not state.dandelion_enabled: + if extend_dandelion_stem and not dandelion_ins.enabled: return True for i in items: - if i in state.Inventory and not dandelion.instance.hasHash(i): + if i in state.Inventory and not dandelion_ins.hasHash(i): continue - if extend_dandelion_stem and not dandelion.instance.hasHash(i): - dandelion.instance.addHash(i, self) + if extend_dandelion_stem and not dandelion_ins.hasHash(i): + dandelion_ins.addHash(i, self) self.handleReceivedInventory(i) return True @@ -437,9 +437,9 @@ class BMProto(AdvancedDispatcher, ObjectTracker): except KeyError: pass - if self.object.inventoryHash in state.Inventory and dandelion.instance.hasHash( + if self.object.inventoryHash in state.Inventory and dandelion_ins.hasHash( self.object.inventoryHash): - dandelion.instance.removeHash( + dandelion_ins.removeHash( self.object.inventoryHash, "cycle detection") if six.PY2: @@ -563,7 +563,7 @@ class BMProto(AdvancedDispatcher, ObjectTracker): if not self.isOutbound: self.append_write_buf(protocol.assembleVersionMessage( self.destination.host, self.destination.port, - network.connectionpool.pool.streams, True, + network.connectionpool.pool.streams, dandelion_ins.enabled, True, nodeid=self.nodeid)) logger.debug( '%(host)s:%(port)i sending version', diff --git a/src/network/connectionpool.py b/src/network/connectionpool.py index 7611d87d..62e12884 100644 --- a/src/network/connectionpool.py +++ b/src/network/connectionpool.py @@ -7,9 +7,9 @@ import re import socket import sys import time +import random from network import asyncore_pollchoose as asyncore -import helper_random from network import knownnodes import protocol import state @@ -216,7 +216,7 @@ class BMConnectionPool(object): connection_base = TCPConnection elif proxy_type == 'SOCKS5': connection_base = Socks5BMConnection - hostname = helper_random.randomchoice([ + hostname = random.choice([ # nosec B311 'quzwelsuziwqgpt2.onion', None ]) elif proxy_type == 'SOCKS4a': @@ -228,7 +228,7 @@ class BMConnectionPool(object): bootstrapper = bootstrap(connection_base) if not hostname: - port = helper_random.randomchoice([8080, 8444]) + port = random.choice([8080, 8444]) # nosec B311 hostname = 'bootstrap%s.bitmessage.org' % port else: port = 8444 @@ -295,7 +295,7 @@ class BMConnectionPool(object): state.maximumNumberOfHalfOpenConnections - pending): try: chosen = self.trustedPeer or chooseConnection( - helper_random.randomchoice(self.streams)) + random.choice(self.streams)) # nosec B311 except ValueError: continue if chosen in self.outboundConnections: diff --git a/src/network/dandelion.py b/src/network/dandelion.py index cbc6729b..ce87653c 100644 --- a/src/network/dandelion.py +++ b/src/network/dandelion.py @@ -9,9 +9,6 @@ from time import time import six from binascii import hexlify -import network.connectionpool # use long name to address recursive import -import state -from queues import invQueue # randomise routes after 600 seconds REASSIGN_INTERVAL = 600 @@ -39,6 +36,8 @@ class Dandelion: # pylint: disable=old-style-class # when to rerandomise routes self.refresh = time() + REASSIGN_INTERVAL self.lock = RLock() + self.enabled = None + self.pool = None @staticmethod def poissonTimeout(start=None, average=0): @@ -49,10 +48,23 @@ class Dandelion: # pylint: disable=old-style-class average = FLUFF_TRIGGER_MEAN_DELAY return start + expovariate(1.0 / average) + FLUFF_TRIGGER_FIXED_DELAY + def init_pool(self, pool): + """pass pool instance""" + self.pool = pool + + def init_dandelion_enabled(self, config): + """Check if Dandelion is enabled and set value in enabled attribute""" + dandelion_enabled = config.safeGetInt('network', 'dandelion') + # dandelion requires outbound connections, without them, + # stem objects will get stuck forever + if not config.safeGetBoolean( + 'bitmessagesettings', 'sendoutgoingconnections'): + dandelion_enabled = 0 + self.enabled = dandelion_enabled + def addHash(self, hashId, source=None, stream=1): - """Add inventory vector to dandelion stem""" - if not state.dandelion_enabled: - return + """Add inventory vector to dandelion stem return status of dandelion enabled""" + assert self.enabled is not None with self.lock: self.hashMap[bytes(hashId)] = Stem( self.getNodeStem(source), @@ -92,7 +104,7 @@ class Dandelion: # pylint: disable=old-style-class """Child (i.e. next) node for an inventory vector during stem mode""" return self.hashMap[bytes(hashId)].child - def maybeAddStem(self, connection): + def maybeAddStem(self, connection, invQueue): """ If we had too few outbound connections, add the current one to the current stem list. Dandelion as designed by the authors should @@ -166,7 +178,7 @@ class Dandelion: # pylint: disable=old-style-class self.nodeMap[node] = self.pickStem(node) return self.nodeMap[node] - def expire(self): + def expire(self, invQueue): """Switch expired objects from stem to fluff mode""" with self.lock: deadline = time() @@ -182,19 +194,18 @@ class Dandelion: # pylint: disable=old-style-class def reRandomiseStems(self): """Re-shuffle stem mapping (parent <-> child pairs)""" + assert self.pool is not None + if self.refresh > time(): + return + with self.lock: try: # random two connections self.stem = sample( - sorted(network.connectionpool.BMConnectionPool( - ).outboundConnections.values()), MAX_STEMS) + sorted(self.pool.outboundConnections.values()), MAX_STEMS) # not enough stems available except ValueError: - self.stem = list(network.connectionpool.BMConnectionPool( - ).outboundConnections.values()) + self.stem = list(self.pool.outboundConnections.values()) self.nodeMap = {} # hashMap stays to cater for pending stems self.refresh = time() + REASSIGN_INTERVAL - - -instance = Dandelion() diff --git a/src/network/downloadthread.py b/src/network/downloadthread.py index 0962ee14..74e1afc8 100644 --- a/src/network/downloadthread.py +++ b/src/network/downloadthread.py @@ -2,15 +2,15 @@ `DownloadThread` class definition """ import time +import random import state import six import addresses -import helper_random import protocol from network import connectionpool +from network import dandelion_ins from .objectracker import missingObjects from .threads import StoppableThread -from network import dandelion class DownloadThread(StoppableThread): @@ -44,7 +44,7 @@ class DownloadThread(StoppableThread): requested = 0 # Choose downloading peers randomly connections = connectionpool.pool.establishedConnections() - helper_random.randomshuffle(connections) + random.shuffle(connections) requestChunk = max(int( min(self.maxRequestChunk, len(missingObjects)) / len(connections)), 1) if connections else 1 @@ -61,7 +61,7 @@ class DownloadThread(StoppableThread): payload = bytearray() chunkCount = 0 for chunk in request: - if chunk in state.Inventory and not dandelion.instance.hasHash(chunk): + if chunk in state.Inventory and not dandelion_ins.hasHash(chunk): try: del i.objectsNewToMe[chunk] except KeyError: diff --git a/src/network/invthread.py b/src/network/invthread.py index 9705b79a..0a6842d8 100644 --- a/src/network/invthread.py +++ b/src/network/invthread.py @@ -9,7 +9,7 @@ import addresses import protocol import state from network import connectionpool -from network import dandelion +from network import dandelion_ins from queues import invQueue from .threads import StoppableThread @@ -40,10 +40,10 @@ class InvThread(StoppableThread): @staticmethod def handleLocallyGenerated(stream, hashId): """Locally generated inventory items require special handling""" - dandelion.instance.addHash(hashId, stream=stream) + dandelion_ins.addHash(hashId, stream=stream) for connection in connectionpool.pool.connections(): - if state.dandelion_enabled and connection != \ - dandelion.instance.objectChildStem(hashId): + if dandelion_ins.enabled and connection != \ + dandelion_ins.objectChildStem(hashId): continue connection.objectsNewToThem[hashId] = time() @@ -52,7 +52,7 @@ class InvThread(StoppableThread): chunk = [] while True: # Dandelion fluff trigger by expiration - handleExpiredDandelion(dandelion.instance.expire()) + handleExpiredDandelion(dandelion_ins.expire(invQueue)) try: data = invQueue.get(False) chunk.append((data[0], data[1])) @@ -75,10 +75,10 @@ class InvThread(StoppableThread): except KeyError: continue try: - if connection == dandelion.instance.objectChildStem(inv[1]): + if connection == dandelion_ins.objectChildStem(inv[1]): # Fluff trigger by RNG # auto-ignore if config set to 0, i.e. dandelion is off - if random.randint(1, 100) >= state.dandelion_enabled: # nosec B311 + if random.randint(1, 100) >= dandelion_ins.enabled: # nosec B311 fluffs.append(inv[1]) # send a dinv only if the stem node supports dandelion elif connection.services & protocol.NODE_DANDELION > 0: @@ -105,7 +105,6 @@ class InvThread(StoppableThread): for _ in range(len(chunk)): invQueue.task_done() - if dandelion.instance.refresh < time(): - dandelion.instance.reRandomiseStems() + dandelion_ins.reRandomiseStems() self.stop.wait(1) diff --git a/src/network/objectracker.py b/src/network/objectracker.py index be2b4219..b14452ea 100644 --- a/src/network/objectracker.py +++ b/src/network/objectracker.py @@ -6,7 +6,7 @@ from threading import RLock import six import network.connectionpool # use long name to address recursive import -from network import dandelion +from network import dandelion_ins from randomtrackingdict import RandomTrackingDict haveBloom = False @@ -111,14 +111,14 @@ class ObjectTracker(object): del i.objectsNewToMe[hashid] except KeyError: if streamNumber in i.streams and ( - not dandelion.instance.hasHash(hashid) - or dandelion.instance.objectChildStem(hashid) == i): + not dandelion_ins.hasHash(hashid) + or dandelion_ins.objectChildStem(hashid) == i): with i.objectsNewToThemLock: i.objectsNewToThem[hashid_bytes] = time.time() # update stream number, # which we didn't have when we just received the dinv # also resets expiration of the stem mode - dandelion.instance.setHashStream(hashid, streamNumber) + dandelion_ins.setHashStream(hashid, streamNumber) if i == self: try: diff --git a/src/network/tcp.py b/src/network/tcp.py index 3dbc15d2..2928a294 100644 --- a/src/network/tcp.py +++ b/src/network/tcp.py @@ -12,14 +12,13 @@ import six # magic imports! import addresses -import helper_random import l10n import protocol import state import network.connectionpool # use long name to address recursive import -from network import dandelion from bmconfigparser import config from highlevelcrypto import randomBytes +from network import dandelion_ins from queues import invQueue, receiveDataQueue, UISignalQueue from tr import _translate @@ -176,7 +175,7 @@ class TCPConnection(BMProto, TLSDispatcher): knownnodes.increaseRating(self.destination) knownnodes.addKnownNode( self.streams, self.destination, time.time()) - dandelion.instance.maybeAddStem(self) + dandelion_ins.maybeAddStem(self, invQueue) self.sendAddr() self.sendBigInv() @@ -208,7 +207,7 @@ class TCPConnection(BMProto, TLSDispatcher): elemCount = min( len(filtered), maxAddrCount / 2 if n else maxAddrCount) - addrs[s] = helper_random.randomsample(filtered, elemCount) + addrs[s] = random.sample(filtered, elemCount) for substream in addrs: for peer, params in addrs[substream]: templist.append((substream, peer, params["lastseen"])) @@ -238,7 +237,7 @@ class TCPConnection(BMProto, TLSDispatcher): with self.objectsNewToThemLock: for objHash in state.Inventory.unexpired_hashes_by_stream(stream): # don't advertise stem objects on bigInv - if dandelion.instance.hasHash(objHash): + if dandelion_ins.hasHash(objHash): continue bigInvList[objHash] = 0 objectCount = 0 @@ -275,7 +274,7 @@ class TCPConnection(BMProto, TLSDispatcher): self.append_write_buf( protocol.assembleVersionMessage( self.destination.host, self.destination.port, - network.connectionpool.pool.streams, + network.connectionpool.pool.streams, dandelion_ins.enabled, False, nodeid=self.nodeid)) self.connectedAt = time.time() receiveDataQueue.put(self.destination) @@ -300,7 +299,7 @@ class TCPConnection(BMProto, TLSDispatcher): if host_is_global: knownnodes.addKnownNode( self.streams, self.destination, time.time()) - dandelion.instance.maybeRemoveStem(self) + dandelion_ins.maybeRemoveStem(self) else: self.checkTimeOffsetNotification() if host_is_global: @@ -326,7 +325,7 @@ class Socks5BMConnection(Socks5Connection, TCPConnection): self.append_write_buf( protocol.assembleVersionMessage( self.destination.host, self.destination.port, - network.connectionpool.pool.streams, + network.connectionpool.pool.streams, dandelion_ins.enabled, False, nodeid=self.nodeid)) self.set_state("bm_header", expectBytes=protocol.Header.size) return True @@ -350,7 +349,7 @@ class Socks4aBMConnection(Socks4aConnection, TCPConnection): self.append_write_buf( protocol.assembleVersionMessage( self.destination.host, self.destination.port, - network.connectionpool.pool.streams, + network.connectionpool.pool.streams, dandelion_ins.enabled, False, nodeid=self.nodeid)) self.set_state("bm_header", expectBytes=protocol.Header.size) return True diff --git a/src/network/uploadthread.py b/src/network/uploadthread.py index bd5f2d67..b303a63e 100644 --- a/src/network/uploadthread.py +++ b/src/network/uploadthread.py @@ -3,12 +3,12 @@ """ import time -import helper_random +import random import protocol import state from network import connectionpool -from network import dandelion from randomtrackingdict import RandomTrackingDict +from network import dandelion_ins from .threads import StoppableThread @@ -24,7 +24,7 @@ class UploadThread(StoppableThread): uploaded = 0 # Choose uploading peers randomly connections = connectionpool.pool.establishedConnections() - helper_random.randomshuffle(connections) + random.shuffle(connections) for i in connections: now = time.time() # avoid unnecessary delay @@ -41,8 +41,8 @@ class UploadThread(StoppableThread): chunk_count = 0 for chunk in request: del i.pendingUpload[chunk] - if dandelion.instance.hasHash(chunk) and \ - i != dandelion.instance.objectChildStem(chunk): + if dandelion_ins.hasHash(chunk) and \ + i != dandelion_ins.objectChildStem(chunk): i.antiIntersectionDelay() self.logger.info( '%s asked for a stem object we didn\'t offer to it.', diff --git a/src/protocol.py b/src/protocol.py index 8fd6c8fd..cf656bc8 100644 --- a/src/protocol.py +++ b/src/protocol.py @@ -339,8 +339,8 @@ def assembleAddrMessage(peerList): return retval -def assembleVersionMessage( - remoteHost, remotePort, participatingStreams, server=False, nodeid=None +def assembleVersionMessage( # pylint: disable=too-many-arguments + remoteHost, remotePort, participatingStreams, dandelion_enabled=True, server=False, nodeid=None, ): """ Construct the payload of a version message, @@ -353,7 +353,7 @@ def assembleVersionMessage( '>q', NODE_NETWORK | (NODE_SSL if haveSSL(server) else 0) - | (NODE_DANDELION if state.dandelion_enabled else 0) + | (NODE_DANDELION if dandelion_enabled else 0) ) payload += pack('>q', int(time.time())) @@ -377,7 +377,7 @@ def assembleVersionMessage( '>q', NODE_NETWORK | (NODE_SSL if haveSSL(server) else 0) - | (NODE_DANDELION if state.dandelion_enabled else 0) + | (NODE_DANDELION if dandelion_enabled else 0) ) # = 127.0.0.1. This will be ignored by the remote host. # The actual remote connected IP will be used. diff --git a/src/state.py b/src/state.py index a40ebbc2..90c9cf0d 100644 --- a/src/state.py +++ b/src/state.py @@ -43,8 +43,6 @@ ownAddresses = {} discoveredPeers = {} -dandelion_enabled = 0 - kivy = False kivyapp = None diff --git a/src/tests/core.py b/src/tests/core.py index 69d4e43d..836117aa 100644 --- a/src/tests/core.py +++ b/src/tests/core.py @@ -322,16 +322,17 @@ class TestCore(unittest.TestCase): def test_version(self): """check encoding/decoding of the version message""" + dandelion_enabled = True # with single stream - msg = protocol.assembleVersionMessage('127.0.0.1', 8444, [1]) + msg = protocol.assembleVersionMessage('127.0.0.1', 8444, [1], dandelion_enabled) decoded = self._decode_msg(msg, "IQQiiQlsLv") peer, _, ua, streams = self._decode_msg(msg, "IQQiiQlsLv")[4:] self.assertEqual( - peer, Node(11 if state.dandelion_enabled else 3, '127.0.0.1', 8444)) + peer, Node(11 if dandelion_enabled else 3, '127.0.0.1', 8444)) self.assertEqual(ua, '/PyBitmessage:' + softwareVersion + '/') self.assertEqual(streams, [1]) # with multiple streams - msg = protocol.assembleVersionMessage('127.0.0.1', 8444, [1, 2, 3]) + msg = protocol.assembleVersionMessage('127.0.0.1', 8444, [1, 2, 3], dandelion_enabled) decoded = self._decode_msg(msg, "IQQiiQlslv") peer, _, ua = decoded[4:7] streams = decoded[7:] diff --git a/src/tests/test_addressgenerator.py b/src/tests/test_addressgenerator.py index ea4e6e44..d7366fe4 100644 --- a/src/tests/test_addressgenerator.py +++ b/src/tests/test_addressgenerator.py @@ -2,8 +2,8 @@ from binascii import unhexlify -from six.moves import queue import six +from six.moves import queue from .partial import TestPartialRun from .samples import ( diff --git a/src/tests/test_api.py b/src/tests/test_api.py index 3ae547d3..94fc8e23 100644 --- a/src/tests/test_api.py +++ b/src/tests/test_api.py @@ -5,12 +5,11 @@ Tests using API. import base64 import json import time - from binascii import hexlify -from six.moves import xmlrpc_client # nosec -import six import psutil +import six +from six.moves import xmlrpc_client # nosec from .samples import ( sample_deterministic_addr3, sample_deterministic_addr4, sample_seed, @@ -181,29 +180,29 @@ class TestAPI(TestAPIProto): self.assertEqual( self.api.getDeterministicAddress(self._seed, 3, 1), sample_deterministic_addr3) - six.assertRegex(self, - self.api.getDeterministicAddress(self._seed, 2, 1), + six.assertRegex( + self, self.api.getDeterministicAddress(self._seed, 2, 1), r'^API Error 0002:') # This is here until the streams will be implemented - six.assertRegex(self, - self.api.getDeterministicAddress(self._seed, 3, 2), + six.assertRegex( + self, self.api.getDeterministicAddress(self._seed, 3, 2), r'API Error 0003:') - six.assertRegex(self, - self.api.createDeterministicAddresses(self._seed, 1, 4, 2), + six.assertRegex( + self, self.api.createDeterministicAddresses(self._seed, 1, 4, 2), r'API Error 0003:') - six.assertRegex(self, - self.api.createDeterministicAddresses('', 1), + six.assertRegex( + self, self.api.createDeterministicAddresses('', 1), r'API Error 0001:') - six.assertRegex(self, - self.api.createDeterministicAddresses(self._seed, 1, 2), + six.assertRegex( + self, self.api.createDeterministicAddresses(self._seed, 1, 2), r'API Error 0002:') - six.assertRegex(self, - self.api.createDeterministicAddresses(self._seed, 0), + six.assertRegex( + self, self.api.createDeterministicAddresses(self._seed, 0), r'API Error 0004:') - six.assertRegex(self, - self.api.createDeterministicAddresses(self._seed, 1000), + six.assertRegex( + self, self.api.createDeterministicAddresses(self._seed, 1000), r'API Error 0005:') addresses = json.loads( @@ -442,7 +441,7 @@ class TestAPI(TestAPIProto): self.assertEqual(self.api.joinChan(self._seed, addr), 'success') self.assertEqual(self.api.leaveChan(addr), 'success') # Joining with wrong address should fail - six.assertRegex(self, - self.api.joinChan(self._seed, 'BM-2cWzSnwjJ7yRP3nLEW'), + six.assertRegex( + self, self.api.joinChan(self._seed, 'BM-2cWzSnwjJ7yRP3nLEW'), r'^API Error 0008:' ) diff --git a/src/tests/test_inventory.py b/src/tests/test_inventory.py index 6e698411..d0b9ff6d 100644 --- a/src/tests/test_inventory.py +++ b/src/tests/test_inventory.py @@ -6,6 +6,7 @@ import struct import tempfile import time import unittest + import six from pybitmessage import highlevelcrypto @@ -51,8 +52,8 @@ class TestStorageAbstract(unittest.TestCase): def test_inventory_storage(self): """Check inherited abstract methods""" - with six.assertRaisesRegex(self, - TypeError, "^Can't instantiate abstract class.*" + with six.assertRaisesRegex( + self, TypeError, "^Can't instantiate abstract class.*" "methods __contains__, __delitem__, __getitem__, __iter__," " __len__, __setitem__" ): # pylint: disable=abstract-class-instantiated diff --git a/src/tests/test_logger.py b/src/tests/test_logger.py index 6e4068fc..bf63a014 100644 --- a/src/tests/test_logger.py +++ b/src/tests/test_logger.py @@ -4,6 +4,7 @@ Testing the logger configuration import os import tempfile + import six from .test_process import TestProcessProto