From dd780f8d806afae29db5c7173d3e3a9eb1bcfb2e Mon Sep 17 00:00:00 2001 From: sandakersmann Date: Thu, 7 Dec 2017 00:27:59 +0100 Subject: [PATCH 01/18] This type of data is called metadata --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7a161d04..0ea6144b 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ Bitmessage is a P2P communications protocol used to send encrypted messages to another person or to many subscribers. It is decentralized and trustless, meaning that you need-not inherently trust any entities like root certificate authorities. It uses strong authentication, which means that the sender of a -message cannot be spoofed, and it aims to hide "non-content" data, like the +message cannot be spoofed, and it aims to hide metadata, like the sender and receiver of messages, from passive eavesdroppers like those running warrantless wiretapping programs. From 395812c0f8b50bcc56b411d4c293b7735382c32f Mon Sep 17 00:00:00 2001 From: Peter Surda Date: Wed, 20 Dec 2017 09:20:24 +0100 Subject: [PATCH 02/18] Systemd config file - tested on Debian 9, you may have to adjust paths/uids if your deployment differs --- packages/systemd/bitmessage.service | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 packages/systemd/bitmessage.service diff --git a/packages/systemd/bitmessage.service b/packages/systemd/bitmessage.service new file mode 100644 index 00000000..1a9f7f47 --- /dev/null +++ b/packages/systemd/bitmessage.service @@ -0,0 +1,18 @@ +[Unit] +Description=Bitmessage Daemon +After=network.target auditd.service + +[Service] +ExecStart=/usr/bin/python2 /usr/src/PyBitmessage/src/bitmessagemain.py +ExecReload=/bin/kill -HUP $MAINPID +KillMode=process +Restart=on-failure +Type=forking +PIDFile=/var/lib/bitmessage/.config/PyBitmessage/singleton.lock +User=bitmessage +Group=nogroup +WorkingDirectory=/var/lib/bitmessage +Environment="HOME=/var/lib/bitmessage" + +[Install] +WantedBy=multi-user.target From 6fb5a751c624efce8bc41736d219b0acda7cebaa Mon Sep 17 00:00:00 2001 From: Peter Surda Date: Wed, 20 Dec 2017 09:41:36 +0100 Subject: [PATCH 03/18] Add collectd monitoring script --- packages/collectd/pybitmessagestatus.py | 60 +++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 packages/collectd/pybitmessagestatus.py diff --git a/packages/collectd/pybitmessagestatus.py b/packages/collectd/pybitmessagestatus.py new file mode 100644 index 00000000..1db9f5b1 --- /dev/null +++ b/packages/collectd/pybitmessagestatus.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python2.7 + +import collectd +import json +import xmlrpclib + +pybmurl = "" +api = "" + +def init_callback(): + global api + api = xmlrpclib.ServerProxy(pybmurl) + collectd.info('pybitmessagestatus.py init done') + +def config_callback(ObjConfiguration): + global pybmurl + apiUsername = "" + apiPassword = "" + apiInterface = "127.0.0.1" + apiPort = 8445 + for node in ObjConfiguration.children: + key = node.key.lower() + if key.lower() == "apiusername" and node.values: + apiUsername = node.values[0] + elif key.lower() == "apipassword" and node.values: + apiPassword = node.values[0] + elif key.lower() == "apiinterface" and node.values: + apiInterface = node.values[0] + elif key.lower() == "apiport" and node.values: + apiPort = node.values[0] + pybmurl = "http://" + apiUsername + ":" + apiPassword + "@" + apiInterface+ ":" + str(int(apiPort)) + "/" + collectd.info('pybitmessagestatus.py config done') + +def read_callback(): + try: + clientStatus = json.loads(api.clientStatus()) + except: + collectd.info("Exception loading or parsing JSON") + return + + for i in ["networkConnections", "numberOfPubkeysProcessed", "numberOfMessagesProcessed", "numberOfBroadcastsProcessed"]: + metric = collectd.Values() + metric.plugin = "pybitmessagestatus" + if i[0:6] == "number": + metric.type = 'counter' + else: + metric.type = 'gauge' + metric.type_instance = i.lower() + try: + metric.values = [clientStatus[i]] + except: + collectd.info("Value for %s missing" % (i)) + metric.dispatch() + +if __name__ == "__main__": + main() +else: + collectd.register_init(init_callback) + collectd.register_config(config_callback) + collectd.register_read(read_callback) From 3cb9547389e41bbf8b7cd8e258e91db57614e4b6 Mon Sep 17 00:00:00 2001 From: Peter Surda Date: Thu, 21 Dec 2017 14:26:51 +0100 Subject: [PATCH 04/18] Only write PID after last fork - should fix systemd integration --- src/bitmessagemain.py | 2 +- src/singleinstance.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/bitmessagemain.py b/src/bitmessagemain.py index 83a41919..91032fe5 100755 --- a/src/bitmessagemain.py +++ b/src/bitmessagemain.py @@ -363,7 +363,7 @@ class Main: # fork not implemented pass else: - shared.thisapp.lock() # relock + shared.thisapp.lock(True) # relock and write pid shared.thisapp.lockPid = None # indicate we're the final child sys.stdout.flush() sys.stderr.flush() diff --git a/src/singleinstance.py b/src/singleinstance.py index 7a025945..883c83cc 100644 --- a/src/singleinstance.py +++ b/src/singleinstance.py @@ -36,7 +36,7 @@ class singleinstance: self.initialized = True atexit.register(self.cleanup) - def lock(self): + def lock(self, writePid = False): if self.lockPid is None: self.lockPid = os.getpid() if sys.platform == 'win32': @@ -68,9 +68,10 @@ class singleinstance: sys.exit(-1) else: pidLine = "%i\n" % self.lockPid - self.fp.truncate(0) - self.fp.write(pidLine) - self.fp.flush() + if writePid: + self.fp.truncate(0) + self.fp.write(pidLine) + self.fp.flush() def cleanup(self): if not self.initialized: From 02490e3286ad1d865bdc9ec4552766a0304f6b37 Mon Sep 17 00:00:00 2001 From: Peter Surda Date: Fri, 29 Dec 2017 08:41:15 +0100 Subject: [PATCH 05/18] Don't break if over 50k messages - typo if there were over 50k messages in inventory caused PyBM to stall --- src/network/tcp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/network/tcp.py b/src/network/tcp.py index 70e22e08..922aa79d 100644 --- a/src/network/tcp.py +++ b/src/network/tcp.py @@ -180,7 +180,7 @@ class TCPConnection(BMProto, TLSDispatcher): payload += hash objectCount += 1 if objectCount >= BMProto.maxObjectCount: - self.sendChunk() + sendChunk() payload = b'' objectCount = 0 From e9b1aa48a91624db922d832a1d342e63ad896b4e Mon Sep 17 00:00:00 2001 From: Peter Surda Date: Fri, 29 Dec 2017 08:49:08 +0100 Subject: [PATCH 06/18] Protocol error handler fixes - was broken if there was no error message in "raise" - added default texts for network exceptions --- src/network/bmobject.py | 17 +++++++++++------ src/network/bmproto.py | 9 ++++++--- src/network/proxy.py | 2 +- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/src/network/bmobject.py b/src/network/bmobject.py index 249ec2ab..267cac58 100644 --- a/src/network/bmobject.py +++ b/src/network/bmobject.py @@ -8,23 +8,28 @@ from network.dandelion import Dandelion import protocol import state -class BMObjectInsufficientPOWError(Exception): pass +class BMObjectInsufficientPOWError(Exception): + errorCodes = ("Insufficient proof of work") -class BMObjectInvalidDataError(Exception): pass +class BMObjectInvalidDataError(Exception): + errorCodes = ("Data invalid") -class BMObjectExpiredError(Exception): pass +class BMObjectExpiredError(Exception): + errorCodes = ("Object expired") -class BMObjectUnwantedStreamError(Exception): pass +class BMObjectUnwantedStreamError(Exception): + errorCodes = ("Object in unwanted stream") -class BMObjectInvalidError(Exception): pass +class BMObjectInvalidError(Exception): + errorCodes = ("Invalid object") class BMObjectAlreadyHaveError(Exception): - pass + errorCodes = ("Already have this object") class BMObject(object): diff --git a/src/network/bmproto.py b/src/network/bmproto.py index 17b2c761..47c6c858 100644 --- a/src/network/bmproto.py +++ b/src/network/bmproto.py @@ -24,13 +24,16 @@ import shared import state import protocol -class BMProtoError(ProxyError): pass +class BMProtoError(ProxyError): + errorCodes = ("Protocol error") -class BMProtoInsufficientDataError(BMProtoError): pass +class BMProtoInsufficientDataError(BMProtoError): + errorCodes = ("Insufficient data") -class BMProtoExcessiveDataError(BMProtoError): pass +class BMProtoExcessiveDataError(BMProtoError): + errorCodes = ("Too much data") class BMProto(AdvancedDispatcher, ObjectTracker): diff --git a/src/network/proxy.py b/src/network/proxy.py index 7d46cd86..96930c18 100644 --- a/src/network/proxy.py +++ b/src/network/proxy.py @@ -10,7 +10,7 @@ import state class ProxyError(Exception): errorCodes = ("UnknownError") - def __init__(self, code): + def __init__(self, code=-1): self.code = code try: self.message = self.__class__.errorCodes[self.code] From bcc5a210a484dd76b7e9ad3b033f47e704026770 Mon Sep 17 00:00:00 2001 From: Peter Surda Date: Fri, 29 Dec 2017 09:13:41 +0100 Subject: [PATCH 07/18] Fix PID file if not daemonized --- src/singleinstance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/singleinstance.py b/src/singleinstance.py index 883c83cc..fdb5ee98 100644 --- a/src/singleinstance.py +++ b/src/singleinstance.py @@ -31,7 +31,7 @@ class singleinstance: import bitmessageqt bitmessageqt.init() - self.lock() + self.lock(not daemon) self.initialized = True atexit.register(self.cleanup) From 1864762a0a3af4f32ba7fc5994bcb51d76643da3 Mon Sep 17 00:00:00 2001 From: Peter Surda Date: Mon, 1 Jan 2018 12:49:08 +0100 Subject: [PATCH 08/18] Apply bandwidth limits without restart - also minor style fixes --- src/bitmessageqt/__init__.py | 10 +++++----- src/network/asyncore_pollchoose.py | 4 ++-- src/network/connectionpool.py | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/bitmessageqt/__init__.py b/src/bitmessageqt/__init__.py index c731b469..e6ff0cd0 100644 --- a/src/bitmessageqt/__init__.py +++ b/src/bitmessageqt/__init__.py @@ -69,7 +69,7 @@ import queues import shutdown import state from statusbar import BMStatusBar -import throttle +from network.asyncore_pollchoose import set_rates from version import softwareVersion import sound @@ -2288,16 +2288,16 @@ class MyForm(settingsmixin.SMainWindow): int(float(self.settingsDialogInstance.ui.lineEditMaxDownloadRate.text())))) BMConfigParser().set('bitmessagesettings', 'maxuploadrate', str( int(float(self.settingsDialogInstance.ui.lineEditMaxUploadRate.text())))) - except: + except ValueError: QMessageBox.about(self, _translate("MainWindow", "Number needed"), _translate( "MainWindow", "Your maximum download and upload rate must be numbers. Ignoring what you typed.")) + else: + set_rates(BMConfigParser().safeGetInt("bitmessagesettings", "maxdownloadrate"), + BMConfigParser().safeGetInt("bitmessagesettings", "maxuploadrate")) BMConfigParser().set('bitmessagesettings', 'maxoutboundconnections', str( int(float(self.settingsDialogInstance.ui.lineEditMaxOutboundConnections.text())))) - throttle.SendThrottle().resetLimit() - throttle.ReceiveThrottle().resetLimit() - BMConfigParser().set('bitmessagesettings', 'namecoinrpctype', self.settingsDialogInstance.getNamecoinType()) BMConfigParser().set('bitmessagesettings', 'namecoinrpchost', str( diff --git a/src/network/asyncore_pollchoose.py b/src/network/asyncore_pollchoose.py index caa9d650..c5586a91 100644 --- a/src/network/asyncore_pollchoose.py +++ b/src/network/asyncore_pollchoose.py @@ -129,8 +129,8 @@ def write(obj): def set_rates(download, upload): global maxDownloadRate, maxUploadRate, downloadBucket, uploadBucket, downloadTimestamp, uploadTimestamp - maxDownloadRate = float(download) - maxUploadRate = float(upload) + maxDownloadRate = float(download) * 1024 + maxUploadRate = float(upload) * 1024 downloadBucket = maxDownloadRate uploadBucket = maxUploadRate downloadTimestamp = time.time() diff --git a/src/network/connectionpool.py b/src/network/connectionpool.py index 2f937a15..3b817e65 100644 --- a/src/network/connectionpool.py +++ b/src/network/connectionpool.py @@ -22,8 +22,8 @@ import state class BMConnectionPool(object): def __init__(self): asyncore.set_rates( - BMConfigParser().safeGetInt("bitmessagesettings", "maxdownloadrate") * 1024, - BMConfigParser().safeGetInt("bitmessagesettings", "maxuploadrate") * 1024) + BMConfigParser().safeGetInt("bitmessagesettings", "maxdownloadrate"), + BMConfigParser().safeGetInt("bitmessagesettings", "maxuploadrate")) self.outboundConnections = {} self.inboundConnections = {} self.listeningSockets = {} From 36cc5b9cf5e122b736854079b78e6b0dfb903bbc Mon Sep 17 00:00:00 2001 From: Peter Surda Date: Mon, 1 Jan 2018 12:51:35 +0100 Subject: [PATCH 09/18] Download optimisations - don't make empty download requests - use smaller chunks when they can be spread across multiple connections --- src/network/downloadthread.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/network/downloadthread.py b/src/network/downloadthread.py index 5fe1ee25..35616f1b 100644 --- a/src/network/downloadthread.py +++ b/src/network/downloadthread.py @@ -43,7 +43,7 @@ class DownloadThread(threading.Thread, StoppableThread): connections = BMConnectionPool().inboundConnections.values() + BMConnectionPool().outboundConnections.values() random.shuffle(connections) try: - requestChunk = max(int(DownloadThread.maxRequestChunk / len(connections)), 1) + requestChunk = max(int(min(DownloadThread.maxRequestChunk, len(missingObjects)) / len(connections)), 1) except ZeroDivisionError: requestChunk = 1 for i in connections: @@ -63,15 +63,14 @@ class DownloadThread(threading.Thread, StoppableThread): except KeyError: continue random.shuffle(request) + if len(request) > requestChunk - downloadPending: + request = request[:max(1, requestChunk - downloadPending)] if not request: continue - if len(request) > requestChunk - downloadPending: - request = request[:requestChunk - downloadPending] # mark them as pending for k in request: i.objectsNewToMe[k] = False missingObjects[k] = now - payload = bytearray() payload.extend(addresses.encodeVarint(len(request))) for chunk in request: From baba0ae206b6cc8f6d81725fb1184a5d70225ed9 Mon Sep 17 00:00:00 2001 From: Peter Surda Date: Mon, 1 Jan 2018 13:04:58 +0100 Subject: [PATCH 10/18] Remove obsolete code --- src/shutdown.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/shutdown.py b/src/shutdown.py index 278759e5..49c2fb9b 100644 --- a/src/shutdown.py +++ b/src/shutdown.py @@ -9,14 +9,12 @@ from helper_sql import sqlQuery, sqlStoredProcedure from helper_threading import StoppableThread from knownnodes import saveKnownNodes from inventory import Inventory -import protocol from queues import addressGeneratorQueue, objectProcessorQueue, UISignalQueue, workerQueue import shared import state def doCleanShutdown(): state.shutdown = 1 #Used to tell proof of work worker threads and the objectProcessorThread to exit. - protocol.broadcastToSendDataQueues((0, 'shutdown', 'no data')) objectProcessorQueue.put(('checkShutdownVariable', 'no data')) for thread in threading.enumerate(): if thread.isAlive() and isinstance(thread, StoppableThread): From d9a42630830eef597021179924351bba3b959386 Mon Sep 17 00:00:00 2001 From: Peter Surda Date: Mon, 1 Jan 2018 13:08:12 +0100 Subject: [PATCH 11/18] Daemonising fixes - change forking exit order as systemd expects (wait until child is ready, then exit parent, then grandparent) - fix signal handler if prctl not installed - revert recent PID file changes --- src/bitmessagemain.py | 19 ++++++++++++++++++- src/helper_generic.py | 2 +- src/singleinstance.py | 18 +++++++++++++----- 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/src/bitmessagemain.py b/src/bitmessagemain.py index 91032fe5..6f796939 100755 --- a/src/bitmessagemain.py +++ b/src/bitmessagemain.py @@ -342,13 +342,21 @@ class Main: sleep(1) def daemonize(self): + grandfatherPid = os.getpid() + parentPid = None try: if os.fork(): + # unlock + shared.thisapp.cleanup() + # wait until grandchild ready + while True: + sleep(1) os._exit(0) except AttributeError: # fork not implemented pass else: + parentPid = os.getpid() shared.thisapp.lock() # relock os.umask(0) try: @@ -358,12 +366,17 @@ class Main: pass try: if os.fork(): + # unlock + shared.thisapp.cleanup() + # wait until child ready + while True: + sleep(1) os._exit(0) except AttributeError: # fork not implemented pass else: - shared.thisapp.lock(True) # relock and write pid + shared.thisapp.lock() # relock shared.thisapp.lockPid = None # indicate we're the final child sys.stdout.flush() sys.stderr.flush() @@ -374,6 +387,10 @@ class Main: os.dup2(si.fileno(), sys.stdin.fileno()) os.dup2(so.fileno(), sys.stdout.fileno()) os.dup2(se.fileno(), sys.stderr.fileno()) + if parentPid: + # signal ready + os.kill(parentPid, signal.SIGTERM) + os.kill(grandfatherPid, signal.SIGTERM) def setSignalHandler(self): signal.signal(signal.SIGINT, helper_generic.signal_handler) diff --git a/src/helper_generic.py b/src/helper_generic.py index b750e519..4f7a1299 100644 --- a/src/helper_generic.py +++ b/src/helper_generic.py @@ -51,7 +51,7 @@ def signal_handler(signal, frame): raise SystemExit if "PoolWorker" in current_process().name: raise SystemExit - if current_thread().name != "PyBitmessage": + if current_thread().name not in ("PyBitmessage", "MainThread"): return logger.error("Got signal %i", signal) if BMConfigParser().safeGetBoolean('bitmessagesettings', 'daemon'): diff --git a/src/singleinstance.py b/src/singleinstance.py index fdb5ee98..d7cc0ab3 100644 --- a/src/singleinstance.py +++ b/src/singleinstance.py @@ -36,7 +36,7 @@ class singleinstance: self.initialized = True atexit.register(self.cleanup) - def lock(self, writePid = False): + def lock(self): if self.lockPid is None: self.lockPid = os.getpid() if sys.platform == 'win32': @@ -68,16 +68,24 @@ class singleinstance: sys.exit(-1) else: pidLine = "%i\n" % self.lockPid - if writePid: - self.fp.truncate(0) - self.fp.write(pidLine) - self.fp.flush() + self.fp.truncate(0) + self.fp.write(pidLine) + self.fp.flush() def cleanup(self): if not self.initialized: return if self.daemon and self.lockPid == os.getpid(): # these are the two initial forks while daemonizing + try: + if sys.platform == 'win32': + if hasattr(self, 'fd'): + os.close(self.fd) + else: + fcntl.lockf(self.fp, fcntl.LOCK_UN) + except Exception, e: + pass + return print "Cleaning up lockfile" try: From 6b54a4ab0e1ddfc39d06dca2d5aea010be7e8a5a Mon Sep 17 00:00:00 2001 From: Peter Surda Date: Mon, 1 Jan 2018 13:10:19 +0100 Subject: [PATCH 12/18] Daemonize fix - forgot to revert a line in previous commit --- src/singleinstance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/singleinstance.py b/src/singleinstance.py index d7cc0ab3..ed1048ba 100644 --- a/src/singleinstance.py +++ b/src/singleinstance.py @@ -31,7 +31,7 @@ class singleinstance: import bitmessageqt bitmessageqt.init() - self.lock(not daemon) + self.lock() self.initialized = True atexit.register(self.cleanup) From bb5f1d6f98b3dabec7a432bcde399bbec12c0b8a Mon Sep 17 00:00:00 2001 From: Peter Surda Date: Tue, 2 Jan 2018 10:29:21 +0100 Subject: [PATCH 13/18] Setup.py typo - surprisingly, it only was broken on some systems, e.g. Debian 8 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0851b78f..ba34f6df 100644 --- a/setup.py +++ b/setup.py @@ -83,7 +83,7 @@ if __name__ == "__main__": 'qrcode': ['qrcode'], 'pyopencl': ['pyopencl'], 'notify2': ['notify2'], - 'sound:platform_system=="Windows"': ['winsound'] + 'sound;platform_system=="Windows"': ['winsound'] }, classifiers=[ "License :: OSI Approved :: MIT License" From 9b58f35b80e1ece4f6d8fdefcc8ec13fce687b08 Mon Sep 17 00:00:00 2001 From: Peter Surda Date: Tue, 2 Jan 2018 13:56:03 +0100 Subject: [PATCH 14/18] App name in version --- src/version.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/version.py b/src/version.py index bd06ca9b..694f71e4 100644 --- a/src/version.py +++ b/src/version.py @@ -1 +1,2 @@ +softwareName = 'PyBitmessage' softwareVersion = '0.6.2' From 8788f2d3497e061d14e9ae4bd0864c6ea2373b39 Mon Sep 17 00:00:00 2001 From: Peter Surda Date: Tue, 2 Jan 2018 14:29:21 +0100 Subject: [PATCH 15/18] Server full and duplicate handling - will try to report "Server full" over protocol for 10 extra connections over limit, instead of simply dropping them - if connected to the same host inbound and outbound, handle as server full (prevents duplicate connections) --- src/network/bmproto.py | 14 ++++++++++++++ src/network/tcp.py | 5 ++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/network/bmproto.py b/src/network/bmproto.py index 47c6c858..21ec692c 100644 --- a/src/network/bmproto.py +++ b/src/network/bmproto.py @@ -497,6 +497,20 @@ class BMProto(AdvancedDispatcher, ObjectTracker): return False except: pass + if not self.isOutbound: + # incoming from a peer we're connected to as outbound, or server full + # report the same error to counter deanonymisation + if state.Peer(self.destination.host, self.peerNode.port) in \ + network.connectionpool.BMConnectionPool().inboundConnections or \ + len(network.connectionpool.BMConnectionPool().inboundConnections) + \ + len(network.connectionpool.BMConnectionPool().outboundConnections) > \ + BMConfigParser().safeGetInt("bitmessagesettings", "maxtotalconnections") + \ + BMConfigParser().safeGetInt("bitmessagesettings", "maxbootstrapconnections"): + self.append_write_buf(protocol.assembleErrorMessage(fatal=2, + errorText="Server full, please try again later.")) + logger.debug ("Closed connection to %s due to server full or duplicate inbound/outbound.", + str(self.destination)) + return False if network.connectionpool.BMConnectionPool().isAlreadyConnected(self.nonce): self.append_write_buf(protocol.assembleErrorMessage(fatal=2, errorText="I'm connected to myself. Closing connection.")) diff --git a/src/network/tcp.py b/src/network/tcp.py index 922aa79d..5a27aca3 100644 --- a/src/network/tcp.py +++ b/src/network/tcp.py @@ -292,7 +292,10 @@ class TCPServer(AdvancedDispatcher): if len(network.connectionpool.BMConnectionPool().inboundConnections) + \ len(network.connectionpool.BMConnectionPool().outboundConnections) > \ BMConfigParser().safeGetInt("bitmessagesettings", "maxtotalconnections") + \ - BMConfigParser().safeGetInt("bitmessagesettings", "maxbootstrapconnections"): + BMConfigParser().safeGetInt("bitmessagesettings", "maxbootstrapconnections") + 10: + # 10 is a sort of buffer, in between it will go through the version handshake + # and return an error to the peer + logger.warning("Server full, dropping connection") sock.close() return try: From 4086253730e2551a9b3aa3df5af13ff9be322bee Mon Sep 17 00:00:00 2001 From: Peter Surda Date: Tue, 2 Jan 2018 15:24:47 +0100 Subject: [PATCH 16/18] Bandwidth limit optimisation - should be slightly more accurate and use slightly fewer resources --- src/network/advanceddispatcher.py | 4 ++-- src/network/asyncore_pollchoose.py | 18 ++++++++++++++---- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/network/advanceddispatcher.py b/src/network/advanceddispatcher.py index c50b6a43..3e84ed85 100644 --- a/src/network/advanceddispatcher.py +++ b/src/network/advanceddispatcher.py @@ -75,7 +75,7 @@ class AdvancedDispatcher(asyncore.dispatcher): def writable(self): self.uploadChunk = AdvancedDispatcher._buf_len if asyncore.maxUploadRate > 0: - self.uploadChunk = asyncore.uploadBucket + self.uploadChunk = int(asyncore.uploadBucket) self.uploadChunk = min(self.uploadChunk, len(self.write_buf)) return asyncore.dispatcher.writable(self) and \ (self.connecting or (self.connected and self.uploadChunk > 0)) @@ -83,7 +83,7 @@ class AdvancedDispatcher(asyncore.dispatcher): def readable(self): self.downloadChunk = AdvancedDispatcher._buf_len if asyncore.maxDownloadRate > 0: - self.downloadChunk = asyncore.downloadBucket + self.downloadChunk = int(asyncore.downloadBucket) try: if self.expectBytes > 0 and not self.fullyEstablished: self.downloadChunk = min(self.downloadChunk, self.expectBytes - len(self.read_buf)) diff --git a/src/network/asyncore_pollchoose.py b/src/network/asyncore_pollchoose.py index c5586a91..5717ff78 100644 --- a/src/network/asyncore_pollchoose.py +++ b/src/network/asyncore_pollchoose.py @@ -112,6 +112,8 @@ uploadBucket = 0 sentBytes = 0 def read(obj): + if not can_receive(): + return try: obj.handle_read_event() except _reraised_exceptions: @@ -120,6 +122,8 @@ def read(obj): obj.handle_error() def write(obj): + if not can_send(): + return try: obj.handle_write_event() except _reraised_exceptions: @@ -136,12 +140,18 @@ def set_rates(download, upload): downloadTimestamp = time.time() uploadTimestamp = time.time() +def can_receive(): + return maxDownloadRate == 0 or downloadBucket > 0 + +def can_send(): + return maxUploadRate == 0 or uploadBucket > 0 + def update_received(download=0): global receivedBytes, downloadBucket, downloadTimestamp currentTimestamp = time.time() receivedBytes += download if maxDownloadRate > 0: - bucketIncrease = int(maxDownloadRate * (currentTimestamp - downloadTimestamp)) + bucketIncrease = maxDownloadRate * (currentTimestamp - downloadTimestamp) downloadBucket += bucketIncrease if downloadBucket > maxDownloadRate: downloadBucket = int(maxDownloadRate) @@ -153,7 +163,7 @@ def update_sent(upload=0): currentTimestamp = time.time() sentBytes += upload if maxUploadRate > 0: - bucketIncrease = int(maxUploadRate * (currentTimestamp - uploadTimestamp)) + bucketIncrease = maxUploadRate * (currentTimestamp - uploadTimestamp) uploadBucket += bucketIncrease if uploadBucket > maxUploadRate: uploadBucket = int(maxUploadRate) @@ -170,9 +180,9 @@ def _exception(obj): def readwrite(obj, flags): try: - if flags & select.POLLIN: + if flags & select.POLLIN and can_receive(): obj.handle_read_event() - if flags & select.POLLOUT: + if flags & select.POLLOUT and can_send(): obj.handle_write_event() if flags & select.POLLPRI: obj.handle_expt_event() From f74f82e54f97a9075e95ae59474e4948ce1d6e82 Mon Sep 17 00:00:00 2001 From: Peter Surda Date: Tue, 2 Jan 2018 22:20:33 +0100 Subject: [PATCH 17/18] Start downloading earlier --- src/network/objectracker.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/network/objectracker.py b/src/network/objectracker.py index f846e7d5..62f01e4f 100644 --- a/src/network/objectracker.py +++ b/src/network/objectracker.py @@ -29,6 +29,7 @@ class ObjectTracker(object): invInitialCapacity = 50000 invErrorRate = 0.03 trackingExpires = 3600 + initialTimeOffset = 60 def __init__(self): self.objectsNewToMe = {} @@ -87,7 +88,7 @@ class ObjectTracker(object): if hashId in Dandelion().hashMap: Dandelion().fluffTrigger(hashId) if hashId not in missingObjects: - missingObjects[hashId] = time.time() + missingObjects[hashId] = time.time() - ObjectTracker.initialTimeOffset with self.objectsNewToMeLock: self.objectsNewToMe[hashId] = True From c9851b9f41da4db4365ef080b1533ab845992bd1 Mon Sep 17 00:00:00 2001 From: Peter Surda Date: Tue, 2 Jan 2018 22:23:03 +0100 Subject: [PATCH 18/18] Connection lookups invalid data handling - shouldn't throw an exception if argument is a string rather than Peer --- src/network/connectionpool.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/network/connectionpool.py b/src/network/connectionpool.py index 3b817e65..44534a76 100644 --- a/src/network/connectionpool.py +++ b/src/network/connectionpool.py @@ -65,12 +65,18 @@ class BMConnectionPool(object): def getConnectionByAddr(self, addr): if addr in self.inboundConnections: return self.inboundConnections[addr] - if addr.host in self.inboundConnections: - return self.inboundConnections[addr.host] + try: + if addr.host in self.inboundConnections: + return self.inboundConnections[addr.host] + except AttributeError: + pass if addr in self.outboundConnections: return self.outboundConnections[addr] - if addr.host in self.udpSockets: - return self.udpSockets[addr.host] + try: + if addr.host in self.udpSockets: + return self.udpSockets[addr.host] + except AttributeError: + pass raise KeyError def isAlreadyConnected(self, nodeid):