From d6c1845b711e763efa382432ec6d2a92438eb22f Mon Sep 17 00:00:00 2001 From: Dmitri Bogomolov <4glitch@gmail.com> Date: Sun, 3 Nov 2019 17:11:52 +0200 Subject: [PATCH] Moved Peer from state to network.node and trustedPeer to network.connectionpool.BMConnectionPool attribute --- src/class_objectProcessor.py | 3 ++- src/helper_startup.py | 26 ++---------------- src/knownnodes.py | 46 +++++++++++++++++--------------- src/network/announcethread.py | 5 +++- src/network/bmproto.py | 24 +++++++++-------- src/network/connectionchooser.py | 2 -- src/network/connectionpool.py | 31 +++++++++++++++++++-- src/network/node.py | 4 +-- src/network/proxy.py | 11 ++++---- src/network/socks5.py | 4 +-- src/network/tcp.py | 9 ++++--- src/network/udp.py | 9 ++++--- src/state.py | 17 ------------ src/tests/core.py | 7 ++--- src/upnp.py | 8 +++--- 15 files changed, 102 insertions(+), 104 deletions(-) diff --git a/src/class_objectProcessor.py b/src/class_objectProcessor.py index e2b95447..b22876e8 100644 --- a/src/class_objectProcessor.py +++ b/src/class_objectProcessor.py @@ -21,6 +21,7 @@ import helper_sent from helper_sql import SqlBulkExecute, sqlExecute, sqlQuery from helper_ackPayload import genAckPayload from network import bmproto +from network.node import Peer import protocol import queues import state @@ -161,7 +162,7 @@ class objectProcessor(threading.Thread): if not host: return - peer = state.Peer(host, port) + peer = Peer(host, port) with knownnodes.knownNodesLock: knownnodes.addKnownNode( stream, peer, is_self=state.ownAddresses.get(peer)) diff --git a/src/helper_startup.py b/src/helper_startup.py index 1a1119f5..9aaad5ef 100644 --- a/src/helper_startup.py +++ b/src/helper_startup.py @@ -1,13 +1,9 @@ """ -src/helper_startup.py -===================== - -Helper Start performs all the startup operations. +Startup operations. """ # pylint: disable=too-many-branches,too-many-statements from __future__ import print_function -import ConfigParser import os import platform import sys @@ -19,28 +15,12 @@ import paths import state from bmconfigparser import BMConfigParser + # The user may de-select Portable Mode in the settings if they want # the config files to stay in the application data folder. StoreConfigFilesInSameDirectoryAsProgramByDefault = False -def _loadTrustedPeer(): - try: - trustedPeer = BMConfigParser().get('bitmessagesettings', 'trustedpeer') - except ConfigParser.Error: - # This probably means the trusted peer wasn't specified so we - # can just leave it as None - return - try: - host, port = trustedPeer.split(':') - except ValueError: - sys.exit( - 'Bad trustedpeer config setting! It should be set as' - ' trustedpeer=:' - ) - state.trustedPeer = state.Peer(host, int(port)) - - def loadConfig(): """Load the config""" config = BMConfigParser() @@ -134,8 +114,6 @@ def loadConfig(): else: updateConfig() - _loadTrustedPeer() - def updateConfig(): """Save the config""" diff --git a/src/knownnodes.py b/src/knownnodes.py index 1d9e6897..bb588fcb 100644 --- a/src/knownnodes.py +++ b/src/knownnodes.py @@ -3,6 +3,7 @@ Manipulations with knownNodes dictionary. """ import json +import logging import os import pickle import threading @@ -10,28 +11,33 @@ import time import state from bmconfigparser import BMConfigParser -from debug import logger +from network.node import Peer knownNodesLock = threading.Lock() +"""Thread lock for knownnodes modification""" knownNodes = {stream: {} for stream in range(1, 4)} +"""The dict of known nodes for each stream""" knownNodesTrimAmount = 2000 +"""trim stream knownnodes dict to this length""" -# forget a node after rating is this low knownNodesForgetRating = -0.5 +"""forget a node after rating is this low""" knownNodesActual = False +logger = logging.getLogger('default') + DEFAULT_NODES = ( - state.Peer('5.45.99.75', 8444), - state.Peer('75.167.159.54', 8444), - state.Peer('95.165.168.168', 8444), - state.Peer('85.180.139.241', 8444), - state.Peer('158.222.217.190', 8080), - state.Peer('178.62.12.187', 8448), - state.Peer('24.188.198.204', 8111), - state.Peer('109.147.204.113', 1195), - state.Peer('178.11.46.221', 8444) + Peer('5.45.99.75', 8444), + Peer('75.167.159.54', 8444), + Peer('95.165.168.168', 8444), + Peer('85.180.139.241', 8444), + Peer('158.222.217.190', 8080), + Peer('178.62.12.187', 8448), + Peer('24.188.198.204', 8111), + Peer('109.147.204.113', 1195), + Peer('178.11.46.221', 8444) ) @@ -57,19 +63,17 @@ def json_deserialize_knownnodes(source): for node in json.load(source): peer = node['peer'] info = node['info'] - peer = state.Peer(str(peer['host']), peer.get('port', 8444)) + peer = Peer(str(peer['host']), peer.get('port', 8444)) knownNodes[node['stream']][peer] = info - if ( - not (knownNodesActual or info.get('self')) and - peer not in DEFAULT_NODES - ): + if not (knownNodesActual + or info.get('self')) and peer not in DEFAULT_NODES: knownNodesActual = True def pickle_deserialize_old_knownnodes(source): """ - Unpickle source and reorganize knownnodes dict if it's in old format + Unpickle source and reorganize knownnodes dict if it has old format the old format was {Peer:lastseen, ...} the new format is {Peer:{"lastseen":i, "rating":f}} """ @@ -129,7 +133,7 @@ def readKnownNodes(): if onionhostname and ".onion" in onionhostname: onionport = config.safeGetInt('bitmessagesettings', 'onionport') if onionport: - self_peer = state.Peer(onionhostname, onionport) + self_peer = Peer(onionhostname, onionport) addKnownNode(1, self_peer, is_self=True) state.ownAddresses[self_peer] = True @@ -182,7 +186,7 @@ def dns(): """Add DNS names to knownnodes""" for port in [8080, 8444]: addKnownNode( - 1, state.Peer('bootstrap%s.bitmessage.org' % port, port)) + 1, Peer('bootstrap%s.bitmessage.org' % port, port)) def cleanupKnownNodes(): @@ -208,8 +212,8 @@ def cleanupKnownNodes(): del knownNodes[stream][node] continue # scrap old nodes (age > 3 hours) with low rating - if (age > 10800 and knownNodes[stream][node]["rating"] <= - knownNodesForgetRating): + if (age > 10800 and knownNodes[stream][node]["rating"] + <= knownNodesForgetRating): needToWriteKnownNodesToDisk = True del knownNodes[stream][node] continue diff --git a/src/network/announcethread.py b/src/network/announcethread.py index 5cd27ede..f635fc90 100644 --- a/src/network/announcethread.py +++ b/src/network/announcethread.py @@ -10,6 +10,7 @@ from bmconfigparser import BMConfigParser from network.bmproto import BMProto from network.connectionpool import BMConnectionPool from network.udp import UDPSocket +from node import Peer from threads import StoppableThread @@ -36,6 +37,8 @@ class AnnounceThread(StoppableThread): for stream in state.streamsInWhichIAmParticipating: addr = ( stream, - state.Peer('127.0.0.1', BMConfigParser().safeGetInt("bitmessagesettings", "port")), + Peer( + '127.0.0.1', + BMConfigParser().safeGetInt('bitmessagesettings', 'port')), time.time()) connection.append_write_buf(BMProto.assembleAddr([addr])) diff --git a/src/network/bmproto.py b/src/network/bmproto.py index 86295b87..bf0b5742 100644 --- a/src/network/bmproto.py +++ b/src/network/bmproto.py @@ -24,8 +24,8 @@ from network.bmobject import ( BMObject, BMObjectInsufficientPOWError, BMObjectInvalidDataError, BMObjectExpiredError, BMObjectUnwantedStreamError, BMObjectInvalidError, BMObjectAlreadyHaveError) -from network.node import Node from network.proxy import ProxyError +from node import Node, Peer from objectracker import missingObjects, ObjectTracker from queues import objectProcessorQueue, portCheckerQueue, invQueue, addrQueue from randomtrackingdict import RandomTrackingDict @@ -443,7 +443,7 @@ class BMProto(AdvancedDispatcher, ObjectTracker): seenTime > time.time() - BMProto.addressAlive and port > 0 ): - peer = state.Peer(decodedIP, port) + peer = Peer(decodedIP, port) try: if knownnodes.knownNodes[stream][peer]["lastseen"] > seenTime: continue @@ -464,7 +464,7 @@ class BMProto(AdvancedDispatcher, ObjectTracker): def bm_command_portcheck(self): """Incoming port check request, queue it.""" - portCheckerQueue.put(state.Peer(self.destination, self.peerNode.port)) + portCheckerQueue.put(Peer(self.destination, self.peerNode.port)) return True def bm_command_ping(self): @@ -594,12 +594,14 @@ class BMProto(AdvancedDispatcher, ObjectTracker): # incoming from a peer we're connected to as outbound, # or server full report the same error to counter deanonymisation if ( - state.Peer(self.destination.host, self.peerNode.port) in - connectionpool.BMConnectionPool().inboundConnections or - len(connectionpool.BMConnectionPool().inboundConnections) + - len(connectionpool.BMConnectionPool().outboundConnections) > - BMConfigParser().safeGetInt("bitmessagesettings", "maxtotalconnections") + - BMConfigParser().safeGetInt("bitmessagesettings", "maxbootstrapconnections") + Peer(self.destination.host, self.peerNode.port) + in connectionpool.BMConnectionPool().inboundConnections + or len(connectionpool.BMConnectionPool().inboundConnections) + + len(connectionpool.BMConnectionPool().outboundConnections) + > BMConfigParser().safeGetInt( + 'bitmessagesettings', 'maxtotalconnections') + + BMConfigParser().safeGetInt( + 'bitmessagesettings', 'maxbootstrapconnections') ): self.append_write_buf(protocol.assembleErrorMessage( errorText="Server full, please try again later.", fatal=2)) @@ -622,7 +624,7 @@ class BMProto(AdvancedDispatcher, ObjectTracker): @staticmethod def assembleAddr(peerList): """Build up a packed address""" - if isinstance(peerList, state.Peer): + if isinstance(peerList, Peer): peerList = (peerList) if not peerList: return b'' @@ -686,7 +688,7 @@ class BMStringParser(BMProto): """ def __init__(self): super(BMStringParser, self).__init__() - self.destination = state.Peer('127.0.0.1', 8444) + self.destination = Peer('127.0.0.1', 8444) self.payload = None ObjectTracker.__init__(self) diff --git a/src/network/connectionchooser.py b/src/network/connectionchooser.py index 838ca45d..9d2f85d6 100644 --- a/src/network/connectionchooser.py +++ b/src/network/connectionchooser.py @@ -28,8 +28,6 @@ def chooseConnection(stream): "bitmessagesettings", "socksproxytype")[0:5] == 'SOCKS' onionOnly = BMConfigParser().safeGetBoolean( "bitmessagesettings", "onionservicesonly") - if state.trustedPeer: - return state.trustedPeer try: retval = portCheckerQueue.get(False) portCheckerQueue.task_done() diff --git a/src/network/connectionpool.py b/src/network/connectionpool.py index 8f959356..654b74a1 100644 --- a/src/network/connectionpool.py +++ b/src/network/connectionpool.py @@ -5,6 +5,7 @@ import errno import logging import re import socket +import sys import time import asyncore_pollchoose as asyncore @@ -14,6 +15,7 @@ import protocol import state from bmconfigparser import BMConfigParser from connectionchooser import chooseConnection +from node import Peer from proxy import Proxy from singleton import Singleton from tcp import ( @@ -29,6 +31,19 @@ class BMConnectionPool(object): """Pool of all existing connections""" # pylint: disable=too-many-instance-attributes + trustedPeer = None + """ + If the trustedpeer option is specified in keys.dat then this will + contain a Peer which will be connected to instead of using the + addresses advertised by other peers. + + The expected use case is where the user has a trusted server where + they run a Bitmessage daemon permanently. If they then run a second + instance of the client on a local machine periodically when they want + to check for messages it will sync with the network a lot faster + without compromising security. + """ + def __init__(self): asyncore.set_rates( BMConfigParser().safeGetInt( @@ -45,6 +60,18 @@ class BMConnectionPool(object): self._spawnWait = 2 self._bootstrapped = False + trustedPeer = BMConfigParser().safeGet( + 'bitmessagesettings', 'trustedpeer') + try: + if trustedPeer: + host, port = trustedPeer.split(':') + self.trustedPeer = Peer(host, int(port)) + except ValueError: + sys.exit( + 'Bad trustedpeer config setting! It should be set as' + ' trustedpeer=:' + ) + def connections(self): """ Shortcut for combined list of connections from @@ -112,7 +139,7 @@ class BMConnectionPool(object): if isinstance(connection, UDPSocket): del self.udpSockets[connection.listening.host] elif isinstance(connection, TCPServer): - del self.listeningSockets[state.Peer( + del self.listeningSockets[Peer( connection.destination.host, connection.destination.port)] elif connection.isOutbound: try: @@ -259,7 +286,7 @@ class BMConnectionPool(object): for i in range( state.maximumNumberOfHalfOpenConnections - pending): try: - chosen = chooseConnection( + chosen = self.trustedPeer or chooseConnection( helper_random.randomchoice(self.streams)) except ValueError: continue diff --git a/src/network/node.py b/src/network/node.py index 0bfda653..4c532b81 100644 --- a/src/network/node.py +++ b/src/network/node.py @@ -1,7 +1,7 @@ """ -src/network/node.py -=================== +Named tuples representing the network peers """ import collections +Peer = collections.namedtuple('Peer', ['host', 'port']) Node = collections.namedtuple('Node', ['services', 'host', 'port']) diff --git a/src/network/proxy.py b/src/network/proxy.py index e65ac6a7..e0bb5e78 100644 --- a/src/network/proxy.py +++ b/src/network/proxy.py @@ -8,9 +8,9 @@ import socket import time import asyncore_pollchoose as asyncore -import state from advanceddispatcher import AdvancedDispatcher from bmconfigparser import BMConfigParser +from node import Peer logger = logging.getLogger('default') @@ -90,9 +90,10 @@ class Proxy(AdvancedDispatcher): def onion_proxy(self, address): """Set onion proxy address""" if address is not None and ( - not isinstance(address, tuple) or len(address) < 2 or - not isinstance(address[0], str) or - not isinstance(address[1], int)): + not isinstance(address, tuple) or len(address) < 2 + or not isinstance(address[0], str) + or not isinstance(address[1], int) + ): raise ValueError self.__class__._onion_proxy = address @@ -107,7 +108,7 @@ class Proxy(AdvancedDispatcher): self.__class__._onion_auth = authTuple def __init__(self, address): - if not isinstance(address, state.Peer): + if not isinstance(address, Peer): raise ValueError AdvancedDispatcher.__init__(self) self.destination = address diff --git a/src/network/socks5.py b/src/network/socks5.py index e0cb7202..f0241744 100644 --- a/src/network/socks5.py +++ b/src/network/socks5.py @@ -8,7 +8,7 @@ src/network/socks5.py import socket import struct -import state +from node import Peer from proxy import GeneralProxyError, Proxy, ProxyError @@ -200,7 +200,7 @@ class Socks5Resolver(Socks5): def __init__(self, host): self.host = host self.port = 8444 - Socks5.__init__(self, address=state.Peer(self.host, self.port)) + Socks5.__init__(self, address=Peer(self.host, self.port)) def state_auth_done(self): """Perform resolving""" diff --git a/src/network/tcp.py b/src/network/tcp.py index a1691ceb..97b00784 100644 --- a/src/network/tcp.py +++ b/src/network/tcp.py @@ -28,6 +28,7 @@ from network.objectracker import ObjectTracker from network.socks4a import Socks4aConnection from network.socks5 import Socks5Connection from network.tls import TLSDispatcher +from node import Peer from queues import UISignalQueue, invQueue, receiveDataQueue logger = logging.getLogger('default') @@ -49,7 +50,7 @@ class TCPConnection(BMProto, TLSDispatcher): self.connectedAt = 0 self.skipUntil = 0 if address is None and sock is not None: - self.destination = state.Peer(*sock.getpeername()) + self.destination = Peer(*sock.getpeername()) self.isOutbound = False TLSDispatcher.__init__(self, sock, server_side=True) self.connectedAt = time.time() @@ -334,7 +335,7 @@ def bootstrap(connection_class): _connection_base = connection_class def __init__(self, host, port): - self._connection_base.__init__(self, state.Peer(host, port)) + self._connection_base.__init__(self, Peer(host, port)) self.close_reason = self._succeed = False def bm_command_addr(self): @@ -384,7 +385,7 @@ class TCPServer(AdvancedDispatcher): 'bitmessagesettings', 'port', str(port)) BMConfigParser().save() break - self.destination = state.Peer(host, port) + self.destination = Peer(host, port) self.bound = True self.listen(5) @@ -402,7 +403,7 @@ class TCPServer(AdvancedDispatcher): except (TypeError, IndexError): return - state.ownAddresses[state.Peer(*sock.getsockname())] = True + state.ownAddresses[Peer(*sock.getsockname())] = True if ( len(connectionpool.BMConnectionPool().inboundConnections) + len(connectionpool.BMConnectionPool().outboundConnections) > diff --git a/src/network/udp.py b/src/network/udp.py index 97c6aee5..cf694567 100644 --- a/src/network/udp.py +++ b/src/network/udp.py @@ -9,6 +9,7 @@ import socket import state import protocol from bmproto import BMProto +from node import Peer from objectracker import ObjectTracker from queues import receiveDataQueue @@ -43,8 +44,8 @@ class UDPSocket(BMProto): # pylint: disable=too-many-instance-attributes else: self.socket = sock self.set_socket_reuse() - self.listening = state.Peer(*self.socket.getsockname()) - self.destination = state.Peer(*self.socket.getsockname()) + self.listening = Peer(*self.socket.getsockname()) + self.destination = Peer(*self.socket.getsockname()) ObjectTracker.__init__(self) self.connecting = False self.connected = True @@ -96,7 +97,7 @@ class UDPSocket(BMProto): # pylint: disable=too-many-instance-attributes self.destination.host, self.destination.port, remoteport) if self.local: state.discoveredPeers[ - state.Peer(self.destination.host, remoteport) + Peer(self.destination.host, remoteport) ] = time.time() return True @@ -131,7 +132,7 @@ class UDPSocket(BMProto): # pylint: disable=too-many-instance-attributes logger.error("socket error: %s", e) return - self.destination = state.Peer(*addr) + self.destination = Peer(*addr) encodedAddr = protocol.encodeHost(addr[0]) self.local = bool(protocol.checkIPAddress(encodedAddr, True)) # overwrite the old buffer to avoid mixing data and so that diff --git a/src/state.py b/src/state.py index a3b930ab..f5526029 100644 --- a/src/state.py +++ b/src/state.py @@ -1,7 +1,6 @@ """ Global runtime variables. """ -import collections neededPubkeys = {} streamsInWhichIAmParticipating = [] @@ -47,24 +46,8 @@ uploadThread = None ownAddresses = {} -trustedPeer = None -""" - If the trustedpeer option is specified in keys.dat then this will - contain a Peer which will be connected to instead of using the - addresses advertised by other peers. The client will only connect to - this peer and the timing attack mitigation will be disabled in order - to download data faster. The expected use case is where the user has - a fast connection to a trusted server where they run a BitMessage - daemon permanently. If they then run a second instance of the client - on a local machine periodically when they want to check for messages - it will sync with the network a lot faster without compromising - security. -""" - discoveredPeers = {} -Peer = collections.namedtuple('Peer', ['host', 'port']) - dandelion = 0 testmode = False diff --git a/src/tests/core.py b/src/tests/core.py index 8d24a768..3871946d 100644 --- a/src/tests/core.py +++ b/src/tests/core.py @@ -17,6 +17,7 @@ from bmconfigparser import BMConfigParser from helper_msgcoding import MsgEncode, MsgDecode from network import asyncore_pollchoose as asyncore from network.connectionpool import BMConnectionPool +from network.node import Peer from network.tcp import Socks4aBMConnection, Socks5BMConnection, TCPConnection from queues import excQueue @@ -30,7 +31,7 @@ def pickle_knownnodes(): with open(knownnodes_file, 'wb') as dst: pickle.dump({ stream: { - state.Peer( + Peer( '%i.%i.%i.%i' % tuple([ random.randint(1, 255) for i in range(4)]), 8444): {'lastseen': now, 'rating': 0.1} @@ -90,7 +91,7 @@ class TestCore(unittest.TestCase): """initial fill script from network.tcp""" BMConfigParser().set('bitmessagesettings', 'dontconnect', 'true') try: - for peer in (state.Peer("127.0.0.1", 8448),): + for peer in (Peer("127.0.0.1", 8448),): direct = TCPConnection(peer) while asyncore.socket_map: print("loop, state = %s" % direct.state) @@ -147,7 +148,7 @@ class TestCore(unittest.TestCase): def _initiate_bootstrap(self): BMConfigParser().set('bitmessagesettings', 'dontconnect', 'true') self._outdate_knownnodes() - knownnodes.addKnownNode(1, state.Peer('127.0.0.1', 8444), is_self=True) + knownnodes.addKnownNode(1, Peer('127.0.0.1', 8444), is_self=True) knownnodes.cleanupKnownNodes() time.sleep(2) diff --git a/src/upnp.py b/src/upnp.py index 979b4186..99000413 100644 --- a/src/upnp.py +++ b/src/upnp.py @@ -1,9 +1,6 @@ # pylint: disable=too-many-statements,too-many-branches,protected-access,no-self-use """ -src/upnp.py -=========== - -A simple upnp module to forward port for BitMessage +Complete UPnP port forwarding implementation in separate thread. Reference: http://mattscodecave.com/posts/using-python-and-upnp-to-forward-a-port """ @@ -22,6 +19,7 @@ import tr from bmconfigparser import BMConfigParser from debug import logger from network import BMConnectionPool, StoppableThread +from network.node import Peer def createRequestXML(service, action, arguments=None): @@ -262,7 +260,7 @@ class uPnPThread(StoppableThread): self.routers.append(newRouter) self.createPortMapping(newRouter) try: - self_peer = state.Peer( + self_peer = Peer( newRouter.GetExternalIPAddress(), self.extPort )