diff --git a/minode/connection.py b/minode/connection.py index b7f9a75..05d552a 100644 --- a/minode/connection.py +++ b/minode/connection.py @@ -411,7 +411,8 @@ class ConnectionBase(threading.Thread): addr = message.Addr.from_message(m) logging.debug('%s:%s -> %s', self.host_print, self.port, addr) for a in addr.addresses: - shared.unchecked_node_pool.add((a.host, a.port)) + if (a.host, a.port) not in shared.core_nodes: + shared.unchecked_node_pool.add((a.host, a.port)) def _request_objects(self): if self.vectors_to_get and len(self.vectors_requested) < 100: @@ -507,4 +508,13 @@ class Connection(ConnectionBase): self.vectors_to_send.update(getdata.vectors) +class Bootstrapper(ConnectionBase): + """A special type of connection to find IP nodes""" + def _process_msg_addr(self, m): + super()._process_msg_addr(m) + shared.node_pool.discard((self.host, self.port)) + self.status = 'disconnecting' + self.send_queue.put(None) + + shared.connection = Connection diff --git a/minode/main.py b/minode/main.py index 7b52796..72cebe0 100644 --- a/minode/main.py +++ b/minode/main.py @@ -101,18 +101,22 @@ def parse_arguments(): # pylint: disable=too-many-branches,too-many-statements def bootstrap_from_dns(): - """Addes addresses of bootstrap servers to known nodes""" + """Addes addresses of bootstrap servers to core nodes""" try: - for item in socket.getaddrinfo('bootstrap8080.bitmessage.org', 80): - shared.unchecked_node_pool.add((item[4][0], 8080)) - logging.debug( - 'Adding %s to unchecked_node_pool' - ' based on DNS bootstrap method', item[4][0]) - for item in socket.getaddrinfo('bootstrap8444.bitmessage.org', 80): - shared.unchecked_node_pool.add((item[4][0], 8444)) - logging.debug( - 'Adding %s to unchecked_node_pool' - ' based on DNS bootstrap method', item[4][0]) + for port in (8080, 8444): + for item in socket.getaddrinfo( + 'bootstrap{}.bitmessage.org'.format(port), 80, + proto=socket.IPPROTO_TCP + ): + try: + addr = item[4][0] + socket.inet_pton(item[0], addr) + except (TypeError, socket.error): + continue + else: + shared.core_nodes.add((addr, port)) + except socket.gaierror: + logging.info('Failed to do a DNS query') except Exception: logging.info('Error during DNS bootstrap', exc_info=True) diff --git a/minode/manager.py b/minode/manager.py index a03f9d6..71f5873 100644 --- a/minode/manager.py +++ b/minode/manager.py @@ -11,7 +11,7 @@ import threading import time from . import proofofwork, shared, structure -from .connection import Connection +from .connection import Bootstrapper, Connection from .i2p import I2PDialer @@ -20,6 +20,7 @@ class Manager(threading.Thread): def __init__(self): super().__init__(name='Manager') self.q = queue.Queue() + self.bootstrap_pool = [] self.last_cleaned_objects = time.time() self.last_cleaned_connections = time.time() self.last_pickled_objects = time.time() @@ -28,9 +29,15 @@ class Manager(threading.Thread): self.last_published_i2p_destination = \ time.time() - 50 * 60 + random.uniform(-1, 1) * 300 # nosec B311 + def fill_bootstrap_pool(self): + """Populate the bootstrap pool by core nodes and checked ones""" + self.bootstrap_pool = list(shared.core_nodes.union(shared.node_pool)) + random.shuffle(self.bootstrap_pool) + def run(self): self.load_data() self.clean_objects() + self.fill_bootstrap_pool() while True: time.sleep(0.8) now = time.time() @@ -68,9 +75,32 @@ class Manager(threading.Thread): with shared.objects_lock: del shared.objects[vector] - @staticmethod - def manage_connections(): + def manage_connections(self): + """Open new connections if needed, remove closed ones""" hosts = set() + + def connect(target, connection_class=Connection): + """ + Open a connection of *connection_class* + to the *target* (host, port) + """ + c = connection_class(*target) + c.start() + with shared.connections_lock: + shared.connections.add(c) + + def bootstrap(): + """Bootstrap from DNS seed-nodes and known nodes""" + try: + target = self.bootstrap_pool.pop() + except IndexError: + logging.warning( + 'Ran out of bootstrap nodes, refilling') + self.fill_bootstrap_pool() + return + logging.info('Starting a bootstrapper for %s:%s', *target) + connect(target, Bootstrapper) + outgoing_connections = 0 for c in shared.connections.copy(): if not c.is_alive() or c.status == 'disconnected': @@ -101,6 +131,8 @@ class Manager(threading.Thread): tuple(shared.unchecked_node_pool), 16)) else: to_connect.update(shared.unchecked_node_pool) + if outgoing_connections < shared.outgoing_connections / 2: + bootstrap() shared.unchecked_node_pool.difference_update(to_connect) if len(shared.node_pool) > 8: to_connect.update(random.sample( @@ -142,11 +174,8 @@ class Manager(threading.Thread): else: continue else: - c = Connection(host, port) - c.start() + connect((host, port)) hosts.add(group) - with shared.connections_lock: - shared.connections.add(c) shared.hosts = hosts @staticmethod @@ -190,7 +219,7 @@ class Manager(threading.Thread): 'r', newline='', encoding='ascii' ) as src: reader = csv.reader(src) - shared.core_nodes = {tuple(row) for row in reader} + shared.core_nodes = {(row[0], int(row[1])) for row in reader} shared.node_pool.update(shared.core_nodes) with open( diff --git a/minode/tests/test_network.py b/minode/tests/test_network.py index c8cdd31..c683637 100644 --- a/minode/tests/test_network.py +++ b/minode/tests/test_network.py @@ -1,4 +1,5 @@ """Tests for network connections""" +import ipaddress import logging import os import random @@ -73,10 +74,43 @@ class TestNetwork(unittest.TestCase): def _make_initial_nodes(self): Manager.load_data() - self.assertGreaterEqual(len(shared.core_nodes), 3) + core_nodes_len = len(shared.core_nodes) + self.assertGreaterEqual(core_nodes_len, 3) main.bootstrap_from_dns() - self.assertGreaterEqual(len(shared.unchecked_node_pool), 3) + self.assertGreaterEqual(len(shared.core_nodes), core_nodes_len) + for host, _ in shared.core_nodes: + try: + ipaddress.IPv4Address(host) + except ipaddress.AddressValueError: + try: + ipaddress.IPv6Address(host) + except ipaddress.AddressValueError: + self.fail('Found not an IP address in the core nodes') + break + else: + self.fail('No IPv6 address found in the core nodes') + + def test_bootstrap(self): + """Start bootstrappers and check node pool""" + if shared.core_nodes: + shared.core_nodes = set() + if shared.unchecked_node_pool: + shared.unchecked_node_pool = set() + + self._make_initial_nodes() + self.assertEqual(len(shared.unchecked_node_pool), 0) + + for node in shared.core_nodes: + c = connection.Bootstrapper(*node) + c.start() + c.join() + if len(shared.unchecked_node_pool) > 2: + break + else: + self.fail( + 'Failed to find at least 3 nodes' + ' after running %s bootstrappers' % len(shared.core_nodes)) def test_connection(self): """Check a normal connection - should receive objects""" @@ -215,3 +249,31 @@ class TestListener(TestProcessProto): if c.status == 'fully_established': self.fail('Established a connection') time.sleep(0.5) + + +class TestBootstrapProcess(TestProcessProto): + """A separate test case for bootstrapping with a minode process""" + _listen = True + _connection_limit = 24 + + def test_bootstrap(self): + """Start a bootstrapper for the local process and check node pool""" + if shared.unchecked_node_pool: + shared.unchecked_node_pool = set() + + started = time.time() + while not self.connections(): + if time.time() - started > 60: + self.fail('Failed to establish a connection') + time.sleep(1) + + for _ in range(3): + c = connection.Bootstrapper('127.0.0.1', 8444) + c.start() + c.join() + if len(shared.unchecked_node_pool) > 2: + break + else: + self.fail( + 'Failed to find at least 3 nodes' + ' after 3 tries to bootstrap with the local process')