Bitmessage/MiNode
3
0
Fork 1
Code Pull Requests 8 Actions Activity

Implementing proper bootstrap procedure #7

Open
lee.miller wants to merge 6 commits from lee.miller/MiNode:bootstrap into v0.3
pull from: lee.miller/MiNode:bootstrap
merge into: Bitmessage:v0.3
Conversation 1 Commits 6 Files Changed 4 +119 -22
4 changed files with 119 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
minode/connection.py
View File

@ -411,7 +411,8 @@ class ConnectionBase(threading.Thread):
addr = message.Addr.from_message(m)
logging.debug('%s:%s -> %s', self.host_print, self.port, addr)
for a in addr.addresses:
shared.unchecked_node_pool.add((a.host, a.port))
if (a.host, a.port) not in shared.core_nodes:
shared.unchecked_node_pool.add((a.host, a.port))
def _request_objects(self):
if self.vectors_to_get and len(self.vectors_requested) < 100:
@ -507,4 +508,12 @@ class Connection(ConnectionBase):
self.vectors_to_send.update(getdata.vectors)
class Bootstrapper(ConnectionBase):
def _process_msg_addr(self, m):
super()._process_msg_addr(m)
shared.node_pool.discard((self.host, self.port))
self.status = 'disconnecting'
self.send_queue.put(None)
shared.connection = Connection

26
minode/main.py
View File

@ -101,18 +101,22 @@ def parse_arguments(): # pylint: disable=too-many-branches,too-many-statements
def bootstrap_from_dns():
"""Addes addresses of bootstrap servers to known nodes"""
"""Addes addresses of bootstrap servers to core nodes"""
try:
for item in socket.getaddrinfo('bootstrap8080.bitmessage.org', 80):
shared.unchecked_node_pool.add((item[4][0], 8080))
logging.debug(
'Adding %s to unchecked_node_pool'
' based on DNS bootstrap method', item[4][0])
for item in socket.getaddrinfo('bootstrap8444.bitmessage.org', 80):
shared.unchecked_node_pool.add((item[4][0], 8444))
logging.debug(
'Adding %s to unchecked_node_pool'
' based on DNS bootstrap method', item[4][0])
for port in (8080, 8444):
for item in socket.getaddrinfo(
'bootstrap{}.bitmessage.org'.format(port), 80,
proto=socket.IPPROTO_TCP
):
try:
addr = item[4][0]
socket.inet_pton(item[0], addr)
except (TypeError, socket.error):
continue
else:
shared.core_nodes.add((addr, port))
except socket.gaierror:
logging.info('Failed to do a DNS query')
except Exception:
logging.info('Error during DNS bootstrap', exc_info=True)

38
minode/manager.py
View File

@ -11,7 +11,7 @@ import threading
import time
from . import proofofwork, shared, structure
from .connection import Connection
from .connection import Bootstrapper, Connection
from .i2p import I2PDialer
@ -20,6 +20,7 @@ class Manager(threading.Thread):
def __init__(self):
super().__init__(name='Manager')
self.q = queue.Queue()
self.bootstrap_pool = []
self.last_cleaned_objects = time.time()
self.last_cleaned_connections = time.time()
self.last_pickled_objects = time.time()
@ -28,9 +29,14 @@ class Manager(threading.Thread):
self.last_published_i2p_destination = \
time.time() - 50 * 60 + random.uniform(-1, 1) * 300 # nosec B311
def fill_bootstrap_pool(self):
self.bootstrap_pool = list(shared.core_nodes.union(shared.node_pool))
random.shuffle(self.bootstrap_pool)
def run(self):
self.load_data()
self.clean_objects()
self.fill_bootstrap_pool()
while True:
time.sleep(0.8)
now = time.time()
@ -68,9 +74,21 @@ class Manager(threading.Thread):
with shared.objects_lock:
del shared.objects[vector]
@staticmethod
def manage_connections():
def manage_connections(self):
hosts = set()
def connect(target, connection_class=Connection):
c = connection_class(*target)
c.start()
with shared.connections_lock:
shared.connections.add(c)
def bootstrap():
"""Bootstrap from DNS seed-nodes and known nodes"""
target = self.bootstrap_pool.pop()
logging.info('Starting a bootstrapper for %s:%s', *target)
connect(target, Bootstrapper)
outgoing_connections = 0
for c in shared.connections.copy():
if not c.is_alive() or c.status == 'disconnected':
@ -101,6 +119,13 @@ class Manager(threading.Thread):
tuple(shared.unchecked_node_pool), 16))
else:
to_connect.update(shared.unchecked_node_pool)
if outgoing_connections < shared.outgoing_connections / 2:
try:
bootstrap()
except IndexError:
logging.warning(
'Ran out of bootstrap nodes, refilling')
self.fill_bootstrap_pool()
shared.unchecked_node_pool.difference_update(to_connect)
if len(shared.node_pool) > 8:
to_connect.update(random.sample(
@ -142,11 +167,8 @@ class Manager(threading.Thread):
else:
continue
else:
c = Connection(host, port)
c.start()
connect((host, port))
hosts.add(group)
with shared.connections_lock:
shared.connections.add(c)
shared.hosts = hosts
@staticmethod
@ -190,7 +212,7 @@ class Manager(threading.Thread):
'r', newline='', encoding='ascii'
) as src:
reader = csv.reader(src)
shared.core_nodes = {tuple(row) for row in reader}
shared.core_nodes = {(row[0], int(row[1])) for row in reader}
shared.node_pool.update(shared.core_nodes)
with open(

66
minode/tests/test_network.py
View File

@ -1,4 +1,5 @@
"""Tests for network connections"""
import ipaddress
import logging
import os
import random
@ -73,10 +74,43 @@ class TestNetwork(unittest.TestCase):
def _make_initial_nodes(self):
Manager.load_data()
self.assertGreaterEqual(len(shared.core_nodes), 3)
core_nodes_len = len(shared.core_nodes)
self.assertGreaterEqual(core_nodes_len, 3)
main.bootstrap_from_dns()
self.assertGreaterEqual(len(shared.unchecked_node_pool), 3)
self.assertGreaterEqual(len(shared.core_nodes), core_nodes_len)
for host, _ in shared.core_nodes:
try:
ipaddress.IPv4Address(host)
except ipaddress.AddressValueError:
try:
ipaddress.IPv6Address(host)
except ipaddress.AddressValueError:
self.fail('Found not an IP address in the core nodes')
break
else:
self.fail('No IPv6 address found in the core nodes')
def test_bootstrap(self):
"""Start bootstrappers and check node pool"""
if shared.core_nodes:
shared.core_nodes = set()
if shared.unchecked_node_pool:
shared.unchecked_node_pool = set()
self._make_initial_nodes()
self.assertEqual(len(shared.unchecked_node_pool), 0)
for node in shared.core_nodes:
c = connection.Bootstrapper(*node)
c.start()
c.join()
if len(shared.unchecked_node_pool) > 2:
break
else:
self.fail(
'Failed to find at least 3 nodes'
' after running %s bootstrappers' % len(shared.core_nodes))
def test_connection(self):
"""Check a normal connection - should receive objects"""
@ -215,3 +249,31 @@ class TestListener(TestProcessProto):
if c.status == 'fully_established':
self.fail('Established a connection')
time.sleep(0.5)
class TestBootstrapProcess(TestProcessProto):
"""A separate test case for bootstrapping with a minode process"""
_listen = True
_connection_limit = 24
def test_bootstrap(self):
"""Start a bootstrapper for the local process and check node pool"""
if shared.unchecked_node_pool:
shared.unchecked_node_pool = set()
started = time.time()
while not self.connections():
if time.time() - started > 60:
self.fail('Failed to establish a connection')
time.sleep(1)
for _ in range(3):
c = connection.Bootstrapper('127.0.0.1', 8444)
c.start()
c.join()
if len(shared.unchecked_node_pool) > 2:
break
else:
self.fail(
'Failed to find at least 3 nodes'
' after 3 tries to bootstrap with the local process')