From 452fe8d5f1b4d13749662a243f53eccf30aea730 Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Tue, 28 Mar 2023 07:01:29 +0300 Subject: [PATCH 01/21] A minimal implementation of proxy for outgoing connections using PySocks, special arg --tor currently just sets host and port for the socks_proxy. --- minode/connection.py | 45 +++++++++++++++++++++++++++++++++++++++++++- minode/main.py | 26 ++++++++++++++++++++++++- minode/manager.py | 6 ++++-- minode/shared.py | 3 +++ 4 files changed, 76 insertions(+), 4 deletions(-) diff --git a/minode/connection.py b/minode/connection.py index 05d552a..9a4cd2f 100644 --- a/minode/connection.py +++ b/minode/connection.py @@ -5,6 +5,7 @@ import errno import logging import math import random +import re import select import socket import ssl @@ -78,7 +79,9 @@ class ConnectionBase(threading.Thread): self.s.settimeout(0) if not self.server: if self.network == 'ip': - self.send_queue.put(message.Version(self.host, self.port)) + self.send_queue.put(message.Version( + ('127.0.0.1' if shared.socks_proxy else self.host), + self.port)) else: self.send_queue.put(message.Version('127.0.0.1', 7656)) while True: @@ -517,4 +520,44 @@ class Bootstrapper(ConnectionBase): self.send_queue.put(None) +class SocksConnection(Connection): + """The socks proxied connection""" + def _connect(self): + peer_str = '{0.host_print}:{0.port}'.format(self) + logging.debug('Connecting to %s', peer_str) + + import socks # pylint: disable=import-outside-toplevel + + try: + self.s = socks.create_connection( + (self.host, self.port), 30, None, socks.PROXY_TYPE_SOCKS5, + shared.socks_proxy[0], shared.socks_proxy[1], True, + None, None, None) + self.status = 'connected' + logging.debug('Established SOCKS connection to %s', peer_str) + except socket.timeout: + pass + except socks.GeneralProxyError as e: + e = e.socket_err + if isinstance(e, socket.timeout) or ( + # general failure, unreachable, refused + not e.errno and re.match(r'^0x0[1,4,5].*', e.msg) + ): + logcall = logging.debug + else: + logcall = logging.info + logcall('Connection to %s failed. Reason: %s', peer_str, e) + except OSError as e: + # unreachable, refused, no route + (logging.info if e.errno not in (0, 101, 111, 113) + else logging.debug)( + 'Connection to %s failed. Reason: %s', peer_str, e) + except Exception: + logging.info( + 'Connection to %s failed.', peer_str, exc_info=True) + + if self.status != 'connected': + self.status = 'failed' + + shared.connection = Connection diff --git a/minode/main.py b/minode/main.py index 72cebe0..2fabfd5 100644 --- a/minode/main.py +++ b/minode/main.py @@ -8,6 +8,11 @@ import os import signal import socket +try: + import socks +except ImportError: + socks = None + from . import i2p, shared from .advertiser import Advertiser from .manager import Manager @@ -52,6 +57,14 @@ def parse_arguments(): # pylint: disable=too-many-branches,too-many-statements '--i2p-transient', action='store_true', help='Generate new I2P destination on start') + if socks is not None: + parser.add_argument( + '--socks-proxy', + help='SOCKS proxy address in the form :') + parser.add_argument( + '--tor', action='store_true', + help='The SOCKS proxy is tor, use 127.0.0.1:9050 if not specified') + args = parser.parse_args() if args.port: shared.listening_port = args.port @@ -71,7 +84,8 @@ def parse_arguments(): # pylint: disable=too-many-branches,too-many-statements if args.no_ip: shared.ip_enabled = False if args.trusted_peer: - if len(args.trusted_peer) > 50: + if len(args.trusted_peer + ) > 50 and not args.trusted_peer.endswith('onion'): # I2P shared.trusted_peer = (args.trusted_peer.encode(), 'i2p') else: @@ -99,6 +113,16 @@ def parse_arguments(): # pylint: disable=too-many-branches,too-many-statements if args.i2p_transient: shared.i2p_transient = True + if socks is None: + return + if args.tor: + shared.tor = True + if not args.socks_proxy: + shared.socks_proxy = ('127.0.0.1', 9050) + if args.socks_proxy: + addr = args.socks_proxy.split(':') + shared.socks_proxy = (addr[0], int(addr[1])) + def bootstrap_from_dns(): """Addes addresses of bootstrap servers to core nodes""" diff --git a/minode/manager.py b/minode/manager.py index caf223b..8ac71cc 100644 --- a/minode/manager.py +++ b/minode/manager.py @@ -11,7 +11,7 @@ import threading import time from . import proofofwork, shared, structure -from .connection import Bootstrapper, Connection +from .connection import Bootstrapper, Connection, SocksConnection from .i2p import I2PDialer @@ -171,7 +171,9 @@ class Manager(threading.Thread): else: continue else: - connect((host, port)) + connect( + (host, port), + Connection if not shared.socks_proxy else SocksConnection) hosts.add(group) @staticmethod diff --git a/minode/shared.py b/minode/shared.py index 72864ec..f6fd81e 100644 --- a/minode/shared.py +++ b/minode/shared.py @@ -27,6 +27,9 @@ header_length = 24 i2p_dest_obj_type = 0x493250 i2p_dest_obj_version = 1 +socks_proxy = None +tor = False + i2p_enabled = False i2p_transient = False i2p_sam_host = '127.0.0.1' From bdfd39a163c4e671ee9b1c18f8443845721af23e Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Sun, 20 Aug 2023 03:00:40 +0300 Subject: [PATCH 02/21] Implement decoding and connection to onion peer: make a separate a bit controversial class structure.OnionPeer(), having .from_object() and .to_object() instead of .from_bytes() etc. --- minode/connection.py | 17 +++++++++++++--- minode/manager.py | 46 +++++++++++++++++++++++++++++++++++++++----- minode/shared.py | 5 +++++ minode/structure.py | 41 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 101 insertions(+), 8 deletions(-) diff --git a/minode/connection.py b/minode/connection.py index 9a4cd2f..4ae1aef 100644 --- a/minode/connection.py +++ b/minode/connection.py @@ -266,6 +266,7 @@ class ConnectionBase(threading.Thread): addr = { structure.NetAddr(c.remote_version.services, c.host, c.port) for c in shared.connections if c.network != 'i2p' + and not c.host.endswith('.onion') and c.server is False and c.status == 'fully_established'} # pylint: disable=unsubscriptable-object # https://github.com/pylint-dev/pylint/issues/3637 @@ -396,9 +397,12 @@ class ConnectionBase(threading.Thread): if not self.server: self.send_queue.put('fully_established') if self.network == 'ip': - shared.address_advertise_queue.put(structure.NetAddr( - version.services, self.host, self.port)) - shared.node_pool.add((self.host, self.port)) + if self.host.endswith('.onion'): + shared.onion_pool.add((self.host, self.port)) + else: + shared.address_advertise_queue.put(structure.NetAddr( + version.services, self.host, self.port)) + shared.node_pool.add((self.host, self.port)) elif self.network == 'i2p': shared.i2p_node_pool.add((self.host, 'i2p')) if self.network == 'ip': @@ -503,6 +507,13 @@ class Connection(ConnectionBase): ' adding to i2p_unchecked_node_pool') logging.debug(dest) shared.i2p_unchecked_node_pool.add((dest, 'i2p')) + elif ( + obj.object_type == shared.onion_obj_type + and obj.version == shared.onion_obj_version + ): + peer = structure.OnionPeer.from_object(obj) + logging.debug('Received onion peer object: %s', peer) + shared.onion_unchecked_pool.add((peer.host, peer.port)) shared.vector_advertise_queue.put(obj.vector) def _process_msg_getdata(self, m): diff --git a/minode/manager.py b/minode/manager.py index 8ac71cc..2249dc7 100644 --- a/minode/manager.py +++ b/minode/manager.py @@ -123,17 +123,31 @@ class Manager(threading.Thread): ): if shared.ip_enabled: - if len(shared.unchecked_node_pool) > 16: + sample_length = 16 + if shared.tor: + if len(shared.onion_unchecked_pool) > 4: + to_connect.update(random.sample( + tuple(shared.onion_unchecked_pool), 4)) + else: + to_connect.update(shared.onion_unchecked_pool) + shared.onion_unchecked_pool.difference_update(to_connect) + if len(shared.onion_pool) > 2: + to_connect.update(random.sample( + tuple(shared.onion_pool), 2)) + else: + to_connect.update(shared.onion_pool) + sample_length = 8 + if len(shared.unchecked_node_pool) > sample_length: to_connect.update(random.sample( - tuple(shared.unchecked_node_pool), 16)) + tuple(shared.unchecked_node_pool), sample_length)) else: to_connect.update(shared.unchecked_node_pool) if outgoing_connections < shared.outgoing_connections / 2: bootstrap() shared.unchecked_node_pool.difference_update(to_connect) - if len(shared.node_pool) > 8: + if len(shared.node_pool) > sample_length / 2: to_connect.update(random.sample( - tuple(shared.node_pool), 8)) + tuple(shared.node_pool), int(sample_length / 2))) else: to_connect.update(shared.node_pool) @@ -212,6 +226,17 @@ class Manager(threading.Thread): logging.warning( 'Error while loading nodes from disk.', exc_info=True) + try: + with open( + os.path.join(shared.data_directory, 'onion_nodes.pickle'), 'br' + ) as src: + shared.onion_pool = pickle.load(src) + except FileNotFoundError: + pass + except Exception: + logging.warning( + 'Error while loading nodes from disk.', exc_info=True) + with open( os.path.join(shared.source_directory, 'core_nodes.csv'), 'r', newline='', encoding='ascii' @@ -259,6 +284,13 @@ class Manager(threading.Thread): shared.i2p_unchecked_node_pool = set(random.sample( tuple(shared.i2p_unchecked_node_pool), 100)) + if len(shared.onion_pool) > 1000: + shared.onion_pool = set( + random.sample(shared.onion_pool, 1000)) + if len(shared.onion_unchecked_pool) > 100: + shared.onion_unchecked_pool = set( + random.sample(shared.onion_unchecked_pool, 100)) + try: with open( os.path.join(shared.data_directory, 'nodes.pickle'), 'bw' @@ -268,7 +300,11 @@ class Manager(threading.Thread): os.path.join(shared.data_directory, 'i2p_nodes.pickle'), 'bw' ) as dst: pickle.dump(shared.i2p_node_pool, dst, protocol=3) - logging.debug('Saved nodes') + with open( + os.path.join(shared.data_directory, 'onion_nodes.pickle'), 'bw' + ) as dst: + pickle.dump(shared.onion_pool, dst, protocol=3) + logging.debug('Saved nodes') except Exception: logging.warning('Error while saving nodes', exc_info=True) diff --git a/minode/shared.py b/minode/shared.py index f6fd81e..0fa0335 100644 --- a/minode/shared.py +++ b/minode/shared.py @@ -26,6 +26,8 @@ timeout = 600 header_length = 24 i2p_dest_obj_type = 0x493250 i2p_dest_obj_version = 1 +onion_obj_type = 0x746f72 +onion_obj_version = 3 socks_proxy = None tor = False @@ -62,6 +64,9 @@ i2p_core_nodes = set() i2p_node_pool = set() i2p_unchecked_node_pool = set() +onion_pool = set() +onion_unchecked_pool = set() + outgoing_connections = 8 connection_limit = 250 diff --git a/minode/structure.py b/minode/structure.py index 3c01537..d2d238a 100644 --- a/minode/structure.py +++ b/minode/structure.py @@ -1,8 +1,10 @@ # -*- coding: utf-8 -*- """Protocol structures""" import base64 +import binascii import hashlib import logging +import re import socket import struct import time @@ -237,3 +239,42 @@ class NetAddr(IStructure): stream, net_addr = struct.unpack('>QI26s', b)[1:] n = NetAddrNoPrefix.from_bytes(net_addr) return cls(n.services, n.host, n.port, stream) + + +class OnionPeer(): + """An object, containing onion peer info""" + def __init__(self, host, port=8444, stream=None, dest_pub=None): + self.stream = stream or shared.stream + self.host = host + self.port = port + try: + self.dest_pub = dest_pub or base64.b32decode( + re.search(r'(.*)\.onion', host).groups()[0], True) + except (AttributeError, binascii.Error) as e: + raise ValueError('Malformed hostname') from e + + def __repr__(self): + return 'onion_peer, stream: {}, host: {}, port {}'.format( + self.stream, self.host, self.port) + + def to_object(self): + """Encode the onion peer to the `Object`""" + payload = b'' + payload += VarInt(self.port).to_bytes() + payload += b'\xfd\x87\xd8\x7e\xeb\x43' + payload += self.dest_pub + return Object( + b'\x00' * 8, int(time.time() + 8 * 3600), shared.onion_obj_type, + shared.onion_obj_version, self.stream, payload) + + @classmethod + def from_object(cls, obj): + """Decode the onion peer from an `Object` instance""" + payload = obj.object_payload + port_length = VarInt.length(payload[0]) + port = VarInt.from_bytes(payload[:port_length]).n + if payload[port_length:port_length + 6] != b'\xfd\x87\xd8\x7e\xeb\x43': + raise ValueError('Malformed onion peer object') + dest_pub = payload[port_length + 6:] + host = base64.b32encode(dest_pub).lower().decode() + '.onion' + return cls(host, port, obj.stream_number, dest_pub) From a10a90540758596a35cd19ddcb1526ed720fdfed Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Thu, 13 Apr 2023 17:36:15 +0300 Subject: [PATCH 03/21] Do not start TLS in onion connections --- minode/connection.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/minode/connection.py b/minode/connection.py index 4ae1aef..9920749 100644 --- a/minode/connection.py +++ b/minode/connection.py @@ -79,9 +79,11 @@ class ConnectionBase(threading.Thread): self.s.settimeout(0) if not self.server: if self.network == 'ip': + version_kwargs = ( + {'services': 1} if self.host.endswith('.onion') else {}) self.send_queue.put(message.Version( ('127.0.0.1' if shared.socks_proxy else self.host), - self.port)) + self.port, **version_kwargs)) else: self.send_queue.put(message.Version('127.0.0.1', 7656)) while True: @@ -260,8 +262,11 @@ class ConnectionBase(threading.Thread): 'Established Bitmessage protocol connection to %s:%s', self.host_print, self.port) self.on_connection_fully_established_scheduled = False - if self.remote_version.services & 2 and self.network == 'ip': - self._do_tls_handshake() # NODE_SSL + if ( # NODE_SSL + self.remote_version.services & 2 and self.network == 'ip' + and not self.host.endswith('.onion') + ): + self._do_tls_handshake() addr = { structure.NetAddr(c.remote_version.services, c.host, c.port) From 63700885a01ad67e52fc83cbbbaa80d67163c4ab Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Thu, 13 Apr 2023 20:35:13 +0300 Subject: [PATCH 04/21] Add an extra 'proxy' with a requirement of PySocks --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 7a40bd5..f2210cb 100644 --- a/setup.py +++ b/setup.py @@ -24,6 +24,7 @@ setup( packages=find_packages(exclude=('*tests',)), package_data={'': ['*.csv', 'tls/*.pem']}, entry_points={'console_scripts': ['minode = minode.main:main']}, + extras_require={'proxy': ['PySocks']}, classifiers=[ "License :: OSI Approved :: MIT License" "Operating System :: OS Independent", From cd6f82bc2a1785ac8ef9600973913f2ba05e860b Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Thu, 13 Apr 2023 21:00:31 +0300 Subject: [PATCH 05/21] Add a simple blind test for process running with --socks-proxy --- minode/tests/test_process.py | 23 +++++++++++++++++++++++ requirements.txt | 1 + 2 files changed, 24 insertions(+) diff --git a/minode/tests/test_process.py b/minode/tests/test_process.py index 3d7b51e..386e40a 100644 --- a/minode/tests/test_process.py +++ b/minode/tests/test_process.py @@ -18,6 +18,11 @@ try: i2p_port_free = True except (OSError, socket.error): i2p_port_free = False +try: + socket.socket().bind(('127.0.0.1', 9050)) + tor_port_free = True +except (OSError, socket.error): + tor_port_free = False class TestProcessProto(unittest.TestCase): @@ -185,3 +190,21 @@ class TestProcessI2P(TestProcess): class TestProcessNoI2P(TestProcessShutdown): """Test minode process shutdown with --i2p and no IP""" _process_cmd = ['minode', '--i2p', '--no-ip'] + + +@unittest.skipIf(tor_port_free, 'No running tor detected') +class TestProcessTor(TestProcessProto): + """A test case for minode process running with tor enabled""" + _process_cmd = ['minode', '--socks-proxy', '127.0.0.1:9050'] + _wait_time = 60 + + def test_connections(self): + """Check minode process connections""" + for _ in range(self._wait_time): + time.sleep(0.5) + connections = self.connections() + for c in connections: + self.assertEqual(c.raddr[0], '127.0.0.1') + self.assertEqual(c.raddr[1], 9050) + if len(connections) > self._connection_limit / 2: + break diff --git a/requirements.txt b/requirements.txt index e560573..8082633 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ coverage psutil +PySocks From fffb5e60520eaa2e29cedbfd73717b4872ceb9d8 Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Thu, 13 Apr 2023 21:07:10 +0300 Subject: [PATCH 06/21] Install and start tor in buildbot --- .buildbot/ubuntu/Dockerfile | 3 ++- .buildbot/ubuntu/build.sh | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.buildbot/ubuntu/Dockerfile b/.buildbot/ubuntu/Dockerfile index c61c378..89de9e3 100644 --- a/.buildbot/ubuntu/Dockerfile +++ b/.buildbot/ubuntu/Dockerfile @@ -9,7 +9,8 @@ RUN apt-add-repository ppa:purplei2p/i2pd && apt-get update -qq RUN apt-get install -yq --no-install-suggests --no-install-recommends \ python3-dev python3-pip python-is-python3 python3.11-dev python3.11-venv -RUN apt-get install -yq --no-install-suggests --no-install-recommends sudo i2pd +RUN apt-get install -yq --no-install-suggests --no-install-recommends \ + sudo i2pd tor RUN echo 'builder ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers diff --git a/.buildbot/ubuntu/build.sh b/.buildbot/ubuntu/build.sh index bda6c27..81c3274 100755 --- a/.buildbot/ubuntu/build.sh +++ b/.buildbot/ubuntu/build.sh @@ -1,3 +1,4 @@ #!/bin/sh sudo service i2pd start +sudo service tor start From 1400486b22c980fbd54ab73c1f3c35535e7c4efe Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Tue, 6 Jun 2023 06:22:00 +0300 Subject: [PATCH 07/21] Parse socks_proxy arg with urllib.parse and support more parameters --- minode/connection.py | 8 +++++--- minode/main.py | 16 +++++++++++++--- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/minode/connection.py b/minode/connection.py index 9920749..ada5b8f 100644 --- a/minode/connection.py +++ b/minode/connection.py @@ -544,11 +544,13 @@ class SocksConnection(Connection): import socks # pylint: disable=import-outside-toplevel + proxy_type = socks.PROXY_TYPES[shared.socks_proxy.scheme.upper()] + try: self.s = socks.create_connection( - (self.host, self.port), 30, None, socks.PROXY_TYPE_SOCKS5, - shared.socks_proxy[0], shared.socks_proxy[1], True, - None, None, None) + (self.host, self.port), 30, None, proxy_type, + shared.socks_proxy.hostname, shared.socks_proxy.port, True, + shared.socks_proxy.username, shared.socks_proxy.password, None) self.status = 'connected' logging.debug('Established SOCKS connection to %s', peer_str) except socket.timeout: diff --git a/minode/main.py b/minode/main.py index 2fabfd5..53cadb4 100644 --- a/minode/main.py +++ b/minode/main.py @@ -5,8 +5,10 @@ import base64 import logging import multiprocessing import os +import re import signal import socket +from urllib import parse try: import socks @@ -118,10 +120,11 @@ def parse_arguments(): # pylint: disable=too-many-branches,too-many-statements if args.tor: shared.tor = True if not args.socks_proxy: - shared.socks_proxy = ('127.0.0.1', 9050) + args.socks_proxy = '127.0.0.1:9050' if args.socks_proxy: - addr = args.socks_proxy.split(':') - shared.socks_proxy = (addr[0], int(addr[1])) + if not re.match(r'^.*://', args.socks_proxy): + args.socks_proxy = '//' + args.socks_proxy + shared.socks_proxy = parse.urlparse(args.socks_proxy, scheme='socks5') def bootstrap_from_dns(): @@ -266,6 +269,13 @@ def main(): 'Error while creating data directory in: %s', shared.data_directory, exc_info=True) + if shared.socks_proxy: + try: + socks.PROXY_TYPES[shared.socks_proxy.scheme.upper()] + except KeyError: + logging.error('Unsupported proxy schema!') + return + if shared.ip_enabled and not shared.trusted_peer: bootstrap_from_dns() From 59fcd9eb2bc90f955e854f1e5aae1093221d22bf Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Wed, 30 Aug 2023 01:47:49 +0300 Subject: [PATCH 08/21] A test for encoding and decoding of onion peer object --- minode/tests/test_structure.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/minode/tests/test_structure.py b/minode/tests/test_structure.py index 970c152..5faf893 100644 --- a/minode/tests/test_structure.py +++ b/minode/tests/test_structure.py @@ -23,6 +23,9 @@ sample_addr_data = unhexlify( sample_object_data = unhexlify( '000000000000000000000000652724030000002a010248454c4c4f') +sample_onion_host = \ + 'bmtestlmgmvpbsg7kzmrxu47chs3cdou2tj4t5iloocgujzsf3e7rbqd.onion' + logging.basicConfig( level=shared.log_level, format='[%(asctime)s] [%(levelname)s] %(message)s') @@ -192,3 +195,21 @@ class TestStructure(unittest.TestCase): nonce, obj.expires_time, obj.object_type, obj.version, obj.stream_number, obj.object_payload) self.assertTrue(obj.is_valid()) + + def test_onion_peer(self): + """Make an onion peer object and decode it back""" + with self.assertRaises(ValueError): + onion_peer = structure.OnionPeer('testing2') + with self.assertRaises(ValueError): + onion_peer = structure.OnionPeer('testing.onion') + onion_peer = structure.OnionPeer(sample_onion_host) + self.assertEqual(onion_peer.stream, shared.stream) + obj = onion_peer.to_object() + self.assertEqual(obj.object_type, shared.onion_obj_type) + self.assertEqual(obj.version, shared.onion_obj_version) + decoded = structure.OnionPeer.from_object(obj) + self.assertEqual(decoded.dest_pub, onion_peer.dest_pub) + self.assertEqual(decoded.port, onion_peer.port) + obj.object_payload = obj.object_payload[0:1] + obj.object_payload[2:] + with self.assertRaises(ValueError): + structure.OnionPeer.from_object(obj) From d6de7c8d1e07ead34ea255fd0c9f98a2fbc45c0f Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Tue, 5 Sep 2023 04:55:58 +0300 Subject: [PATCH 09/21] A rough implementation of onion service based on pybitmessage plugin; publishing interval for onionpeer object is a half of its TTL, as for I2P destination. --- minode/main.py | 9 +++ minode/manager.py | 16 ++++++ minode/shared.py | 1 + minode/tor.py | 137 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 163 insertions(+) create mode 100644 minode/tor.py diff --git a/minode/main.py b/minode/main.py index 53cadb4..bf8be62 100644 --- a/minode/main.py +++ b/minode/main.py @@ -276,6 +276,15 @@ def main(): logging.error('Unsupported proxy schema!') return + if shared.tor: + try: + from . import tor # pylint: disable=import-outside-toplevel + except ImportError: + logging.info('Failed to import tor module.', exc_info=True) + else: + if not tor.start_tor_service(): + logging.warning('Failed to start tor service.') + if shared.ip_enabled and not shared.trusted_peer: bootstrap_from_dns() diff --git a/minode/manager.py b/minode/manager.py index 2249dc7..53d6edf 100644 --- a/minode/manager.py +++ b/minode/manager.py @@ -28,6 +28,10 @@ class Manager(threading.Thread): # Publish destination 5-15 minutes after start self.last_published_i2p_destination = \ time.time() - 50 * 60 + random.uniform(-1, 1) * 300 # nosec B311 + # Publish onion 4-8 min later + self.last_published_onion_peer = \ + self.last_published_i2p_destination - 3 * 3600 + \ + random.uniform(1, 2) * 240 # nosec B311 def fill_bootstrap_pool(self): """Populate the bootstrap pool by core nodes and checked ones""" @@ -59,6 +63,9 @@ class Manager(threading.Thread): if now - self.last_published_i2p_destination > 3600: self.publish_i2p_destination() self.last_published_i2p_destination = now + if now - self.last_published_onion_peer > 3600 * 4: + self.publish_onion_peer() + self.last_published_onion_peer = now @staticmethod def clean_objects(): @@ -320,3 +327,12 @@ class Manager(threading.Thread): shared.i2p_dest_obj_type, shared.i2p_dest_obj_version, shared.stream, dest_pub_raw) proofofwork.do_pow_and_publish(obj) + + @staticmethod + def publish_onion_peer(): + """Make and publish a `structure.OnionPeer`""" + if shared.onion_hostname: + logging.info('Publishing our onion peer') + obj = structure.OnionPeer( + shared.onion_hostname, shared.listening_port).to_object() + proofofwork.do_pow_and_publish(obj) diff --git a/minode/shared.py b/minode/shared.py index 0fa0335..f3d97f0 100644 --- a/minode/shared.py +++ b/minode/shared.py @@ -31,6 +31,7 @@ onion_obj_version = 3 socks_proxy = None tor = False +onion_hostname = '' i2p_enabled = False i2p_transient = False diff --git a/minode/tor.py b/minode/tor.py new file mode 100644 index 0000000..3e5ec1c --- /dev/null +++ b/minode/tor.py @@ -0,0 +1,137 @@ +"""Tor specific procedures""" +import logging +import os +import stat +import random +import tempfile + +import stem +import stem.control +import stem.process +import stem.util +import stem.version + +from . import shared + + +def logwrite(line): + """A simple log writing handler for tor messages""" + try: + level, line = line.split('[', 1)[1].split(']', 1) + except (IndexError, ValueError): + logging.warning(line) + else: + if level in ('err', 'warn'): + logging.info('(tor)%s', line) + + +def start_tor_service(): + """Start own tor instance and configure a hidden service""" + try: + socket_dir = os.path.join(shared.data_directory, 'tor') + os.makedirs(socket_dir, exist_ok=True) + except OSError: + try: + socket_dir = tempfile.mkdtemp() + except OSError: + logging.info('Failed to create a temp dir.') + return + + try: + present_permissions = os.stat(socket_dir)[0] + disallowed_permissions = stat.S_IRWXG | stat.S_IRWXO + allowed_permissions = ((1 << 32) - 1) ^ disallowed_permissions + os.chmod(socket_dir, allowed_permissions & present_permissions) + except OSError: + logging.debug('Failed to set dir permissions.') + return + + stem.util.log.get_logger().setLevel(logging.WARNING) + + control_socket = os.path.abspath(os.path.join(socket_dir, 'tor_control')) + tor_config = { + 'SocksPort': str(shared.socks_proxy.port), + 'ControlSocket': control_socket} + + for attempt in range(50): + if attempt > 0: + port = random.randint(32767, 65535) # nosec B311 + tor_config['SocksPort'] = str(port) + try: + stem.process.launch_tor_with_config( + tor_config, take_ownership=True, timeout=20, + init_msg_handler=logwrite) + except OSError: + if not attempt: + try: + stem.version.get_system_tor_version() + except IOError: + return + continue + else: + logging.info('Started tor on port %s', port) + break + else: + logging.debug('Failed to start tor.') + return + + try: + controller = stem.control.Controller.from_socket_file(control_socket) + controller.authenticate() + except stem.SocketError: + logging.debug('Failed to instantiate or authenticate on controller.') + return + + onionkey = onionkeytype = None + try: + with open( + os.path.join(shared.data_directory, 'onion_dest_priv.key'), + 'r', encoding='ascii' + ) as src: + onionkey = src.read() + logging.debug('Loaded onion service private key.') + onionkeytype = 'ED25519-V3' + except FileNotFoundError: + pass + except Exception: + logging.info( + 'Error while loading onion service private key.', exc_info=True) + + response = controller.create_ephemeral_hidden_service( + shared.listening_port, key_type=onionkeytype or 'NEW', + key_content=onionkey or 'BEST' + ) + + if not response.is_ok(): + logging.info('Bad response from controller ):') + return + + shared.onion_hostname = '{}.onion'.format(response.service_id) + logging.info('Started hidden service %s', shared.onion_hostname) + + if onionkey: + return True + + try: + with open( + os.path.join(shared.data_directory, 'onion_dest_priv.key'), + 'w', encoding='ascii' + ) as src: + src.write(response.private_key) + logging.debug('Saved onion service private key.') + except Exception: + logging.warning( + 'Error while saving onion service private key.', exc_info=True) + + try: + with open( + os.path.join(shared.data_directory, 'onion_dest.pub'), + 'w', encoding='ascii' + ) as src: + src.write(response.service_id) + logging.debug('Saved onion service public key.') + except Exception: + logging.warning( + 'Error while saving onion service public key.', exc_info=True) + + return True From b465ddff859053302aa86ef5883f7de492c61f48 Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Tue, 5 Sep 2023 05:00:56 +0300 Subject: [PATCH 10/21] Add a stem requirement for testing and a new extra - 'tor' --- requirements.txt | 1 + setup.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8082633..1cf55db 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ coverage psutil PySocks +stem diff --git a/setup.py b/setup.py index f2210cb..b4ee8f6 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ setup( packages=find_packages(exclude=('*tests',)), package_data={'': ['*.csv', 'tls/*.pem']}, entry_points={'console_scripts': ['minode = minode.main:main']}, - extras_require={'proxy': ['PySocks']}, + extras_require={'proxy': ['PySocks'], 'tor': ['PySocks', 'stem>1.8.0']}, classifiers=[ "License :: OSI Approved :: MIT License" "Operating System :: OS Independent", From 947607937cb1f5e634ea946be8bbd3acf28622ed Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Tue, 5 Sep 2023 20:55:00 +0300 Subject: [PATCH 11/21] Skip TLS also for incoming connections using tor --- minode/connection.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/minode/connection.py b/minode/connection.py index ada5b8f..dd78143 100644 --- a/minode/connection.py +++ b/minode/connection.py @@ -265,6 +265,7 @@ class ConnectionBase(threading.Thread): if ( # NODE_SSL self.remote_version.services & 2 and self.network == 'ip' and not self.host.endswith('.onion') + and not (self.server and shared.tor) ): self._do_tls_handshake() @@ -415,7 +416,9 @@ class ConnectionBase(threading.Thread): shared.services, version.host, shared.listening_port)) if self.server: if self.network == 'ip': - self.send_queue.put(message.Version(self.host, self.port)) + version_kwargs = {'services': 1} if shared.tor else {} + self.send_queue.put(message.Version( + self.host, self.port, **version_kwargs)) else: self.send_queue.put(message.Version('127.0.0.1', 7656)) From ff0df703881a37ce05c500c9ff58cecd68580aee Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Wed, 6 Sep 2023 01:39:40 +0300 Subject: [PATCH 12/21] Complete help string on the --tor arg --- minode/main.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/minode/main.py b/minode/main.py index bf8be62..cbd3db7 100644 --- a/minode/main.py +++ b/minode/main.py @@ -65,7 +65,9 @@ def parse_arguments(): # pylint: disable=too-many-branches,too-many-statements help='SOCKS proxy address in the form :') parser.add_argument( '--tor', action='store_true', - help='The SOCKS proxy is tor, use 127.0.0.1:9050 if not specified') + help='The SOCKS proxy is tor, use 127.0.0.1:9050 if not specified,' + ' start tor and setup a hidden service' + ) args = parser.parse_args() if args.port: From 651a4f9681e14c6cd5bc428e58f2df82ef4ea036 Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Fri, 28 Jul 2023 00:17:15 +0300 Subject: [PATCH 13/21] Update command line dump in README --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 77f42f2..72c7a34 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ usage: main.py [-h] [-p PORT] [--host HOST] [--debug] [--data-dir DATA_DIR] [--connection-limit CONNECTION_LIMIT] [--i2p] [--i2p-tunnel-length I2P_TUNNEL_LENGTH] [--i2p-sam-host I2P_SAM_HOST] [--i2p-sam-port I2P_SAM_PORT] - [--i2p-transient] + [--i2p-transient] [--socks-proxy SOCKS_PROXY] [--tor] optional arguments: -h, --help show this help message and exit @@ -53,6 +53,10 @@ optional arguments: --i2p-sam-port I2P_SAM_PORT Port of I2P SAMv3 bridge --i2p-transient Generate new I2P destination on start + --socks-proxy SOCKS_PROXY + SOCKS proxy address in the form : + --tor The SOCKS proxy is tor, use 127.0.0.1:9050 if not + specified, start tor and setup a hidden service ``` From b1749c368c412f50081d5f499bc64262e289a65a Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Fri, 8 Sep 2023 04:03:23 +0300 Subject: [PATCH 14/21] Test process with --tor, set env variable HOME for tor --- minode/tests/test_process.py | 2 +- tox.ini | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/minode/tests/test_process.py b/minode/tests/test_process.py index 386e40a..e76504f 100644 --- a/minode/tests/test_process.py +++ b/minode/tests/test_process.py @@ -195,7 +195,7 @@ class TestProcessNoI2P(TestProcessShutdown): @unittest.skipIf(tor_port_free, 'No running tor detected') class TestProcessTor(TestProcessProto): """A test case for minode process running with tor enabled""" - _process_cmd = ['minode', '--socks-proxy', '127.0.0.1:9050'] + _process_cmd = ['minode', '--tor'] _wait_time = 60 def test_connections(self): diff --git a/tox.ini b/tox.ini index 6843306..97ade46 100644 --- a/tox.ini +++ b/tox.ini @@ -3,6 +3,8 @@ envlist = reset,py3{6,7,8,9,10,11},stats skip_missing_interpreters = true [testenv] +setenv = + HOME = {envtmpdir} deps = -rrequirements.txt commands = coverage run -a -m tests From a0537a9e66c46a9f2170c76a7be2fd4f46a939ae Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Thu, 26 Oct 2023 04:21:46 +0300 Subject: [PATCH 15/21] Don't try to resolve DNS seeds if tor is enabled --- minode/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/minode/main.py b/minode/main.py index cbd3db7..7cd80ad 100644 --- a/minode/main.py +++ b/minode/main.py @@ -287,7 +287,7 @@ def main(): if not tor.start_tor_service(): logging.warning('Failed to start tor service.') - if shared.ip_enabled and not shared.trusted_peer: + if shared.ip_enabled and not shared.trusted_peer and not shared.tor: bootstrap_from_dns() if shared.i2p_enabled: From c5a1310083509eb4aef4d7888c04d31d30b73841 Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Thu, 26 Oct 2023 03:32:55 +0300 Subject: [PATCH 16/21] Conform tor setup to the restrictions for incoming and outgoing connections; don't start the tor service with --no-incoming if there is a system one. --- minode/main.py | 18 +++++++++--------- minode/tor.py | 5 +++++ 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/minode/main.py b/minode/main.py index 7cd80ad..5499314 100644 --- a/minode/main.py +++ b/minode/main.py @@ -271,21 +271,21 @@ def main(): 'Error while creating data directory in: %s', shared.data_directory, exc_info=True) - if shared.socks_proxy: + if shared.socks_proxy and shared.send_outgoing_connections: try: socks.PROXY_TYPES[shared.socks_proxy.scheme.upper()] except KeyError: logging.error('Unsupported proxy schema!') return - if shared.tor: - try: - from . import tor # pylint: disable=import-outside-toplevel - except ImportError: - logging.info('Failed to import tor module.', exc_info=True) - else: - if not tor.start_tor_service(): - logging.warning('Failed to start tor service.') + if shared.tor: + try: + from . import tor # pylint: disable=import-outside-toplevel + except ImportError: + logging.info('Failed to import tor module.', exc_info=True) + else: + if not tor.start_tor_service(): + logging.warning('The tor service has not started.') if shared.ip_enabled and not shared.trusted_peer and not shared.tor: bootstrap_from_dns() diff --git a/minode/tor.py b/minode/tor.py index 3e5ec1c..db9dacb 100644 --- a/minode/tor.py +++ b/minode/tor.py @@ -63,6 +63,8 @@ def start_tor_service(): init_msg_handler=logwrite) except OSError: if not attempt: + if not shared.listen_for_connections: + return try: stem.version.get_system_tor_version() except IOError: @@ -75,6 +77,9 @@ def start_tor_service(): logging.debug('Failed to start tor.') return + if not shared.listen_for_connections: + return True + try: controller = stem.control.Controller.from_socket_file(control_socket) controller.authenticate() From d17de19aca877d1eba9ca3b3b6e78afb06ea546c Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Sat, 25 Nov 2023 16:09:32 +0200 Subject: [PATCH 17/21] Improve the logic of starting tor: set missing initial port, exit on failure, increase timeout to 90 reducing number of attempts to 20. --- minode/main.py | 8 ++++++++ minode/tor.py | 11 ++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/minode/main.py b/minode/main.py index 5499314..461adee 100644 --- a/minode/main.py +++ b/minode/main.py @@ -283,9 +283,17 @@ def main(): from . import tor # pylint: disable=import-outside-toplevel except ImportError: logging.info('Failed to import tor module.', exc_info=True) + tor = None else: if not tor.start_tor_service(): logging.warning('The tor service has not started.') + tor = None + if not tor: + try: + socket.socket().bind(('127.0.0.1', 9050)) + return + except (OSError, socket.error): + pass if shared.ip_enabled and not shared.trusted_peer and not shared.tor: bootstrap_from_dns() diff --git a/minode/tor.py b/minode/tor.py index db9dacb..e42d33c 100644 --- a/minode/tor.py +++ b/minode/tor.py @@ -37,6 +37,10 @@ def start_tor_service(): logging.info('Failed to create a temp dir.') return + if os.getuid() == 0: + logging.info('Tor is not going to start as root') + return + try: present_permissions = os.stat(socket_dir)[0] disallowed_permissions = stat.S_IRWXG | stat.S_IRWXO @@ -49,17 +53,18 @@ def start_tor_service(): stem.util.log.get_logger().setLevel(logging.WARNING) control_socket = os.path.abspath(os.path.join(socket_dir, 'tor_control')) + port = str(shared.socks_proxy.port) tor_config = { - 'SocksPort': str(shared.socks_proxy.port), + 'SocksPort': port, 'ControlSocket': control_socket} - for attempt in range(50): + for attempt in range(20): if attempt > 0: port = random.randint(32767, 65535) # nosec B311 tor_config['SocksPort'] = str(port) try: stem.process.launch_tor_with_config( - tor_config, take_ownership=True, timeout=20, + tor_config, take_ownership=True, timeout=90, init_msg_handler=logwrite) except OSError: if not attempt: From ec4e24185ad39d0ba7166365e62fbe5d0fced7c8 Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Thu, 4 Jul 2024 02:47:52 +0300 Subject: [PATCH 18/21] Don't advertise own peer if run with --no-incoming or reported IP is 127.0.0.1 --- minode/connection.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/minode/connection.py b/minode/connection.py index dd78143..2158f3b 100644 --- a/minode/connection.py +++ b/minode/connection.py @@ -411,7 +411,10 @@ class ConnectionBase(threading.Thread): shared.node_pool.add((self.host, self.port)) elif self.network == 'i2p': shared.i2p_node_pool.add((self.host, 'i2p')) - if self.network == 'ip': + if ( + self.network == 'ip' and shared.listen_for_connections + and version.host != '127.0.0.1' + ): shared.address_advertise_queue.put(structure.NetAddr( shared.services, version.host, shared.listening_port)) if self.server: From cb239c7d68a80cf072db37af361b139bc9a76219 Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Thu, 10 Aug 2023 19:58:43 +0300 Subject: [PATCH 19/21] Bump version to 0.3.4 --- minode/shared.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/minode/shared.py b/minode/shared.py index f3d97f0..ca867f5 100644 --- a/minode/shared.py +++ b/minode/shared.py @@ -21,7 +21,7 @@ protocol_version = 3 services = 3 # NODE_NETWORK, NODE_SSL stream = 1 nonce = os.urandom(8) -user_agent = b'/MiNode:0.3.3/' +user_agent = b'/MiNode:0.3.4/' timeout = 600 header_length = 24 i2p_dest_obj_type = 0x493250 From fe6a6af1dbb26bd32ee432ffa9202046d6b1cd8f Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Sat, 9 Nov 2024 01:11:14 +0200 Subject: [PATCH 20/21] Started a test case for the tor service --- minode/tests/common.py | 9 +++++ minode/tests/test_process.py | 7 ++-- minode/tests/test_tor.py | 64 ++++++++++++++++++++++++++++++++++++ minode/tor.py | 4 +-- 4 files changed, 77 insertions(+), 7 deletions(-) create mode 100644 minode/tests/common.py create mode 100644 minode/tests/test_tor.py diff --git a/minode/tests/common.py b/minode/tests/common.py new file mode 100644 index 0000000..87ec48a --- /dev/null +++ b/minode/tests/common.py @@ -0,0 +1,9 @@ +"""Common expressions for the tests""" + +import socket + +try: + socket.socket().bind(('127.0.0.1', 9050)) + tor_port_free = True +except (OSError, socket.error): + tor_port_free = False diff --git a/minode/tests/test_process.py b/minode/tests/test_process.py index e76504f..0f22c36 100644 --- a/minode/tests/test_process.py +++ b/minode/tests/test_process.py @@ -13,16 +13,13 @@ import psutil from minode.i2p import util from minode.structure import NetAddrNoPrefix +from .common import tor_port_free + try: socket.socket().bind(('127.0.0.1', 7656)) i2p_port_free = True except (OSError, socket.error): i2p_port_free = False -try: - socket.socket().bind(('127.0.0.1', 9050)) - tor_port_free = True -except (OSError, socket.error): - tor_port_free = False class TestProcessProto(unittest.TestCase): diff --git a/minode/tests/test_tor.py b/minode/tests/test_tor.py new file mode 100644 index 0000000..21549cc --- /dev/null +++ b/minode/tests/test_tor.py @@ -0,0 +1,64 @@ +"""Tests for tor module""" +import collections +import os +import tempfile +import unittest + +from minode import shared + +from .common import tor_port_free + +try: + from minode import tor +except ImportError: + tor = None + + +Proxy = collections.namedtuple('Proxy', ['hostname', 'port']) + + +@unittest.skipIf( + tor_port_free or tor is None, 'Inapropriate environment for tor service') +class TestTor(unittest.TestCase): + """A test case running the tor service""" + tor = None + _files = ['onion_dest_priv.key', 'onion_dest.pub'] + + @classmethod + def cleanup(cls): + """Remove used files""" + for f in cls._files: + try: + os.remove(os.path.join(shared.data_directory, f)) + except FileNotFoundError: + pass + + @classmethod + def setUpClass(cls): + shared.data_directory = tempfile.gettempdir() + shared.socks_proxy = Proxy('127.0.0.1', 9050) + cls.cleanup() + + @classmethod + def tearDownClass(cls): + if cls.tor: + cls.tor.close() + cls.cleanup() + + def test_tor(self): + """Start the tor service as in main and check the environment""" + self.tor = tor.start_tor_service() + if not self.tor: + self.fail('The tor service has hot started.') + + with open( + os.path.join(shared.data_directory, 'onion_dest.pub'), + 'r', encoding='ascii' + ) as key_file: + onion_key = key_file.read() + self.assertEqual(onion_key + '.onion', shared.onion_hostname) + + # with open( + # os.path.join(shared.data_directory, 'onion_dest_priv.key'), 'rb' + # ) as key_file: + # private_key = key_file.read() diff --git a/minode/tor.py b/minode/tor.py index e42d33c..2fe14b9 100644 --- a/minode/tor.py +++ b/minode/tor.py @@ -120,7 +120,7 @@ def start_tor_service(): logging.info('Started hidden service %s', shared.onion_hostname) if onionkey: - return True + return controller try: with open( @@ -144,4 +144,4 @@ def start_tor_service(): logging.warning( 'Error while saving onion service public key.', exc_info=True) - return True + return controller From 9732f0c7f5a88dea6d63d2c85363219289948a22 Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Fri, 21 Feb 2025 03:26:10 +0200 Subject: [PATCH 21/21] Handle pylint too-many-branches in main.main() --- minode/main.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/minode/main.py b/minode/main.py index 461adee..050e639 100644 --- a/minode/main.py +++ b/minode/main.py @@ -281,21 +281,20 @@ def main(): if shared.tor: try: from . import tor # pylint: disable=import-outside-toplevel - except ImportError: - logging.info('Failed to import tor module.', exc_info=True) - tor = None - else: if not tor.start_tor_service(): logging.warning('The tor service has not started.') tor = None + except ImportError: + logging.info('Failed to import tor module.', exc_info=True) + tor = None + if not tor: try: socket.socket().bind(('127.0.0.1', 9050)) return except (OSError, socket.error): pass - - if shared.ip_enabled and not shared.trusted_peer and not shared.tor: + elif shared.ip_enabled and not shared.trusted_peer: bootstrap_from_dns() if shared.i2p_enabled: