A test case for incoming connection, needs debugging

- same multiprocessing start method for start.sh and setuptools entry point,
 - make it possible to run outside of the main thread.

.buildbot/ubuntu/Dockerfile

@@ -5,5 +5,15 @@ RUN apt-get update
 RUN apt-get install -yq --no-install-suggests --no-install-recommends \
     python3-dev python3-pip python3.9 python3.9-dev python3.9-venv
 
+RUN apt-get install -yq --no-install-suggests --no-install-recommends \
+    software-properties-common build-essential libcap-dev libffi-dev \
+    libssl-dev python-all-dev python-setuptools python-six git
+
 RUN python3.9 -m pip install setuptools wheel
 RUN python3.9 -m pip install --upgrade pip tox virtualenv
+
+RUN git clone https://github.com/Bitmessage/PyBitmessage.git
+
+RUN cd PyBitmessage; python2 setup.py install; python3 setup.py install
+
+ADD . .

minode/main.py

@@ -263,9 +263,10 @@ def start_i2p_listener():
 
 
 def main():
-    signal.signal(signal.SIGINT, handler)
-    signal.signal(signal.SIGTERM, handler)
-
+    try:
+        multiprocessing.set_start_method('spawn')
+    except RuntimeError:
+        pass
     parse_arguments()
 
     logging.basicConfig(
@@ -273,6 +274,12 @@ def main():
         format='[%(asctime)s] [%(levelname)s] %(message)s')
     logging.info('Starting MiNode')
 
+    try:
+        signal.signal(signal.SIGINT, handler)
+        signal.signal(signal.SIGTERM, handler)
+    except ValueError:
+        logging.warning('Working outside of the main thread!')
+
     logging.info('Data directory: %s', shared.data_directory)
     if not os.path.exists(shared.data_directory):
         try:
@@ -315,5 +322,4 @@ def main():
 
 
 if __name__ == '__main__':
-    multiprocessing.set_start_method('spawn')
     main()

minode/manager.py

@@ -128,7 +128,7 @@ class Manager(threading.Thread):
                             ' an I2P connection', exc_info=True)
                 else:
                     continue
-            else:
+            elif outgoing_connections < shared.outgoing_connections:
                 c = Connection(addr[0], addr[1])
                 c.start()
                 hosts.add(c.host)

minode/tests/test_app.py

@@ -0,0 +1,83 @@
+import unittest
+import sys
+import tempfile
+import time
+import threading
+
+from minode import shared
+from minode.main import main as app
+
+
+class TestAppProto(unittest.TestCase):
+    """Import and start the application"""
+    _process_cmd = ['minode']
+    _connection_limit = 4 if sys.platform.startswith('win') else 6
+    _listen = False
+    _listening_port = None
+
+    home = None
+
+    @classmethod
+    def _build_app_args(cls):
+        if not cls.home:
+            cls.home = tempfile.gettempdir()
+        args = cls._process_cmd + [
+            '--data-dir', cls.home,
+            '--connection-limit', str(cls._connection_limit)
+        ]
+        if not cls._listen:
+            args += ['--no-incoming']
+        elif cls._listening_port:
+            args += ['-p', str(cls._listening_port)]
+
+        return args
+
+    def _connections(self):
+        return [
+            c for c in shared.connections.copy()
+            if c.status == 'fully_established']
+
+    @classmethod
+    def setUpClass(cls):
+        sys.argv = cls._build_app_args()
+        cls.app = threading.Thread(name="minode", target=app, daemon=True)
+        cls.app.start()
+
+    @classmethod
+    def tearDownClass(cls):
+        shared.shutting_down = True
+
+
+class TestApp(TestAppProto):
+    """Check the app parameters"""
+    _wait_time = 120
+    _check_limit = True
+
+    def test_connections(self):
+        """Check connections"""
+        _started = time.time()
+
+        def continue_check_limit(extra_time):
+            for t in range(extra_time * 4):
+                self.assertLessEqual(
+                    len(self._connections()),
+                    # shared.outgoing_connections, one listening
+                    # TODO: find the cause of one extra
+                    (min(self._connection_limit, 8) if not self._listen
+                     else self._connection_limit) + 1,
+                    'Opened more connections than required'
+                    ' by --connection-limit')
+                time.sleep(0.5)
+
+        for t in range(self._wait_time * 2):
+            if len(self._connections()) > self._connection_limit / 2:
+                _time_to_connect = round(time.time() - _started)
+                break
+            time.sleep(0.5)
+        else:
+            self.fail(
+                'Failed establish at least %s connections in %s sec'
+                % (self._connection_limit / 2, self._wait_time))
+
+        if self._check_limit:
+            continue_check_limit(_time_to_connect)

minode/tests/test_process.py

@@ -1,12 +1,18 @@
 import unittest
+import os
+import shutil
 import signal
 import subprocess
 import sys
 import tempfile
+import threading
 import time
 
 import psutil
 
+from minode import shared
+from minode.main import main as app
+
 
 class TestProcessProto(unittest.TestCase):
     """Test process attributes, common flow"""
@@ -18,18 +24,23 @@ class TestProcessProto(unittest.TestCase):
     home = None
 
     @classmethod
-    def setUpClass(cls):
+    def _build_app_args(cls, extra=None):
         if not cls.home:
             cls.home = tempfile.gettempdir()
-        cmd = cls._process_cmd + [
+        args = cls._process_cmd + [
             '--data-dir', cls.home,
             '--connection-limit', str(cls._connection_limit)
         ]
         if not cls._listen:
-            cmd += ['--no-incoming']
+            args += ['--no-incoming']
         elif cls._listening_port:
-            cmd += ['-p', str(cls._listening_port)]
-        cls.process = psutil.Popen(cmd, stderr=subprocess.STDOUT)  # nosec
+            args += ['-p', str(cls._listening_port)]
+
+        if extra:
+            args += extra
+
+        print('ARGS: %r' % args)
+        return args
 
     @classmethod
     def _stop_process(cls, timeout=5):
@@ -40,9 +51,14 @@ class TestProcessProto(unittest.TestCase):
             return False
         return True
 
+    @classmethod
+    def setUpClass(cls):
+        cmd = cls._build_app_args()
+        cls.process = psutil.Popen(cmd, stderr=subprocess.STDOUT)  # nosec
+
     @classmethod
     def tearDownClass(cls):
-        """Ensures that pybitmessage stopped and removes files"""
+        """Ensures the process is stopped and removes files"""
         try:
             if not cls._stop_process(10):
                 try:
@@ -112,3 +128,41 @@ class TestProcess(TestProcessProto):
         else:
             if self._listen:
                 self.fail('No listening connection found')
+
+
+class TestConnectivity(TestProcessProto):
+    """Check connectivity between instances"""
+    _process_cmd = [
+        'minode', '--trusted-peer', '127.0.0.1:8445']
+    # _listen = True
+
+    @classmethod
+    def setUpClass(cls):
+        super(TestConnectivity, cls).setUpClass()
+
+        cls._listen = True
+        cls._listening_port = 8445
+        cls.home = os.path.join(cls.home, 'client')
+        os.mkdir(cls.home)
+        # sys.argv = cls._build_app_args(['--trusted-peer', '127.0.0.1:8444'])
+        cls._process_cmd = ['minode', '--no-outgoing']
+        sys.argv = cls._build_app_args()
+        cls.app = threading.Thread(name="minode", target=app, daemon=True)
+        cls.app.start()
+
+    @classmethod
+    def tearDownClass(cls):
+        super(TestConnectivity, cls).tearDownClass()
+        shared.shutting_down = True
+        shutil.rmtree(cls.home)
+
+    def test_connections(self):
+        """Check the connection with trusted peer"""
+        time.sleep(5)
+        for t in range(10):
+            time.sleep(1)
+            connection_count = len([
+                c for c in shared.connections.copy()
+                if c.status == 'fully_established'])
+            if connection_count != 1:
+                self.fail("Unexpected connection count: %i" % connection_count)