Merge branch 'v0.6' into py3

.buildbot/android/Dockerfile

@@ -22,7 +22,7 @@ RUN apt-get -y update -qq \
 RUN apt-get -y install -qq --no-install-recommends openjdk-17-jdk \
     && apt-get -y autoremove
 
-RUN pip install pip install buildozer cython virtualenv
+RUN pip install buildozer cython virtualenv
 
 
 ENV ANDROID_NDK_HOME="${ANDROID_HOME}/android-ndk"

.buildbot/android/build.sh

@@ -1,9 +1,32 @@
 #!/bin/bash
 export LC_ALL=en_US.UTF-8
 export LANG=en_US.UTF-8
+
+# buildozer OOM workaround
+mkdir -p ~/.gradle
+echo "org.gradle.jvmargs=-Xmx2g -XX:MaxMetaspaceSize=512m -XX:+HeapDumpOnOutOfMemoryError -Dfile.encoding=UTF-8" \
+    > ~/.gradle/gradle.properties
+
+# workaround for symlink
+rm -rf src/pybitmessage
+mkdir -p src/pybitmessage
+cp src/*.py src/pybitmessage
+cp -r src/bitmessagekivy src/backend src/mockbm src/pybitmessage
+
 pushd packages/android
-buildozer android debug || exit $?
+
+BUILDMODE=debug
+
+if [ "$BUILDBOT_JOBNAME" = "android" -a \
+     "$BUILDBOT_REPOSITORY" = "https://github.com/Bitmessage/PyBitmessage" -a \
+     "$BUILDBOT_BRANCH" = "v0.6" ]; then
+   sed -e 's/android.release_artifact *=.*/release_artifact = aab/' -i "" buildozer.spec
+   BUILDMODE=release
+fi 
+
+buildozer android $BUILDMODE || exit $?
 popd
 
 mkdir -p ../out
-cp packages/android/bin/*.apk ../out
+RELEASE_ARTIFACT=$(grep release_artifact packages/android/buildozer.spec |cut -d= -f2|tr -Cd 'a-z')
+cp packages/android/bin/*.${RELEASE_ARTIFACT} ../out

.buildbot/android/test.sh

@@ -1,5 +1,11 @@
 #!/bin/bash
 
+RELEASE_ARTIFACT=$(grep release_artifact packages/android/buildozer.spec |cut -d= -f2|tr -Cd 'a-z')
+
+if [ $RELEASE_ARTIFACT = "aab" ]; then
+	exit
+fi
+
 unzip -p packages/android/bin/*.apk assets/private.tar \
     | tar --list -z > package.list
 cat package.list

.buildbot/appimage/Dockerfile

@@ -24,3 +24,5 @@ RUN wget -qO appimage-builder-x86_64.AppImage \
     https://github.com/AppImageCrafters/appimage-builder/releases/download/v1.1.0/appimage-builder-1.1.0-x86_64.AppImage
 
 ADD . .
+
+CMD .buildbot/tox-bionic/build.sh

.buildbot/tox-bionic/Dockerfile

@@ -20,3 +20,7 @@ RUN python3.8 -m pip install --upgrade pip tox virtualenv
 ENV LANG en_US.UTF-8
 ENV LANGUAGE en_US:en
 ENV LC_ALL en_US.UTF-8
+
+ADD . .
+
+CMD .buildbot/tox-bionic/test.sh

.buildbot/tox-focal/Dockerfile

@@ -13,3 +13,5 @@ RUN apt-get install -yq --no-install-suggests --no-install-recommends \
 RUN python3.9 -m pip install --upgrade pip tox virtualenv
 
 ADD . .
+
+CMD .buildbot/tox-focal/test.sh

.buildbot/tox-jammy/Dockerfile

@@ -10,3 +10,7 @@ RUN apt-get install -yq --no-install-suggests --no-install-recommends \
     python3-dev python3-pip language-pack-en qt5dxcb-plugin tor xvfb
 
 RUN pip install tox
+
+ADD . .
+
+CMD .buildbot/tox-jammy/test.sh

docker-test.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+DOCKERFILE=.buildbot/tox-bionic/Dockerfile
+
+docker build -t pybm/tox -f $DOCKERFILE .
+
+if [ $? -gt 0 ]; then
+    docker build --no-cache -t pybm/tox -f $DOCKERFILE .
+fi
+
+docker run --rm -it pybm/tox

packages/android/buildozer.spec

@@ -1,19 +1,19 @@
 [app]
 
 # (str) Title of your application
-title = mockone
+title = PyBitmessage Mock
 
 # (str) Package name
-package.name = mock
+package.name = pybitmessagemock
 
 # (str) Package domain (needed for android/ios packaging)
-package.domain = org.mock
+package.domain = at.bitmessage
 
 # (str) Source code where the main.py live
 source.dir = ../../src
 
 # (list) Source files to include (let empty to include all the files)
-source.include_exts = py,png,jpg,kv,atlas,tflite,sql
+source.include_exts = py,png,jpg,kv,atlas,tflite,sql,json
 
 # (list) List of inclusions using pattern matching
 #source.include_patterns = assets/*,images/*.png
@@ -28,7 +28,7 @@ source.include_exts = py,png,jpg,kv,atlas,tflite,sql
 #source.exclude_patterns = license,images/*/*.jpg
 
 # (str) Application versioning (method 1)
-version = 0.1
+version = 0.1.1
 
 # (str) Application versioning (method 2)
 # version.regex = __version__ = ['"](.*)['"]
@@ -36,7 +36,7 @@ version = 0.1
 
 # (list) Application requirements
 # comma separated e.g. requirements = sqlite3,kivy
-requirements = python3,kivy
+requirements = python3,kivy,sqlite3,kivymd==1.0.2,Pillow,opencv,kivy-garden.qrcode,qrcode,typing_extensions,libpng
 
 # (str) Custom source folders for requirements
 # Sets custom source for any requirements with recipes
@@ -92,7 +92,7 @@ fullscreen = 0
 
 # (int) Android API to use  (targetSdkVersion AND compileSdkVersion)
 # note: when changing, Dockerfile also needs to be changed to install corresponding build tools
-android.api = 28
+android.api = 33
 
 # (int) Minimum API required. You will need to set the android.ndk_api to be as low as this value.
 android.minapi = 21
@@ -243,6 +243,8 @@ android.allow_backup = True
 # Usage example : android.manifest_placeholders = [myCustomUrl:\"org.kivy.customurl\"]
 # android.manifest_placeholders = [:]
 
+android.release_artifact = apk
+
 #
 # Python for android (p4a) specific
 #

requirements.txt

@@ -1,7 +1,7 @@
 coverage
 psutil
 pycryptodome
-PyQt5;python_version>="3.7"
+PyQt5;python_version>="3.7" and platform_machine=="x86_64"
 mock;python_version<="2.7"
 python_prctl;platform_system=="Linux"
 six

run-tests-in-docker.sh

@@ -1,13 +0,0 @@
-#!/bin/sh
-
-DOCKERFILE=packages/docker/Dockerfile.bionic
-
-# explicitly mark appimage stage because it builds in any case
-docker build --target appimage -t pybm/appimage -f $DOCKERFILE .
-
-if [ $? -gt 0 ]; then
-    docker build --no-cache --target appimage -t pybm/appimage -f $DOCKERFILE .
-fi
-
-docker build --target tox -t pybm/tox -f $DOCKERFILE .
-docker run --rm -t pybm/tox

src/api.py

@@ -103,7 +103,7 @@ try:
 except ImportError:
     connectionpool = None
 
-from network import stats, StoppableThread
+from network import stats, StoppableThread, invQueue
 from version import softwareVersion
 
 try:  # TODO: write tests for XML vulnerabilities
@@ -1383,7 +1383,7 @@ class BMRPCDispatcher(object):
         logger.info(
             'Broadcasting inv for msg(API disseminatePreEncryptedMsg'
             ' command): %s', hexlify(inventoryHash))
-        queues.invQueue.put((toStreamNumber, inventoryHash))
+        invQueue.put((toStreamNumber, inventoryHash))
         return hexlify(inventoryHash).decode()
 
     @command('trashSentMessageByAckData')
@@ -1440,7 +1440,7 @@ class BMRPCDispatcher(object):
         logger.info(
             'broadcasting inv within API command disseminatePubkey with'
             ' hash: %s', hexlify(inventoryHash))
-        queues.invQueue.put((pubkeyStreamNumber, inventoryHash))
+        invQueue.put((pubkeyStreamNumber, inventoryHash))
 
     @command(
         'getMessageDataByDestinationHash', 'getMessageDataByDestinationTag')

src/bitmessagekivy/tests/telenium_process.py

@@ -8,6 +8,8 @@ import tempfile
 from time import time, sleep
 from six.moves import getcwdb
 
+from requests.exceptions import ChunkedEncodingError
+
 from telenium.tests import TeleniumTestCase
 from telenium.client import TeleniumHttpException
 
@@ -55,7 +57,10 @@ class TeleniumTestProcess(TeleniumTestCase):
     def tearDownClass(cls):
         """Ensures that pybitmessage stopped and removes files"""
         # pylint: disable=no-member
-        super(TeleniumTestProcess, cls).tearDownClass()
+        try:
+            super(TeleniumTestProcess, cls).tearDownClass()
+        except ChunkedEncodingError:
+            pass
         cleanup()
 
     def assert_wait_no_except(self, selector, timeout=-1, value='inbox'):

src/class_objectProcessor.py

@@ -31,7 +31,7 @@ from addresses import (
 from bmconfigparser import config
 from helper_sql import (
     sql_ready, sql_timeout, SqlBulkExecute, sqlExecute, sqlQuery)
-from network import knownnodes
+from network import knownnodes, invQueue
 from network.node import Peer
 from tr import _translate
 from dbcompat import dbstr
@@ -739,7 +739,7 @@ class objectProcessor(threading.Thread):
             inventoryHash = highlevelcrypto.calculateInventoryHash(ackPayload)
             state.Inventory[inventoryHash] = (
                 objectType, toStreamNumber, ackPayload, expiresTime, b'')
-            queues.invQueue.put((toStreamNumber, inventoryHash))
+            invQueue.put((toStreamNumber, inventoryHash))
 
         # Display timing data
         timeRequiredToAttemptToDecryptMessage = time.time(

src/class_singleWorker.py

@@ -29,7 +29,7 @@ import tr
 from addresses import decodeAddress, decodeVarint, encodeVarint
 from bmconfigparser import config
 from helper_sql import sqlExecute, sqlQuery
-from network import knownnodes, StoppableThread
+from network import knownnodes, StoppableThread, invQueue
 from six.moves import configparser, queue
 from six.moves.reprlib import repr
 import six
@@ -303,7 +303,7 @@ class singleWorker(StoppableThread):
         self.logger.info(
             'broadcasting inv with hash: %s', hexlify(inventoryHash))
 
-        queues.invQueue.put((streamNumber, inventoryHash))
+        invQueue.put((streamNumber, inventoryHash))
         queues.UISignalQueue.put(('updateStatusBar', ''))
         try:
             config.set(
@@ -391,7 +391,7 @@ class singleWorker(StoppableThread):
         self.logger.info(
             'broadcasting inv with hash: %s', hexlify(inventoryHash))
 
-        queues.invQueue.put((streamNumber, inventoryHash))
+        invQueue.put((streamNumber, inventoryHash))
         queues.UISignalQueue.put(('updateStatusBar', ''))
         try:
             config.set(
@@ -484,7 +484,7 @@ class singleWorker(StoppableThread):
         self.logger.info(
             'broadcasting inv with hash: %s', hexlify(inventoryHash))
 
-        queues.invQueue.put((streamNumber, inventoryHash))
+        invQueue.put((streamNumber, inventoryHash))
         queues.UISignalQueue.put(('updateStatusBar', ''))
         try:
             config.set(
@@ -538,7 +538,7 @@ class singleWorker(StoppableThread):
         self.logger.info(
             'sending inv (within sendOnionPeerObj function) for object: %s',
             hexlify(inventoryHash))
-        queues.invQueue.put((streamNumber, inventoryHash))
+        invQueue.put((streamNumber, inventoryHash))
 
     def sendBroadcast(self):
         """Send a broadcast-type object (assemble the object, perform PoW and put it to the inv announcement queue)"""
@@ -716,7 +716,7 @@ class singleWorker(StoppableThread):
                 ' for object: %s',
                 hexlify(inventoryHash)
             )
-            queues.invQueue.put((streamNumber, inventoryHash))
+            invQueue.put((streamNumber, inventoryHash))
 
             queues.UISignalQueue.put((
                 'updateSentItemStatusByAckdata', (
@@ -1374,7 +1374,7 @@ class singleWorker(StoppableThread):
                 'Broadcasting inv for my msg(within sendmsg function): %s',
                 hexlify(inventoryHash)
             )
-            queues.invQueue.put((toStreamNumber, inventoryHash))
+            invQueue.put((toStreamNumber, inventoryHash))
 
             # Update the sent message in the sent table with the
             # necessary information.
@@ -1515,7 +1515,7 @@ class singleWorker(StoppableThread):
         state.Inventory[inventoryHash] = (
             objectType, streamNumber, payload, embeddedTime, '')
         self.logger.info('sending inv (for the getpubkey message)')
-        queues.invQueue.put((streamNumber, inventoryHash))
+        invQueue.put((streamNumber, inventoryHash))
 
         # wait 10% past expiration
         sleeptill = int(time.time() + TTL * 1.1)

src/highlevelcrypto.py

@@ -11,11 +11,17 @@ import hashlib
 import os
 from binascii import hexlify
 
-import pyelliptic
+try:
-from pyelliptic import OpenSSL
+    import pyelliptic
-from pyelliptic import arithmetic as a
+    from fallback import RIPEMD160Hash
+    from pyelliptic import OpenSSL
+    from pyelliptic import arithmetic as a
+except ImportError:
+    from pybitmessage import pyelliptic
+    from pybitmessage.fallback import RIPEMD160Hash
+    from pybitmessage.pyelliptic import OpenSSL
+    from pybitmessage.pyelliptic import arithmetic as a
 
-from fallback import RIPEMD160Hash
 
 __all__ = [
     'decodeWalletImportFormat', 'deterministic_keys',

src/main-android-live.py

@@ -0,0 +1,13 @@
+"""This module is for thread start."""
+import state
+import sys
+from bitmessagemain import main
+from termcolor import colored
+print(colored('kivy is not supported at the moment for this version..', 'red'))
+sys.exit()
+
+
+if __name__ == '__main__':
+    state.kivy = True
+    print("Kivy Loading......")
+    main()

src/main.py

@@ -1,13 +1,31 @@
-"""This module is for thread start."""
+# pylint: disable=unused-import, wrong-import-position, ungrouped-imports
+# flake8: noqa:E401, E402
+
+"""Mock kivy app with mock threads."""
+
+import os
+from kivy.config import Config
+from mockbm import multiqueue
 import state
-import sys
+
-from bitmessagemain import main
+from mockbm.class_addressGenerator import FakeAddressGenerator  # noqa:E402
-from termcolor import colored
+from bitmessagekivy.mpybit import NavigateApp  # noqa:E402
-print(colored('kivy is not supported at the moment for this version..', 'red'))
+from mockbm import network  # noqa:E402
-sys.exit()
+
+stats = network.stats
+objectracker = network.objectracker
 
 
-if __name__ == '__main__':
+def main():
-    state.kivy = True
+    """main method for starting threads"""
-    print("Kivy Loading......")
+    addressGeneratorThread = FakeAddressGenerator()
+    addressGeneratorThread.daemon = True
+    addressGeneratorThread.start()
+    state.kivyapp = NavigateApp()
+    state.kivyapp.run()
+    addressGeneratorThread.stopThread()
+
+
+if __name__ == "__main__":
+    os.environ['INSTALL_TESTS'] = "True"
     main()

src/network/__init__.py

@@ -1,17 +1,24 @@
 """
 Network subsystem package
 """
+from six.moves import queue
 from .dandelion import Dandelion
 from .threads import StoppableThread
+from .multiqueue import MultiQueue
 
 dandelion_ins = Dandelion()
 
+# network queues
+invQueue = MultiQueue()
+addrQueue = MultiQueue()
+portCheckerQueue = queue.Queue()
+receiveDataQueue = queue.Queue()
+
 __all__ = ["StoppableThread"]
 
 
 def start(config, state):
     """Start network threads"""
-    import state
     from .announcethread import AnnounceThread
     from network import connectionpool
     from .addrthread import AddrThread

src/network/addrthread.py

@@ -7,7 +7,7 @@ from six.moves import queue
 # magic imports!
 from network import connectionpool
 from protocol import assembleAddrMessage
-from queues import addrQueue  # FIXME: init with queue
+from network import addrQueue  # FIXME: init with queue
 
 from .threads import StoppableThread
 

src/network/bmproto.py

@@ -18,7 +18,7 @@ import protocol
 import state
 import network.connectionpool  # use long name to address recursive import
 from bmconfigparser import config
-from queues import invQueue, objectProcessorQueue, portCheckerQueue
+from queues import objectProcessorQueue
 from randomtrackingdict import RandomTrackingDict
 from network.advanceddispatcher import AdvancedDispatcher
 from network.bmobject import (
@@ -28,7 +28,7 @@ from network.bmobject import (
 )
 from network.proxy import ProxyError
 
-from network import dandelion_ins
+from network import dandelion_ins, invQueue, portCheckerQueue
 from .node import Node, Peer
 from .objectracker import ObjectTracker, missingObjects
 

src/network/connectionchooser.py

@@ -5,11 +5,14 @@ Select which node to connect to
 import logging
 import random
 
+from six.moves import queue
+
 from network import knownnodes
 import protocol
 import state
+
 from bmconfigparser import config
-from queues import queue, portCheckerQueue
+from network import portCheckerQueue
 
 logger = logging.getLogger('default')
 

src/network/invthread.py

@@ -9,8 +9,7 @@ import addresses
 import protocol
 import state
 from network import connectionpool
-from network import dandelion_ins
+from network import dandelion_ins, invQueue
-from queues import invQueue
 from .threads import StoppableThread
 
 

src/network/multiqueue.py

@@ -2,16 +2,11 @@
 A queue with multiple internal subqueues.
 Elements are added into a random subqueue, and retrieval rotates
 """
-
+import random
 from collections import deque
 
 from six.moves import queue
 
-try:
-    import helper_random
-except ImportError:
-    from . import helper_random
-
 
 class MultiQueue(queue.Queue):
     """A base queue class"""
@@ -38,7 +33,7 @@ class MultiQueue(queue.Queue):
     # Put a new item in the queue
     def _put(self, item):
         # self.queue.append(item)
-        self.queues[helper_random.randomrandrange(self.queueCount)].append(
+        self.queues[random.randrange(self.queueCount)].append(  # nosec B311
             (item))
 
     # Get an item from the queue

src/network/receivequeuethread.py

@@ -7,7 +7,7 @@ import socket
 
 from network import connectionpool
 from network.advanceddispatcher import UnknownStateError
-from queues import receiveDataQueue
+from network import receiveDataQueue
 from .threads import StoppableThread
 
 

src/network/tcp.py

@@ -18,8 +18,8 @@ import state
 import network.connectionpool  # use long name to address recursive import
 from bmconfigparser import config
 from highlevelcrypto import randomBytes
-from network import dandelion_ins
+from network import dandelion_ins, invQueue, receiveDataQueue
-from queues import invQueue, receiveDataQueue, UISignalQueue
+from queues import UISignalQueue
 from tr import _translate
 
 from network import asyncore_pollchoose as asyncore

src/network/tls.py

@@ -11,7 +11,7 @@ import six
 import network.asyncore_pollchoose as asyncore
 import paths
 from network.advanceddispatcher import AdvancedDispatcher
-from queues import receiveDataQueue
+from network import receiveDataQueue
 
 logger = logging.getLogger('default')
 

src/network/udp.py

@@ -9,8 +9,8 @@ import time
 import protocol
 import state
 import network.connectionpool  # use long name to address recursive import
-from queues import receiveDataQueue
 
+from network import receiveDataQueue
 from .bmproto import BMProto
 from .node import Peer
 from .objectracker import ObjectTracker

src/pyelliptic/openssl.py

@@ -806,6 +806,10 @@ def loadOpenSSL():
             'libcrypto.dylib', '/usr/local/opt/openssl/lib/libcrypto.dylib'])
     elif 'win32' in sys.platform or 'win64' in sys.platform:
         libdir.append('libeay32.dll')
+    # kivy
+    elif 'ANDROID_ARGUMENT' in environ:
+        libdir.append('libcrypto1.1.so')
+        libdir.append('libssl1.1.so')
     else:
         libdir.append('libcrypto.so')
         libdir.append('libssl.so')

src/queues.py

@@ -5,11 +5,6 @@ import time
 
 from six.moves import queue
 
-try:
-    from multiqueue import MultiQueue
-except ImportError:
-    from .multiqueue import MultiQueue
-
 
 class ObjectProcessorQueue(queue.Queue):
     """Special queue class using lock for `.threads.objectProcessor`"""
@@ -44,10 +39,6 @@ addressGeneratorQueue = queue.Queue()
 #: `.network.ReceiveQueueThread` instances dump objects they hear
 #: on the network into this queue to be processed.
 objectProcessorQueue = ObjectProcessorQueue()
-invQueue = MultiQueue()
-addrQueue = MultiQueue()
-portCheckerQueue = queue.Queue()
-receiveDataQueue = queue.Queue()
 #: The address generator thread uses this queue to get information back
 #: to the API thread.
 apiAddressGeneratorReturnQueue = queue.Queue()

src/tests/test_inventory.py

@@ -38,7 +38,7 @@ class TestFilesystemInventory(TestPartialRun):
         embedded_time = int(time.time() + TTL)
         msg = struct.pack('>Q', embedded_time) + os.urandom(166)
         invhash = highlevelcrypto.calculateInventoryHash(msg)
-        self.inventory[invhash] = (2, 1, msg, embedded_time, b'')
+        self.inventory[bytes(invhash)] = (2, 1, msg, embedded_time, b'')
 
     @classmethod
     def tearDownClass(cls):

src/tests/test_multiqueue.py

@@ -1,7 +1,7 @@
 """Test cases for multiqueue"""
 
 import unittest
-from pybitmessage.multiqueue import MultiQueue
+from pybitmessage.network.multiqueue import MultiQueue
 
 
 class TestMultiQueue(unittest.TestCase):

src/tests/test_protocol.py

@@ -63,6 +63,8 @@ class TestProtocol(TestSocketInet):
 
     def test_check_local(self):
         """Check the logic of TCPConnection.local"""
+        self.assertFalse(
+            protocol.checkIPAddress(protocol.encodeHost('127.0.0.1')))
         self.assertTrue(
             protocol.checkIPAddress(protocol.encodeHost('127.0.0.1'), True))
         self.assertTrue(