From df34857d6a83b2bf96c7b53ed8b8e1c3999ab86f Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Thu, 5 Oct 2023 12:57:53 +0300 Subject: [PATCH 1/8] Integrate bitmsghash.bmpow - PyBitmessage's CPoW extension, define a threading.Thread subclass Worker() for that. --- minode/bitmsghash/bitmsghash.cpp | 166 +++++++++++++++++++++++++++++++ minode/proofofwork.py | 89 +++++++++++++---- setup.py | 17 +++- 3 files changed, 250 insertions(+), 22 deletions(-) create mode 100644 minode/bitmsghash/bitmsghash.cpp diff --git a/minode/bitmsghash/bitmsghash.cpp b/minode/bitmsghash/bitmsghash.cpp new file mode 100644 index 0000000..2cc107e --- /dev/null +++ b/minode/bitmsghash/bitmsghash.cpp @@ -0,0 +1,166 @@ +// bitmessage cracker, build with g++ or MSVS to a shared library, +// use via minode.proofofwork module +#ifdef _WIN32 +#include "winsock.h" +#include "windows.h" +#define uint64_t unsigned __int64 +#else +#include +#include +#include +#endif +#include +#include +#include +#if defined(__APPLE__) || defined(__FreeBSD__) || defined (__DragonFly__) || defined (__OpenBSD__) || defined (__NetBSD__) +#include +#include +#endif + +#include "openssl/sha.h" + +#define HASH_SIZE 64 +#define BUFLEN 16384 + +#if defined(__GNUC__) + #define EXPORT __attribute__ ((__visibility__("default"))) +#elif defined(_WIN32) + #define EXPORT __declspec(dllexport) +#endif + +#ifndef __APPLE__ +#define ntohll(x) ( ( (uint64_t)(ntohl( (unsigned int)((x << 32) >> 32) )) << 32) | ntohl( ((unsigned int)(x >> 32)) ) ) +#endif + +unsigned long long max_val; +unsigned char *initialHash; +unsigned long long successval = 0; +unsigned int numthreads = 0; + +#ifdef _WIN32 +DWORD WINAPI threadfunc(LPVOID param) { +#else +void * threadfunc(void* param) { +#endif + unsigned int incamt = *((unsigned int*)param); + SHA512_CTX sha; + unsigned char buf[HASH_SIZE + sizeof(uint64_t)] = { 0 }; + unsigned char output[HASH_SIZE] = { 0 }; + + memcpy(buf + sizeof(uint64_t), initialHash, HASH_SIZE); + + unsigned long long tmpnonce = incamt; + unsigned long long * nonce = (unsigned long long *)buf; + unsigned long long * hash = (unsigned long long *)output; + while (successval == 0) { + tmpnonce += numthreads; + + (*nonce) = ntohll(tmpnonce); /* increment nonce */ + SHA512_Init(&sha); + SHA512_Update(&sha, buf, HASH_SIZE + sizeof(uint64_t)); + SHA512_Final(output, &sha); + SHA512_Init(&sha); + SHA512_Update(&sha, output, HASH_SIZE); + SHA512_Final(output, &sha); + + if (ntohll(*hash) < max_val) { + successval = tmpnonce; + } + } +#ifdef _WIN32 + return 0; +#else + return NULL; +#endif +} + +void getnumthreads() +{ +#ifdef _WIN32 + DWORD_PTR dwProcessAffinity, dwSystemAffinity; +#elif __linux__ + cpu_set_t dwProcessAffinity; +#elif __OpenBSD__ + int mib[2], core_count = 0; + int dwProcessAffinity = 0; + size_t len2; +#else + int dwProcessAffinity = 0; + int32_t core_count = 0; +#endif + size_t len = sizeof(dwProcessAffinity); + if (numthreads > 0) + return; +#ifdef _WIN32 + GetProcessAffinityMask(GetCurrentProcess(), &dwProcessAffinity, &dwSystemAffinity); +#elif __linux__ + sched_getaffinity(0, len, &dwProcessAffinity); +#elif __OpenBSD__ + len2 = sizeof(core_count); + mib[0] = CTL_HW; + mib[1] = HW_NCPU; + if (sysctl(mib, 2, &core_count, &len2, 0, 0) == 0) + numthreads = core_count; +#else + if (sysctlbyname("hw.logicalcpu", &core_count, &len, 0, 0) == 0) + numthreads = core_count; + else if (sysctlbyname("hw.ncpu", &core_count, &len, 0, 0) == 0) + numthreads = core_count; +#endif + for (unsigned int i = 0; i < len * 8; i++) +#if defined(_WIN32) +#if defined(_MSC_VER) + if (dwProcessAffinity & (1i64 << i)) +#else // CYGWIN/MINGW + if (dwProcessAffinity & (1ULL << i)) +#endif +#elif defined __linux__ + if (CPU_ISSET(i, &dwProcessAffinity)) +#else + if (dwProcessAffinity & (1 << i)) +#endif + numthreads++; + if (numthreads == 0) // something failed + numthreads = 1; + printf("Number of threads: %i\n", (int)numthreads); +} + +extern "C" EXPORT unsigned long long BitmessagePOW(unsigned char * starthash, unsigned long long target) +{ + successval = 0; + max_val = target; + getnumthreads(); + initialHash = (unsigned char *)starthash; +# ifdef _WIN32 + HANDLE* threads = (HANDLE*)calloc(sizeof(HANDLE), numthreads); +# else + pthread_t* threads = (pthread_t*)calloc(sizeof(pthread_t), numthreads); + struct sched_param schparam; + schparam.sched_priority = 0; +# endif + unsigned int *threaddata = (unsigned int *)calloc(sizeof(unsigned int), numthreads); + for (unsigned int i = 0; i < numthreads; i++) { + threaddata[i] = i; +# ifdef _WIN32 + threads[i] = CreateThread(NULL, 0, threadfunc, (LPVOID)&threaddata[i], 0, NULL); + SetThreadPriority(threads[i], THREAD_PRIORITY_IDLE); +# else + pthread_create(&threads[i], NULL, threadfunc, (void*)&threaddata[i]); +# ifdef __linux__ + pthread_setschedparam(threads[i], SCHED_IDLE, &schparam); +# else + pthread_setschedparam(threads[i], SCHED_RR, &schparam); +# endif +# endif + } +# ifdef _WIN32 + WaitForMultipleObjects(numthreads, threads, TRUE, INFINITE); +# else + for (unsigned int i = 0; i < numthreads; i++) { + pthread_join(threads[i], NULL); + } +# endif + free(threads); + free(threaddata); + return successval; +} diff --git a/minode/proofofwork.py b/minode/proofofwork.py index f8aa69f..713f511 100644 --- a/minode/proofofwork.py +++ b/minode/proofofwork.py @@ -1,6 +1,8 @@ """Doing proof of work""" import base64 +import ctypes import hashlib +import importlib import logging import multiprocessing import struct @@ -25,30 +27,75 @@ def _pow_worker(target, initial_hash, q): q.put(struct.pack('>Q', nonce)) -def _worker(obj): - q = multiprocessing.Queue() - p = multiprocessing.Process( - target=_pow_worker, args=(obj.pow_target(), obj.pow_initial_hash(), q)) +class Worker(threading.Thread): + def __init__(self, obj=None): + self.obj = obj + super().__init__( + name='Worker' if not self.obj + else 'Worker-%s' % (base64.b16encode(obj.vector).decode()) + ) + self.bmpow = bso = None + try: + bitmsglib = importlib.util.find_spec('minode.bitmsghash').origin + except AttributeError: + return - logging.debug('Starting POW process') - t = time.time() - p.start() - nonce = q.get() - p.join() + try: + bso = ctypes.CDLL(bitmsglib) + except Exception: # pylint: disable=broad-exception-caught + logging.warning('Error loading bitmsghash', exc_info=True) + try: + # MSVS on Windows + bso = ctypes.WinDLL(bitmsglib) + except (AttributeError, TypeError, ValueError): + return - logging.debug( - 'Finished doing POW, nonce: %s, time: %ss', nonce, time.time() - t) - obj = structure.Object( - nonce, obj.expires_time, obj.object_type, obj.version, - obj.stream_number, obj.object_payload) - logging.debug( - 'Object vector is %s', base64.b16encode(obj.vector).decode()) + self.bmpow = bso.BitmessagePOW + self.bmpow.restype = ctypes.c_ulonglong - with shared.objects_lock: - shared.objects[obj.vector] = obj - shared.vector_advertise_queue.put(obj.vector) + @staticmethod + def _worker_result(obj): + q = multiprocessing.Queue() + p = multiprocessing.Process(target=_pow_worker, args=( + obj.pow_target(), obj.pow_initial_hash(), q)) + + p.start() + nonce = q.get() + p.join() + + return nonce + + def add_pow(self, obj): + t = time.time() + if not self.bmpow: + logging.debug('Starting PoW process') + nonce = self._worker_result(obj) + else: + logging.debug('Calling PoW extension') + nonce = struct.pack('>Q', self.bmpow( + ctypes.pointer( + ctypes.create_string_buffer(obj.pow_initial_hash(), 64)), + ctypes.c_ulonglong(obj.pow_target()) + )) + + logging.debug( + 'Finished doing PoW, nonce: %s, time: %ss', nonce, time.time() - t) + obj = structure.Object( + nonce, obj.expires_time, obj.object_type, obj.version, + obj.stream_number, obj.object_payload) + logging.debug( + 'Object vector is %s', base64.b16encode(obj.vector).decode()) + + return obj + + def run(self): + if self.obj: + obj = self.add_pow(self.obj) + + with shared.objects_lock: + shared.objects[obj.vector] = obj + shared.vector_advertise_queue.put(obj.vector) def do_pow_and_publish(obj): - t = threading.Thread(target=_worker, args=(obj, )) - t.start() + Worker(obj).start() diff --git a/setup.py b/setup.py index 7a40bd5..4f72204 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,9 @@ #!/usr/bin/env python import os +import sys -from setuptools import setup, find_packages +from setuptools import Extension, find_packages, setup from minode import shared @@ -12,6 +13,19 @@ README = open(os.path.join( name, version = shared.user_agent.strip(b'/').split(b':') +bitmsghash = Extension( + 'minode.bitmsghash', + sources=['minode/bitmsghash/bitmsghash.cpp'], + libraries=['pthread', 'crypto'], +) + +if sys.platform[:3] == 'win': + bitmsghash.libraries = ['libeay32', 'ws2_32'] + openssl_dir = os.getenv('OPENSSL_DIR') + if openssl_dir: + bitmsghash.library_dirs = [os.path.join(openssl_dir, 'lib')] + bitmsghash.include_dirs = [os.path.join(openssl_dir, 'include')] + setup( name=name.decode('utf-8'), version=version.decode('utf-8'), @@ -23,6 +37,7 @@ setup( url='https://git.bitmessage.org/lee.miller/MiNode', packages=find_packages(exclude=('*tests',)), package_data={'': ['*.csv', 'tls/*.pem']}, + ext_modules=[bitmsghash], entry_points={'console_scripts': ['minode = minode.main:main']}, classifiers=[ "License :: OSI Approved :: MIT License" From 904a554631a707213d904c63090f92e2a773940f Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Thu, 5 Oct 2023 15:07:03 +0300 Subject: [PATCH 2/8] Update pow check approach in test_object using proofofwork.Worker --- minode/tests/test_structure.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/minode/tests/test_structure.py b/minode/tests/test_structure.py index 16e41fe..5e241fa 100644 --- a/minode/tests/test_structure.py +++ b/minode/tests/test_structure.py @@ -122,8 +122,7 @@ class TestStructure(unittest.TestCase): obj = structure.Object( b'\x00' * 8, int(time.time() + 300), 42, 1, 2, b'HELLO') vector = obj.vector - proofofwork._worker(obj) # pylint: disable=protected-access - obj = shared.objects.popitem()[1] + obj = proofofwork.Worker().add_pow(obj) self.assertNotEqual(obj.vector, vector) self.assertFalse(obj.is_expired()) self.assertFalse(obj.is_valid()) From 224feb28d0e15a0271330071325941c82f5e6589 Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Thu, 5 Oct 2023 15:17:27 +0300 Subject: [PATCH 3/8] Update buildbot_multibuild Dockerfile with debs needed for compiling C ext --- .buildbot/ubuntu/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.buildbot/ubuntu/Dockerfile b/.buildbot/ubuntu/Dockerfile index e9ab2d2..fc3ef6f 100644 --- a/.buildbot/ubuntu/Dockerfile +++ b/.buildbot/ubuntu/Dockerfile @@ -1,5 +1,7 @@ FROM ubuntu:focal +ENV DEBIAN_FRONTEND=noninteractive + RUN apt-get update RUN apt-get install -yq software-properties-common @@ -8,6 +10,7 @@ RUN apt-add-repository ppa:purplei2p/i2pd RUN apt-get update RUN apt-get install -yq --no-install-suggests --no-install-recommends \ + build-essential libcap-dev libffi-dev libssl-dev \ python3-dev python3-pip python3.9 python3.9-dev python3.9-venv sudo i2pd RUN echo 'builder ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers From 450839079cde98b7e32fb4f77386ad491a8b8c98 Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Thu, 5 Oct 2023 23:37:27 +0300 Subject: [PATCH 4/8] Add a glob fallback searching bitmsghash*.so (e.g. bitmsghash.cpython-39-x86_64-linux-gnu.so) --- minode/proofofwork.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/minode/proofofwork.py b/minode/proofofwork.py index 713f511..18f43ca 100644 --- a/minode/proofofwork.py +++ b/minode/proofofwork.py @@ -1,10 +1,12 @@ """Doing proof of work""" import base64 import ctypes +import glob import hashlib import importlib import logging import multiprocessing +import os import struct import threading import time @@ -37,7 +39,12 @@ class Worker(threading.Thread): self.bmpow = bso = None try: bitmsglib = importlib.util.find_spec('minode.bitmsghash').origin - except AttributeError: + if not bitmsglib: + bitmsglib = glob.glob(os.path.join( + os.path.dirname(os.path.abspath(__file__)), + 'bitmsghash*.so' + ))[0] + except (AttributeError, IndexError): return try: From 794cf2657c92c5721a83e4c5999944d869a6d5bf Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Sat, 7 Oct 2023 02:12:35 +0300 Subject: [PATCH 5/8] Initial efforts to make bitmsghash a valid python extension --- minode/bitmsghash/bitmsghash.cpp | 37 ++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/minode/bitmsghash/bitmsghash.cpp b/minode/bitmsghash/bitmsghash.cpp index 2cc107e..c1a68c1 100644 --- a/minode/bitmsghash/bitmsghash.cpp +++ b/minode/bitmsghash/bitmsghash.cpp @@ -19,6 +19,9 @@ #include "openssl/sha.h" +#define PY_SSIZE_T_CLEAN +#include + #define HASH_SIZE 64 #define BUFLEN 16384 @@ -164,3 +167,37 @@ extern "C" EXPORT unsigned long long BitmessagePOW(unsigned char * starthash, un free(threaddata); return successval; } + +// python module definitions +static PyObject * +bitmessage_pow(PyObject *self, PyObject *args) +{ + const char *initial_hash; + unsigned long long target; + unsigned long long nonce; + + if (!PyArg_ParseTuple(args, "yK", &initial_hash, target)) + return NULL; + nonce = BitmessagePOW((unsigned char *)initial_hash, target); + return PyLong_FromUnsignedLongLong(nonce); +}; + +static PyMethodDef BitmsghashMethods[] = { + {"bitmessage_pow", bitmessage_pow, METH_VARARGS, + "Do the Bitmessage PoW and return nonce."}, + {NULL, NULL, 0, NULL} +}; + +static struct PyModuleDef bitmsghashmodule = { + PyModuleDef_HEAD_INIT, + "bitmsghash", + "A C extension for PoW borrowed from PyBitmessage", + -1, + BitmsghashMethods +}; + +PyMODINIT_FUNC +PyInit_bitmsghash(void) +{ + return PyModule_Create(&bitmsghashmodule); +} From 1139bdb8b7274c5cc4adab8ee16b61425b9f95a0 Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Sat, 7 Oct 2023 03:06:56 +0300 Subject: [PATCH 6/8] Use pip install -e directly to help tox install the extension. use_develop doesn't work for some reason. Other envs will lack the bitmsghash*.so and more code in the proofofwork will be covered. --- tox.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tox.ini b/tox.ini index caeb4c6..dc82b8f 100644 --- a/tox.ini +++ b/tox.ini @@ -12,6 +12,9 @@ deps = flake8 commands = flake8 minode --count --select=E9,F63,F7,F82 --show-source --statistics +[testenv:py39] +commands_pre = pip install -e . + [testenv:reset] deps = -rrequirements.txt From ff63139d7825b6b651d60062f8ec1ac442607f60 Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Fri, 8 Sep 2023 22:15:44 +0300 Subject: [PATCH 7/8] Document proofofwork public calls --- minode/proofofwork.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/minode/proofofwork.py b/minode/proofofwork.py index 18f43ca..a929cbb 100644 --- a/minode/proofofwork.py +++ b/minode/proofofwork.py @@ -30,6 +30,7 @@ def _pow_worker(target, initial_hash, q): class Worker(threading.Thread): + """The thread doind PoW for objects and publishing them""" def __init__(self, obj=None): self.obj = obj super().__init__( @@ -73,6 +74,10 @@ class Worker(threading.Thread): return nonce def add_pow(self, obj): + """ + Do PoW for the given object and return a new `.structure.Object` + - with the proper nonce and vector. + """ t = time.time() if not self.bmpow: logging.debug('Starting PoW process') @@ -105,4 +110,9 @@ class Worker(threading.Thread): def do_pow_and_publish(obj): + """ + Start a worker thread doing PoW for the given object + and putting a new object and its vector into appropriate places in `shared` + to advertize to the network. + """ Worker(obj).start() From 2e19e129332650c6595ced679d5a76b2c8cb434c Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Mon, 16 Oct 2023 13:59:36 +0300 Subject: [PATCH 8/8] Implement permanently running worker thread --- minode/proofofwork.py | 23 ++++++++++++++++++----- minode/shared.py | 2 ++ 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/minode/proofofwork.py b/minode/proofofwork.py index a929cbb..d22dc34 100644 --- a/minode/proofofwork.py +++ b/minode/proofofwork.py @@ -11,7 +11,7 @@ import struct import threading import time -from . import shared, structure +from . import message, shared, structure def _pow_worker(target, initial_hash, q): @@ -100,13 +100,26 @@ class Worker(threading.Thread): return obj + @staticmethod + def _publish(obj): + with shared.objects_lock: + shared.objects[obj.vector] = obj + shared.vector_advertise_queue.put(obj.vector) + def run(self): if self.obj: - obj = self.add_pow(self.obj) + self._publish(self.add_pow(self.obj)) + return - with shared.objects_lock: - shared.objects[obj.vector] = obj - shared.vector_advertise_queue.put(obj.vector) + while not shared.shutting_down: + data = shared.objects_queue.get() + + obj = structure.Object.from_message( + message.Message.from_bytes(data)) + if int.from_bytes(obj.nonce, 'big') == 0: + obj = self.add_pow(obj) + + self._publish(obj) def do_pow_and_publish(obj): diff --git a/minode/shared.py b/minode/shared.py index d49786c..6991650 100644 --- a/minode/shared.py +++ b/minode/shared.py @@ -64,3 +64,5 @@ connection_limit = 250 objects = {} objects_lock = threading.Lock() + +objects_queue = queue.Queue()