diff --git a/.buildbot/ubuntu/Dockerfile b/.buildbot/ubuntu/Dockerfile index e9ab2d2..fc3ef6f 100644 --- a/.buildbot/ubuntu/Dockerfile +++ b/.buildbot/ubuntu/Dockerfile @@ -1,5 +1,7 @@ FROM ubuntu:focal +ENV DEBIAN_FRONTEND=noninteractive + RUN apt-get update RUN apt-get install -yq software-properties-common @@ -8,6 +10,7 @@ RUN apt-add-repository ppa:purplei2p/i2pd RUN apt-get update RUN apt-get install -yq --no-install-suggests --no-install-recommends \ + build-essential libcap-dev libffi-dev libssl-dev \ python3-dev python3-pip python3.9 python3.9-dev python3.9-venv sudo i2pd RUN echo 'builder ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers diff --git a/minode/bitmsghash/bitmsghash.cpp b/minode/bitmsghash/bitmsghash.cpp new file mode 100644 index 0000000..c1a68c1 --- /dev/null +++ b/minode/bitmsghash/bitmsghash.cpp @@ -0,0 +1,203 @@ +// bitmessage cracker, build with g++ or MSVS to a shared library, +// use via minode.proofofwork module +#ifdef _WIN32 +#include "winsock.h" +#include "windows.h" +#define uint64_t unsigned __int64 +#else +#include +#include +#include +#endif +#include +#include +#include +#if defined(__APPLE__) || defined(__FreeBSD__) || defined (__DragonFly__) || defined (__OpenBSD__) || defined (__NetBSD__) +#include +#include +#endif + +#include "openssl/sha.h" + +#define PY_SSIZE_T_CLEAN +#include + +#define HASH_SIZE 64 +#define BUFLEN 16384 + +#if defined(__GNUC__) + #define EXPORT __attribute__ ((__visibility__("default"))) +#elif defined(_WIN32) + #define EXPORT __declspec(dllexport) +#endif + +#ifndef __APPLE__ +#define ntohll(x) ( ( (uint64_t)(ntohl( (unsigned int)((x << 32) >> 32) )) << 32) | ntohl( ((unsigned int)(x >> 32)) ) ) +#endif + +unsigned long long max_val; +unsigned char *initialHash; +unsigned long long successval = 0; +unsigned int numthreads = 0; + +#ifdef _WIN32 +DWORD WINAPI threadfunc(LPVOID param) { +#else +void * threadfunc(void* param) { +#endif + unsigned int incamt = *((unsigned int*)param); + SHA512_CTX sha; + unsigned char buf[HASH_SIZE + sizeof(uint64_t)] = { 0 }; + unsigned char output[HASH_SIZE] = { 0 }; + + memcpy(buf + sizeof(uint64_t), initialHash, HASH_SIZE); + + unsigned long long tmpnonce = incamt; + unsigned long long * nonce = (unsigned long long *)buf; + unsigned long long * hash = (unsigned long long *)output; + while (successval == 0) { + tmpnonce += numthreads; + + (*nonce) = ntohll(tmpnonce); /* increment nonce */ + SHA512_Init(&sha); + SHA512_Update(&sha, buf, HASH_SIZE + sizeof(uint64_t)); + SHA512_Final(output, &sha); + SHA512_Init(&sha); + SHA512_Update(&sha, output, HASH_SIZE); + SHA512_Final(output, &sha); + + if (ntohll(*hash) < max_val) { + successval = tmpnonce; + } + } +#ifdef _WIN32 + return 0; +#else + return NULL; +#endif +} + +void getnumthreads() +{ +#ifdef _WIN32 + DWORD_PTR dwProcessAffinity, dwSystemAffinity; +#elif __linux__ + cpu_set_t dwProcessAffinity; +#elif __OpenBSD__ + int mib[2], core_count = 0; + int dwProcessAffinity = 0; + size_t len2; +#else + int dwProcessAffinity = 0; + int32_t core_count = 0; +#endif + size_t len = sizeof(dwProcessAffinity); + if (numthreads > 0) + return; +#ifdef _WIN32 + GetProcessAffinityMask(GetCurrentProcess(), &dwProcessAffinity, &dwSystemAffinity); +#elif __linux__ + sched_getaffinity(0, len, &dwProcessAffinity); +#elif __OpenBSD__ + len2 = sizeof(core_count); + mib[0] = CTL_HW; + mib[1] = HW_NCPU; + if (sysctl(mib, 2, &core_count, &len2, 0, 0) == 0) + numthreads = core_count; +#else + if (sysctlbyname("hw.logicalcpu", &core_count, &len, 0, 0) == 0) + numthreads = core_count; + else if (sysctlbyname("hw.ncpu", &core_count, &len, 0, 0) == 0) + numthreads = core_count; +#endif + for (unsigned int i = 0; i < len * 8; i++) +#if defined(_WIN32) +#if defined(_MSC_VER) + if (dwProcessAffinity & (1i64 << i)) +#else // CYGWIN/MINGW + if (dwProcessAffinity & (1ULL << i)) +#endif +#elif defined __linux__ + if (CPU_ISSET(i, &dwProcessAffinity)) +#else + if (dwProcessAffinity & (1 << i)) +#endif + numthreads++; + if (numthreads == 0) // something failed + numthreads = 1; + printf("Number of threads: %i\n", (int)numthreads); +} + +extern "C" EXPORT unsigned long long BitmessagePOW(unsigned char * starthash, unsigned long long target) +{ + successval = 0; + max_val = target; + getnumthreads(); + initialHash = (unsigned char *)starthash; +# ifdef _WIN32 + HANDLE* threads = (HANDLE*)calloc(sizeof(HANDLE), numthreads); +# else + pthread_t* threads = (pthread_t*)calloc(sizeof(pthread_t), numthreads); + struct sched_param schparam; + schparam.sched_priority = 0; +# endif + unsigned int *threaddata = (unsigned int *)calloc(sizeof(unsigned int), numthreads); + for (unsigned int i = 0; i < numthreads; i++) { + threaddata[i] = i; +# ifdef _WIN32 + threads[i] = CreateThread(NULL, 0, threadfunc, (LPVOID)&threaddata[i], 0, NULL); + SetThreadPriority(threads[i], THREAD_PRIORITY_IDLE); +# else + pthread_create(&threads[i], NULL, threadfunc, (void*)&threaddata[i]); +# ifdef __linux__ + pthread_setschedparam(threads[i], SCHED_IDLE, &schparam); +# else + pthread_setschedparam(threads[i], SCHED_RR, &schparam); +# endif +# endif + } +# ifdef _WIN32 + WaitForMultipleObjects(numthreads, threads, TRUE, INFINITE); +# else + for (unsigned int i = 0; i < numthreads; i++) { + pthread_join(threads[i], NULL); + } +# endif + free(threads); + free(threaddata); + return successval; +} + +// python module definitions +static PyObject * +bitmessage_pow(PyObject *self, PyObject *args) +{ + const char *initial_hash; + unsigned long long target; + unsigned long long nonce; + + if (!PyArg_ParseTuple(args, "yK", &initial_hash, target)) + return NULL; + nonce = BitmessagePOW((unsigned char *)initial_hash, target); + return PyLong_FromUnsignedLongLong(nonce); +}; + +static PyMethodDef BitmsghashMethods[] = { + {"bitmessage_pow", bitmessage_pow, METH_VARARGS, + "Do the Bitmessage PoW and return nonce."}, + {NULL, NULL, 0, NULL} +}; + +static struct PyModuleDef bitmsghashmodule = { + PyModuleDef_HEAD_INIT, + "bitmsghash", + "A C extension for PoW borrowed from PyBitmessage", + -1, + BitmsghashMethods +}; + +PyMODINIT_FUNC +PyInit_bitmsghash(void) +{ + return PyModule_Create(&bitmsghashmodule); +} diff --git a/minode/proofofwork.py b/minode/proofofwork.py index f8aa69f..d22dc34 100644 --- a/minode/proofofwork.py +++ b/minode/proofofwork.py @@ -1,13 +1,17 @@ """Doing proof of work""" import base64 +import ctypes +import glob import hashlib +import importlib import logging import multiprocessing +import os import struct import threading import time -from . import shared, structure +from . import message, shared, structure def _pow_worker(target, initial_hash, q): @@ -25,30 +29,103 @@ def _pow_worker(target, initial_hash, q): q.put(struct.pack('>Q', nonce)) -def _worker(obj): - q = multiprocessing.Queue() - p = multiprocessing.Process( - target=_pow_worker, args=(obj.pow_target(), obj.pow_initial_hash(), q)) +class Worker(threading.Thread): + """The thread doind PoW for objects and publishing them""" + def __init__(self, obj=None): + self.obj = obj + super().__init__( + name='Worker' if not self.obj + else 'Worker-%s' % (base64.b16encode(obj.vector).decode()) + ) + self.bmpow = bso = None + try: + bitmsglib = importlib.util.find_spec('minode.bitmsghash').origin + if not bitmsglib: + bitmsglib = glob.glob(os.path.join( + os.path.dirname(os.path.abspath(__file__)), + 'bitmsghash*.so' + ))[0] + except (AttributeError, IndexError): + return - logging.debug('Starting POW process') - t = time.time() - p.start() - nonce = q.get() - p.join() + try: + bso = ctypes.CDLL(bitmsglib) + except Exception: # pylint: disable=broad-exception-caught + logging.warning('Error loading bitmsghash', exc_info=True) + try: + # MSVS on Windows + bso = ctypes.WinDLL(bitmsglib) + except (AttributeError, TypeError, ValueError): + return - logging.debug( - 'Finished doing POW, nonce: %s, time: %ss', nonce, time.time() - t) - obj = structure.Object( - nonce, obj.expires_time, obj.object_type, obj.version, - obj.stream_number, obj.object_payload) - logging.debug( - 'Object vector is %s', base64.b16encode(obj.vector).decode()) + self.bmpow = bso.BitmessagePOW + self.bmpow.restype = ctypes.c_ulonglong - with shared.objects_lock: - shared.objects[obj.vector] = obj - shared.vector_advertise_queue.put(obj.vector) + @staticmethod + def _worker_result(obj): + q = multiprocessing.Queue() + p = multiprocessing.Process(target=_pow_worker, args=( + obj.pow_target(), obj.pow_initial_hash(), q)) + + p.start() + nonce = q.get() + p.join() + + return nonce + + def add_pow(self, obj): + """ + Do PoW for the given object and return a new `.structure.Object` + - with the proper nonce and vector. + """ + t = time.time() + if not self.bmpow: + logging.debug('Starting PoW process') + nonce = self._worker_result(obj) + else: + logging.debug('Calling PoW extension') + nonce = struct.pack('>Q', self.bmpow( + ctypes.pointer( + ctypes.create_string_buffer(obj.pow_initial_hash(), 64)), + ctypes.c_ulonglong(obj.pow_target()) + )) + + logging.debug( + 'Finished doing PoW, nonce: %s, time: %ss', nonce, time.time() - t) + obj = structure.Object( + nonce, obj.expires_time, obj.object_type, obj.version, + obj.stream_number, obj.object_payload) + logging.debug( + 'Object vector is %s', base64.b16encode(obj.vector).decode()) + + return obj + + @staticmethod + def _publish(obj): + with shared.objects_lock: + shared.objects[obj.vector] = obj + shared.vector_advertise_queue.put(obj.vector) + + def run(self): + if self.obj: + self._publish(self.add_pow(self.obj)) + return + + while not shared.shutting_down: + data = shared.objects_queue.get() + + obj = structure.Object.from_message( + message.Message.from_bytes(data)) + if int.from_bytes(obj.nonce, 'big') == 0: + obj = self.add_pow(obj) + + self._publish(obj) def do_pow_and_publish(obj): - t = threading.Thread(target=_worker, args=(obj, )) - t.start() + """ + Start a worker thread doing PoW for the given object + and putting a new object and its vector into appropriate places in `shared` + to advertize to the network. + """ + Worker(obj).start() diff --git a/minode/shared.py b/minode/shared.py index d49786c..6991650 100644 --- a/minode/shared.py +++ b/minode/shared.py @@ -64,3 +64,5 @@ connection_limit = 250 objects = {} objects_lock = threading.Lock() + +objects_queue = queue.Queue() diff --git a/minode/tests/test_structure.py b/minode/tests/test_structure.py index 970c152..c461251 100644 --- a/minode/tests/test_structure.py +++ b/minode/tests/test_structure.py @@ -143,8 +143,7 @@ class TestStructure(unittest.TestCase): obj = structure.Object( b'\x00' * 8, int(time.time() + 300), 42, 1, 2, b'HELLO') vector = obj.vector - proofofwork._worker(obj) # pylint: disable=protected-access - obj = shared.objects.popitem()[1] + obj = proofofwork.Worker().add_pow(obj) self.assertNotEqual(obj.vector, vector) self.assertFalse(obj.is_expired()) self.assertFalse(obj.is_valid()) diff --git a/setup.py b/setup.py index 7a40bd5..4f72204 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,9 @@ #!/usr/bin/env python import os +import sys -from setuptools import setup, find_packages +from setuptools import Extension, find_packages, setup from minode import shared @@ -12,6 +13,19 @@ README = open(os.path.join( name, version = shared.user_agent.strip(b'/').split(b':') +bitmsghash = Extension( + 'minode.bitmsghash', + sources=['minode/bitmsghash/bitmsghash.cpp'], + libraries=['pthread', 'crypto'], +) + +if sys.platform[:3] == 'win': + bitmsghash.libraries = ['libeay32', 'ws2_32'] + openssl_dir = os.getenv('OPENSSL_DIR') + if openssl_dir: + bitmsghash.library_dirs = [os.path.join(openssl_dir, 'lib')] + bitmsghash.include_dirs = [os.path.join(openssl_dir, 'include')] + setup( name=name.decode('utf-8'), version=version.decode('utf-8'), @@ -23,6 +37,7 @@ setup( url='https://git.bitmessage.org/lee.miller/MiNode', packages=find_packages(exclude=('*tests',)), package_data={'': ['*.csv', 'tls/*.pem']}, + ext_modules=[bitmsghash], entry_points={'console_scripts': ['minode = minode.main:main']}, classifiers=[ "License :: OSI Approved :: MIT License" diff --git a/tox.ini b/tox.ini index caeb4c6..dc82b8f 100644 --- a/tox.ini +++ b/tox.ini @@ -12,6 +12,9 @@ deps = flake8 commands = flake8 minode --count --select=E9,F63,F7,F82 --show-source --statistics +[testenv:py39] +commands_pre = pip install -e . + [testenv:reset] deps = -rrequirements.txt