From da0b771e358fdc2539c9537c7f94871c78ffdd0b Mon Sep 17 00:00:00 2001 From: Arceliar Date: Thu, 30 May 2013 17:38:02 +0200 Subject: [PATCH 1/7] Workers get idle priority. --- src/proofofwork.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/proofofwork.py b/src/proofofwork.py index 03d7c22d..d8140707 100644 --- a/src/proofofwork.py +++ b/src/proofofwork.py @@ -1,6 +1,21 @@ +def _set_idle(): + import sys + try: + sys.getwindowsversion() + import win32api,win32process,win32con + pid = win32api.GetCurrentProcessId() + handle = win32api.OpenProcess(win32con.PROCESS_ALL_ACCESS, True, pid) + win32process.SetPriorityClass(handle, win32process.IDLE_PRIORITY_CLASS) + except: + import os + os.nice(20) + def _pool_worker(nonce, initialHash, target, pool_size): import hashlib + import sys + import os from struct import unpack, pack + _set_idle() trialValue = 99999999999999999999 while trialValue > target: nonce += pool_size @@ -8,7 +23,7 @@ def _pool_worker(nonce, initialHash, target, pool_size): return [trialValue, nonce] def run(target, initialHash): - from multiprocessing import Pool, cpu_count + from multiprocessing import Pool, cpu_count, Value import time try: pool_size = cpu_count() @@ -25,4 +40,3 @@ def run(target, initialHash): pool.terminate() return result[0], result[1] time.sleep(1) - From 74b94e757198786b586aded87cc81e7213c6cfeb Mon Sep 17 00:00:00 2001 From: Arceliar Date: Thu, 30 May 2013 17:41:08 +0200 Subject: [PATCH 2/7] Removed extraneous import. --- src/proofofwork.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/proofofwork.py b/src/proofofwork.py index d8140707..cb20e667 100644 --- a/src/proofofwork.py +++ b/src/proofofwork.py @@ -23,7 +23,7 @@ def _pool_worker(nonce, initialHash, target, pool_size): return [trialValue, nonce] def run(target, initialHash): - from multiprocessing import Pool, cpu_count, Value + from multiprocessing import Pool, cpu_count import time try: pool_size = cpu_count() From 2ae1ef0d505fb18156bbc5ca11e25ae922cd0383 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Thu, 30 May 2013 19:54:27 +0200 Subject: [PATCH 3/7] Messy hack to kill busy child processes on exit. --- src/shared.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/shared.py b/src/shared.py index 850c84f3..2edaf616 100644 --- a/src/shared.py +++ b/src/shared.py @@ -7,6 +7,7 @@ import highlevelcrypto import Queue import pickle import os +import signal myECCryptorObjects = {} MyECSubscriptionCryptorObjects = {} @@ -173,12 +174,18 @@ def doCleanShutdown(): print 'Finished flushing inventory.' printLock.release() - + if safeConfigGetBoolean('bitmessagesettings','daemon'): printLock.acquire() print 'Done.' printLock.release() os._exit(0) + + #Messy hack to kill child processes immediately. May not work on all platforms. + try: + os.killpg(os.getpgid(os.getpid()), signal.SIGTERM) + except: + os.kill(os.getpid(), signal.SIGTERM) #Wen you want to command a sendDataThread to do something, like shutdown or send some data, this function puts your data into the queues for each of the sendDataThreads. The sendDataThreads are responsible for putting their queue into (and out of) the sendDataQueues list. def broadcastToSendDataQueues(data): @@ -197,4 +204,4 @@ def flushInventory(): sqlReturnQueue.get() del inventory[hash] sqlSubmitQueue.put('commit') - sqlLock.release() \ No newline at end of file + sqlLock.release() From 2762de52e261317acf91dcbf4117467db40898bc Mon Sep 17 00:00:00 2001 From: Arceliar Date: Thu, 30 May 2013 19:58:05 +0200 Subject: [PATCH 4/7] Moved workaround code slightly. --- src/shared.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/shared.py b/src/shared.py index 2edaf616..9ca07e69 100644 --- a/src/shared.py +++ b/src/shared.py @@ -173,20 +173,19 @@ def doCleanShutdown(): printLock.acquire() print 'Finished flushing inventory.' printLock.release() - - - if safeConfigGetBoolean('bitmessagesettings','daemon'): - printLock.acquire() - print 'Done.' - printLock.release() - os._exit(0) - + #Messy hack to kill child processes immediately. May not work on all platforms. try: os.killpg(os.getpgid(os.getpid()), signal.SIGTERM) except: os.kill(os.getpid(), signal.SIGTERM) + if safeConfigGetBoolean('bitmessagesettings','daemon'): + printLock.acquire() + print 'Done.' + printLock.release() + os._exit(0) + #Wen you want to command a sendDataThread to do something, like shutdown or send some data, this function puts your data into the queues for each of the sendDataThreads. The sendDataThreads are responsible for putting their queue into (and out of) the sendDataQueues list. def broadcastToSendDataQueues(data): #print 'running broadcastToSendDataQueues' From 279f38ff97508297a69956785ee5f4311e05227e Mon Sep 17 00:00:00 2001 From: Arceliar Date: Wed, 5 Jun 2013 23:20:34 +0200 Subject: [PATCH 5/7] Trying to enable multicore PoW with single core as a fallback when it's a problem. --- src/proofofwork.py | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/src/proofofwork.py b/src/proofofwork.py index 5fd7e43a..062fada6 100644 --- a/src/proofofwork.py +++ b/src/proofofwork.py @@ -3,23 +3,23 @@ #from multiprocessing import Pool, cpu_count import hashlib from struct import unpack, pack -#import sys +import sys #import os def _set_idle(): - try: + if 'linux' in sys.platform: + import os + os.nice(20) + else: + try: sys.getwindowsversion() import win32api,win32process,win32con pid = win32api.GetCurrentProcessId() handle = win32api.OpenProcess(win32con.PROCESS_ALL_ACCESS, True, pid) win32process.SetPriorityClass(handle, win32process.IDLE_PRIORITY_CLASS) - except: - try: - #Linux - os.nice(20) - except: - #Windows 64-bit - pass + except: + #Windows 64-bit + pass def _pool_worker(nonce, initialHash, target, pool_size): _set_idle() @@ -29,25 +29,29 @@ def _pool_worker(nonce, initialHash, target, pool_size): trialValue, = unpack('>Q',hashlib.sha512(hashlib.sha512(pack('>Q',nonce) + initialHash).digest()).digest()[0:8]) return [trialValue, nonce] -def run(target, initialHash): +def _doSafePoW(target, initialHash): nonce = 0 trialValue = 99999999999999999999 while trialValue > target: nonce += 1 trialValue, = unpack('>Q',hashlib.sha512(hashlib.sha512(pack('>Q',nonce) + initialHash).digest()).digest()[0:8]) return [trialValue, nonce] - """try: + +def _doFastPoW(target, initialHash): + import shared + import time + from multiprocessing import Pool, cpu_count + import os + try: pool_size = cpu_count() except: pool_size = 4 - try: maxCores = config.getint('bitmessagesettings', 'maxcores') except: maxCores = 99999 if pool_size > maxCores: pool_size = maxCores - pool = Pool(processes=pool_size) result = [] for i in range(pool_size): @@ -55,12 +59,17 @@ def run(target, initialHash): while True: if shared.shutdown: pool.terminate() - time.sleep(5) #Don't return anything (doing so will cause exceptions because we'll return an unusable response). Sit here and wait for this thread to close. + pool.join() #Don't return anything (doing so will cause exceptions because we'll return an unusable response). Sit here and wait for this thread to close. return for i in range(pool_size): if result[i].ready(): result = result[i].get() pool.terminate() return result[0], result[1] - time.sleep(0.2)""" + time.sleep(0.2) +def run(target, initialHash): + if linux in sys.platform: + return _doFastPoW(target, initialHash) + else: + return _doSafePoW(target, initialHash) From 7f4fee40fbb5ed42c9b17597a82a6002eb9ef672 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Wed, 5 Jun 2013 23:29:40 +0200 Subject: [PATCH 6/7] Removed an extraneous conditional import. --- src/proofofwork.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/proofofwork.py b/src/proofofwork.py index 062fada6..36c5563e 100644 --- a/src/proofofwork.py +++ b/src/proofofwork.py @@ -41,7 +41,6 @@ def _doFastPoW(target, initialHash): import shared import time from multiprocessing import Pool, cpu_count - import os try: pool_size = cpu_count() except: @@ -69,7 +68,7 @@ def _doFastPoW(target, initialHash): time.sleep(0.2) def run(target, initialHash): - if linux in sys.platform: + if 'linux' in sys.platform: return _doFastPoW(target, initialHash) else: return _doSafePoW(target, initialHash) From 30d829d0d476c73d3663749a1fb7529300791c0a Mon Sep 17 00:00:00 2001 From: Arceliar Date: Thu, 6 Jun 2013 19:30:57 +0200 Subject: [PATCH 7/7] Wait for PoW processes to exit before returning a result. Hopefully not needed. --- src/proofofwork.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/proofofwork.py b/src/proofofwork.py index 36c5563e..058e4bf3 100644 --- a/src/proofofwork.py +++ b/src/proofofwork.py @@ -64,6 +64,7 @@ def _doFastPoW(target, initialHash): if result[i].ready(): result = result[i].get() pool.terminate() + pool.join() #Wait for the workers to exit... return result[0], result[1] time.sleep(0.2)