Allow multiple ReceiveQueue threads

- defaults to 3
This commit is contained in:
Peter Šurda 2017-07-10 07:08:10 +02:00
parent f088e0ae21
commit bdf61489ae
Signed by untrusted user: PeterSurda
GPG Key ID: 0C5F50C0B5F37D87
5 changed files with 32 additions and 10 deletions

View File

@ -267,9 +267,10 @@ class Main:
asyncoreThread = BMNetworkThread() asyncoreThread = BMNetworkThread()
asyncoreThread.daemon = True asyncoreThread.daemon = True
asyncoreThread.start() asyncoreThread.start()
receiveQueueThread = ReceiveQueueThread() for i in range(BMConfigParser().getint("threads", "receive")):
receiveQueueThread.daemon = True receiveQueueThread = ReceiveQueueThread(i)
receiveQueueThread.start() receiveQueueThread.daemon = True
receiveQueueThread.start()
announceThread = AnnounceThread() announceThread = AnnounceThread()
announceThread.daemon = True announceThread.daemon = True
announceThread.start() announceThread.start()

View File

@ -15,6 +15,9 @@ BMConfigDefaults = {
"maxtotalconnections": 200, "maxtotalconnections": 200,
"maxuploadrate": 0, "maxuploadrate": 0,
}, },
"threads": {
"receive": 3,
},
"network": { "network": {
"asyncore": True, "asyncore": True,
"bind": None, "bind": None,

View File

@ -1,4 +1,6 @@
from contextlib import contextmanager
import threading import threading
try: try:
import prctl import prctl
def set_thread_name(name): prctl.set_name(name) def set_thread_name(name): prctl.set_name(name)
@ -20,3 +22,16 @@ class StoppableThread(object):
def stopThread(self): def stopThread(self):
self._stopped = True self._stopped = True
self.stop.set() self.stop.set()
class BusyError(threading.ThreadError):
pass
@contextmanager
def nonBlocking(lock):
locked = lock.acquire(False)
if not locked:
raise BusyError
try:
yield
finally:
lock.release()

View File

@ -6,6 +6,7 @@ import time
import asyncore_pollchoose as asyncore import asyncore_pollchoose as asyncore
from debug import logger from debug import logger
from helper_threading import BusyError, nonBlocking
class AdvancedDispatcher(asyncore.dispatcher): class AdvancedDispatcher(asyncore.dispatcher):
_buf_len = 2097152 # 2MB _buf_len = 2097152 # 2MB
@ -22,6 +23,7 @@ class AdvancedDispatcher(asyncore.dispatcher):
self.expectBytes = 0 self.expectBytes = 0
self.readLock = threading.RLock() self.readLock = threading.RLock()
self.writeLock = threading.RLock() self.writeLock = threading.RLock()
self.processingLock = threading.RLock()
def append_write_buf(self, data): def append_write_buf(self, data):
if data: if data:
@ -43,8 +45,9 @@ class AdvancedDispatcher(asyncore.dispatcher):
return False return False
while len(self.read_buf) >= self.expectBytes: while len(self.read_buf) >= self.expectBytes:
try: try:
if getattr(self, "state_" + str(self.state))() is False: with nonBlocking(self.processingLock):
break if getattr(self, "state_" + str(self.state))() is False:
break
except AttributeError: except AttributeError:
raise raise
return False return False

View File

@ -15,17 +15,16 @@ import protocol
import state import state
class ReceiveQueueThread(threading.Thread, StoppableThread): class ReceiveQueueThread(threading.Thread, StoppableThread):
def __init__(self): def __init__(self, num=0):
threading.Thread.__init__(self, name="ReceiveQueueThread") threading.Thread.__init__(self, name="ReceiveQueue_%i" %(num))
self.initStop() self.initStop()
self.name = "ReceiveQueueThread" self.name = "ReceiveQueue_%i" % (num)
logger.info("init receive queue thread") logger.info("init receive queue thread %i", num)
def run(self): def run(self):
while not self._stopped and state.shutdown == 0: while not self._stopped and state.shutdown == 0:
try: try:
dest = receiveDataQueue.get(block=True, timeout=1) dest = receiveDataQueue.get(block=True, timeout=1)
receiveDataQueue.task_done()
except Queue.Empty: except Queue.Empty:
continue continue
@ -44,3 +43,4 @@ class ReceiveQueueThread(threading.Thread, StoppableThread):
# AttributeError = state isn't implemented # AttributeError = state isn't implemented
except (KeyError, AttributeError): except (KeyError, AttributeError):
pass pass
receiveDataQueue.task_done()