Object processor queue class

Previous commit didn't include the class. This class takes care of queue
size monitoring so that the system doesn't run out of memory.
This commit is contained in:
Peter Šurda 2016-01-22 13:49:28 +01:00
parent e781420f4d
commit fe0106bdce
Signed by untrusted user: PeterSurda
GPG Key ID: 0C5F50C0B5F37D87

View File

@ -0,0 +1,28 @@
import shared
import Queue
import threading
import time
class ObjectProcessorQueue(Queue.Queue):
maxSize = 32000000
def __init__(self):
Queue.Queue.__init__(self)
self.sizeLock = threading.Lock()
self.curSize = 0 # in Bytes. We maintain this to prevent nodes from flooing us with objects which take up too much memory. If this gets too big we'll sleep before asking for further objects.
def put(self, item, block = True, timeout = None):
while self.curSize >= self.maxSize and not shared.shutdown:
time.sleep(1)
if shared.shutdown:
return
with self.sizeLock:
self.curSize += len(item[1])
Queue.Queue.put(self, item, block, timeout)
def get(self, block = True, timeout = None):
item = Queue.Queue.get(self, block, timeout)
with self.sizeLock:
self.curSize -= len(item[1])
return item