2019-08-18 04:24:26 +00:00
from queue import Queue
2016-01-22 12:49:28 +00:00
import threading
import time
2019-08-18 04:24:26 +00:00
class ObjectProcessorQueue ( Queue ) :
2016-01-22 12:49:28 +00:00
maxSize = 32000000
def __init__ ( self ) :
2019-08-18 04:24:26 +00:00
Queue . __init__ ( self )
2016-01-22 12:49:28 +00:00
self . sizeLock = threading . Lock ( )
self . curSize = 0 # in Bytes. We maintain this to prevent nodes from flooing us with objects which take up too much memory. If this gets too big we'll sleep before asking for further objects.
def put ( self , item , block = True , timeout = None ) :
2016-01-22 13:12:57 +00:00
while self . curSize > = self . maxSize :
2016-01-22 12:49:28 +00:00
time . sleep ( 1 )
with self . sizeLock :
self . curSize + = len ( item [ 1 ] )
2019-08-18 04:24:26 +00:00
Queue . put ( self , item , block , timeout )
2016-01-22 12:49:28 +00:00
def get ( self , block = True , timeout = None ) :
2019-08-18 04:24:26 +00:00
item = Queue . get ( self , block , timeout )
2016-01-22 12:49:28 +00:00
with self . sizeLock :
self . curSize - = len ( item [ 1 ] )
return item