Peter Surda
ca7becb921
- in corner cases, download request could have contained an incorrect request length. I haven't actually checked if this can be triggered though - wait for downloading until anti intersection delay expires. Doesn't necessarily mean that it will always avoid peer's anti intersection delay, but it's close enough - tracks last time an object was received. If it was too long time ago, reset the download request queue. This avoid situations like when a request gets ignored during the anti intersection delay, but it will keep thinking there are still pending requests as long as not all missing objects have been requested. This caused staggered download (request 1000 items, wait 1 minute, request 1000 more, wait another minute, ...) - with these fixes, you should end up downloading as fast as your network and CPU allow - best tested with trustedpeer
141 lines
4.2 KiB
Python
141 lines
4.2 KiB
Python
import random
|
|
from threading import RLock
|
|
from time import time
|
|
import helper_random
|
|
|
|
class RandomTrackingDict(object):
|
|
maxPending = 10
|
|
pendingTimeout = 60
|
|
def __init__(self): # O(1)
|
|
self.dictionary = {}
|
|
self.indexDict = []
|
|
self.len = 0
|
|
self.pendingLen = 0
|
|
self.lastPoll = 0
|
|
self.lastObject = 0
|
|
self.lock = RLock()
|
|
|
|
def __len__(self):
|
|
return self.len
|
|
|
|
def __contains__(self, key):
|
|
return key in self.dictionary
|
|
|
|
def __getitem__(self, key):
|
|
return self.dictionary[key][1]
|
|
|
|
def _swap(self, i1, i2):
|
|
with self.lock:
|
|
key1 = self.indexDict[i1]
|
|
key2 = self.indexDict[i2]
|
|
self.indexDict[i1] = key2
|
|
self.indexDict[i2] = key1
|
|
self.dictionary[key1][0] = i2
|
|
self.dictionary[key2][0] = i1
|
|
# for quick reassignment
|
|
return i2
|
|
|
|
def __setitem__(self, key, value):
|
|
with self.lock:
|
|
if key in self.dictionary:
|
|
self.dictionary[key][1] = value
|
|
else:
|
|
self.indexDict.append(key)
|
|
self.dictionary[key] = [self.len, value]
|
|
self._swap(self.len, self.len - self.pendingLen)
|
|
self.len += 1
|
|
|
|
def __delitem__(self, key):
|
|
if not key in self.dictionary:
|
|
raise KeyError
|
|
with self.lock:
|
|
index = self.dictionary[key][0]
|
|
# not pending
|
|
if index < self.len - self.pendingLen:
|
|
# left of pending part
|
|
index = self._swap(index, self.len - self.pendingLen - 1)
|
|
# pending
|
|
else:
|
|
self.pendingLen -= 1
|
|
# end
|
|
self._swap(index, self.len - 1)
|
|
# if the following del is batched, performance of this single
|
|
# operation can improve 4x, but it's already very fast so we'll
|
|
# ignore it for the time being
|
|
del self.indexDict[-1]
|
|
del self.dictionary[key]
|
|
self.len -= 1
|
|
|
|
def setMaxPending(self, maxPending):
|
|
self.maxPending = maxPending
|
|
|
|
def setPendingTimeout(self, pendingTimeout):
|
|
self.pendingTimeout = pendingTimeout
|
|
|
|
def setLastObject(self):
|
|
self.lastObject = time()
|
|
|
|
def randomKeys(self, count=1):
|
|
if self.len == 0 or ((self.pendingLen >= self.maxPending or
|
|
self.pendingLen == self.len) and self.lastPoll +
|
|
self.pendingTimeout > time()):
|
|
raise KeyError
|
|
with self.lock:
|
|
# reset if we've requested all
|
|
# or if last object received too long time ago
|
|
if self.pendingLen == self.len or self.lastObject + self.pendingTimeout > time():
|
|
self.pendingLen = 0
|
|
available = self.len - self.pendingLen
|
|
if count > available:
|
|
count = available
|
|
randomIndex = helper_random.randomsample(range(self.len - self.pendingLen), count)
|
|
retval = [self.indexDict[i] for i in randomIndex]
|
|
|
|
for i in sorted(randomIndex, reverse=True):
|
|
# swap with one below lowest pending
|
|
self._swap(i, self.len - self.pendingLen - 1)
|
|
self.pendingLen += 1
|
|
self.lastPoll = time()
|
|
return retval
|
|
|
|
if __name__ == '__main__':
|
|
def randString():
|
|
retval = b''
|
|
for _ in range(32):
|
|
retval += chr(random.randint(0,255))
|
|
return retval
|
|
|
|
a = []
|
|
k = RandomTrackingDict()
|
|
d = {}
|
|
|
|
# print "populating normal dict"
|
|
# a.append(time())
|
|
# for i in range(50000):
|
|
# d[randString()] = True
|
|
# a.append(time())
|
|
print "populating random tracking dict"
|
|
a.append(time())
|
|
for i in range(50000):
|
|
k[randString()] = True
|
|
a.append(time())
|
|
print "done"
|
|
while len(k) > 0:
|
|
retval = k.randomKeys(1000)
|
|
if not retval:
|
|
print "error getting random keys"
|
|
#a.append(time())
|
|
try:
|
|
k.randomKeys(100)
|
|
print "bad"
|
|
except KeyError:
|
|
pass
|
|
#a.append(time())
|
|
for i in retval:
|
|
del k[i]
|
|
#a.append(time())
|
|
a.append(time())
|
|
|
|
for x in range(len(a) - 1):
|
|
print "%i: %.3f" % (x, a[x+1] - a[x])
|