2019-09-06 12:28:06 +02:00
|
|
|
"""
|
|
|
|
src/network/objectracker.py
|
|
|
|
===========================
|
|
|
|
"""
|
2017-05-24 16:51:49 +02:00
|
|
|
import time
|
2017-06-02 07:09:35 +02:00
|
|
|
from threading import RLock
|
2017-05-24 16:51:49 +02:00
|
|
|
|
2018-02-03 11:46:39 +01:00
|
|
|
import network.connectionpool
|
2017-10-20 01:21:49 +02:00
|
|
|
from network.dandelion import Dandelion
|
2018-02-01 12:20:41 +01:00
|
|
|
from randomtrackingdict import RandomTrackingDict
|
2017-05-24 16:51:49 +02:00
|
|
|
|
|
|
|
haveBloom = False
|
|
|
|
|
|
|
|
try:
|
|
|
|
# pybloomfiltermmap
|
|
|
|
from pybloomfilter import BloomFilter
|
|
|
|
haveBloom = True
|
|
|
|
except ImportError:
|
|
|
|
try:
|
|
|
|
# pybloom
|
|
|
|
from pybloom import BloomFilter
|
|
|
|
haveBloom = True
|
|
|
|
except ImportError:
|
|
|
|
pass
|
|
|
|
|
|
|
|
# it isn't actually implemented yet so no point in turning it on
|
|
|
|
haveBloom = False
|
|
|
|
|
2018-12-10 13:33:07 +01:00
|
|
|
# tracking pending downloads globally, for stats
|
|
|
|
missingObjects = {}
|
|
|
|
|
|
|
|
|
2017-05-27 19:09:21 +02:00
|
|
|
class ObjectTracker(object):
|
2019-09-06 12:28:06 +02:00
|
|
|
"""Object tracker mixin"""
|
2017-05-24 16:51:49 +02:00
|
|
|
invCleanPeriod = 300
|
|
|
|
invInitialCapacity = 50000
|
|
|
|
invErrorRate = 0.03
|
2017-07-05 09:25:49 +02:00
|
|
|
trackingExpires = 3600
|
2018-01-02 22:20:33 +01:00
|
|
|
initialTimeOffset = 60
|
2017-05-24 16:51:49 +02:00
|
|
|
|
|
|
|
def __init__(self):
|
2018-02-01 12:19:39 +01:00
|
|
|
self.objectsNewToMe = RandomTrackingDict()
|
2017-05-24 16:51:49 +02:00
|
|
|
self.objectsNewToThem = {}
|
2017-06-02 07:09:35 +02:00
|
|
|
self.objectsNewToThemLock = RLock()
|
2017-05-24 16:51:49 +02:00
|
|
|
self.initInvBloom()
|
|
|
|
self.initAddrBloom()
|
2017-06-02 07:09:35 +02:00
|
|
|
self.lastCleaned = time.time()
|
2017-05-24 16:51:49 +02:00
|
|
|
|
|
|
|
def initInvBloom(self):
|
2019-09-06 12:28:06 +02:00
|
|
|
"""Init bloom filter for tracking. WIP."""
|
2017-05-24 16:51:49 +02:00
|
|
|
if haveBloom:
|
|
|
|
# lock?
|
2017-05-27 19:09:21 +02:00
|
|
|
self.invBloom = BloomFilter(capacity=ObjectTracker.invInitialCapacity,
|
|
|
|
error_rate=ObjectTracker.invErrorRate)
|
2017-05-24 16:51:49 +02:00
|
|
|
|
|
|
|
def initAddrBloom(self):
|
2019-09-06 12:28:06 +02:00
|
|
|
"""Init bloom filter for tracking addrs, WIP. This either needs to be moved to addrthread.py or removed."""
|
2017-05-24 16:51:49 +02:00
|
|
|
if haveBloom:
|
|
|
|
# lock?
|
2017-05-27 19:09:21 +02:00
|
|
|
self.addrBloom = BloomFilter(capacity=ObjectTracker.invInitialCapacity,
|
|
|
|
error_rate=ObjectTracker.invErrorRate)
|
2017-05-24 16:51:49 +02:00
|
|
|
|
|
|
|
def clean(self):
|
2019-09-06 12:28:06 +02:00
|
|
|
"""Clean up tracking to prevent memory bloat"""
|
2017-06-02 07:09:35 +02:00
|
|
|
if self.lastCleaned < time.time() - ObjectTracker.invCleanPeriod:
|
2017-05-24 16:51:49 +02:00
|
|
|
if haveBloom:
|
2019-09-06 12:28:06 +02:00
|
|
|
if missingObjects == 0:
|
2017-05-24 16:51:49 +02:00
|
|
|
self.initInvBloom()
|
|
|
|
self.initAddrBloom()
|
2017-06-02 07:09:35 +02:00
|
|
|
else:
|
|
|
|
# release memory
|
2017-07-05 09:25:49 +02:00
|
|
|
deadline = time.time() - ObjectTracker.trackingExpires
|
2017-06-02 07:09:35 +02:00
|
|
|
with self.objectsNewToThemLock:
|
2017-07-05 09:25:49 +02:00
|
|
|
self.objectsNewToThem = {k: v for k, v in self.objectsNewToThem.iteritems() if v >= deadline}
|
2017-06-02 07:09:35 +02:00
|
|
|
self.lastCleaned = time.time()
|
2017-05-24 16:51:49 +02:00
|
|
|
|
|
|
|
def hasObj(self, hashid):
|
2019-09-06 12:28:06 +02:00
|
|
|
"""Do we already have object?"""
|
2017-05-24 16:51:49 +02:00
|
|
|
if haveBloom:
|
|
|
|
return hashid in self.invBloom
|
2019-09-06 12:28:06 +02:00
|
|
|
return hashid in self.objectsNewToMe
|
2017-05-24 16:51:49 +02:00
|
|
|
|
2017-05-27 19:09:21 +02:00
|
|
|
def handleReceivedInventory(self, hashId):
|
2019-09-06 12:28:06 +02:00
|
|
|
"""Handling received inventory"""
|
2017-05-24 16:51:49 +02:00
|
|
|
if haveBloom:
|
2017-05-27 19:09:21 +02:00
|
|
|
self.invBloom.add(hashId)
|
2017-06-21 12:16:33 +02:00
|
|
|
try:
|
|
|
|
with self.objectsNewToThemLock:
|
|
|
|
del self.objectsNewToThem[hashId]
|
|
|
|
except KeyError:
|
|
|
|
pass
|
2018-02-03 11:46:39 +01:00
|
|
|
if hashId not in missingObjects:
|
|
|
|
missingObjects[hashId] = time.time()
|
|
|
|
self.objectsNewToMe[hashId] = True
|
|
|
|
|
|
|
|
def handleReceivedObject(self, streamNumber, hashid):
|
2019-09-06 12:28:06 +02:00
|
|
|
"""Handling received object"""
|
2019-11-03 13:09:00 +01:00
|
|
|
for i in network.connectionpool.BMConnectionPool().connections():
|
2018-02-03 11:46:39 +01:00
|
|
|
if not i.fullyEstablished:
|
|
|
|
continue
|
|
|
|
try:
|
|
|
|
del i.objectsNewToMe[hashid]
|
|
|
|
except KeyError:
|
2019-09-05 16:39:11 +02:00
|
|
|
if streamNumber in i.streams and (
|
|
|
|
not Dandelion().hasHash(hashid) or Dandelion().objectChildStem(hashid) == i):
|
2018-02-03 11:46:39 +01:00
|
|
|
with i.objectsNewToThemLock:
|
|
|
|
i.objectsNewToThem[hashid] = time.time()
|
|
|
|
# update stream number, which we didn't have when we just received the dinv
|
|
|
|
# also resets expiration of the stem mode
|
|
|
|
Dandelion().setHashStream(hashid, streamNumber)
|
|
|
|
|
|
|
|
if i == self:
|
|
|
|
try:
|
|
|
|
with i.objectsNewToThemLock:
|
|
|
|
del i.objectsNewToThem[hashid]
|
|
|
|
except KeyError:
|
|
|
|
pass
|
2018-04-02 19:33:41 +02:00
|
|
|
self.objectsNewToMe.setLastObject()
|
2017-05-24 16:51:49 +02:00
|
|
|
|
|
|
|
def hasAddr(self, addr):
|
2019-09-06 12:28:06 +02:00
|
|
|
"""WIP, should be moved to addrthread.py or removed"""
|
2017-05-24 16:51:49 +02:00
|
|
|
if haveBloom:
|
|
|
|
return addr in self.invBloom
|
2019-09-06 12:28:06 +02:00
|
|
|
return None
|
2017-05-24 16:51:49 +02:00
|
|
|
|
|
|
|
def addAddr(self, hashid):
|
2019-09-06 12:28:06 +02:00
|
|
|
"""WIP, should be moved to addrthread.py or removed"""
|
2017-05-24 16:51:49 +02:00
|
|
|
if haveBloom:
|
|
|
|
self.addrBloom.add(hashid)
|