Asyncore performance optimisation
- use bytearray instead of strings for buffers
This commit is contained in:
parent
1eb0dd6f01
commit
4c9006a632
|
@ -12,8 +12,8 @@ class AdvancedDispatcher(asyncore.dispatcher):
|
|||
def __init__(self, sock=None):
|
||||
if not hasattr(self, '_map'):
|
||||
asyncore.dispatcher.__init__(self, sock)
|
||||
self.read_buf = b""
|
||||
self.write_buf = b""
|
||||
self.read_buf = bytearray()
|
||||
self.write_buf = bytearray()
|
||||
self.state = "init"
|
||||
self.lastTx = time.time()
|
||||
self.sentBytes = 0
|
||||
|
@ -25,18 +25,23 @@ class AdvancedDispatcher(asyncore.dispatcher):
|
|||
|
||||
def append_write_buf(self, data):
|
||||
if data:
|
||||
if isinstance(data, list):
|
||||
with self.writeLock:
|
||||
self.write_buf += data
|
||||
for chunk in data:
|
||||
self.write_buf.extend(chunk)
|
||||
else:
|
||||
with self.writeLock:
|
||||
self.write_buf.extend(data)
|
||||
|
||||
def slice_write_buf(self, length=0):
|
||||
if length > 0:
|
||||
with self.writeLock:
|
||||
self.write_buf = self.write_buf[length:]
|
||||
del self.write_buf[0:length]
|
||||
|
||||
def slice_read_buf(self, length=0):
|
||||
if length > 0:
|
||||
with self.readLock:
|
||||
self.read_buf = self.read_buf[length:]
|
||||
del self.read_buf[0:length]
|
||||
|
||||
def process(self):
|
||||
if not self.connected:
|
||||
|
@ -85,7 +90,7 @@ class AdvancedDispatcher(asyncore.dispatcher):
|
|||
self.receivedBytes += len(newData)
|
||||
asyncore.update_received(len(newData))
|
||||
with self.readLock:
|
||||
self.read_buf += newData
|
||||
self.read_buf.extend(newData)
|
||||
|
||||
def handle_write(self):
|
||||
self.lastTx = time.time()
|
||||
|
@ -108,7 +113,9 @@ class AdvancedDispatcher(asyncore.dispatcher):
|
|||
return False
|
||||
|
||||
def handle_close(self):
|
||||
self.read_buf = b""
|
||||
self.write_buf = b""
|
||||
with self.readLock:
|
||||
self.read_buf = bytearray()
|
||||
with self.writeLock:
|
||||
self.write_buf = bytearray()
|
||||
self.state = "close"
|
||||
asyncore.dispatcher.close(self)
|
||||
|
|
|
@ -138,16 +138,16 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
|
|||
def decode_payload_node(self):
|
||||
services, host, port = self.decode_payload_content("Q16sH")
|
||||
if host[0:12] == '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xFF\xFF':
|
||||
host = socket.inet_ntop(socket.AF_INET, host[12:])
|
||||
host = socket.inet_ntop(socket.AF_INET, buffer(host, 12, 4))
|
||||
elif host[0:6] == '\xfd\x87\xd8\x7e\xeb\x43':
|
||||
# Onion, based on BMD/bitcoind
|
||||
host = base64.b32encode(host[6:]).lower() + ".onion"
|
||||
else:
|
||||
host = socket.inet_ntop(socket.AF_INET6, host)
|
||||
host = socket.inet_ntop(socket.AF_INET6, buffer(host))
|
||||
if host == "":
|
||||
# This can happen on Windows systems which are not 64-bit compatible
|
||||
# so let us drop the IPv6 address.
|
||||
host = socket.inet_ntop(socket.AF_INET, host[12:])
|
||||
host = socket.inet_ntop(socket.AF_INET, buffer(host, 12, 4))
|
||||
|
||||
return Node(services, host, port)
|
||||
|
||||
|
@ -272,7 +272,7 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
|
|||
return True
|
||||
#TODO make this more asynchronous
|
||||
random.shuffle(items)
|
||||
for i in items:
|
||||
for i in map(str, items):
|
||||
if i in DandelionStems().stem and \
|
||||
self != DandelionStems().stem[i]:
|
||||
self.antiIntersectionDelay()
|
||||
|
@ -298,7 +298,7 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
|
|||
else:
|
||||
pass
|
||||
|
||||
for i in items:
|
||||
for i in map(str, items):
|
||||
self.handleReceivedInventory(i)
|
||||
|
||||
return True
|
||||
|
@ -323,7 +323,7 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
|
|||
self.dandelionRoutes = BMConnectionPool.dandelionRouteSelector(self)
|
||||
self.dandelionRefresh = time.time() + REASSIGN_INTERVAL
|
||||
|
||||
for i in items:
|
||||
for i in map(str, items):
|
||||
DandelionStems().add(i, self, self.dandelionRoutes)
|
||||
self.handleReceivedInventory(i)
|
||||
|
||||
|
@ -354,12 +354,12 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
|
|||
|
||||
try:
|
||||
self.object.checkObjectByType()
|
||||
objectProcessorQueue.put((self.object.objectType, self.object.data))
|
||||
objectProcessorQueue.put((self.object.objectType, buffer(self.object.data)))
|
||||
except BMObjectInvalidError as e:
|
||||
BMProto.stopDownloadingObject(self.object.inventoryHash, True)
|
||||
|
||||
Inventory()[self.object.inventoryHash] = (
|
||||
self.object.objectType, self.object.streamNumber, self.payload[objectOffset:], self.object.expiresTime, self.object.tag)
|
||||
self.object.objectType, self.object.streamNumber, buffer(self.payload[objectOffset:]), self.object.expiresTime, buffer(self.object.tag))
|
||||
invQueue.put((self.object.streamNumber, self.object.inventoryHash, self.destination))
|
||||
return True
|
||||
|
||||
|
@ -370,7 +370,7 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
|
|||
addresses = self._decode_addr()
|
||||
for i in addresses:
|
||||
seenTime, stream, services, ip, port = i
|
||||
decodedIP = protocol.checkIPAddress(ip)
|
||||
decodedIP = protocol.checkIPAddress(buffer(ip))
|
||||
if stream not in state.streamsInWhichIAmParticipating:
|
||||
continue
|
||||
if decodedIP is not False and seenTime > time.time() - BMProto.addressAlive:
|
||||
|
|
|
@ -55,7 +55,10 @@ class DownloadThread(threading.Thread, StoppableThread):
|
|||
i.objectsNewToMe[k] = False
|
||||
self.pending[k] = now
|
||||
|
||||
payload = addresses.encodeVarint(len(request)) + ''.join(request)
|
||||
payload = bytearray()
|
||||
payload.extend(addresses.encodeVarint(len(request)))
|
||||
for chunk in request:
|
||||
payload.extend(chunk)
|
||||
i.append_write_buf(protocol.CreatePacket('getdata', payload))
|
||||
logger.debug("%s:%i Requesting %i objects", i.destination.host, i.destination.port, len(request))
|
||||
requested += len(request)
|
||||
|
|
|
@ -89,7 +89,7 @@ class UDPSocket(BMProto):
|
|||
remoteport = False
|
||||
for i in addresses:
|
||||
seenTime, stream, services, ip, port = i
|
||||
decodedIP = protocol.checkIPAddress(ip)
|
||||
decodedIP = protocol.checkIPAddress(buffer(ip))
|
||||
if stream not in state.streamsInWhichIAmParticipating:
|
||||
continue
|
||||
if seenTime < time.time() - BMProto.maxTimeOffset or seenTime > time.time() + BMProto.maxTimeOffset:
|
||||
|
@ -142,7 +142,7 @@ class UDPSocket(BMProto):
|
|||
else:
|
||||
self.local = False
|
||||
# overwrite the old buffer to avoid mixing data and so that self.local works correctly
|
||||
self.read_buf = recdata
|
||||
self.read_buf[0:] = recdata
|
||||
self.bm_proto_reset()
|
||||
receiveDataQueue.put(self.listening)
|
||||
|
||||
|
|
Reference in New Issue
Block a user