Downloading fixes

- able to request more objects with one command
- fixes to logic and error handling

src/class_receiveDataThread.py

@@ -9,6 +9,7 @@ import threading
 import shared
 import hashlib
 import os
+import Queue
 import select
 import socket
 import random
@@ -217,15 +218,10 @@ class receiveDataThread(threading.Thread):
         self.data = self.data[payloadLength + protocol.Header.size:] # take this message out and then process the next message
 
         if self.data == '': # if there are no more messages
-            for objectHash in Missing().pull(100):
+            try:
-                if self.sendDataThreadQueue.full():
+                self.sendgetdata(Missing().pull(100))
-                    break
+            except Queue.full:
-                if objectHash in Inventory():
+                pass
-                    logger.debug('Inventory already has object listed in inv message.')
-                    Missing().delete(objectHash)
-                else:
-                    # We don't have the object in our inventory. Let's request it.
-                    self.sendgetdata(objectHash)
         self.processData()
 
 
@@ -418,10 +414,10 @@ class receiveDataThread(threading.Thread):
 
     # Send a getdata message to our peer to request the object with the given
     # hash
-    def sendgetdata(self, hash):
+    def sendgetdata(self, hashes):
-        logger.debug('sending getdata to retrieve object with hash: ' + hexlify(hash))
+        logger.debug('sending getdata to retrieve %i objects', len(hashes))
-        payload = '\x01' + hash
+        payload = encodeVarint(len(hashes)) + ''.join(hashes)
-        self.sendDataThreadQueue.put((0, 'sendRawData', protocol.CreatePacket('getdata', payload)))
+        self.sendDataThreadQueue.put((0, 'sendRawData', protocol.CreatePacket('getdata', payload)), False)
 
 
     # We have received a getdata request from our peer

src/inventory.py

@@ -37,7 +37,7 @@ class Inventory(collections.MutableMapping):
             value = self.InventoryItem(*value)
             self._inventory[hash] = value
             self._streams[value.stream].add(hash)
-        Missing().delete(hash, True)
+        Missing().delete(hash)
 
     def __delitem__(self, hash):
         raise NotImplementedError
@@ -126,10 +126,8 @@ class Missing(object):
                 return
             except ValueError:
                 pass
-            if len(self.hashes[objectHash]['peers']) == 0 and self.hashes[objectHash]['requested'] < time.time() - self.frequency:
+            if len(self.hashes[objectHash]['peers']) == 0:
-                self.delete(objectHash)
+                del self.hashes[objectHash]
-            else:
-                self.hashes[objectHash]['requested'] = time.time()
 
     def pull(self, count=1):
         if count < 1:
@@ -139,6 +137,7 @@ class Missing(object):
             return objectHashes
         try:
             for objectHash in self.hashes.keys():
+                with self.lock:
                     if len(objectHashes) >= count:
                         break
                     if current_thread().peer not in self.pending:
@@ -161,7 +160,6 @@ class Missing(object):
                             continue
                         # already requested too many times, remove all signs of this object
                         if self.hashes[objectHash]['requestedCount'] >= self.maxRequestCount:
-                        with self.lock:
                             del self.hashes[objectHash]
                             for thread in self.pending.keys():
                                 if objectHash in self.pending[thread]['objects']:
@@ -170,7 +168,6 @@ class Missing(object):
                         # all ok, request
                         objectHashes.append(objectHash)
                         self.hashes[objectHash]['requested'] = time.time()
-                    with self.lock:
                         self.hashes[objectHash]['requestedCount'] += 1
                         self.pending[current_thread().peer]['requested'] = time.time()
                         self.addPending(objectHash)
@@ -183,10 +180,18 @@ class Missing(object):
         with self.lock:
             if objectHash in self.hashes:
                 del self.hashes[objectHash]
-            if objectHash in self.pending[current_thread().peer]['objects']:
-                self.pending[current_thread().peer]['objects'].remove(objectHash)
-            if justReceived:
             self.pending[current_thread().peer]['received'] = time.time()
+        while True:
+            try:
+                for thread in self.pending.keys():
+                    with self.lock:
+                        if objectHash in self.pending[thread]['objects']:
+                            self.pending[thread]['objects'].remove(objectHash)
+            except (KeyError, RuntimeError):
+                pass
+            else:
+                break
+
 
     def stop(self):
         with self.lock:
@@ -194,10 +199,13 @@ class Missing(object):
             self.pending = {}
 
     def threadEnd(self):
-        with self.lock:
+        while True:
+            try:
                 for objectHash in self.hashes:
                     self.removeObjectFromCurrentThread(objectHash)
-            try:
+                with self.lock:
                     del self.pending[current_thread().peer]
-            except KeyError:
+            except (KeyError, RuntimeError):
                 pass
+            else:
+                break