Sudden KeyError in addKnownNode()

to track multiport configurations / port change events

src/network/knownnodes.py

@@ -32,6 +32,9 @@ knownNodesForgetRating = -0.5
 
 knownNodesActual = False
 
+outages = {}
+"""a dict with all hosts"""
+
 logger = logging.getLogger('default')
 
 DEFAULT_NODES = (
@@ -69,7 +72,13 @@ def json_deserialize_knownnodes(source):
     for node in json.load(source):
         peer = node['peer']
         info = node['info']
-        peer = Peer(str(peer['host']), peer.get('port', 8444))
+        port = peer.get('port', 8444)
+        peer = Peer(str(peer['host']), port)
+        outages[peer.host] = {
+            'lastseen': info.get('lastseen', time.time()),
+            'port': port,
+            'stream': info.get('stream', 1)
+        }
         knownNodes[node['stream']][peer] = info
         if not (knownNodesActual
                 or info.get('self')) and peer not in DEFAULT_NODES:
@@ -119,11 +128,18 @@ def addKnownNode(stream, peer, lastseen=None, is_self=False):
     else:
         lastseen = int(lastseen)
         try:
+            prev = outages[peer.host]
+            if peer.port == prev['port']:
+                outages[peer.host]['lastseen'] = lastseen
             info = knownNodes[stream].get(peer)
-            if lastseen > info['lastseen']:
             info['lastseen'] = lastseen
-        except (KeyError, TypeError):
+        except KeyError:
             pass
+        except TypeError:
+            # don't update expired node if have enough nodes in that stream
+            if len(knownNodes[stream]) > 64:
+                return
+            rating = -0.2
         else:
             return
 
@@ -131,6 +147,25 @@ def addKnownNode(stream, peer, lastseen=None, is_self=False):
         if len(knownNodes[stream]) > BMConfigParser().safeGetInt(
                 "knownnodes", "maxnodes"):
             return
+        try:
+            prev = outages[peer.host]
+        except KeyError:
+            outages[peer.host] = {
+                'stream': stream,
+                'port': peer.port,
+                'lastseen': lastseen
+            }
+        else:
+            if stream == prev['stream']:
+                if lastseen - prev['lastseen'] > 3600 * 24:
+                    # more than a day ago, this should be port change
+                    try:
+                        del knownNodes[stream][Peer(peer.host, prev['port'])]
+                    except KeyError:
+                        pass
+                    outages[peer.host]['port'] = peer.port
+                else:
+                    rating = -0.2
 
     knownNodes[stream][peer] = {
         'lastseen': lastseen,

src/shutdown.py

@@ -8,8 +8,7 @@ import state
 from debug import logger
 from helper_sql import sqlQuery, sqlStoredProcedure
 from inventory import Inventory
-from network import StoppableThread
+from network import knownnodes, StoppableThread
-from network.knownnodes import saveKnownNodes
 from queues import (
     addressGeneratorQueue, objectProcessorQueue, UISignalQueue, workerQueue)
 
@@ -29,7 +28,7 @@ def doCleanShutdown():
         'updateStatusBar',
         'Saving the knownNodes list of peers to disk...'))
     logger.info('Saving knownNodes list of peers to disk')
-    saveKnownNodes()
+    knownnodes.saveKnownNodes()
     logger.info('Done saving knownNodes list of peers to disk')
     UISignalQueue.put((
         'updateStatusBar',
@@ -79,6 +78,37 @@ def doCleanShutdown():
             except Queue.Empty:
                 break
 
+    # Log knownnodes/outages/multiport statistics
+    dupes = {}
+    nodes_counter = 0
+    for stream in knownnodes.knownNodes.itervalues():
+        for peer in stream:
+            nodes_counter += 1
+            dup = dupes.get(peer.host)
+            port = str(peer.port)
+            if dup:
+                dupes[peer.host].append(port)
+            else:
+                dupes[peer.host] = [port]
+
+    dup_counter = 0
+    max_ports = (1, '')
+    for dup, ports in dupes.iteritems():
+        ports_len = len(ports)
+        if ports_len > 1:
+            dup_counter += 1
+            if ports_len > max_ports[0]:
+                max_ports = (ports_len, dup)
+            logger.warning(
+                'Multiport host: %s, ports: %s',
+                dup, ', '.join(ports))
+    logger.warning(
+        'Knownnodes len: %s, outages: %s, multiport: %s',
+        nodes_counter, len(knownnodes.outages), dup_counter)
+    if max_ports[0] > 0:
+        logger.warning(
+            'Maximum number of ports %s - %s', *max_ports)
+
     if state.thisapp.daemon or not state.enableGUI:
         logger.info('Clean shutdown complete.')
         state.thisapp.cleanup()