filesystem quality fixes

src/storage/filesystem.py

@@ -1,6 +1,5 @@
 """
-src/storage/filesystem.py
+Module for using filesystem (directory with files) for inventory storage
-=========================
 """
 from binascii import hexlify, unhexlify
 from os import listdir, makedirs, path, remove, rmdir
@@ -12,8 +11,9 @@ from paths import lookupAppdataFolder
 from storage import InventoryStorage, InventoryItem
 
 
-class FilesystemInventory(InventoryStorage):    # pylint: disable=too-many-ancestors, abstract-method
+class FilesystemInventory(InventoryStorage):
-    """Module for using filesystem (directory with files) for inventory storage"""
+    """Filesystem for inventory storage"""
+    # pylint: disable=too-many-ancestors, abstract-method
     topDir = "inventory"
     objectDir = "objects"
     metadataFilename = "metadata"
@@ -21,21 +21,23 @@ class FilesystemInventory(InventoryStorage):    # pylint: disable=too-many-ances
 
     def __init__(self):
         super(FilesystemInventory, self).__init__()
-        self.baseDir = path.join(lookupAppdataFolder(), FilesystemInventory.topDir)
+        self.baseDir = path.join(
+            lookupAppdataFolder(), FilesystemInventory.topDir)
         for createDir in [self.baseDir, path.join(self.baseDir, "objects")]:
             if path.exists(createDir):
                 if not path.isdir(createDir):
-                    raise IOError("%s exists but it's not a directory" % (createDir))
+                    raise IOError(
+                        "%s exists but it's not a directory" % createDir)
             else:
                 makedirs(createDir)
-        # Guarantees that two receiveDataThreads don't receive and process the same message
+        # Guarantees that two receiveDataThreads
+        # don't receive and process the same message
         # concurrently (probably sent by a malicious individual)
         self.lock = RLock()
         self._inventory = {}
         self._load()
 
     def __contains__(self, hashval):
-        retval = False
         for streamDict in self._inventory.values():
             if hashval in streamDict:
                 return True
@@ -48,7 +50,12 @@ class FilesystemInventory(InventoryStorage):    # pylint: disable=too-many-ances
             except KeyError:
                 continue
             if retval.payload is None:
-                retval = InventoryItem(retval.type, retval.stream, self.getData(hashval), retval.expires, retval.tag)
+                retval = InventoryItem(
+                    retval.type,
+                    retval.stream,
+                    self.getData(hashval),
+                    retval.expires,
+                    retval.tag)
             return retval
         raise KeyError(hashval)
 
@@ -56,7 +63,10 @@ class FilesystemInventory(InventoryStorage):    # pylint: disable=too-many-ances
         with self.lock:
             value = InventoryItem(*value)
             try:
-                makedirs(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hashval)))
+                makedirs(path.join(
+                    self.baseDir,
+                    FilesystemInventory.objectDir,
+                    hexlify(hashval)))
             except OSError:
                 pass
             try:
@@ -69,7 +79,11 @@ class FilesystemInventory(InventoryStorage):    # pylint: disable=too-many-ances
                     ),
                     "w",
                 ) as f:
-                    f.write("%s,%s,%s,%s," % (value.type, value.stream, value.expires, hexlify(value.tag)))
+                    f.write("%s,%s,%s,%s," % (
+                        value.type,
+                        value.stream,
+                        value.expires,
+                        hexlify(value.tag)))
                 with open(
                     path.join(
                         self.baseDir,
@@ -115,7 +129,10 @@ class FilesystemInventory(InventoryStorage):    # pylint: disable=too-many-ances
             except IOError:
                 pass
             try:
-                rmdir(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hashval)))
+                rmdir(path.join(
+                    self.baseDir,
+                    FilesystemInventory.objectDir,
+                    hexlify(hashval)))
             except IOError:
                 pass
 
@@ -135,7 +152,8 @@ class FilesystemInventory(InventoryStorage):    # pylint: disable=too-many-ances
         newInventory = {}
         for hashId in self.object_list():
             try:
-                objectType, streamNumber, expiresTime, tag = self.getMetadata(hashId)
+                objectType, streamNumber, expiresTime, tag = self.getMetadata(
+                    hashId)
                 try:
                     newInventory[streamNumber][hashId] = InventoryItem(
                         objectType, streamNumber, None, expiresTime, tag)
@@ -155,7 +173,8 @@ class FilesystemInventory(InventoryStorage):    # pylint: disable=too-many-ances
 
     def object_list(self):
         """Return inventory vectors (hashes) from a directory"""
-        return [unhexlify(x) for x in listdir(path.join(self.baseDir, FilesystemInventory.objectDir))]
+        return [unhexlify(x) for x in listdir(path.join(
+            self.baseDir, FilesystemInventory.objectDir))]
 
     def getData(self, hashId):
         """Get object data"""
@@ -185,15 +204,20 @@ class FilesystemInventory(InventoryStorage):    # pylint: disable=too-many-ances
                 ),
                 "r",
             ) as f:
-                objectType, streamNumber, expiresTime, tag, undef = string.split(f.read(), ",", 4)
+                objectType, streamNumber, expiresTime, tag = string.split(
-                return [int(objectType), int(streamNumber), int(expiresTime), unhexlify(tag)]
+                    f.read(), ",", 4)[:4]
+                return [
+                    int(objectType),
+                    int(streamNumber),
+                    int(expiresTime),
+                    unhexlify(tag)]
         except IOError:
             raise KeyError
 
     def by_type_and_tag(self, objectType, tag):
         """Get a list of objects filtered by object type and tag"""
         retval = []
-        for stream, streamDict in self._inventory:
+        for streamDict in self._inventory.values():
             for hashId, item in streamDict:
                 if item.type == objectType and item.tag == tag:
                     try:
@@ -201,7 +225,12 @@ class FilesystemInventory(InventoryStorage):    # pylint: disable=too-many-ances
                             item.payload = self.getData(hashId)
                     except IOError:
                         continue
-                    retval.append(InventoryItem(item.type, item.stream, item.payload, item.expires, item.tag))
+                    retval.append(InventoryItem(
+                        item.type,
+                        item.stream,
+                        item.payload,
+                        item.expires,
+                        item.tag))
         return retval
 
     def hashes_by_stream(self, stream):
@@ -215,7 +244,8 @@ class FilesystemInventory(InventoryStorage):    # pylint: disable=too-many-ances
         """Return unexpired hashes in the inventory for a particular stream"""
         t = int(time.time())
         try:
-            return [x for x, value in self._inventory[stream].items() if value.expires > t]
+            return [x for x, value in self._inventory[stream].items()
+                    if value.expires > t]
         except KeyError:
             return []
 
@@ -227,7 +257,7 @@ class FilesystemInventory(InventoryStorage):    # pylint: disable=too-many-ances
         """Clean out old items from the inventory"""
         minTime = int(time.time()) - (60 * 60 * 30)
         deletes = []
-        for stream, streamDict in self._inventory.items():
+        for streamDict in self._inventory.values():
             for hashId, item in streamDict.items():
                 if item.expires < minTime:
                     deletes.append(hashId)