From 6fad4f56657c4f44d1bc30298928b12fe5011bbc Mon Sep 17 00:00:00 2001 From: lakshyacis Date: Tue, 24 Dec 2019 18:23:22 +0530 Subject: [PATCH] filesystem quality fixes --- src/storage/filesystem.py | 70 ++++++++++++++++++++++++++++----------- 1 file changed, 50 insertions(+), 20 deletions(-) diff --git a/src/storage/filesystem.py b/src/storage/filesystem.py index 43ba03fc..571a16c1 100644 --- a/src/storage/filesystem.py +++ b/src/storage/filesystem.py @@ -1,6 +1,5 @@ """ -src/storage/filesystem.py -========================= +Module for using filesystem (directory with files) for inventory storage """ from binascii import hexlify, unhexlify from os import listdir, makedirs, path, remove, rmdir @@ -12,8 +11,9 @@ from paths import lookupAppdataFolder from storage import InventoryStorage, InventoryItem -class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ancestors, abstract-method - """Module for using filesystem (directory with files) for inventory storage""" +class FilesystemInventory(InventoryStorage): + """Filesystem for inventory storage""" + # pylint: disable=too-many-ancestors, abstract-method topDir = "inventory" objectDir = "objects" metadataFilename = "metadata" @@ -21,21 +21,23 @@ class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ances def __init__(self): super(FilesystemInventory, self).__init__() - self.baseDir = path.join(lookupAppdataFolder(), FilesystemInventory.topDir) + self.baseDir = path.join( + lookupAppdataFolder(), FilesystemInventory.topDir) for createDir in [self.baseDir, path.join(self.baseDir, "objects")]: if path.exists(createDir): if not path.isdir(createDir): - raise IOError("%s exists but it's not a directory" % (createDir)) + raise IOError( + "%s exists but it's not a directory" % createDir) else: makedirs(createDir) - # Guarantees that two receiveDataThreads don't receive and process the same message + # Guarantees that two receiveDataThreads + # don't receive and process the same message # concurrently (probably sent by a malicious individual) self.lock = RLock() self._inventory = {} self._load() def __contains__(self, hashval): - retval = False for streamDict in self._inventory.values(): if hashval in streamDict: return True @@ -48,7 +50,12 @@ class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ances except KeyError: continue if retval.payload is None: - retval = InventoryItem(retval.type, retval.stream, self.getData(hashval), retval.expires, retval.tag) + retval = InventoryItem( + retval.type, + retval.stream, + self.getData(hashval), + retval.expires, + retval.tag) return retval raise KeyError(hashval) @@ -56,7 +63,10 @@ class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ances with self.lock: value = InventoryItem(*value) try: - makedirs(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hashval))) + makedirs(path.join( + self.baseDir, + FilesystemInventory.objectDir, + hexlify(hashval))) except OSError: pass try: @@ -69,7 +79,11 @@ class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ances ), "w", ) as f: - f.write("%s,%s,%s,%s," % (value.type, value.stream, value.expires, hexlify(value.tag))) + f.write("%s,%s,%s,%s," % ( + value.type, + value.stream, + value.expires, + hexlify(value.tag))) with open( path.join( self.baseDir, @@ -115,7 +129,10 @@ class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ances except IOError: pass try: - rmdir(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hashval))) + rmdir(path.join( + self.baseDir, + FilesystemInventory.objectDir, + hexlify(hashval))) except IOError: pass @@ -135,7 +152,8 @@ class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ances newInventory = {} for hashId in self.object_list(): try: - objectType, streamNumber, expiresTime, tag = self.getMetadata(hashId) + objectType, streamNumber, expiresTime, tag = self.getMetadata( + hashId) try: newInventory[streamNumber][hashId] = InventoryItem( objectType, streamNumber, None, expiresTime, tag) @@ -155,7 +173,8 @@ class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ances def object_list(self): """Return inventory vectors (hashes) from a directory""" - return [unhexlify(x) for x in listdir(path.join(self.baseDir, FilesystemInventory.objectDir))] + return [unhexlify(x) for x in listdir(path.join( + self.baseDir, FilesystemInventory.objectDir))] def getData(self, hashId): """Get object data""" @@ -185,15 +204,20 @@ class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ances ), "r", ) as f: - objectType, streamNumber, expiresTime, tag, undef = string.split(f.read(), ",", 4) - return [int(objectType), int(streamNumber), int(expiresTime), unhexlify(tag)] + objectType, streamNumber, expiresTime, tag = string.split( + f.read(), ",", 4)[:4] + return [ + int(objectType), + int(streamNumber), + int(expiresTime), + unhexlify(tag)] except IOError: raise KeyError def by_type_and_tag(self, objectType, tag): """Get a list of objects filtered by object type and tag""" retval = [] - for stream, streamDict in self._inventory: + for streamDict in self._inventory.values(): for hashId, item in streamDict: if item.type == objectType and item.tag == tag: try: @@ -201,7 +225,12 @@ class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ances item.payload = self.getData(hashId) except IOError: continue - retval.append(InventoryItem(item.type, item.stream, item.payload, item.expires, item.tag)) + retval.append(InventoryItem( + item.type, + item.stream, + item.payload, + item.expires, + item.tag)) return retval def hashes_by_stream(self, stream): @@ -215,7 +244,8 @@ class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ances """Return unexpired hashes in the inventory for a particular stream""" t = int(time.time()) try: - return [x for x, value in self._inventory[stream].items() if value.expires > t] + return [x for x, value in self._inventory[stream].items() + if value.expires > t] except KeyError: return [] @@ -227,7 +257,7 @@ class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ances """Clean out old items from the inventory""" minTime = int(time.time()) - (60 * 60 * 30) deletes = [] - for stream, streamDict in self._inventory.items(): + for streamDict in self._inventory.values(): for hashId, item in streamDict.items(): if item.expires < minTime: deletes.append(hashId)