From 6f910f67c0ebb45f6b7455922f33c710a515ae7b Mon Sep 17 00:00:00 2001 From: lakshyacis Date: Tue, 24 Sep 2019 14:50:20 +0530 Subject: [PATCH] filesystem flake8 fixes --- src/storage/filesystem.py | 71 ++++++++++++++++++++++++++++++++------- src/storage/sqlite.py | 14 ++++++-- src/storage/storage.py | 5 +++ 3 files changed, 75 insertions(+), 15 deletions(-) diff --git a/src/storage/filesystem.py b/src/storage/filesystem.py index d64894a9..4f7e717d 100644 --- a/src/storage/filesystem.py +++ b/src/storage/filesystem.py @@ -3,11 +3,11 @@ from os import listdir, makedirs, path, remove, rmdir import string from threading import RLock import time -import traceback from paths import lookupAppdataFolder from storage import InventoryStorage, InventoryItem + class FilesystemInventory(InventoryStorage): topDir = "inventory" objectDir = "objects" @@ -23,7 +23,9 @@ class FilesystemInventory(InventoryStorage): raise IOError("%s exists but it's not a directory" % (createDir)) else: makedirs(createDir) - self.lock = RLock() # Guarantees that two receiveDataThreads don't receive and process the same message concurrently (probably sent by a malicious individual) + # Guarantees that two receiveDataThreads don't receive and process the same message + # concurrently (probably sent by a malicious individual) + self.lock = RLock() self._inventory = {} self._load() @@ -53,9 +55,25 @@ class FilesystemInventory(InventoryStorage): except OSError: pass try: - with open(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hash), FilesystemInventory.metadataFilename), 'w') as f: + with open( + path.join( + self.baseDir, + FilesystemInventory.objectDir, + hexlify(hash), + FilesystemInventory.metadataFilename, + ), + "w", + ) as f: f.write("%s,%s,%s,%s," % (value.type, value.stream, value.expires, hexlify(value.tag))) - with open(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hash), FilesystemInventory.dataFilename), 'w') as f: + with open( + path.join( + self.baseDir, + FilesystemInventory.objectDir, + hexlify(hash), + FilesystemInventory.dataFilename, + ), + "w", + ) as f: f.write(value.payload) except IOError: raise KeyError @@ -73,11 +91,21 @@ class FilesystemInventory(InventoryStorage): pass with self.lock: try: - remove(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hash), FilesystemInventory.metadataFilename)) + remove( + path.join( + self.baseDir, + FilesystemInventory.objectDir, + hexlify(hash), + FilesystemInventory.metadataFilename)) except IOError: pass try: - remove(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hash), FilesystemInventory.dataFilename)) + remove( + path.join( + self.baseDir, + FilesystemInventory.objectDir, + hexlify(hash), + FilesystemInventory.dataFilename)) except IOError: pass try: @@ -88,7 +116,7 @@ class FilesystemInventory(InventoryStorage): def __iter__(self): elems = [] for streamDict in self._inventory.values(): - elems.extend (streamDict.keys()) + elems.extend(streamDict.keys()) return elems.__iter__() def __len__(self): @@ -103,10 +131,12 @@ class FilesystemInventory(InventoryStorage): try: objectType, streamNumber, expiresTime, tag = self.getMetadata(hashId) try: - newInventory[streamNumber][hashId] = InventoryItem(objectType, streamNumber, None, expiresTime, tag) + newInventory[streamNumber][hashId] = InventoryItem( + objectType, streamNumber, None, expiresTime, tag) except KeyError: newInventory[streamNumber] = {} - newInventory[streamNumber][hashId] = InventoryItem(objectType, streamNumber, None, expiresTime, tag) + newInventory[streamNumber][hashId] = InventoryItem( + objectType, streamNumber, None, expiresTime, tag) except KeyError: print "error loading %s" % (hexlify(hashId)) pass @@ -122,25 +152,42 @@ class FilesystemInventory(InventoryStorage): def getData(self, hashId): try: - with open(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hashId), FilesystemInventory.dataFilename), 'r') as f: + with open( + path.join( + self.baseDir, + FilesystemInventory.objectDir, + hexlify(hashId), + FilesystemInventory.dataFilename, + ), + "r", + ) as f: return f.read() except IOError: raise AttributeError def getMetadata(self, hashId): try: - with open(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hashId), FilesystemInventory.metadataFilename), 'r') as f: + with open( + path.join( + self.baseDir, + FilesystemInventory.objectDir, + hexlify(hashId), + FilesystemInventory.metadataFilename, + ), + "r", + ) as f: objectType, streamNumber, expiresTime, tag, undef = string.split(f.read(), ",", 4) return [int(objectType), int(streamNumber), int(expiresTime), unhexlify(tag)] except IOError: raise KeyError def by_type_and_tag(self, objectType, tag): + """Get a list of objects filtered by object type and tag""" retval = [] for stream, streamDict in self._inventory: for hashId, item in streamDict: if item.type == objectType and item.tag == tag: - try: + try: if item.payload is None: item.payload = self.getData(hashId) except IOError: diff --git a/src/storage/sqlite.py b/src/storage/sqlite.py index 438cbdcb..d1ec220f 100644 --- a/src/storage/sqlite.py +++ b/src/storage/sqlite.py @@ -10,9 +10,17 @@ from storage import InventoryStorage, InventoryItem class SqliteInventory(InventoryStorage): def __init__(self): super(self.__class__, self).__init__() - self._inventory = {} #of objects (like msg payloads and pubkey payloads) Does not include protocol headers (the first 24 bytes of each packet). - self._objects = {} # cache for existing objects, used for quick lookups if we have an object. This is used for example whenever we receive an inv message from a peer to check to see what items are new to us. We don't delete things out of it; instead, the singleCleaner thread clears and refills it. - self.lock = RLock() # Guarantees that two receiveDataThreads don't receive and process the same message concurrently (probably sent by a malicious individual) + # of objects (like msg payloads and pubkey payloads) + # Does not include protocol headers (the first 24 bytes of each packet). + self._inventory = {} + # cache for existing objects, used for quick lookups if we have an object. + # This is used for example whenever we receive an inv message from a peer + # to check to see what items are new to us. + # We don't delete things out of it; instead, the singleCleaner thread clears and refills it. + self._objects = {} + # Guarantees that two receiveDataThreads don't receive and process the same message concurrently + # (probably sent by a malicious individual) + self.lock = RLock() def __contains__(self, hash): with self.lock: diff --git a/src/storage/storage.py b/src/storage/storage.py index 08c85708..050197b8 100644 --- a/src/storage/storage.py +++ b/src/storage/storage.py @@ -3,6 +3,7 @@ import collections InventoryItem = collections.namedtuple('InventoryItem', 'type stream payload expires tag') class Storage(object): + """Base class for storing inventory (extendable for other items to store)""" pass # def __init__(self): # super(self.__class__, self).__init__() @@ -31,15 +32,19 @@ class InventoryStorage(Storage, collections.MutableMapping): raise NotImplementedError def by_type_and_tag(self, objectType, tag): + """Return objects filtered by object type and tag""" raise NotImplementedError def unexpired_hashes_by_stream(self, stream): + """Return unexpired inventory vectors filtered by stream""" raise NotImplementedError def flush(self): + """Flush cache""" raise NotImplementedError def clean(self): + """Free memory / perform garbage collection""" raise NotImplementedError class MailboxStorage(Storage, collections.MutableMapping):