citizenaspirant
/
PyBitmessage
1
0
Fork 0
Code Issues 1 Pull Requests Releases Wiki Activity

filesystem pylint fixes

This commit is contained in:
lakshyacis 2019-09-24 14:51:53 +05:30
parent 6f910f67c0
commit ac341482d4
No known key found for this signature in database
GPG Key ID: D2C539C8EC63E9EB
1 changed files with 34 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
src/storage/filesystem.py
View File

@ -1,3 +1,7 @@
"""
src/storage/filesystem.py
=========================
"""
from binascii import hexlify, unhexlify from binascii import hexlify, unhexlify
from os import listdir, makedirs, path, remove, rmdir from os import listdir, makedirs, path, remove, rmdir
import string import string
@ -8,14 +12,15 @@ from paths import lookupAppdataFolder
from storage import InventoryStorage, InventoryItem from storage import InventoryStorage, InventoryItem
class FilesystemInventory(InventoryStorage): class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ancestors, abstract-method
"""Module for using filesystem (directory with files) for inventory storage"""
topDir = "inventory" topDir = "inventory"
objectDir = "objects" objectDir = "objects"
metadataFilename = "metadata" metadataFilename = "metadata"
dataFilename = "data" dataFilename = "data"
def __init__(self): def __init__(self):
super(self.__class__, self).__init__() super(FilesystemInventory, self).__init__()
self.baseDir = path.join(lookupAppdataFolder(), FilesystemInventory.topDir) self.baseDir = path.join(lookupAppdataFolder(), FilesystemInventory.topDir)
for createDir in [self.baseDir, path.join(self.baseDir, "objects")]: for createDir in [self.baseDir, path.join(self.baseDir, "objects")]:
if path.exists(createDir): if path.exists(createDir):
@ -29,29 +34,29 @@ class FilesystemInventory(InventoryStorage):
self._inventory = {} self._inventory = {}
self._load() self._load()
def __contains__(self, hash): def __contains__(self, hashval):
retval = False retval = False
for streamDict in self._inventory.values(): for streamDict in self._inventory.values():
if hash in streamDict: if hashval in streamDict:
return True return True
return False return False
def __getitem__(self, hash): def __getitem__(self, hashval):
for streamDict in self._inventory.values(): for streamDict in self._inventory.values():
try: try:
retval = streamDict[hash] retval = streamDict[hashval]
except KeyError: except KeyError:
continue continue
if retval.payload is None: if retval.payload is None:
retval = InventoryItem(retval.type, retval.stream, self.getData(hash), retval.expires, retval.tag) retval = InventoryItem(retval.type, retval.stream, self.getData(hashval), retval.expires, retval.tag)
return retval return retval
raise KeyError(hash) raise KeyError(hashval)
def __setitem__(self, hash, value): def __setitem__(self, hashval, value):
with self.lock: with self.lock:
value = InventoryItem(*value) value = InventoryItem(*value)
try: try:
makedirs(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hash))) makedirs(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hashval)))
except OSError: except OSError:
pass pass
try: try:
@ -59,7 +64,7 @@ class FilesystemInventory(InventoryStorage):
path.join( path.join(
self.baseDir, self.baseDir,
FilesystemInventory.objectDir, FilesystemInventory.objectDir,
hexlify(hash), hexlify(hashval),
FilesystemInventory.metadataFilename, FilesystemInventory.metadataFilename,
), ),
"w", "w",
@ -69,7 +74,7 @@ class FilesystemInventory(InventoryStorage):
path.join( path.join(
self.baseDir, self.baseDir,
FilesystemInventory.objectDir, FilesystemInventory.objectDir,
hexlify(hash), hexlify(hashval),
FilesystemInventory.dataFilename, FilesystemInventory.dataFilename,
), ),
"w", "w",
@ -78,15 +83,16 @@ class FilesystemInventory(InventoryStorage):
except IOError: except IOError:
raise KeyError raise KeyError
try: try:
self._inventory[value.stream][hash] = value self._inventory[value.stream][hashval] = value
except KeyError: except KeyError:
self._inventory[value.stream] = {} self._inventory[value.stream] = {}
self._inventory[value.stream][hash] = value self._inventory[value.stream][hashval] = value
def delHashId(self, hash): def delHashId(self, hashval):
for stream in self._inventory.keys(): """Remove object from inventory"""
for stream in self._inventory:
try: try:
del self._inventory[stream][hash] del self._inventory[stream][hashval]
except KeyError: except KeyError:
pass pass
with self.lock: with self.lock:
@ -95,7 +101,7 @@ class FilesystemInventory(InventoryStorage):
path.join( path.join(
self.baseDir, self.baseDir,
FilesystemInventory.objectDir, FilesystemInventory.objectDir,
hexlify(hash), hexlify(hashval),
FilesystemInventory.metadataFilename)) FilesystemInventory.metadataFilename))
except IOError: except IOError:
pass pass
@ -104,12 +110,12 @@ class FilesystemInventory(InventoryStorage):
path.join( path.join(
self.baseDir, self.baseDir,
FilesystemInventory.objectDir, FilesystemInventory.objectDir,
hexlify(hash), hexlify(hashval),
FilesystemInventory.dataFilename)) FilesystemInventory.dataFilename))
except IOError: except IOError:
pass pass
try: try:
rmdir(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hash))) rmdir(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hashval)))
except IOError: except IOError:
pass pass
@ -139,18 +145,20 @@ class FilesystemInventory(InventoryStorage):
objectType, streamNumber, None, expiresTime, tag) objectType, streamNumber, None, expiresTime, tag)
except KeyError: except KeyError:
print "error loading %s" % (hexlify(hashId)) print "error loading %s" % (hexlify(hashId))
pass
self._inventory = newInventory self._inventory = newInventory
# for i, v in self._inventory.items(): # for i, v in self._inventory.items():
# print "loaded stream: %s, %i items" % (i, len(v)) # print "loaded stream: %s, %i items" % (i, len(v))
def stream_list(self): def stream_list(self):
"""Return list of streams"""
return self._inventory.keys() return self._inventory.keys()
def object_list(self): def object_list(self):
"""Return inventory vectors (hashes) from a directory"""
return [unhexlify(x) for x in listdir(path.join(self.baseDir, FilesystemInventory.objectDir))] return [unhexlify(x) for x in listdir(path.join(self.baseDir, FilesystemInventory.objectDir))]
def getData(self, hashId): def getData(self, hashId):
"""Get object data"""
try: try:
with open( with open(
path.join( path.join(
@ -166,6 +174,7 @@ class FilesystemInventory(InventoryStorage):
raise AttributeError raise AttributeError
def getMetadata(self, hashId): def getMetadata(self, hashId):
"""Get object metadata"""
try: try:
with open( with open(
path.join( path.join(
@ -196,12 +205,14 @@ class FilesystemInventory(InventoryStorage):
return retval return retval
def hashes_by_stream(self, stream): def hashes_by_stream(self, stream):
"""Return inventory vectors (hashes) for a stream"""
try: try:
return self._inventory[stream].keys() return self._inventory[stream].keys()
except KeyError: except KeyError:
return [] return []
def unexpired_hashes_by_stream(self, stream): def unexpired_hashes_by_stream(self, stream):
"""Return unexpired hashes in the inventory for a particular stream"""
t = int(time.time()) t = int(time.time())
try: try:
return [x for x, value in self._inventory[stream].items() if value.expires > t] return [x for x, value in self._inventory[stream].items() if value.expires > t]
@ -209,9 +220,11 @@ class FilesystemInventory(InventoryStorage):
return [] return []
def flush(self): def flush(self):
"""Flush the inventory and create a new, empty one"""
self._load() self._load()
def clean(self): def clean(self):
"""Clean out old items from the inventory"""
minTime = int(time.time()) - (60 * 60 * 30) minTime = int(time.time()) - (60 * 60 * 30)
deletes = [] deletes = []
for stream, streamDict in self._inventory.items(): for stream, streamDict in self._inventory.items():