citizenaspirant
/
PyBitmessage
1
0
Fork 0
Code Issues 1 Pull Requests Releases Wiki Activity

filesystem quality fixes

This commit is contained in:
lakshyacis 2019-12-24 18:23:22 +05:30
parent 81872c7f2f
commit 6fad4f5665
No known key found for this signature in database
GPG Key ID: D2C539C8EC63E9EB
1 changed files with 50 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
src/storage/filesystem.py
View File

@ -1,6 +1,5 @@
""" """
src/storage/filesystem.py Module for using filesystem (directory with files) for inventory storage
=========================
""" """
from binascii import hexlify, unhexlify from binascii import hexlify, unhexlify
from os import listdir, makedirs, path, remove, rmdir from os import listdir, makedirs, path, remove, rmdir
@ -12,8 +11,9 @@ from paths import lookupAppdataFolder
from storage import InventoryStorage, InventoryItem from storage import InventoryStorage, InventoryItem
class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ancestors, abstract-method class FilesystemInventory(InventoryStorage):
"""Module for using filesystem (directory with files) for inventory storage""" """Filesystem for inventory storage"""
# pylint: disable=too-many-ancestors, abstract-method
topDir = "inventory" topDir = "inventory"
objectDir = "objects" objectDir = "objects"
metadataFilename = "metadata" metadataFilename = "metadata"
@ -21,21 +21,23 @@ class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ances
def __init__(self): def __init__(self):
super(FilesystemInventory, self).__init__() super(FilesystemInventory, self).__init__()
self.baseDir = path.join(lookupAppdataFolder(), FilesystemInventory.topDir) self.baseDir = path.join(
lookupAppdataFolder(), FilesystemInventory.topDir)
for createDir in [self.baseDir, path.join(self.baseDir, "objects")]: for createDir in [self.baseDir, path.join(self.baseDir, "objects")]:
if path.exists(createDir): if path.exists(createDir):
if not path.isdir(createDir): if not path.isdir(createDir):
raise IOError("%s exists but it's not a directory" % (createDir)) raise IOError(
"%s exists but it's not a directory" % createDir)
else: else:
makedirs(createDir) makedirs(createDir)
# Guarantees that two receiveDataThreads don't receive and process the same message # Guarantees that two receiveDataThreads
# don't receive and process the same message
# concurrently (probably sent by a malicious individual) # concurrently (probably sent by a malicious individual)
self.lock = RLock() self.lock = RLock()
self._inventory = {} self._inventory = {}
self._load() self._load()
def __contains__(self, hashval): def __contains__(self, hashval):
retval = False
for streamDict in self._inventory.values(): for streamDict in self._inventory.values():
if hashval in streamDict: if hashval in streamDict:
return True return True
@ -48,7 +50,12 @@ class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ances
except KeyError: except KeyError:
continue continue
if retval.payload is None: if retval.payload is None:
retval = InventoryItem(retval.type, retval.stream, self.getData(hashval), retval.expires, retval.tag) retval = InventoryItem(
retval.type,
retval.stream,
self.getData(hashval),
retval.expires,
retval.tag)
return retval return retval
raise KeyError(hashval) raise KeyError(hashval)
@ -56,7 +63,10 @@ class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ances
with self.lock: with self.lock:
value = InventoryItem(*value) value = InventoryItem(*value)
try: try:
makedirs(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hashval))) makedirs(path.join(
self.baseDir,
FilesystemInventory.objectDir,
hexlify(hashval)))
except OSError: except OSError:
pass pass
try: try:
@ -69,7 +79,11 @@ class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ances
), ),
"w", "w",
) as f: ) as f:
f.write("%s,%s,%s,%s," % (value.type, value.stream, value.expires, hexlify(value.tag))) f.write("%s,%s,%s,%s," % (
value.type,
value.stream,
value.expires,
hexlify(value.tag)))
with open( with open(
path.join( path.join(
self.baseDir, self.baseDir,
@ -115,7 +129,10 @@ class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ances
except IOError: except IOError:
pass pass
try: try:
rmdir(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hashval))) rmdir(path.join(
self.baseDir,
FilesystemInventory.objectDir,
hexlify(hashval)))
except IOError: except IOError:
pass pass
@ -135,7 +152,8 @@ class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ances
newInventory = {} newInventory = {}
for hashId in self.object_list(): for hashId in self.object_list():
try: try:
objectType, streamNumber, expiresTime, tag = self.getMetadata(hashId) objectType, streamNumber, expiresTime, tag = self.getMetadata(
hashId)
try: try:
newInventory[streamNumber][hashId] = InventoryItem( newInventory[streamNumber][hashId] = InventoryItem(
objectType, streamNumber, None, expiresTime, tag) objectType, streamNumber, None, expiresTime, tag)
@ -155,7 +173,8 @@ class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ances
def object_list(self): def object_list(self):
"""Return inventory vectors (hashes) from a directory""" """Return inventory vectors (hashes) from a directory"""
return [unhexlify(x) for x in listdir(path.join(self.baseDir, FilesystemInventory.objectDir))] return [unhexlify(x) for x in listdir(path.join(
self.baseDir, FilesystemInventory.objectDir))]
def getData(self, hashId): def getData(self, hashId):
"""Get object data""" """Get object data"""
@ -185,15 +204,20 @@ class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ances
), ),
"r", "r",
) as f: ) as f:
objectType, streamNumber, expiresTime, tag, undef = string.split(f.read(), ",", 4) objectType, streamNumber, expiresTime, tag = string.split(
return [int(objectType), int(streamNumber), int(expiresTime), unhexlify(tag)] f.read(), ",", 4)[:4]
return [
int(objectType),
int(streamNumber),
int(expiresTime),
unhexlify(tag)]
except IOError: except IOError:
raise KeyError raise KeyError
def by_type_and_tag(self, objectType, tag): def by_type_and_tag(self, objectType, tag):
"""Get a list of objects filtered by object type and tag""" """Get a list of objects filtered by object type and tag"""
retval = [] retval = []
for stream, streamDict in self._inventory: for streamDict in self._inventory.values():
for hashId, item in streamDict: for hashId, item in streamDict:
if item.type == objectType and item.tag == tag: if item.type == objectType and item.tag == tag:
try: try:
@ -201,7 +225,12 @@ class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ances
item.payload = self.getData(hashId) item.payload = self.getData(hashId)
except IOError: except IOError:
continue continue
retval.append(InventoryItem(item.type, item.stream, item.payload, item.expires, item.tag)) retval.append(InventoryItem(
item.type,
item.stream,
item.payload,
item.expires,
item.tag))
return retval return retval
def hashes_by_stream(self, stream): def hashes_by_stream(self, stream):
@ -215,7 +244,8 @@ class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ances
"""Return unexpired hashes in the inventory for a particular stream""" """Return unexpired hashes in the inventory for a particular stream"""
t = int(time.time()) t = int(time.time())
try: try:
return [x for x, value in self._inventory[stream].items() if value.expires > t] return [x for x, value in self._inventory[stream].items()
if value.expires > t]
except KeyError: except KeyError:
return [] return []
@ -227,7 +257,7 @@ class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ances
"""Clean out old items from the inventory""" """Clean out old items from the inventory"""
minTime = int(time.time()) - (60 * 60 * 30) minTime = int(time.time()) - (60 * 60 * 30)
deletes = [] deletes = []
for stream, streamDict in self._inventory.items(): for streamDict in self._inventory.values():
for hashId, item in streamDict.items(): for hashId, item in streamDict.items():
if item.expires < minTime: if item.expires < minTime:
deletes.append(hashId) deletes.append(hashId)