citizenaspirant
/
PyBitmessage
1
0
Fork 0
Code Issues 1 Pull Requests Releases Wiki Activity
Browse Source

filesystem quality fixes

v0.6
lakshyacis 3 years ago
parent
81872c7f2f
commit
6fad4f5665
No known key found for this signature in database GPG Key ID: D2C539C8EC63E9EB
1 changed files with 50 additions and 20 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 70
      src/storage/filesystem.py

70
src/storage/filesystem.py
View File

@ -1,6 +1,5 @@
"""
src/storage/filesystem.py
=========================
Module for using filesystem (directory with files) for inventory storage
"""
from binascii import hexlify, unhexlify
from os import listdir, makedirs, path, remove, rmdir
@ -12,8 +11,9 @@ from paths import lookupAppdataFolder
from storage import InventoryStorage, InventoryItem
class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ancestors, abstract-method
"""Module for using filesystem (directory with files) for inventory storage"""
class FilesystemInventory(InventoryStorage):
"""Filesystem for inventory storage"""
# pylint: disable=too-many-ancestors, abstract-method
topDir = "inventory"
objectDir = "objects"
metadataFilename = "metadata"
@ -21,21 +21,23 @@ class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ances
def __init__(self):
super(FilesystemInventory, self).__init__()
self.baseDir = path.join(lookupAppdataFolder(), FilesystemInventory.topDir)
self.baseDir = path.join(
lookupAppdataFolder(), FilesystemInventory.topDir)
for createDir in [self.baseDir, path.join(self.baseDir, "objects")]:
if path.exists(createDir):
if not path.isdir(createDir):
raise IOError("%s exists but it's not a directory" % (createDir))
raise IOError(
"%s exists but it's not a directory" % createDir)
else:
makedirs(createDir)
# Guarantees that two receiveDataThreads don't receive and process the same message
# Guarantees that two receiveDataThreads
# don't receive and process the same message
# concurrently (probably sent by a malicious individual)
self.lock = RLock()
self._inventory = {}
self._load()
def __contains__(self, hashval):
retval = False
for streamDict in self._inventory.values():
if hashval in streamDict:
return True
@ -48,7 +50,12 @@ class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ances
except KeyError:
continue
if retval.payload is None:
retval = InventoryItem(retval.type, retval.stream, self.getData(hashval), retval.expires, retval.tag)
retval = InventoryItem(
retval.type,
retval.stream,
self.getData(hashval),
retval.expires,
retval.tag)
return retval
raise KeyError(hashval)
@ -56,7 +63,10 @@ class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ances
with self.lock:
value = InventoryItem(*value)
try:
makedirs(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hashval)))
makedirs(path.join(
self.baseDir,
FilesystemInventory.objectDir,
hexlify(hashval)))
except OSError:
pass
try:
@ -69,7 +79,11 @@ class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ances
),
"w",
) as f:
f.write("%s,%s,%s,%s," % (value.type, value.stream, value.expires, hexlify(value.tag)))
f.write("%s,%s,%s,%s," % (
value.type,
value.stream,
value.expires,
hexlify(value.tag)))
with open(
path.join(
self.baseDir,
@ -115,7 +129,10 @@ class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ances
except IOError:
pass
try:
rmdir(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hashval)))
rmdir(path.join(
self.baseDir,
FilesystemInventory.objectDir,
hexlify(hashval)))
except IOError:
pass
@ -135,7 +152,8 @@ class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ances
newInventory = {}
for hashId in self.object_list():
try:
objectType, streamNumber, expiresTime, tag = self.getMetadata(hashId)
objectType, streamNumber, expiresTime, tag = self.getMetadata(
hashId)
try:
newInventory[streamNumber][hashId] = InventoryItem(
objectType, streamNumber, None, expiresTime, tag)
@ -155,7 +173,8 @@ class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ances
def object_list(self):
"""Return inventory vectors (hashes) from a directory"""
return [unhexlify(x) for x in listdir(path.join(self.baseDir, FilesystemInventory.objectDir))]
return [unhexlify(x) for x in listdir(path.join(
self.baseDir, FilesystemInventory.objectDir))]
def getData(self, hashId):
"""Get object data"""
@ -185,15 +204,20 @@ class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ances
),
"r",
) as f:
objectType, streamNumber, expiresTime, tag, undef = string.split(f.read(), ",", 4)
return [int(objectType), int(streamNumber), int(expiresTime), unhexlify(tag)]
objectType, streamNumber, expiresTime, tag = string.split(
f.read(), ",", 4)[:4]
return [
int(objectType),
int(streamNumber),
int(expiresTime),
unhexlify(tag)]
except IOError:
raise KeyError
def by_type_and_tag(self, objectType, tag):
"""Get a list of objects filtered by object type and tag"""
retval = []
for stream, streamDict in self._inventory:
for streamDict in self._inventory.values():
for hashId, item in streamDict:
if item.type == objectType and item.tag == tag:
try:
@ -201,7 +225,12 @@ class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ances
item.payload = self.getData(hashId)
except IOError:
continue
retval.append(InventoryItem(item.type, item.stream, item.payload, item.expires, item.tag))
retval.append(InventoryItem(
item.type,
item.stream,
item.payload,
item.expires,
item.tag))
return retval
def hashes_by_stream(self, stream):
@ -215,7 +244,8 @@ class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ances
"""Return unexpired hashes in the inventory for a particular stream"""
t = int(time.time())
try:
return [x for x, value in self._inventory[stream].items() if value.expires > t]
return [x for x, value in self._inventory[stream].items()
if value.expires > t]
except KeyError:
return []
@ -227,7 +257,7 @@ class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ances
"""Clean out old items from the inventory"""
minTime = int(time.time()) - (60 * 60 * 30)
deletes = []
for stream, streamDict in self._inventory.items():
for streamDict in self._inventory.values():
for hashId, item in streamDict.items():
if item.expires < minTime:
deletes.append(hashId)

Write Preview
Loading…
Cancel
Save