Bitmessage/PyBitmessage-2024-08-21
Archived
2
0
Fork 0
Code Issues 168 Pull Requests 53 Actions Packages Projects Releases 5 Wiki Activity

Format storage.filesystem a bit

Browse Source
This commit is contained in:
Lee Miller 2023-09-21 03:30:36 +03:00
parent a629c15f7d
commit c34b827f0f
Signed by untrusted user: lee.miller
GPG Key ID: 4F97A5EA88F4AB63
1 changed files with 21 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
src/storage/filesystem.py
View File

@ -2,9 +2,9 @@
Module for using filesystem (directory with files) for inventory storage Module for using filesystem (directory with files) for inventory storage
""" """
import logging import logging
import os
import time import time
from binascii import hexlify, unhexlify from binascii import hexlify, unhexlify
from os import listdir, makedirs, path, remove, rmdir
from threading import RLock from threading import RLock
from paths import lookupAppdataFolder from paths import lookupAppdataFolder
@ -22,15 +22,15 @@ class FilesystemInventory(InventoryStorage):
def __init__(self): def __init__(self):
super(FilesystemInventory, self).__init__() super(FilesystemInventory, self).__init__()
self.baseDir = path.join( self.baseDir = os.path.join(
lookupAppdataFolder(), FilesystemInventory.topDir) lookupAppdataFolder(), FilesystemInventory.topDir)
for createDir in [self.baseDir, path.join(self.baseDir, "objects")]: for createDir in [self.baseDir, os.path.join(self.baseDir, "objects")]:
if path.exists(createDir): if os.path.exists(createDir):
if not path.isdir(createDir): if not os.path.isdir(createDir):
raise IOError( raise IOError(
"%s exists but it's not a directory" % createDir) "%s exists but it's not a directory" % createDir)
else: else:
makedirs(createDir) os.makedirs(createDir)
# Guarantees that two receiveDataThreads # Guarantees that two receiveDataThreads
# don't receive and process the same message # don't receive and process the same message
# concurrently (probably sent by a malicious individual) # concurrently (probably sent by a malicious individual)
@ -67,7 +67,7 @@ class FilesystemInventory(InventoryStorage):
with self.lock: with self.lock:
value = InventoryItem(*value) value = InventoryItem(*value)
try: try:
makedirs(path.join( os.makedirs(os.path.join(
self.baseDir, self.baseDir,
FilesystemInventory.objectDir, FilesystemInventory.objectDir,
hexlify(hashval).decode())) hexlify(hashval).decode()))
@ -75,7 +75,7 @@ class FilesystemInventory(InventoryStorage):
pass pass
try: try:
with open( with open(
path.join( os.path.join(
self.baseDir, self.baseDir,
FilesystemInventory.objectDir, FilesystemInventory.objectDir,
hexlify(hashval).decode(), hexlify(hashval).decode(),
@ -89,7 +89,7 @@ class FilesystemInventory(InventoryStorage):
value.expires, value.expires,
hexlify(value.tag).decode())) hexlify(value.tag).decode()))
with open( with open(
path.join( os.path.join(
self.baseDir, self.baseDir,
FilesystemInventory.objectDir, FilesystemInventory.objectDir,
hexlify(hashval).decode(), hexlify(hashval).decode(),
@ -115,8 +115,8 @@ class FilesystemInventory(InventoryStorage):
pass pass
with self.lock: with self.lock:
try: try:
remove( os.remove(
path.join( os.path.join(
self.baseDir, self.baseDir,
FilesystemInventory.objectDir, FilesystemInventory.objectDir,
hexlify(hashval).decode(), hexlify(hashval).decode(),
@ -124,8 +124,8 @@ class FilesystemInventory(InventoryStorage):
except IOError: except IOError:
pass pass
try: try:
remove( os.remove(
path.join( os.path.join(
self.baseDir, self.baseDir,
FilesystemInventory.objectDir, FilesystemInventory.objectDir,
hexlify(hashval).decode(), hexlify(hashval).decode(),
@ -133,7 +133,7 @@ class FilesystemInventory(InventoryStorage):
except IOError: except IOError:
pass pass
try: try:
rmdir(path.join( os.rmdir(os.path.join(
self.baseDir, self.baseDir,
FilesystemInventory.objectDir, FilesystemInventory.objectDir,
hexlify(hashval).decode())) hexlify(hashval).decode()))
@ -169,8 +169,6 @@ class FilesystemInventory(InventoryStorage):
logger.debug( logger.debug(
'error loading %s', hexlify(hashId), exc_info=True) 'error loading %s', hexlify(hashId), exc_info=True)
self._inventory = newInventory self._inventory = newInventory
# for i, v in self._inventory.items():
# print "loaded stream: %s, %i items" % (i, len(v))
def stream_list(self): def stream_list(self):
"""Return list of streams""" """Return list of streams"""
@ -178,14 +176,14 @@ class FilesystemInventory(InventoryStorage):
def object_list(self): def object_list(self):
"""Return inventory vectors (hashes) from a directory""" """Return inventory vectors (hashes) from a directory"""
return [unhexlify(x) for x in listdir(path.join( return [unhexlify(x) for x in os.listdir(os.path.join(
self.baseDir, FilesystemInventory.objectDir))] self.baseDir, FilesystemInventory.objectDir))]
def getData(self, hashId): def getData(self, hashId):
"""Get object data""" """Get object data"""
try: try:
with open( with open(
path.join( os.path.join(
self.baseDir, self.baseDir,
FilesystemInventory.objectDir, FilesystemInventory.objectDir,
hexlify(hashId).decode(), hexlify(hashId).decode(),
@ -201,7 +199,7 @@ class FilesystemInventory(InventoryStorage):
"""Get object metadata""" """Get object metadata"""
try: try:
with open( with open(
path.join( os.path.join(
self.baseDir, self.baseDir,
FilesystemInventory.objectDir, FilesystemInventory.objectDir,
hexlify(hashId).decode(), hexlify(hashId).decode(),
@ -247,10 +245,10 @@ class FilesystemInventory(InventoryStorage):
def unexpired_hashes_by_stream(self, stream): def unexpired_hashes_by_stream(self, stream):
"""Return unexpired hashes in the inventory for a particular stream""" """Return unexpired hashes in the inventory for a particular stream"""
t = int(time.time())
try: try:
return [x for x, value in self._inventory[stream].items() return [
if value.expires > t] x for x, value in self._inventory[stream].items()
if value.expires > int(time.time())]
except KeyError: except KeyError:
return [] return []
@ -260,7 +258,7 @@ class FilesystemInventory(InventoryStorage):
def clean(self): def clean(self):
"""Clean out old items from the inventory""" """Clean out old items from the inventory"""
minTime = int(time.time()) - (60 * 60 * 30) minTime = int(time.time()) - 60 * 60 * 30
deletes = [] deletes = []
for streamDict in self._inventory.values(): for streamDict in self._inventory.values():
for hashId, item in streamDict.items(): for hashId, item in streamDict.items():