From 12118606ec3afbbfb1b27186a38d398c9be06520 Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Thu, 1 Dec 2022 01:43:46 +0200 Subject: [PATCH 1/8] Separate creating the Inventory instance from singleton --- src/inventory.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/inventory.py b/src/inventory.py index 985f1382..e6d4a24c 100644 --- a/src/inventory.py +++ b/src/inventory.py @@ -7,6 +7,16 @@ from bmconfigparser import config from singleton import Singleton +def create_inventory_instance(backend="sqlite"): + """ + Create an instance of the inventory class + defined in `storage.`. + """ + return getattr( + getattr(storage, backend), + "{}Inventory".format(backend.title()))() + + @Singleton class Inventory(): """ @@ -15,11 +25,7 @@ class Inventory(): """ def __init__(self): self._moduleName = config.safeGet("inventory", "storage") - self._inventoryClass = getattr( - getattr(storage, self._moduleName), - "{}Inventory".format(self._moduleName.title()) - ) - self._realInventory = self._inventoryClass() + self._realInventory = create_inventory_instance(self._moduleName) self.numberOfInventoryLookupsPerformed = 0 # cheap inheritance copied from asyncore -- 2.45.1 From 9e9cffab33cf1a43879ba0d1d9671cec2447e5ba Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Tue, 17 May 2022 19:11:31 +0300 Subject: [PATCH 2/8] Started a test case for inventory, so far using a filesystem backend --- src/tests/test_inventory.py | 44 +++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 src/tests/test_inventory.py diff --git a/src/tests/test_inventory.py b/src/tests/test_inventory.py new file mode 100644 index 00000000..9ecb576f --- /dev/null +++ b/src/tests/test_inventory.py @@ -0,0 +1,44 @@ +"""Tests for inventory""" + +import os +import shutil +import struct +import tempfile +import time +import unittest + +from pybitmessage.addresses import calculateInventoryHash + +from .partial import TestPartialRun + + +class TestFilesystemInventory(TestPartialRun): + """A test case for the inventory using filesystem backend""" + + @classmethod + def setUpClass(cls): + cls.home = os.environ['BITMESSAGE_HOME'] = tempfile.mkdtemp() + super(TestFilesystemInventory, cls).setUpClass() + + from inventory import create_inventory_instance + cls.inventory = create_inventory_instance('filesystem') + + def test_consistency(self): + """Ensure the inventory is of proper class""" + if os.path.isfile(os.path.join(self.home, 'messages.dat')): + # this will likely never happen + self.fail("Failed to configure filesystem inventory!") + + def test_appending(self): + """Add a sample message to the inventory""" + TTL = 24 * 60 * 60 + embedded_time = int(time.time() + TTL) + msg = struct.pack('>Q', embedded_time) + os.urandom(166) + invhash = calculateInventoryHash(msg) + self.inventory[invhash] = (2, 1, msg, embedded_time, b'') + + @classmethod + def tearDownClass(cls): + super(TestFilesystemInventory, cls).tearDownClass() + cls.inventory.flush() + shutil.rmtree(os.path.join(cls.home, cls.inventory.topDir)) -- 2.45.1 From e1592a42607f4b3fca8a43405445b0fcffe8c8b7 Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Tue, 17 May 2022 19:13:02 +0300 Subject: [PATCH 3/8] Use dotted imports in the storage package to run on python3 --- src/storage/filesystem.py | 2 +- src/storage/sqlite.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/storage/filesystem.py b/src/storage/filesystem.py index 150e8d9e..58e8db19 100644 --- a/src/storage/filesystem.py +++ b/src/storage/filesystem.py @@ -9,7 +9,7 @@ from os import listdir, makedirs, path, remove, rmdir from threading import RLock from paths import lookupAppdataFolder -from storage import InventoryItem, InventoryStorage +from .storage import InventoryItem, InventoryStorage logger = logging.getLogger('default') diff --git a/src/storage/sqlite.py b/src/storage/sqlite.py index 50a2034e..10874332 100644 --- a/src/storage/sqlite.py +++ b/src/storage/sqlite.py @@ -6,7 +6,7 @@ import time from threading import RLock from helper_sql import SqlBulkExecute, sqlExecute, sqlQuery -from storage import InventoryItem, InventoryStorage +from .storage import InventoryItem, InventoryStorage class SqliteInventory(InventoryStorage): # pylint: disable=too-many-ancestors -- 2.45.1 From 81f574c618f400ac3ac8e9985efba83ad70625b9 Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Tue, 20 Sep 2022 23:50:28 +0300 Subject: [PATCH 4/8] MutableMapping finally moved to collections.abc in py310 --- src/storage/storage.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/storage/storage.py b/src/storage/storage.py index 0391979a..e60d7dcb 100644 --- a/src/storage/storage.py +++ b/src/storage/storage.py @@ -1,10 +1,15 @@ """ Storing inventory items """ -import collections -InventoryItem = collections.namedtuple( - 'InventoryItem', 'type stream payload expires tag') +from collections import namedtuple +try: + from collections import MutableMapping # pylint: disable=deprecated-class +except ImportError: + from collections.abc import MutableMapping + + +InventoryItem = namedtuple('InventoryItem', 'type stream payload expires tag') class Storage(object): # pylint: disable=too-few-public-methods @@ -13,7 +18,7 @@ class Storage(object): # pylint: disable=too-few-public-methods pass -class InventoryStorage(Storage, collections.MutableMapping): +class InventoryStorage(Storage, MutableMapping): """Module used for inventory storage""" def __init__(self): # pylint: disable=super-init-not-called @@ -54,7 +59,7 @@ class InventoryStorage(Storage, collections.MutableMapping): raise NotImplementedError -class MailboxStorage(Storage, collections.MutableMapping): +class MailboxStorage(Storage, MutableMapping): """Method for storing mails""" def __delitem__(self, key): -- 2.45.1 From 8af1a13e1067ebfe99e4dc3eb307851a400acd27 Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Wed, 21 Sep 2022 01:02:36 +0300 Subject: [PATCH 5/8] Simplify storage abstract base definition --- src/storage/filesystem.py | 4 ++- src/storage/sqlite.py | 2 +- src/storage/storage.py | 71 +++++++++++++------------------------ src/tests/test_inventory.py | 14 ++++++++ 4 files changed, 42 insertions(+), 49 deletions(-) diff --git a/src/storage/filesystem.py b/src/storage/filesystem.py index 58e8db19..09408813 100644 --- a/src/storage/filesystem.py +++ b/src/storage/filesystem.py @@ -16,7 +16,6 @@ logger = logging.getLogger('default') class FilesystemInventory(InventoryStorage): """Filesystem for inventory storage""" - # pylint: disable=too-many-ancestors, abstract-method topDir = "inventory" objectDir = "objects" metadataFilename = "metadata" @@ -46,6 +45,9 @@ class FilesystemInventory(InventoryStorage): return True return False + def __delitem__(self, hash_): + raise NotImplementedError + def __getitem__(self, hashval): for streamDict in self._inventory.values(): try: diff --git a/src/storage/sqlite.py b/src/storage/sqlite.py index 10874332..eb5df098 100644 --- a/src/storage/sqlite.py +++ b/src/storage/sqlite.py @@ -9,7 +9,7 @@ from helper_sql import SqlBulkExecute, sqlExecute, sqlQuery from .storage import InventoryItem, InventoryStorage -class SqliteInventory(InventoryStorage): # pylint: disable=too-many-ancestors +class SqliteInventory(InventoryStorage): """Inventory using SQLite""" def __init__(self): super(SqliteInventory, self).__init__() diff --git a/src/storage/storage.py b/src/storage/storage.py index e60d7dcb..c000132c 100644 --- a/src/storage/storage.py +++ b/src/storage/storage.py @@ -2,77 +2,54 @@ Storing inventory items """ +from abc import ABCMeta, abstractmethod from collections import namedtuple try: from collections import MutableMapping # pylint: disable=deprecated-class except ImportError: from collections.abc import MutableMapping +import six + InventoryItem = namedtuple('InventoryItem', 'type stream payload expires tag') -class Storage(object): # pylint: disable=too-few-public-methods - """Base class for storing inventory - (extendable for other items to store)""" - pass +class InventoryStorage(MutableMapping): + """ + Base class for storing inventory + (extendable for other items to store) + """ - -class InventoryStorage(Storage, MutableMapping): - """Module used for inventory storage""" - - def __init__(self): # pylint: disable=super-init-not-called + def __init__(self): self.numberOfInventoryLookupsPerformed = 0 - def __contains__(self, _): - raise NotImplementedError - - def __getitem__(self, _): - raise NotImplementedError - - def __setitem__(self, _, value): - raise NotImplementedError - - def __delitem__(self, _): - raise NotImplementedError - - def __iter__(self): - raise NotImplementedError - - def __len__(self): - raise NotImplementedError + @abstractmethod + def __contains__(self, item): + pass + @abstractmethod def by_type_and_tag(self, objectType, tag): """Return objects filtered by object type and tag""" - raise NotImplementedError + pass + @abstractmethod def unexpired_hashes_by_stream(self, stream): """Return unexpired inventory vectors filtered by stream""" - raise NotImplementedError + pass + @abstractmethod def flush(self): """Flush cache""" - raise NotImplementedError + pass + @abstractmethod def clean(self): """Free memory / perform garbage collection""" - raise NotImplementedError + pass -class MailboxStorage(Storage, MutableMapping): - """Method for storing mails""" - - def __delitem__(self, key): - raise NotImplementedError - - def __getitem__(self, key): - raise NotImplementedError - - def __iter__(self): - raise NotImplementedError - - def __len__(self): - raise NotImplementedError - - def __setitem__(self, key, value): - raise NotImplementedError +@six.add_metaclass(ABCMeta) +class MailboxStorage(MutableMapping): + """An abstract class for storing mails. TODO""" + pass diff --git a/src/tests/test_inventory.py b/src/tests/test_inventory.py index 9ecb576f..b6d0cc85 100644 --- a/src/tests/test_inventory.py +++ b/src/tests/test_inventory.py @@ -7,6 +7,7 @@ import tempfile import time import unittest +from pybitmessage.storage import storage from pybitmessage.addresses import calculateInventoryHash from .partial import TestPartialRun @@ -42,3 +43,16 @@ class TestFilesystemInventory(TestPartialRun): super(TestFilesystemInventory, cls).tearDownClass() cls.inventory.flush() shutil.rmtree(os.path.join(cls.home, cls.inventory.topDir)) + + +class TestStorageAbstract(unittest.TestCase): + """A test case for refactoring of the storage abstract classes""" + + def test_inventory_storage(self): + """Check inherited abstract methods""" + with self.assertRaisesRegexp( + TypeError, "^Can't instantiate abstract class.*" + "methods __contains__, __delitem__, __getitem__, __iter__," + " __len__, __setitem__" + ): # pylint: disable=abstract-class-instantiated + storage.InventoryStorage() -- 2.45.1 From a629c15f7d08697f564db2197e27747a38005131 Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Thu, 21 Sep 2023 03:06:24 +0300 Subject: [PATCH 6/8] Fix appending in storage.filesystem for python3 --- src/storage/filesystem.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/src/storage/filesystem.py b/src/storage/filesystem.py index 09408813..5fa08ca0 100644 --- a/src/storage/filesystem.py +++ b/src/storage/filesystem.py @@ -2,7 +2,6 @@ Module for using filesystem (directory with files) for inventory storage """ import logging -import string import time from binascii import hexlify, unhexlify from os import listdir, makedirs, path, remove, rmdir @@ -71,7 +70,7 @@ class FilesystemInventory(InventoryStorage): makedirs(path.join( self.baseDir, FilesystemInventory.objectDir, - hexlify(hashval))) + hexlify(hashval).decode())) except OSError: pass try: @@ -79,7 +78,7 @@ class FilesystemInventory(InventoryStorage): path.join( self.baseDir, FilesystemInventory.objectDir, - hexlify(hashval), + hexlify(hashval).decode(), FilesystemInventory.metadataFilename, ), "w", @@ -88,15 +87,15 @@ class FilesystemInventory(InventoryStorage): value.type, value.stream, value.expires, - hexlify(value.tag))) + hexlify(value.tag).decode())) with open( path.join( self.baseDir, FilesystemInventory.objectDir, - hexlify(hashval), + hexlify(hashval).decode(), FilesystemInventory.dataFilename, ), - "w", + "wb", ) as f: f.write(value.payload) except IOError: @@ -120,7 +119,7 @@ class FilesystemInventory(InventoryStorage): path.join( self.baseDir, FilesystemInventory.objectDir, - hexlify(hashval), + hexlify(hashval).decode(), FilesystemInventory.metadataFilename)) except IOError: pass @@ -129,7 +128,7 @@ class FilesystemInventory(InventoryStorage): path.join( self.baseDir, FilesystemInventory.objectDir, - hexlify(hashval), + hexlify(hashval).decode(), FilesystemInventory.dataFilename)) except IOError: pass @@ -137,7 +136,7 @@ class FilesystemInventory(InventoryStorage): rmdir(path.join( self.baseDir, FilesystemInventory.objectDir, - hexlify(hashval))) + hexlify(hashval).decode())) except IOError: pass @@ -189,7 +188,7 @@ class FilesystemInventory(InventoryStorage): path.join( self.baseDir, FilesystemInventory.objectDir, - hexlify(hashId), + hexlify(hashId).decode(), FilesystemInventory.dataFilename, ), "r", @@ -205,13 +204,13 @@ class FilesystemInventory(InventoryStorage): path.join( self.baseDir, FilesystemInventory.objectDir, - hexlify(hashId), + hexlify(hashId).decode(), FilesystemInventory.metadataFilename, ), "r", ) as f: - objectType, streamNumber, expiresTime, tag = string.split( - f.read(), ",", 4)[:4] + objectType, streamNumber, expiresTime, tag = f.read().split( + ",", 4)[:4] return [ int(objectType), int(streamNumber), -- 2.45.1 From c34b827f0fe323c50a0daab429bdff169df655f2 Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Thu, 21 Sep 2023 03:30:36 +0300 Subject: [PATCH 7/8] Format storage.filesystem a bit --- src/storage/filesystem.py | 44 +++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/src/storage/filesystem.py b/src/storage/filesystem.py index 5fa08ca0..e756a820 100644 --- a/src/storage/filesystem.py +++ b/src/storage/filesystem.py @@ -2,9 +2,9 @@ Module for using filesystem (directory with files) for inventory storage """ import logging +import os import time from binascii import hexlify, unhexlify -from os import listdir, makedirs, path, remove, rmdir from threading import RLock from paths import lookupAppdataFolder @@ -22,15 +22,15 @@ class FilesystemInventory(InventoryStorage): def __init__(self): super(FilesystemInventory, self).__init__() - self.baseDir = path.join( + self.baseDir = os.path.join( lookupAppdataFolder(), FilesystemInventory.topDir) - for createDir in [self.baseDir, path.join(self.baseDir, "objects")]: - if path.exists(createDir): - if not path.isdir(createDir): + for createDir in [self.baseDir, os.path.join(self.baseDir, "objects")]: + if os.path.exists(createDir): + if not os.path.isdir(createDir): raise IOError( "%s exists but it's not a directory" % createDir) else: - makedirs(createDir) + os.makedirs(createDir) # Guarantees that two receiveDataThreads # don't receive and process the same message # concurrently (probably sent by a malicious individual) @@ -67,7 +67,7 @@ class FilesystemInventory(InventoryStorage): with self.lock: value = InventoryItem(*value) try: - makedirs(path.join( + os.makedirs(os.path.join( self.baseDir, FilesystemInventory.objectDir, hexlify(hashval).decode())) @@ -75,7 +75,7 @@ class FilesystemInventory(InventoryStorage): pass try: with open( - path.join( + os.path.join( self.baseDir, FilesystemInventory.objectDir, hexlify(hashval).decode(), @@ -89,7 +89,7 @@ class FilesystemInventory(InventoryStorage): value.expires, hexlify(value.tag).decode())) with open( - path.join( + os.path.join( self.baseDir, FilesystemInventory.objectDir, hexlify(hashval).decode(), @@ -115,8 +115,8 @@ class FilesystemInventory(InventoryStorage): pass with self.lock: try: - remove( - path.join( + os.remove( + os.path.join( self.baseDir, FilesystemInventory.objectDir, hexlify(hashval).decode(), @@ -124,8 +124,8 @@ class FilesystemInventory(InventoryStorage): except IOError: pass try: - remove( - path.join( + os.remove( + os.path.join( self.baseDir, FilesystemInventory.objectDir, hexlify(hashval).decode(), @@ -133,7 +133,7 @@ class FilesystemInventory(InventoryStorage): except IOError: pass try: - rmdir(path.join( + os.rmdir(os.path.join( self.baseDir, FilesystemInventory.objectDir, hexlify(hashval).decode())) @@ -169,8 +169,6 @@ class FilesystemInventory(InventoryStorage): logger.debug( 'error loading %s', hexlify(hashId), exc_info=True) self._inventory = newInventory -# for i, v in self._inventory.items(): -# print "loaded stream: %s, %i items" % (i, len(v)) def stream_list(self): """Return list of streams""" @@ -178,14 +176,14 @@ class FilesystemInventory(InventoryStorage): def object_list(self): """Return inventory vectors (hashes) from a directory""" - return [unhexlify(x) for x in listdir(path.join( + return [unhexlify(x) for x in os.listdir(os.path.join( self.baseDir, FilesystemInventory.objectDir))] def getData(self, hashId): """Get object data""" try: with open( - path.join( + os.path.join( self.baseDir, FilesystemInventory.objectDir, hexlify(hashId).decode(), @@ -201,7 +199,7 @@ class FilesystemInventory(InventoryStorage): """Get object metadata""" try: with open( - path.join( + os.path.join( self.baseDir, FilesystemInventory.objectDir, hexlify(hashId).decode(), @@ -247,10 +245,10 @@ class FilesystemInventory(InventoryStorage): def unexpired_hashes_by_stream(self, stream): """Return unexpired hashes in the inventory for a particular stream""" - t = int(time.time()) try: - return [x for x, value in self._inventory[stream].items() - if value.expires > t] + return [ + x for x, value in self._inventory[stream].items() + if value.expires > int(time.time())] except KeyError: return [] @@ -260,7 +258,7 @@ class FilesystemInventory(InventoryStorage): def clean(self): """Clean out old items from the inventory""" - minTime = int(time.time()) - (60 * 60 * 30) + minTime = int(time.time()) - 60 * 60 * 30 deletes = [] for streamDict in self._inventory.values(): for hashId, item in streamDict.items(): -- 2.45.1 From 0f8fdb4b0b221766fee5aac0fd8463d0ba7ebd1d Mon Sep 17 00:00:00 2001 From: Lee Miller Date: Tue, 21 Nov 2023 13:29:08 +0200 Subject: [PATCH 8/8] Remove MailboxStorage TODO --- src/storage/storage.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/storage/storage.py b/src/storage/storage.py index c000132c..9b33eef7 100644 --- a/src/storage/storage.py +++ b/src/storage/storage.py @@ -2,15 +2,13 @@ Storing inventory items """ -from abc import ABCMeta, abstractmethod +from abc import abstractmethod from collections import namedtuple try: from collections import MutableMapping # pylint: disable=deprecated-class except ImportError: from collections.abc import MutableMapping -import six - InventoryItem = namedtuple('InventoryItem', 'type stream payload expires tag') @@ -47,9 +45,3 @@ class InventoryStorage(MutableMapping): def clean(self): """Free memory / perform garbage collection""" pass - - -@six.add_metaclass(ABCMeta) -class MailboxStorage(MutableMapping): - """An abstract class for storing mails. TODO""" - pass -- 2.45.1