diff --git a/minode/sql.py b/minode/sql.py index f1b3cd3..a513aef 100644 --- a/minode/sql.py +++ b/minode/sql.py @@ -1,5 +1,6 @@ """Inventory implementation using sqlite""" +import logging import os import sqlite3 import threading @@ -14,6 +15,7 @@ class Inventory(): """sqlite inventory""" def __init__(self): self._lock = threading.Lock() + self._deleted = 0 self._db = sqlite3.connect( os.path.join(shared.data_directory, 'objects.dat'), check_same_thread=False @@ -29,17 +31,16 @@ class Inventory(): COMMIT; """) self.rowid = len(self) or None - cur = self._db.cursor() - cur.execute("SELECT value FROM status WHERE key='lastvacuumtime'") - now = int(time.time()) try: - vacuumed = cur.fetchone()[0] + self.lastvacuumtime = self._db.execute( + "SELECT value FROM status WHERE key='lastvacuumtime'" + ).fetchone()[0] except TypeError: - pass - else: - if vacuumed < now - 86400: # 24 hours - cur.execute('VACUUM') - cur.execute("INSERT INTO status VALUES ('lastvacuumtime', ?)", (now,)) + self.lastvacuumtime = int(time.time()) + self._db.execute( + "INSERT INTO status VALUES ('lastvacuumtime', ?)", + (self.lastvacuumtime,) + ) self._db.commit() self._db.row_factory = self.__object @@ -55,12 +56,22 @@ class Inventory(): def cleanup(self): """Remove expired objects""" with self._lock: - self._db.execute( - 'DELETE FROM objects WHERE expires < ?', - (int(time.time()) - 3 * 3600,) - ) + now = int(time.time()) + cur = self._db.execute( + 'DELETE FROM objects WHERE expires < ?', (now - 3 * 3600,)) self._db.commit() - # conditional vacuum and validity check + self._deleted += cur.rowcount + logging.debug('Deleted %s expired objects', cur.rowcount) + # conditional vacuum and validity check (TODO) + # every 24 hours or after deleting a lot of items + if self._deleted > 10000 or self.lastvacuumtime < now - 86400: + logging.info('Doing VACUUM for objects') + cur.execute('VACUUM') + cur.execute( + "INSERT INTO status VALUES ('lastvacuumtime', ?)", (now,)) + self._db.commit() + self._deleted = 0 + self.lastvacuumtime = now def filter(self, stream=None, object_type=None, tag=None): """Generator of objects with the given parameters"""