Add VACUUM after deleting 10000 objects, do it in cleanup()

minode/sql.py

@@ -1,5 +1,6 @@
 """Inventory implementation using sqlite"""
 
+import logging
 import os
 import sqlite3
 import threading
@@ -14,6 +15,7 @@ class Inventory():
     """sqlite inventory"""
     def __init__(self):
         self._lock = threading.Lock()
+        self._deleted = 0
         self._db = sqlite3.connect(
             os.path.join(shared.data_directory, 'objects.dat'),
             check_same_thread=False
@@ -29,17 +31,16 @@ class Inventory():
         COMMIT;
         """)
         self.rowid = len(self) or None
-        cur = self._db.cursor()
-        cur.execute("SELECT value FROM status WHERE key='lastvacuumtime'")
-        now = int(time.time())
         try:
-            vacuumed = cur.fetchone()[0]
+            self.lastvacuumtime = self._db.execute(
+                "SELECT value FROM status WHERE key='lastvacuumtime'"
+            ).fetchone()[0]
         except TypeError:
-            pass
-        else:
-            if vacuumed < now - 86400:  # 24 hours
-                cur.execute('VACUUM')
-        cur.execute("INSERT INTO status VALUES ('lastvacuumtime', ?)", (now,))
+            self.lastvacuumtime = int(time.time())
+            self._db.execute(
+                "INSERT INTO status VALUES ('lastvacuumtime', ?)",
+                (self.lastvacuumtime,)
+            )
         self._db.commit()
         self._db.row_factory = self.__object
 
@@ -55,12 +56,22 @@ class Inventory():
     def cleanup(self):
         """Remove expired objects"""
         with self._lock:
-            self._db.execute(
-                'DELETE FROM objects WHERE expires < ?',
-                (int(time.time()) - 3 * 3600,)
-            )
+            now = int(time.time())
+            cur = self._db.execute(
+                'DELETE FROM objects WHERE expires < ?', (now - 3 * 3600,))
             self._db.commit()
-            # conditional vacuum and validity check
+            self._deleted += cur.rowcount
+            logging.debug('Deleted %s expired objects', cur.rowcount)
+            # conditional vacuum and validity check (TODO)
+            # every 24 hours or after deleting a lot of items
+            if self._deleted > 10000 or self.lastvacuumtime < now - 86400:
+                logging.info('Doing VACUUM for objects')
+                cur.execute('VACUUM')
+                cur.execute(
+                    "INSERT INTO status VALUES ('lastvacuumtime', ?)", (now,))
+                self._db.commit()
+                self._deleted = 0
+                self.lastvacuumtime = now
 
     def filter(self, stream=None, object_type=None, tag=None):
         """Generator of objects with the given parameters"""