From cdc297733132a8694140807c371449dccb931e2b Mon Sep 17 00:00:00 2001 From: Muzahid Date: Sat, 6 Feb 2021 19:01:56 +0530 Subject: [PATCH] Migrate Db with respect to their versions --- src/class_sqlThread.py | 490 ++++++++++++++++++++++------------------- 1 file changed, 261 insertions(+), 229 deletions(-) diff --git a/src/class_sqlThread.py b/src/class_sqlThread.py index 84188408..8f77fe15 100644 --- a/src/class_sqlThread.py +++ b/src/class_sqlThread.py @@ -20,7 +20,262 @@ from debug import logger # pylint: disable=attribute-defined-outside-init,protected-access -class sqlThread(threading.Thread): +class UpgradeDB(): + """Upgrade Db with respect to versions""" + + def __init__(self, cur, version): + """ Apply switcher to call methods accordingly """ + + self.cur = cur + self.parameters = "" + + # Switch methods with respect to versions + if not version in range(1, 11): + # return None when its getting wrong input + return None + method_name = 'version_' + str(version) + method = getattr(self, method_name, lambda: "Invalid version") + return method() + + def version_1(self): + """ + For version 1 and 3 + Add a new column to the inventory table to store tags. + """ + + logger.debug( + 'In messages.dat database, adding tag field to' + ' the inventory table.') + item = '''ALTER TABLE inventory ADD tag blob DEFAULT '' ''' + parameters = '' + self.cur.execute(item, parameters) + item = '''update settings set value=? WHERE key='version';''' + parameters = (4,) + self.cur.execute(item, parameters) + + def version_2(self): + """ + For version 2 + Let's get rid of the first20bytesofencryptedmessage field in the inventory table. + """ + + logger.debug( + 'In messages.dat database, removing an obsolete field from' + ' the inventory table.') + self.cur.execute( + '''CREATE TEMPORARY TABLE inventory_backup''' + '''(hash blob, objecttype text, streamnumber int, payload blob,''' + ''' receivedtime integer, UNIQUE(hash) ON CONFLICT REPLACE);''') + self.cur.execute( + '''INSERT INTO inventory_backup SELECT hash, objecttype, streamnumber, payload, receivedtime''' + ''' FROM inventory;''') + self.cur.execute('''DROP TABLE inventory''') + self.cur.execute( + '''CREATE TABLE inventory''' + ''' (hash blob, objecttype text, streamnumber int, payload blob, receivedtime integer,''' + ''' UNIQUE(hash) ON CONFLICT REPLACE)''') + self.cur.execute( + '''INSERT INTO inventory SELECT hash, objecttype, streamnumber, payload, receivedtime''' + ''' FROM inventory_backup;''') + self.cur.execute('''DROP TABLE inventory_backup;''') + item = '''update settings set value=? WHERE key='version';''' + parameters = (3,) + self.cur.execute(item, parameters) + + def version_3(self): + """ + For version 3 + Call method for version 1 + """ + self.version_1() + + def version_4(self): + """ + For version 4 + Add a new column to the pubkeys table to store the address version. + We're going to trash all of our pubkeys and let them be redownloaded. + """ + + self.cur.execute('''DROP TABLE pubkeys''') + self.cur.execute( + '''CREATE TABLE pubkeys (hash blob, addressversion int, transmitdata blob, time int,''' + '''usedpersonally text, UNIQUE(hash, addressversion) ON CONFLICT REPLACE)''') + self.cur.execute( + '''delete from inventory where objecttype = 'pubkey';''') + item = '''update settings set value=? WHERE key='version';''' + parameters = (5,) + self.cur.execute(item, parameters) + + def version_5(self): + """ + For version 5 + Add a new table: objectprocessorqueue with which to hold objects + That have yet to be processed if the user shuts down Bitmessage. + """ + + self.cur.execute('''DROP TABLE knownnodes''') + self.cur.execute( + '''CREATE TABLE objectprocessorqueue''' + ''' (objecttype text, data blob, UNIQUE(objecttype, data) ON CONFLICT REPLACE)''') + item = '''update settings set value=? WHERE key='version';''' + parameters = (6,) + self.cur.execute(item, parameters) + + def version_6(self): + """ + For version 6 + Changes related to protocol v3 + In table inventory and objectprocessorqueue, objecttype is now + an integer (it was a human-friendly string previously) + """ + + logger.debug( + 'In messages.dat database, dropping and recreating' + ' the inventory table.') + self.cur.execute('''DROP TABLE inventory''') + self.cur.execute( + '''CREATE TABLE inventory''' + ''' (hash blob, objecttype int, streamnumber int, payload blob, expirestime integer,''' + ''' tag blob, UNIQUE(hash) ON CONFLICT REPLACE)''') + self.cur.execute('''DROP TABLE objectprocessorqueue''') + self.cur.execute( + '''CREATE TABLE objectprocessorqueue''' + ''' (objecttype int, data blob, UNIQUE(objecttype, data) ON CONFLICT REPLACE)''') + item = '''update settings set value=? WHERE key='version';''' + parameters = (7,) + self.cur.execute(item, parameters) + logger.debug( + 'Finished dropping and recreating the inventory table.') + + def version_7(self): + """ + For version 7 + The format of data stored in the pubkeys table has changed. Let's + clear it, and the pubkeys from inventory, so that they'll + be re-downloaded. + """ + + logger.debug( + 'In messages.dat database, clearing pubkeys table' + ' because the data format has been updated.') + self.cur.execute( + '''delete from inventory where objecttype = 1;''') + self.cur.execute( + '''delete from pubkeys;''') + # Any sending messages for which we *thought* that we had + # the pubkey must be rechecked. + self.cur.execute( + '''UPDATE sent SET status='msgqueued' WHERE status='doingmsgpow' or status='badkey';''') + query = '''update settings set value=? WHERE key='version';''' + parameters = (8,) + self.cur.execute(query, parameters) + logger.debug('Finished clearing currently held pubkeys.') + + def version_8(self): + """ + For version 8 + Add a new column to the inbox table to store the hash of + the message signature. We'll use this as temporary message UUID + in order to detect duplicates. + """ + logger.debug( + 'In messages.dat database, adding sighash field to' + ' the inbox table.') + item = '''ALTER TABLE inbox ADD sighash blob DEFAULT '' ''' + parameters = '' + self.cur.execute(item, parameters) + item = '''update settings set value=? WHERE key='version';''' + parameters = (9,) + self.cur.execute(item, parameters) + + def version_9(self): + """ + For version 9 + We'll also need a `sleeptill` field and a `ttl` field. Also we + can combine the pubkeyretrynumber and msgretrynumber into one. + """ + + logger.info( + 'In messages.dat database, making TTL-related changes:' + ' combining the pubkeyretrynumber and msgretrynumber' + ' fields into the retrynumber field and adding the' + ' sleeptill and ttl fields...') + self.cur.execute( + '''CREATE TEMPORARY TABLE sent_backup''' + ''' (msgid blob, toaddress text, toripe blob, fromaddress text, subject text, message text,''' + ''' ackdata blob, lastactiontime integer, status text, retrynumber integer,''' + ''' folder text, encodingtype int)''') + self.cur.execute( + '''INSERT INTO sent_backup SELECT msgid, toaddress, toripe, fromaddress,''' + ''' subject, message, ackdata, lastactiontime,''' + ''' status, 0, folder, encodingtype FROM sent;''') + self.cur.execute('''DROP TABLE sent''') + self.cur.execute( + '''CREATE TABLE sent''' + ''' (msgid blob, toaddress text, toripe blob, fromaddress text, subject text, message text,''' + ''' ackdata blob, senttime integer, lastactiontime integer, sleeptill int, status text,''' + ''' retrynumber integer, folder text, encodingtype int, ttl int)''') + self.cur.execute( + '''INSERT INTO sent SELECT msgid, toaddress, toripe, fromaddress, subject, message, ackdata,''' + ''' lastactiontime, lastactiontime, 0, status, 0, folder, encodingtype, 216000 FROM sent_backup;''') + self.cur.execute('''DROP TABLE sent_backup''') + logger.info('In messages.dat database, finished making TTL-related changes.') + logger.debug('In messages.dat database, adding address field to the pubkeys table.') + # We're going to have to calculate the address for each row in the pubkeys + # table. Then we can take out the hash field. + self.cur.execute('''ALTER TABLE pubkeys ADD address text DEFAULT '' ''') + self.cur.execute('''SELECT hash, addressversion FROM pubkeys''') + queryResult = self.cur.fetchall() + from addresses import encodeAddress + for row in queryResult: + addressHash, addressVersion = row + address = encodeAddress(addressVersion, 1, hash) + item = '''UPDATE pubkeys SET address=? WHERE hash=?;''' + parameters = (address, addressHash) + self.cur.execute(item, parameters) + # Now we can remove the hash field from the pubkeys table. + self.cur.execute( + '''CREATE TEMPORARY TABLE pubkeys_backup''' + ''' (address text, addressversion int, transmitdata blob, time int,''' + ''' usedpersonally text, UNIQUE(address) ON CONFLICT REPLACE)''') + self.cur.execute( + '''INSERT INTO pubkeys_backup''' + ''' SELECT address, addressversion, transmitdata, time, usedpersonally FROM pubkeys;''') + self.cur.execute('''DROP TABLE pubkeys''') + self.cur.execute( + '''CREATE TABLE pubkeys''' + ''' (address text, addressversion int, transmitdata blob, time int, usedpersonally text,''' + ''' UNIQUE(address) ON CONFLICT REPLACE)''') + self.cur.execute( + '''INSERT INTO pubkeys SELECT''' + ''' address, addressversion, transmitdata, time, usedpersonally FROM pubkeys_backup;''') + self.cur.execute('''DROP TABLE pubkeys_backup''') + logger.debug( + 'In messages.dat database, done adding address field to the pubkeys table' + ' and removing the hash field.') + self.cur.execute('''update settings set value=10 WHERE key='version';''') + + def version_10(self): + """ + For version 10 + Update the address colunm to unique in addressbook table + """ + + logger.debug( + 'In messages.dat database, updating address column to UNIQUE' + ' in the addressbook table.') + self.cur.execute( + '''ALTER TABLE addressbook RENAME TO old_addressbook''') + self.cur.execute( + '''CREATE TABLE addressbook''' + ''' (label text, address text, UNIQUE(address) ON CONFLICT IGNORE)''') + self.cur.execute( + '''INSERT INTO addressbook SELECT label, address FROM old_addressbook;''') + self.cur.execute('''DROP TABLE old_addressbook''') + self.cur.execute('''update settings set value=11 WHERE key='version';''') + + +class sqlThread(threading.Thread, UpgradeDB): """A thread for all SQL operations""" def __init__(self): @@ -47,7 +302,7 @@ class sqlThread(threading.Thread): self.cur.execute( '''CREATE TABLE subscriptions (label text, address text, enabled bool)''') self.cur.execute( - '''CREATE TABLE addressbook (label text, address text, UNIQUE(address) ON CONFLICT IGNORE)''') + '''CREATE TABLE addressbook (label text, address text''') self.cur.execute( '''CREATE TABLE blacklist (label text, address text, enabled bool)''') self.cur.execute( @@ -63,7 +318,7 @@ class sqlThread(threading.Thread): '''('Bitmessage new releases/announcements','BM-GtovgYdgs7qXPkoYaRgrLFuFKz1SFpsw',1)''') self.cur.execute( '''CREATE TABLE settings (key blob, value blob, UNIQUE(key) ON CONFLICT REPLACE)''') - self.cur.execute('''INSERT INTO settings VALUES('version','11')''') + self.cur.execute('''INSERT INTO settings VALUES('version','10')''') self.cur.execute('''INSERT INTO settings VALUES('lastvacuumtime',?)''', ( int(time.time()),)) self.cur.execute( @@ -178,236 +433,13 @@ class sqlThread(threading.Thread): '''update sent set status='broadcastqueued' where status='broadcastpending' ''') self.conn.commit() - # Let's get rid of the first20bytesofencryptedmessage field in - # the inventory table. - item = '''SELECT value FROM settings WHERE key='version';''' - parameters = '' - self.cur.execute(item, parameters) - if int(self.cur.fetchall()[0][0]) == 2: - logger.debug( - 'In messages.dat database, removing an obsolete field from' - ' the inventory table.') - self.cur.execute( - '''CREATE TEMPORARY TABLE inventory_backup''' - '''(hash blob, objecttype text, streamnumber int, payload blob,''' - ''' receivedtime integer, UNIQUE(hash) ON CONFLICT REPLACE);''') - self.cur.execute( - '''INSERT INTO inventory_backup SELECT hash, objecttype, streamnumber, payload, receivedtime''' - ''' FROM inventory;''') - self.cur.execute('''DROP TABLE inventory''') - self.cur.execute( - '''CREATE TABLE inventory''' - ''' (hash blob, objecttype text, streamnumber int, payload blob, receivedtime integer,''' - ''' UNIQUE(hash) ON CONFLICT REPLACE)''') - self.cur.execute( - '''INSERT INTO inventory SELECT hash, objecttype, streamnumber, payload, receivedtime''' - ''' FROM inventory_backup;''') - self.cur.execute('''DROP TABLE inventory_backup;''') - item = '''update settings set value=? WHERE key='version';''' - parameters = (3,) - self.cur.execute(item, parameters) - - # Add a new column to the inventory table to store tags. + # Upgrade Db with respect to their versions item = '''SELECT value FROM settings WHERE key='version';''' parameters = '' self.cur.execute(item, parameters) currentVersion = int(self.cur.fetchall()[0][0]) - if currentVersion == 1 or currentVersion == 3: - logger.debug( - 'In messages.dat database, adding tag field to' - ' the inventory table.') - item = '''ALTER TABLE inventory ADD tag blob DEFAULT '' ''' - parameters = '' - self.cur.execute(item, parameters) - item = '''update settings set value=? WHERE key='version';''' - parameters = (4,) - self.cur.execute(item, parameters) - - # Add a new column to the pubkeys table to store the address version. - # We're going to trash all of our pubkeys and let them be redownloaded. - item = '''SELECT value FROM settings WHERE key='version';''' - parameters = '' - self.cur.execute(item, parameters) - currentVersion = int(self.cur.fetchall()[0][0]) - if currentVersion == 4: - self.cur.execute('''DROP TABLE pubkeys''') - self.cur.execute( - '''CREATE TABLE pubkeys (hash blob, addressversion int, transmitdata blob, time int,''' - '''usedpersonally text, UNIQUE(hash, addressversion) ON CONFLICT REPLACE)''') - self.cur.execute( - '''delete from inventory where objecttype = 'pubkey';''') - item = '''update settings set value=? WHERE key='version';''' - parameters = (5,) - self.cur.execute(item, parameters) - - # Add a new table: objectprocessorqueue with which to hold objects - # that have yet to be processed if the user shuts down Bitmessage. - item = '''SELECT value FROM settings WHERE key='version';''' - parameters = '' - self.cur.execute(item, parameters) - currentVersion = int(self.cur.fetchall()[0][0]) - if currentVersion == 5: - self.cur.execute('''DROP TABLE knownnodes''') - self.cur.execute( - '''CREATE TABLE objectprocessorqueue''' - ''' (objecttype text, data blob, UNIQUE(objecttype, data) ON CONFLICT REPLACE)''') - item = '''update settings set value=? WHERE key='version';''' - parameters = (6,) - self.cur.execute(item, parameters) - - # changes related to protocol v3 - # In table inventory and objectprocessorqueue, objecttype is now - # an integer (it was a human-friendly string previously) - item = '''SELECT value FROM settings WHERE key='version';''' - parameters = '' - self.cur.execute(item, parameters) - currentVersion = int(self.cur.fetchall()[0][0]) - if currentVersion == 6: - logger.debug( - 'In messages.dat database, dropping and recreating' - ' the inventory table.') - self.cur.execute('''DROP TABLE inventory''') - self.cur.execute( - '''CREATE TABLE inventory''' - ''' (hash blob, objecttype int, streamnumber int, payload blob, expirestime integer,''' - ''' tag blob, UNIQUE(hash) ON CONFLICT REPLACE)''') - self.cur.execute('''DROP TABLE objectprocessorqueue''') - self.cur.execute( - '''CREATE TABLE objectprocessorqueue''' - ''' (objecttype int, data blob, UNIQUE(objecttype, data) ON CONFLICT REPLACE)''') - item = '''update settings set value=? WHERE key='version';''' - parameters = (7,) - self.cur.execute(item, parameters) - logger.debug( - 'Finished dropping and recreating the inventory table.') - - # The format of data stored in the pubkeys table has changed. Let's - # clear it, and the pubkeys from inventory, so that they'll - # be re-downloaded. - item = '''SELECT value FROM settings WHERE key='version';''' - parameters = '' - self.cur.execute(item, parameters) - currentVersion = int(self.cur.fetchall()[0][0]) - if currentVersion == 7: - logger.debug( - 'In messages.dat database, clearing pubkeys table' - ' because the data format has been updated.') - self.cur.execute( - '''delete from inventory where objecttype = 1;''') - self.cur.execute( - '''delete from pubkeys;''') - # Any sending messages for which we *thought* that we had - # the pubkey must be rechecked. - self.cur.execute( - '''UPDATE sent SET status='msgqueued' WHERE status='doingmsgpow' or status='badkey';''') - query = '''update settings set value=? WHERE key='version';''' - parameters = (8,) - self.cur.execute(query, parameters) - logger.debug('Finished clearing currently held pubkeys.') - - # Add a new column to the inbox table to store the hash of - # the message signature. We'll use this as temporary message UUID - # in order to detect duplicates. - item = '''SELECT value FROM settings WHERE key='version';''' - parameters = '' - self.cur.execute(item, parameters) - currentVersion = int(self.cur.fetchall()[0][0]) - if currentVersion == 8: - logger.debug( - 'In messages.dat database, adding sighash field to' - ' the inbox table.') - item = '''ALTER TABLE inbox ADD sighash blob DEFAULT '' ''' - parameters = '' - self.cur.execute(item, parameters) - item = '''update settings set value=? WHERE key='version';''' - parameters = (9,) - self.cur.execute(item, parameters) - - # We'll also need a `sleeptill` field and a `ttl` field. Also we - # can combine the pubkeyretrynumber and msgretrynumber into one. - item = '''SELECT value FROM settings WHERE key='version';''' - parameters = '' - self.cur.execute(item, parameters) - currentVersion = int(self.cur.fetchall()[0][0]) - if currentVersion == 9: - logger.info( - 'In messages.dat database, making TTL-related changes:' - ' combining the pubkeyretrynumber and msgretrynumber' - ' fields into the retrynumber field and adding the' - ' sleeptill and ttl fields...') - self.cur.execute( - '''CREATE TEMPORARY TABLE sent_backup''' - ''' (msgid blob, toaddress text, toripe blob, fromaddress text, subject text, message text,''' - ''' ackdata blob, lastactiontime integer, status text, retrynumber integer,''' - ''' folder text, encodingtype int)''') - self.cur.execute( - '''INSERT INTO sent_backup SELECT msgid, toaddress, toripe, fromaddress,''' - ''' subject, message, ackdata, lastactiontime,''' - ''' status, 0, folder, encodingtype FROM sent;''') - self.cur.execute('''DROP TABLE sent''') - self.cur.execute( - '''CREATE TABLE sent''' - ''' (msgid blob, toaddress text, toripe blob, fromaddress text, subject text, message text,''' - ''' ackdata blob, senttime integer, lastactiontime integer, sleeptill int, status text,''' - ''' retrynumber integer, folder text, encodingtype int, ttl int)''') - self.cur.execute( - '''INSERT INTO sent SELECT msgid, toaddress, toripe, fromaddress, subject, message, ackdata,''' - ''' lastactiontime, lastactiontime, 0, status, 0, folder, encodingtype, 216000 FROM sent_backup;''') - self.cur.execute('''DROP TABLE sent_backup''') - logger.info('In messages.dat database, finished making TTL-related changes.') - logger.debug('In messages.dat database, adding address field to the pubkeys table.') - # We're going to have to calculate the address for each row in the pubkeys - # table. Then we can take out the hash field. - self.cur.execute('''ALTER TABLE pubkeys ADD address text DEFAULT '' ''') - self.cur.execute('''SELECT hash, addressversion FROM pubkeys''') - queryResult = self.cur.fetchall() - from addresses import encodeAddress - for row in queryResult: - addressHash, addressVersion = row - address = encodeAddress(addressVersion, 1, hash) - item = '''UPDATE pubkeys SET address=? WHERE hash=?;''' - parameters = (address, addressHash) - self.cur.execute(item, parameters) - # Now we can remove the hash field from the pubkeys table. - self.cur.execute( - '''CREATE TEMPORARY TABLE pubkeys_backup''' - ''' (address text, addressversion int, transmitdata blob, time int,''' - ''' usedpersonally text, UNIQUE(address) ON CONFLICT REPLACE)''') - self.cur.execute( - '''INSERT INTO pubkeys_backup''' - ''' SELECT address, addressversion, transmitdata, time, usedpersonally FROM pubkeys;''') - self.cur.execute('''DROP TABLE pubkeys''') - self.cur.execute( - '''CREATE TABLE pubkeys''' - ''' (address text, addressversion int, transmitdata blob, time int, usedpersonally text,''' - ''' UNIQUE(address) ON CONFLICT REPLACE)''') - self.cur.execute( - '''INSERT INTO pubkeys SELECT''' - ''' address, addressversion, transmitdata, time, usedpersonally FROM pubkeys_backup;''') - self.cur.execute('''DROP TABLE pubkeys_backup''') - logger.debug( - 'In messages.dat database, done adding address field to the pubkeys table' - ' and removing the hash field.') - self.cur.execute('''update settings set value=10 WHERE key='version';''') - - # Update the address colunm to unique in addressbook table - item = '''SELECT value FROM settings WHERE key='version';''' - parameters = '' - self.cur.execute(item, parameters) - currentVersion = int(self.cur.fetchall()[0][0]) - if currentVersion == 10: - logger.debug( - 'In messages.dat database, updating address column to UNIQUE' - ' in the addressbook table.') - self.cur.execute( - '''ALTER TABLE addressbook RENAME TO old_addressbook''') - self.cur.execute( - '''CREATE TABLE addressbook''' - ''' (label text, address text, UNIQUE(address) ON CONFLICT IGNORE)''') - self.cur.execute( - '''INSERT INTO addressbook SELECT label, address FROM old_addressbook;''') - self.cur.execute('''DROP TABLE old_addressbook''') - self.cur.execute('''update settings set value=11 WHERE key='version';''') + # Call upgrade class for upgrade DB + UpgradeDB(self.cur, currentVersion) # Are you hoping to add a new option to the keys.dat file of existing # Bitmessage users or modify the SQLite database? Add it right