From 10250f3730bf8288e29c97da68e88f9158f27c6b Mon Sep 17 00:00:00 2001 From: cis-kuldeep Date: Wed, 14 Jul 2021 23:52:00 +0530 Subject: [PATCH] initial commit --- src/bitmessageqt/settings.py | 9 +- src/bmconfigparser.py | 8 +- src/class_sqlThread.py | 480 ++++++++++++------------------ src/helper_startup.py | 6 +- src/sql/__init__.py | 0 src/sql/init_version_10.sql | 29 ++ src/sql/init_version_2.sql | 55 ++++ src/sql/init_version_3.sql | 6 + src/sql/init_version_4.sql | 26 ++ src/sql/init_version_5.sql | 17 ++ src/sql/init_version_6.sql | 39 +++ src/sql/init_version_7.sql | 18 ++ src/sql/init_version_8.sql | 5 + src/sql/init_version_9.sql | 141 +++++++++ src/sql/run.sql | 126 ++++++++ src/tests/sql/__init__.py | 0 src/tests/sql/create_function.sql | 9 + src/tests/sql/init_version_10.sql | 17 ++ src/tests/sql/init_version_2.sql | 15 + src/tests/sql/init_version_3.sql | 16 + src/tests/sql/init_version_4.sql | 17 ++ src/tests/sql/init_version_5.sql | 10 + src/tests/sql/init_version_6.sql | 15 + src/tests/sql/init_version_7.sql | 45 +++ src/tests/sql/init_version_8.sql | 14 + src/tests/sql/init_version_9.sql | 27 ++ src/tests/test_sqlthread.py | 264 +++++++++++++++- 27 files changed, 1106 insertions(+), 308 deletions(-) create mode 100644 src/sql/__init__.py create mode 100644 src/sql/init_version_10.sql create mode 100644 src/sql/init_version_2.sql create mode 100644 src/sql/init_version_3.sql create mode 100644 src/sql/init_version_4.sql create mode 100644 src/sql/init_version_5.sql create mode 100644 src/sql/init_version_6.sql create mode 100644 src/sql/init_version_7.sql create mode 100644 src/sql/init_version_8.sql create mode 100644 src/sql/init_version_9.sql create mode 100644 src/sql/run.sql create mode 100644 src/tests/sql/__init__.py create mode 100644 src/tests/sql/create_function.sql create mode 100644 src/tests/sql/init_version_10.sql create mode 100644 src/tests/sql/init_version_2.sql create mode 100644 src/tests/sql/init_version_3.sql create mode 100644 src/tests/sql/init_version_4.sql create mode 100644 src/tests/sql/init_version_5.sql create mode 100644 src/tests/sql/init_version_6.sql create mode 100644 src/tests/sql/init_version_7.sql create mode 100644 src/tests/sql/init_version_8.sql create mode 100644 src/tests/sql/init_version_9.sql diff --git a/src/bitmessageqt/settings.py b/src/bitmessageqt/settings.py index 4173ebd2..76b392c4 100644 --- a/src/bitmessageqt/settings.py +++ b/src/bitmessageqt/settings.py @@ -57,16 +57,9 @@ class SettingsDialog(QtGui.QDialog): pass else: # Append proxy types defined in plugins - # FIXME: this should be a function in mod:`plugin` for ep in pkg_resources.iter_entry_points( 'bitmessage.proxyconfig'): - try: - ep.load() - except Exception: # it should add only functional plugins - # many possible exceptions, which are don't matter - pass - else: - self.comboBoxProxyType.addItem(ep.name) + self.comboBoxProxyType.addItem(ep.name) self.lineEditMaxOutboundConnections.setValidator( QtGui.QIntValidator(0, 8, self.lineEditMaxOutboundConnections)) diff --git a/src/bmconfigparser.py b/src/bmconfigparser.py index 1f5d4fce..8c7abae6 100644 --- a/src/bmconfigparser.py +++ b/src/bmconfigparser.py @@ -10,8 +10,12 @@ from datetime import datetime from six import string_types from six.moves import configparser -import state -from singleton import Singleton +try: + import state + from singleton import Singleton +except ImportError: + from . import state + from .singleton import Singleton SafeConfigParser = configparser.SafeConfigParser diff --git a/src/class_sqlThread.py b/src/class_sqlThread.py index d2a0d2ec..37c32e70 100644 --- a/src/class_sqlThread.py +++ b/src/class_sqlThread.py @@ -9,18 +9,7 @@ import sys import threading import time -if sys.version_info[0] == 3: - from . import helper_sql - from . import helper_startup - from . import paths - from . import queues - from . import state - from . import tr - from .bmconfigparser import BMConfigParser - from .debug import logger - # pylint: disable=attribute-defined-outside-init,protected-access - from .addresses import encodeAddress -else: +try: import helper_sql import helper_startup import paths @@ -29,22 +18,133 @@ else: import tr from bmconfigparser import BMConfigParser from debug import logger - # pylint: disable=attribute-defined-outside-init,protected-access from addresses import encodeAddress +except ImportError: + from . import helper_sql + from . import helper_startup + from . import paths + from . import queues + from . import state + from . import tr + from .bmconfigparser import BMConfigParser + from .debug import logger + from .addresses import encodeAddress + +# pylint: disable=attribute-defined-outside-init,protected-access +root_path = os.path.dirname(os.path.dirname(__file__)) -class sqlThread(threading.Thread): +def connection_build(): + """ + Stablish SQL connection + """ + conn = sqlite3.connect(state.appdata + 'messages.dat') + conn.text_factory = str + cur = conn.cursor() + return conn, cur + + +class UpgradeDB(object): + """ + Upgrade Db with respect to versions + """ + + conn, cur = connection_build() + + def __init__(self): + self._current_level = None + self.max_level = 11 + + def __get_current_settings_version(self): + """ + Upgrade Db with respect to their versions + """ + item = '''SELECT value FROM settings WHERE key='version';''' + parameters = () + self.cur.execute(item, parameters) + return int(self.cur.fetchall()[0][0]) + + def _upgrade_one_level_sql_statement(self, file_name): + """ + Execute SQL files and queries + """ + try: + print("=======================") + print(file_name) + if int(file_name) == 8: + res = self.cur.execute('''PRAGMA table_info('inbox');''') + print("""""""""""""""-----------res""""""""""""""") + print(res) + print("=======================") + with open(os.path.join(root_path, "src/sql/init_version_{}.sql".format(file_name))) as sql_file: + sql_as_string = sql_file.read() + self.cur.executescript(sql_as_string) + except Exception as err: + if str(err) == 'table inbox already exists': + return "ERROR trying to upgrade database. Error message: table inbox already exists" + else: + sys.stderr.write( + 'ERROR trying to upgrade database. Error message: %s\n' % str(err)) + os._exit(0) + + @property + def sql_schema_version(self): + self._current_level = self.__get_current_settings_version() + + @sql_schema_version.setter + def sql_schema_version(self, cur, conn, current_level): + """ + Update version with one level + """ + item = '''update settings set value=? WHERE key='version';''' + parameters = (current_level + 1,) + self.cur.execute(item, parameters) + self._current_level = self.__get_current_settings_version() + + # @sql_schema_version.setter + def upgrade_to_latest(self, cur, conn, current_level=None): + """ + Initialise upgrade level + """ + + # Declare variables + self.conn = conn + self.cur = cur + if current_level: + self._current_level = current_level + else: + self._current_level = self.__get_current_settings_version() + self.max_level = 11 + + # call upgrading level in loop + for l in range(self._current_level, self.max_level): + if int(l) == 1: + continue + self._upgrade_one_level_sql_statement(l) + # self.increment_settings_version(l) + self._current_level = l + + def increment_settings_version(self, level): + """ + Update version with one level + """ + item = '''update settings set value=? WHERE key='version';''' + parameters = (level + 1,) + self.cur.execute(item, parameters) + + +class sqlThread(threading.Thread, UpgradeDB): """A thread for all SQL operations""" def __init__(self): + super(sqlThread, self).__init__() threading.Thread.__init__(self, name="SQL") - def run(self): # pylint: disable=too-many-locals, too-many-branches, too-many-statements + def run(self): # pylint: disable=too-many-locals, too-many-branches, too-many-statements, + # Redefinition-of-parameters-type-from-tuple-to-str, R0204, line-too-long, E501 """Process SQL queries from `.helper_sql.sqlSubmitQueue`""" helper_sql.sql_available = True - self.conn = sqlite3.connect(state.appdata + 'messages.dat') - self.conn.text_factory = str - self.cur = self.conn.cursor() + self.conn, self.cur = connection_build() self.cur.execute('PRAGMA secure_delete = true') @@ -93,7 +193,7 @@ class sqlThread(threading.Thread): else: sys.stderr.write( - 'ERROR trying to create database file (message.dat). Error message: %s\n' % str(err)) + 'ERROR trying to create database file (message.dat). in1111 Error message: %s\n' % str(err)) os._exit(0) # If the settings version is equal to 2 or 3 then the @@ -102,34 +202,7 @@ class sqlThread(threading.Thread): settingsversion = BMConfigParser().getint( 'bitmessagesettings', 'settingsversion') - # People running earlier versions of PyBitmessage do not have the - # usedpersonally field in their pubkeys table. Let's add it. - if settingsversion == 2: - item = '''ALTER TABLE pubkeys ADD usedpersonally text DEFAULT 'no' ''' - parameters = '' - self.cur.execute(item, parameters) - self.conn.commit() - - settingsversion = 3 - - # People running earlier versions of PyBitmessage do not have the - # encodingtype field in their inbox and sent tables or the read field - # in the inbox table. Let's add them. - if settingsversion == 3: - item = '''ALTER TABLE inbox ADD encodingtype int DEFAULT '2' ''' - parameters = '' - self.cur.execute(item, parameters) - - item = '''ALTER TABLE inbox ADD read bool DEFAULT '1' ''' - parameters = '' - self.cur.execute(item, parameters) - - item = '''ALTER TABLE sent ADD encodingtype int DEFAULT '2' ''' - parameters = '' - self.cur.execute(item, parameters) - self.conn.commit() - - settingsversion = 4 + settingsversion = self.earlier_setting_version(settingsversion) BMConfigParser().set( 'bitmessagesettings', 'settingsversion', str(settingsversion)) @@ -141,8 +214,34 @@ class sqlThread(threading.Thread): # file so that when we make changes to the database, the database # version we are on can stay embedded in the messages.dat file. Let us # check to see if the settings table exists yet. + + self.embaded_version() + + # apply version migration + + self.upgrade_to_latest(self.cur, self.conn) + + # Are you hoping to add a new option to the keys.dat file of existing + # Bitmessage users or modify the SQLite database? Add it right + # above this line! + + self.add_new_option() + + # Let us check to see the last time we vaccumed the messages.dat file. + # If it has been more than a month let's do it now. + + self.check_vaccumed() + + def embaded_version(self): + """ + From now on, let us keep a 'version' embedded in the messages.dat + file so that when we make changes to the database, the database + version we are on can stay embedded in the messages.dat file. Let us + check to see if the settings table exists yet. + """ + item = '''SELECT name FROM sqlite_master WHERE type='table' AND name='settings';''' - parameters = '' + parameters = () self.cur.execute(item, parameters) if self.cur.fetchall() == []: # The settings table doesn't exist. We need to make it. @@ -194,237 +293,11 @@ class sqlThread(threading.Thread): '''update sent set status='broadcastqueued' where status='broadcastpending' ''') self.conn.commit() - # Let's get rid of the first20bytesofencryptedmessage field in - # the inventory table. - item = '''SELECT value FROM settings WHERE key='version';''' - parameters = '' - self.cur.execute(item, parameters) - if int(self.cur.fetchall()[0][0]) == 2: - logger.debug( - 'In messages.dat database, removing an obsolete field from' - ' the inventory table.') - self.cur.execute( - '''CREATE TEMPORARY TABLE inventory_backup''' - '''(hash blob, objecttype text, streamnumber int, payload blob,''' - ''' receivedtime integer, UNIQUE(hash) ON CONFLICT REPLACE);''') - self.cur.execute( - '''INSERT INTO inventory_backup SELECT hash, objecttype, streamnumber, payload, receivedtime''' - ''' FROM inventory;''') - self.cur.execute('''DROP TABLE inventory''') - self.cur.execute( - '''CREATE TABLE inventory''' - ''' (hash blob, objecttype text, streamnumber int, payload blob, receivedtime integer,''' - ''' UNIQUE(hash) ON CONFLICT REPLACE)''') - self.cur.execute( - '''INSERT INTO inventory SELECT hash, objecttype, streamnumber, payload, receivedtime''' - ''' FROM inventory_backup;''') - self.cur.execute('''DROP TABLE inventory_backup;''') - item = '''update settings set value=? WHERE key='version';''' - parameters = (3,) - self.cur.execute(item, parameters) - - # Add a new column to the inventory table to store tags. - item = '''SELECT value FROM settings WHERE key='version';''' - parameters = '' - self.cur.execute(item, parameters) - currentVersion = int(self.cur.fetchall()[0][0]) - if currentVersion == 1 or currentVersion == 3: - logger.debug( - 'In messages.dat database, adding tag field to' - ' the inventory table.') - item = '''ALTER TABLE inventory ADD tag blob DEFAULT '' ''' - parameters = '' - self.cur.execute(item, parameters) - item = '''update settings set value=? WHERE key='version';''' - parameters = (4,) - self.cur.execute(item, parameters) - - # Add a new column to the pubkeys table to store the address version. - # We're going to trash all of our pubkeys and let them be redownloaded. - item = '''SELECT value FROM settings WHERE key='version';''' - parameters = '' - self.cur.execute(item, parameters) - currentVersion = int(self.cur.fetchall()[0][0]) - if currentVersion == 4: - self.cur.execute('''DROP TABLE pubkeys''') - self.cur.execute( - '''CREATE TABLE pubkeys (hash blob, addressversion int, transmitdata blob, time int,''' - '''usedpersonally text, UNIQUE(hash, addressversion) ON CONFLICT REPLACE)''') - self.cur.execute( - '''delete from inventory where objecttype = 'pubkey';''') - item = '''update settings set value=? WHERE key='version';''' - parameters = (5,) - self.cur.execute(item, parameters) - - # Add a new table: objectprocessorqueue with which to hold objects - # that have yet to be processed if the user shuts down Bitmessage. - item = '''SELECT value FROM settings WHERE key='version';''' - parameters = '' - self.cur.execute(item, parameters) - currentVersion = int(self.cur.fetchall()[0][0]) - if currentVersion == 5: - self.cur.execute('''DROP TABLE knownnodes''') - self.cur.execute( - '''CREATE TABLE objectprocessorqueue''' - ''' (objecttype text, data blob, UNIQUE(objecttype, data) ON CONFLICT REPLACE)''') - item = '''update settings set value=? WHERE key='version';''' - parameters = (6,) - self.cur.execute(item, parameters) - - # changes related to protocol v3 - # In table inventory and objectprocessorqueue, objecttype is now - # an integer (it was a human-friendly string previously) - item = '''SELECT value FROM settings WHERE key='version';''' - parameters = '' - self.cur.execute(item, parameters) - currentVersion = int(self.cur.fetchall()[0][0]) - if currentVersion == 6: - logger.debug( - 'In messages.dat database, dropping and recreating' - ' the inventory table.') - self.cur.execute('''DROP TABLE inventory''') - self.cur.execute( - '''CREATE TABLE inventory''' - ''' (hash blob, objecttype int, streamnumber int, payload blob, expirestime integer,''' - ''' tag blob, UNIQUE(hash) ON CONFLICT REPLACE)''') - self.cur.execute('''DROP TABLE objectprocessorqueue''') - self.cur.execute( - '''CREATE TABLE objectprocessorqueue''' - ''' (objecttype int, data blob, UNIQUE(objecttype, data) ON CONFLICT REPLACE)''') - item = '''update settings set value=? WHERE key='version';''' - parameters = (7,) - self.cur.execute(item, parameters) - logger.debug( - 'Finished dropping and recreating the inventory table.') - - # The format of data stored in the pubkeys table has changed. Let's - # clear it, and the pubkeys from inventory, so that they'll - # be re-downloaded. - item = '''SELECT value FROM settings WHERE key='version';''' - parameters = '' - self.cur.execute(item, parameters) - currentVersion = int(self.cur.fetchall()[0][0]) - if currentVersion == 7: - logger.debug( - 'In messages.dat database, clearing pubkeys table' - ' because the data format has been updated.') - self.cur.execute( - '''delete from inventory where objecttype = 1;''') - self.cur.execute( - '''delete from pubkeys;''') - # Any sending messages for which we *thought* that we had - # the pubkey must be rechecked. - self.cur.execute( - '''UPDATE sent SET status='msgqueued' WHERE status='doingmsgpow' or status='badkey';''') - query = '''update settings set value=? WHERE key='version';''' - parameters = (8,) - self.cur.execute(query, parameters) - logger.debug('Finished clearing currently held pubkeys.') - - # Add a new column to the inbox table to store the hash of - # the message signature. We'll use this as temporary message UUID - # in order to detect duplicates. - item = '''SELECT value FROM settings WHERE key='version';''' - parameters = '' - self.cur.execute(item, parameters) - currentVersion = int(self.cur.fetchall()[0][0]) - if currentVersion == 8: - logger.debug( - 'In messages.dat database, adding sighash field to' - ' the inbox table.') - item = '''ALTER TABLE inbox ADD sighash blob DEFAULT '' ''' - parameters = '' - self.cur.execute(item, parameters) - item = '''update settings set value=? WHERE key='version';''' - parameters = (9,) - self.cur.execute(item, parameters) - - # We'll also need a `sleeptill` field and a `ttl` field. Also we - # can combine the pubkeyretrynumber and msgretrynumber into one. - - item = '''SELECT value FROM settings WHERE key='version';''' - parameters = '' - self.cur.execute(item, parameters) - currentVersion = int(self.cur.fetchall()[0][0]) - if currentVersion == 9: - logger.info( - 'In messages.dat database, making TTL-related changes:' - ' combining the pubkeyretrynumber and msgretrynumber' - ' fields into the retrynumber field and adding the' - ' sleeptill and ttl fields...') - self.cur.execute( - '''CREATE TEMPORARY TABLE sent_backup''' - ''' (msgid blob, toaddress text, toripe blob, fromaddress text, subject text, message text,''' - ''' ackdata blob, lastactiontime integer, status text, retrynumber integer,''' - ''' folder text, encodingtype int)''') - self.cur.execute( - '''INSERT INTO sent_backup SELECT msgid, toaddress, toripe, fromaddress,''' - ''' subject, message, ackdata, lastactiontime,''' - ''' status, 0, folder, encodingtype FROM sent;''') - self.cur.execute('''DROP TABLE sent''') - self.cur.execute( - '''CREATE TABLE sent''' - ''' (msgid blob, toaddress text, toripe blob, fromaddress text, subject text, message text,''' - ''' ackdata blob, senttime integer, lastactiontime integer, sleeptill int, status text,''' - ''' retrynumber integer, folder text, encodingtype int, ttl int)''') - self.cur.execute( - '''INSERT INTO sent SELECT msgid, toaddress, toripe, fromaddress, subject, message, ackdata,''' - ''' lastactiontime, lastactiontime, 0, status, 0, folder, encodingtype, 216000 FROM sent_backup;''') - self.cur.execute('''DROP TABLE sent_backup''') - logger.info('In messages.dat database, finished making TTL-related changes.') - logger.debug('In messages.dat database, adding address field to the pubkeys table.') - # We're going to have to calculate the address for each row in the pubkeys - # table. Then we can take out the hash field. - self.cur.execute('''ALTER TABLE pubkeys ADD address text DEFAULT '' ;''') - - # replica for loop to update hashed address - self.cur.execute('''UPDATE pubkeys SET address=(enaddr(pubkeys.addressversion, 1, hash)); ''') - - # Now we can remove the hash field from the pubkeys table. - self.cur.execute( - '''CREATE TEMPORARY TABLE pubkeys_backup''' - ''' (address text, addressversion int, transmitdata blob, time int,''' - ''' usedpersonally text, UNIQUE(address) ON CONFLICT REPLACE)''') - self.cur.execute( - '''INSERT INTO pubkeys_backup''' - ''' SELECT address, addressversion, transmitdata, time, usedpersonally FROM pubkeys;''') - self.cur.execute('''DROP TABLE pubkeys''') - self.cur.execute( - '''CREATE TABLE pubkeys''' - ''' (address text, addressversion int, transmitdata blob, time int, usedpersonally text,''' - ''' UNIQUE(address) ON CONFLICT REPLACE)''') - self.cur.execute( - '''INSERT INTO pubkeys SELECT''' - ''' address, addressversion, transmitdata, time, usedpersonally FROM pubkeys_backup;''') - self.cur.execute('''DROP TABLE pubkeys_backup''') - logger.debug( - 'In messages.dat database, done adding address field to the pubkeys table' - ' and removing the hash field.') - self.cur.execute('''update settings set value=10 WHERE key='version';''') - - # Update the address colunm to unique in addressbook table - item = '''SELECT value FROM settings WHERE key='version';''' - parameters = '' - self.cur.execute(item, parameters) - currentVersion = int(self.cur.fetchall()[0][0]) - if currentVersion == 10: - logger.debug( - 'In messages.dat database, updating address column to UNIQUE' - ' in the addressbook table.') - self.cur.execute( - '''ALTER TABLE addressbook RENAME TO old_addressbook''') - self.cur.execute( - '''CREATE TABLE addressbook''' - ''' (label text, address text, UNIQUE(address) ON CONFLICT IGNORE)''') - self.cur.execute( - '''INSERT INTO addressbook SELECT label, address FROM old_addressbook;''') - self.cur.execute('''DROP TABLE old_addressbook''') - self.cur.execute('''update settings set value=11 WHERE key='version';''') - - # Are you hoping to add a new option to the keys.dat file of existing - # Bitmessage users or modify the SQLite database? Add it right - # above this line! - + def add_new_option(self): # pylint: disable=too-many-locals, too-many-branches, too-many-statements + """ + Add new option + RUN SQL query + """ try: testpayload = '\x00\x00' t = ('1234', 1, testpayload, '12345678', 'no') @@ -465,10 +338,14 @@ class sqlThread(threading.Thread): else: logger.error(err) - # Let us check to see the last time we vaccumed the messages.dat file. - # If it has been more than a month let's do it now. + def check_vaccumed(self): # pylint: disable=too-many-locals, too-many-branches, too-many-statements, + # Redefinition-of-parameters-type-from-tuple-to-str, R0204, line-too-long, E501 + """ + Check vaccume and apply sql queries for different different conditions + """ + item = '''SELECT value FROM settings WHERE key='lastvacuumtime';''' - parameters = '' + parameters = () self.cur.execute(item, parameters) queryreturn = self.cur.fetchall() for row in queryreturn: @@ -635,8 +512,45 @@ class sqlThread(threading.Thread): helper_sql.sqlReturnQueue.put((self.cur.fetchall(), rowcount)) # helper_sql.sqlSubmitQueue.task_done() + def earlier_setting_version(self, settingsversion): + """ + Upgrade schema with respect setting version + """ + + # People running earlier versions of PyBitmessage do not have the + # usedpersonally field in their pubkeys table. Let's add it. + if settingsversion == 2: + item = '''ALTER TABLE pubkeys ADD usedpersonally text DEFAULT 'no' ''' + parameters = () + self.cur.execute(item, parameters) + self.conn.commit() + + settingsversion = 3 + + # People running earlier versions of PyBitmessage do not have the + # encodingtype field in their inbox and sent tables or the read field + # in the inbox table. Let's add them. + if settingsversion == 3: + item = '''ALTER TABLE inbox ADD encodingtype int DEFAULT '2' ''' + parameters = () + self.cur.execute(item, parameters) + + item = '''ALTER TABLE inbox ADD read bool DEFAULT '1' ''' + parameters = () + self.cur.execute(item, parameters) + + item = '''ALTER TABLE sent ADD encodingtype int DEFAULT '2' ''' + parameters = () + self.cur.execute(item, parameters) + self.conn.commit() + + return 4 + return settingsversion + def create_function(self): - # create_function + """ + Apply create_function to DB + """ try: self.conn.create_function("enaddr", 3, func=encodeAddress, deterministic=True) except (TypeError, sqlite3.NotSupportedError) as err: diff --git a/src/helper_startup.py b/src/helper_startup.py index 332fe058..3630c258 100644 --- a/src/helper_startup.py +++ b/src/helper_startup.py @@ -1,15 +1,11 @@ """ Startup operations. """ -# pylint: disable=too-many-branches,too-many-statements - import logging import os import platform -import sys import time from distutils.version import StrictVersion - import sys if sys.version_info[0] == 3: from . import defaults @@ -29,6 +25,8 @@ try: except ImportError: get_plugin = None +# pylint: disable=too-many-branches,too-many-statements + logger = logging.getLogger('default') diff --git a/src/sql/__init__.py b/src/sql/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/sql/init_version_10.sql b/src/sql/init_version_10.sql new file mode 100644 index 00000000..4211390c --- /dev/null +++ b/src/sql/init_version_10.sql @@ -0,0 +1,29 @@ +-- -- +-- -- Alter table `addressbook` +-- -- + +ALTER TABLE addressbook RENAME TO old_addressbook; + + +-- -- +-- -- Table structure for table `addressbook` +-- -- + +CREATE TABLE `addressbook` ( + `label` text NOT NULL, + `address` text NOT NULL, + UNIQUE(address) ON CONFLICT IGNORE +) ; + +-- -- +-- -- Table Query for `pubkeys_backup` +-- -- + +INSERT INTO addressbook SELECT label, address FROM old_addressbook; + + +-- -- +-- -- Drop table `old_addressbook` +-- -- + +DROP TABLE old_addressbook; diff --git a/src/sql/init_version_2.sql b/src/sql/init_version_2.sql new file mode 100644 index 00000000..80054041 --- /dev/null +++ b/src/sql/init_version_2.sql @@ -0,0 +1,55 @@ + -- + -- Temp Table structure for table `inventory_backup` + -- + + CREATE TEMP TABLE `inventory_backup` ( + `hash` blob NOT NULL, + `objecttype` text DEFAULT NULL, + `streamnumber` int NOT NULL, + `receivedtime` int NOT NULL, + `payload` blob DEFAULT NULL, + -- `integer` integer NOT NULL, + -- `tag` blob DEFAULT NULL, + UNIQUE(hash) ON CONFLICT REPLACE + ) ; + + -- + -- Dumping data for table `inventory_backup` + -- + + INSERT INTO `inventory_backup` SELECT hash, objecttype, streamnumber, payload, receivedtime FROM inventory; + + + -- + -- Drop table `inventory` + -- + + DROP TABLE inventory; + + + -- + -- Table structure for table `inventory` + -- + + + CREATE TABLE `inventory` ( + `hash` blob NOT NULL, + `objecttype` text DEFAULT NULL, + `streamnumber` int NOT NULL, + `receivedtime` int NOT NULL, + `payload` blob DEFAULT NULL, + UNIQUE(hash) ON CONFLICT REPLACE + ) ; + + + -- + -- Dumping data for table `inventory` + -- + + INSERT INTO inventory SELECT hash, objecttype, streamnumber, payload, receivedtime FROM inventory_backup; + + -- + -- Drop data for table `inventory_backup` + -- + + DROP TABLE inventory_backup; diff --git a/src/sql/init_version_3.sql b/src/sql/init_version_3.sql new file mode 100644 index 00000000..b0693eee --- /dev/null +++ b/src/sql/init_version_3.sql @@ -0,0 +1,6 @@ + +-- +-- Alter table `inventory` +-- + +ALTER TABLE inventory ADD tag blob DEFAULT ''; \ No newline at end of file diff --git a/src/sql/init_version_4.sql b/src/sql/init_version_4.sql new file mode 100644 index 00000000..6805d888 --- /dev/null +++ b/src/sql/init_version_4.sql @@ -0,0 +1,26 @@ + -- + -- Drop Table `pubkeys` + -- + + DROP TABLE pubkeys; + + +-- +-- Table structure for table `pubkeys` +-- + + + CREATE TABLE `pubkeys` ( + `hash` blob NOT NULL, + `addressversion` int DEFAULT NULL, + `transmitdata` blob NOT NULL, + `time` int NOT NULL, + `usedpersonally` text DEFAULT NULL, + UNIQUE(hash, addressversion) ON CONFLICT REPLACE + ) ; + + -- + -- Drop from Table `pubkeys` + -- + + DELETE FROM inventory WHERE objecttype = 'pubkey'; diff --git a/src/sql/init_version_5.sql b/src/sql/init_version_5.sql new file mode 100644 index 00000000..f63cadd6 --- /dev/null +++ b/src/sql/init_version_5.sql @@ -0,0 +1,17 @@ + -- + -- Drop Table `knownnodes` + -- + + DROP TABLE knownnodes; + + + -- + -- Table structure for table `objectprocessorqueue` + -- + + + CREATE TABLE `objectprocessorqueue` ( + `objecttype` text DEFAULT NULL, + `data` blob DEFAULT NULL, + UNIQUE(objecttype, data) ON CONFLICT REPLACE + ) ; diff --git a/src/sql/init_version_6.sql b/src/sql/init_version_6.sql new file mode 100644 index 00000000..d53b7438 --- /dev/null +++ b/src/sql/init_version_6.sql @@ -0,0 +1,39 @@ +-- -- +-- -- Drop table `inventory` +-- -- + + DROP TABLE inventory; + + +-- -- +-- -- Table structure for table `inventory` +-- -- + + +CREATE TABLE `inventory` ( + `hash` blob NOT NULL, + `objecttype` int DEFAULT NULL, + `streamnumber` int NOT NULL, + `payload` blob NOT NULL, + `expirestime` integer DEFAULT NULL, + `tag` blob DEFAULT NULL, + UNIQUE(hash) ON CONFLICT REPLACE +) ; + +-- -- +-- -- Drop table `inventory` +-- -- + + DROP TABLE objectprocessorqueue; + + +-- -- +-- -- Table structure for table `objectprocessorqueue` +-- -- + + +CREATE TABLE `objectprocessorqueue` ( + `objecttype` int DEFAULT NULL, + `data` blob DEFAULT NULL, + UNIQUE(objecttype, data) ON CONFLICT REPLACE +) ; diff --git a/src/sql/init_version_7.sql b/src/sql/init_version_7.sql new file mode 100644 index 00000000..ec5913f6 --- /dev/null +++ b/src/sql/init_version_7.sql @@ -0,0 +1,18 @@ +-- -- +-- -- Drop table `inventory` +-- -- + + DELETE FROM inventory WHERE objecttype = 1; + +-- -- +-- -- Drop table `pubkeys` +-- -- + + DELETE FROM pubkeys; + + +-- -- +-- -- Update table `pubkeys` +-- -- + + UPDATE sent SET status='msgqueued' WHERE status='doingmsgpow' or status='badkey'; diff --git a/src/sql/init_version_8.sql b/src/sql/init_version_8.sql new file mode 100644 index 00000000..70963481 --- /dev/null +++ b/src/sql/init_version_8.sql @@ -0,0 +1,5 @@ +-- -- +-- -- Alter table `inbox` +-- -- + +ALTER TABLE inbox ADD sighash blob DEFAULT ''; diff --git a/src/sql/init_version_9.sql b/src/sql/init_version_9.sql new file mode 100644 index 00000000..b2e29d99 --- /dev/null +++ b/src/sql/init_version_9.sql @@ -0,0 +1,141 @@ +-- -- +-- -- Table structure for table `sent_backup` +-- -- + + +CREATE TEMPORARY TABLE `sent_backup` ( + `msgid` blob DEFAULT NULL, + `toaddress` text DEFAULT NULL, + `toripe` blob DEFAULT NULL, + `fromaddress` text DEFAULT NULL, + `subject` text DEFAULT NULL, + `message` text DEFAULT NULL, + `ackdata` blob DEFAULT NULL, + `lastactiontime` integer DEFAULT NULL, + `status` text DEFAULT NULL, + `retrynumber` integer DEFAULT NULL, + `folder` text DEFAULT NULL, + `encodingtype` int DEFAULT NULL +) ; + + + -- -- + -- -- Dumping data for table `sent_backup` + -- -- + +INSERT INTO sent_backup SELECT msgid, toaddress, toripe, fromaddress, subject, message, ackdata, lastactiontime, status, 0, folder, encodingtype FROM sent; + + +-- -- +-- -- Drope table `sent` +-- -- + +DROP TABLE sent; + + +-- -- +-- -- Table structure for table `sent_backup` +-- -- + + +CREATE TABLE `sent` ( + `msgid` blob DEFAULT NULL, + `toaddress` text DEFAULT NULL, + `toripe` blob DEFAULT NULL, + `fromaddress` text DEFAULT NULL, + `subject` text DEFAULT NULL, + `message` text DEFAULT NULL, + `ackdata` blob DEFAULT NULL, + `senttime` integer DEFAULT NULL, + `lastactiontime` integer DEFAULT NULL, + `sleeptill` int DEFAULT NULL, + `status` text DEFAULT NULL, + `retrynumber` integer DEFAULT NULL, + `folder` text DEFAULT NULL, + `encodingtype` int DEFAULT NULL, + `ttl` int DEFAULT NULL +) ; + + +-- -- +-- -- Dumping data for table `sent` +-- -- + + +INSERT INTO sent SELECT msgid, toaddress, toripe, fromaddress, subject, message, ackdata, lastactiontime, lastactiontime, 0, status, 0, folder, encodingtype, 216000 FROM sent_backup; + + +--UPDATE pubkeys SET address= (select enaddr(?, ?, ?)", (addressVersion, 1, addressHash)) WHERE hash=? + +-- -- +-- -- Drop table `sent` +-- -- + +DROP TABLE sent_backup; + +-- -- +-- -- Update Table `pubkeys` +-- -- We're going to have to calculate the address for each row in the pubkeys +-- -- table. Then we can take out the hash field. +-- -- + +ALTER TABLE pubkeys ADD address text DEFAULT '' ; + +-- -- +-- -- Update Table `pubkeys` +-- -- replica for loop to update hashed address +-- -- + +UPDATE pubkeys SET address=(enaddr(pubkeys.addressversion, 1, hash)); + +-- -- +-- -- Table structure for table `pubkeys_backup` +-- -- + +CREATE TEMPORARY TABLE `pubkeys_backup` ( + `address` text DEFAULT NULL, + `addressversion` int DEFAULT NULL, + `transmitdata` blob DEFAULT NULL, + `time` int DEFAULT NULL, + `usedpersonally` text DEFAULT NULL, + UNIQUE(address) ON CONFLICT REPLACE +) ; + + +-- -- +-- -- Dumping data for table `pubkeys_backup` +-- -- + +INSERT INTO pubkeys_backup SELECT address, addressversion, transmitdata, time, usedpersonally FROM pubkeys; + + +-- -- +-- -- Drope table `pubkeys` +-- -- + +DROP TABLE pubkeys; + +-- -- +-- -- Table structure for table `pubkeys` +-- -- + +CREATE TABLE `pubkeys` ( + `address` text DEFAULT NULL, + `addressversion` int DEFAULT NULL, + `transmitdata` blob DEFAULT NULL, + `time` int DEFAULT NULL, + `usedpersonally` text DEFAULT NULL, + UNIQUE(address) ON CONFLICT REPLACE +) ; + +-- -- +-- -- Dumping data for table `pubkeys` +-- -- + +INSERT INTO pubkeys SELECT address, addressversion, transmitdata, time, usedpersonally FROM pubkeys_backup; + +-- -- +-- -- Dropping table `pubkeys_backup` +-- -- + +DROP TABLE pubkeys_backup; diff --git a/src/sql/run.sql b/src/sql/run.sql new file mode 100644 index 00000000..2538b65b --- /dev/null +++ b/src/sql/run.sql @@ -0,0 +1,126 @@ +-- +-- Table structure for table `inbox` +-- + +CREATE TABLE `inbox` ( + `msgid` blob DEFAULT NULL, + `toaddress` text DEFAULT NULL, + `fromaddress` text DEFAULT NULL, + `subject` text DEFAULT NULL, + `received` text DEFAULT NULL, + `message` text DEFAULT NULL, + `folder` text DEFAULT NULL, + `encodingtype` int DEFAULT NULL, + `read` bool DEFAULT NULL, + `sighash` blob DEFAULT NULL, + UNIQUE(msgid) ON CONFLICT REPLACE +) ; + +-- +-- Table structure for table `sent` +-- + +CREATE TABLE `sent` ( + `msgid` blob DEFAULT NULL, + `toaddress` text DEFAULT NULL, + `toripe` blob DEFAULT NULL, + `fromaddress` text DEFAULT NULL, + `subject` text DEFAULT NULL, + `message` text DEFAULT NULL, + `ackdata` blob DEFAULT NULL, + `senttime` integer DEFAULT NULL, + `lastactiontime` integer DEFAULT NULL, + `sleeptill` integer DEFAULT NULL, + `status` text DEFAULT NULL, + `retrynumber` integer DEFAULT NULL, + `folder` text DEFAULT NULL, + `encodingtype` int DEFAULT NULL, + `ttl` int DEFAULT NULL +) ; + + +-- +-- Table structure for table `subscriptions` +-- + +CREATE TABLE `subscriptions` ( + `label` text DEFAULT NULL, + `address` text DEFAULT NULL, + `enabled` bool DEFAULT NULL +) ; + + +-- +-- Table structure for table `addressbook` +-- + +CREATE TABLE `addressbook` ( + `label` text DEFAULT NULL, + `address` text DEFAULT NULL, + UNIQUE(address) ON CONFLICT IGNORE +) ; + +-- +-- Table structure for table `blacklist` +-- + +CREATE TABLE `blacklist` ( + `label` text DEFAULT NULL, + `address` text DEFAULT NULL, + `enabled` bool DEFAULT NULL +) ; + +-- +-- Table structure for table `whitelist` +-- + +CREATE TABLE `whitelist` ( + `label` text DEFAULT NULL, + `address` text DEFAULT NULL, + `enabled` bool DEFAULT NULL +) ; + + +-- +-- Table structure for table `pubkeys` +-- + +CREATE TABLE `pubkeys` ( + `address` text DEFAULT NULL, + `addressversion` int DEFAULT NULL, + `transmitdata` blob DEFAULT NULL, + `time` int DEFAULT NULL, + `usedpersonally` text DEFAULT NULL, + UNIQUE(address) ON CONFLICT REPLACE +) ; + +-- +-- Table structure for table `inventory` +-- + +CREATE TABLE `inventory` ( + `hash` blob DEFAULT NULL, + `objecttype` int DEFAULT NULL, + `streamnumber` int DEFAULT NULL, + `payload` blob DEFAULT NULL, + `expirestime` integer DEFAULT NULL, + `tag` blob DEFAULT NULL, + UNIQUE(hash) ON CONFLICT REPLACE +) ; + +-- +-- Insert data for table `subscriptions` +-- + +INSERT INTO subscriptions VALUES ('Bitmessage new releases/announcements','BM-GtovgYdgs7qXPkoYaRgrLFuFKz1SFpsw',1); + + +-- +-- Table structure for table `settings` +-- + +CREATE TABLE `settings` ( + `key` blob DEFAULT NULL, + `value` blob DEFAULT NULL, + UNIQUE(key) ON CONFLICT REPLACE +) ; diff --git a/src/tests/sql/__init__.py b/src/tests/sql/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/tests/sql/create_function.sql b/src/tests/sql/create_function.sql new file mode 100644 index 00000000..da32b19e --- /dev/null +++ b/src/tests/sql/create_function.sql @@ -0,0 +1,9 @@ + CREATE TABLE `testhash` ( + `addressversion` int DEFAULT NULL, + `hash` blob DEFAULT NULL, + `address` text DEFAULT NULL, + UNIQUE(address) ON CONFLICT IGNORE +); + +INSERT INTO testhash (addressversion, hash) VALUES(4, "21122112211221122112"); + diff --git a/src/tests/sql/init_version_10.sql b/src/tests/sql/init_version_10.sql new file mode 100644 index 00000000..6eccd551 --- /dev/null +++ b/src/tests/sql/init_version_10.sql @@ -0,0 +1,17 @@ +CREATE TABLE IF NOT EXISTS `addressbook` ( + `label` blob NOT NULL, + `address` text DEFAULT NULL, + UNIQUE(address) ON CONFLICT IGNORE +) ; + +ALTER TABLE addressbook RENAME TO old_addressbook; + +CREATE TABLE IF NOT EXISTS `addressbook` ( + `label` text NOT NULL, + `address` text DEFAULT NULL, + UNIQUE(address) ON CONFLICT IGNORE +) ; + +INSERT INTO addressbook SELECT label, address FROM old_addressbook; + +DROP TABLE old_addressbook; diff --git a/src/tests/sql/init_version_2.sql b/src/tests/sql/init_version_2.sql new file mode 100644 index 00000000..68e01012 --- /dev/null +++ b/src/tests/sql/init_version_2.sql @@ -0,0 +1,15 @@ +DROP TABLE inventory; + + + +CREATE TABLE IF NOT EXISTS `inventory` ( + `hash` blob NOT NULL, + `objecttype` int DEFAULT NULL, + `streamnumber` int NOT NULL, + `receivedtime` int NOT NULL, + `payload` blob DEFAULT NULL, + `integer` integer NOT NULL, + UNIQUE(hash) ON CONFLICT REPLACE +) ; + +INSERT INTO `inventory` VALUES ('hash', 1, 1,1, 1,'test'); diff --git a/src/tests/sql/init_version_3.sql b/src/tests/sql/init_version_3.sql new file mode 100644 index 00000000..b4ae1e6e --- /dev/null +++ b/src/tests/sql/init_version_3.sql @@ -0,0 +1,16 @@ +CREATE TABLE IF NOT EXISTS `settings` ( + `key` blob NOT NULL, + `value` text DEFAULT NULL, + UNIQUE(key) ON CONFLICT REPLACE +) ; + +INSERT INTO `settings` VALUES ('version','1'); + +CREATE TABLE IF NOT EXISTS `inventory` ( + `hash` blob NOT NULL, + `objecttype` int DEFAULT NULL, + `streamnumber` int NOT NULL, + `payload` blob DEFAULT NULL, + `integer` integer NOT NULL, + UNIQUE(hash) ON CONFLICT REPLACE +) ; \ No newline at end of file diff --git a/src/tests/sql/init_version_4.sql b/src/tests/sql/init_version_4.sql new file mode 100644 index 00000000..42485f9a --- /dev/null +++ b/src/tests/sql/init_version_4.sql @@ -0,0 +1,17 @@ +DROP TABLE inventory; + +CREATE TABLE IF NOT EXISTS `inventory` ( + `hash` blob NOT NULL, + `objecttype` text DEFAULT NULL, + `streamnumber` int NOT NULL, + `payload` blob DEFAULT NULL, + `integer` integer NOT NULL, + UNIQUE(hash) ON CONFLICT REPLACE +); + +INSERT INTO `inventory` VALUES ('hash', "pubkey", 1, 1,'test'); + +CREATE TABLE IF NOT EXISTS `pubkeys` ( + `objecttype` int, + UNIQUE(objecttype) ON CONFLICT REPLACE +) ; diff --git a/src/tests/sql/init_version_5.sql b/src/tests/sql/init_version_5.sql new file mode 100644 index 00000000..d50deeb3 --- /dev/null +++ b/src/tests/sql/init_version_5.sql @@ -0,0 +1,10 @@ +DROP TABLE objectprocessorqueue; + +CREATE TABLE IF NOT EXISTS `knownnodes` ( + `hash` blob NOT NULL, + `objecttype` int DEFAULT NULL, + `streamnumber` int NOT NULL, + `payload` blob DEFAULT NULL, + `integer` integer NOT NULL, + UNIQUE(hash) ON CONFLICT REPLACE +) ; diff --git a/src/tests/sql/init_version_6.sql b/src/tests/sql/init_version_6.sql new file mode 100644 index 00000000..85793190 --- /dev/null +++ b/src/tests/sql/init_version_6.sql @@ -0,0 +1,15 @@ +CREATE TABLE IF NOT EXISTS `inventory` ( + `hash` blob NOT NULL, + `objecttype` int DEFAULT NULL, + `streamnumber` int NOT NULL, + `payload` blob DEFAULT NULL, + `integer` integer NOT NULL, + UNIQUE(hash) ON CONFLICT REPLACE +) ; + +INSERT INTO `inventory` VALUES ('hash', 1, 1, 1,'test'); + +CREATE TABLE IF NOT EXISTS `objectprocessorqueue` ( + `objecttype` int, + UNIQUE(objecttype) ON CONFLICT REPLACE +) ; \ No newline at end of file diff --git a/src/tests/sql/init_version_7.sql b/src/tests/sql/init_version_7.sql new file mode 100644 index 00000000..a57bfeca --- /dev/null +++ b/src/tests/sql/init_version_7.sql @@ -0,0 +1,45 @@ +DROP TABLE `inventory`; + +CREATE TABLE IF NOT EXISTS `inventory` ( + `hash` blob NOT NULL, + `objecttype` int DEFAULT NULL, + `streamnumber` int NOT NULL, + `payload` blob DEFAULT NULL, + `integer` integer NOT NULL, + UNIQUE(hash) ON CONFLICT REPLACE +) ; + +INSERT INTO `inventory` VALUES ('hash', 1, 1, 1,'test'); + +CREATE TABLE IF NOT EXISTS `pubkeys` ( + `hash` text, + `addressversion` int, + `transmitdata` blob, + `time` int, + `usedpersonally` text, + UNIQUE(hash) ON CONFLICT REPLACE +) ; + +INSERT INTO `pubkeys` VALUES ('hash','1','1','1','test'); + +CREATE TABLE IF NOT EXISTS `sent` ( + `msgid` blob DEFAULT NULL, + `toaddress` text DEFAULT NULL, + `toripe` blob DEFAULT NULL, + `fromaddress` text DEFAULT NULL, + `subject` text DEFAULT NULL, + `message` text DEFAULT NULL, + `ackdata` blob DEFAULT NULL, + `senttime` integer DEFAULT NULL, + `lastactiontime` integer DEFAULT NULL, + `sleeptill` integer DEFAULT NULL, + `status` text DEFAULT NULL, + `retrynumber` integer DEFAULT NULL, + `folder` text DEFAULT NULL, + `encodingtype` int DEFAULT NULL, + `ttl` int DEFAULT NULL +) ; + +INSERT INTO `sent` VALUES +('msgid','toaddress','toripe','fromaddress','subject','message','ackdata','senttime','lastactiontime','sleeptill','doingmsgpow','retrynumber','folder','encodingtype','ttl'), +('msgid','toaddress','toripe','fromaddress','subject','message','ackdata','senttime','lastactiontime','sleeptill','badkey','retrynumber','folder','encodingtype','ttl'); diff --git a/src/tests/sql/init_version_8.sql b/src/tests/sql/init_version_8.sql new file mode 100644 index 00000000..8081addb --- /dev/null +++ b/src/tests/sql/init_version_8.sql @@ -0,0 +1,14 @@ +DROP TABLE `inbox`; + +CREATE TABLE IF NOT EXISTS `inbox` ( + `msgid` blob NOT NULL, + `toaddress` text DEFAULT NULL, + `fromaddress` text DEFAULT NULL, + `subject` text DEFAULT NULL, + `received` text DEFAULT NULL, + `message` text DEFAULT NULL, + `folder` text DEFAULT NULL, + `encodingtype` int DEFAULT NULL, + `read` bool DEFAULT NULL, + UNIQUE(msgid) ON CONFLICT REPLACE +) ; diff --git a/src/tests/sql/init_version_9.sql b/src/tests/sql/init_version_9.sql new file mode 100644 index 00000000..b3f8db44 --- /dev/null +++ b/src/tests/sql/init_version_9.sql @@ -0,0 +1,27 @@ +CREATE TABLE IF NOT EXISTS `sent` ( + `msgid` blob NOT NULL, + `toaddress` text DEFAULT NULL, + `toripe` blob DEFAULT NULL, + `fromaddress` text DEFAULT NULL, + `subject` text DEFAULT NULL, + `message` text DEFAULT NULL, + `ackdata` blob DEFAULT NULL, + `senttime` integer DEFAULT NULL, + `lastactiontime` integer DEFAULT NULL, + `sleeptill` integer DEFAULT NULL, + `status` text DEFAULT NULL, + `retrynumber` integer DEFAULT NULL, + `folder` text DEFAULT NULL, + `encodingtype` int DEFAULT NULL, + `ttl` int DEFAULT NULL, + UNIQUE(msgid) ON CONFLICT REPLACE +) ; + +CREATE TABLE IF NOT EXISTS `pubkeys` ( + `hash` text, + `addressversion` int, + `transmitdata` blob, + `time` int, + `usedpersonally` text, + UNIQUE(hash) ON CONFLICT REPLACE +) ; diff --git a/src/tests/test_sqlthread.py b/src/tests/test_sqlthread.py index a612df3a..6d6660f7 100644 --- a/src/tests/test_sqlthread.py +++ b/src/tests/test_sqlthread.py @@ -5,40 +5,282 @@ import tempfile import threading import unittest -from .common import skip_python3 - -skip_python3() +# from .common import skip_python3 +# +# skip_python3() os.environ['BITMESSAGE_HOME'] = tempfile.gettempdir() - from pybitmessage.helper_sql import ( - sqlQuery, sql_ready, sqlStoredProcedure) # noqa:E402 -from pybitmessage.class_sqlThread import sqlThread # noqa:E402 + sqlQuery, sql_ready, sqlStoredProcedure, SqlBulkExecute, sqlExecuteScript, sqlExecute) # noqa:E402 +from pybitmessage.class_sqlThread import sqlThread, UpgradeDB # noqa:E402 from pybitmessage.addresses import encodeAddress # noqa:E402 +# try: +# os.environ['BITMESSAGE_HOME'] = tempfile.gettempdir() +# from pybitmessage.helper_sql import ( +# sqlQuery, sql_ready, sqlStoredProcedure, SqlBulkExecute, sqlExecuteScript, sqlExecute) # noqa:E402 +# from pybitmessage.class_sqlThread import sqlThread, UpgradeDB # noqa:E402 +# from pybitmessage.addresses import encodeAddress # noqa:E402 +# except: +# from ..helper_sql import sqlStoredProcedure, sql_ready, sqlExecute, SqlBulkExecute, sqlQuery, sqlExecuteScript # noqa:E402 +# from ..class_sqlThread import (sqlThread, UpgradeDB) # noqa:E402 +# from ..addresses import encodeAddress # noqa:E402 +# from ..helper_sql import sqlStoredProcedure, sql_ready, sqlExecute, SqlBulkExecute, sqlQuery, sqlExecuteScript # noqa:E402 +# from ..class_sqlThread import (sqlThread, UpgradeDB) # noqa:E402 +# from ..addresses import encodeAddress # noqa:E402 + + +def filter_table_column(schema, column): + """ + Filter column from schema + """ + for x in schema: + for y in x: + if y == column: + yield y + class TestSqlThread(unittest.TestCase): """Test case for SQL thread""" + # query file path + __name__ = None + root_path = os.path.dirname(os.path.dirname(__file__)) + @classmethod def setUpClass(cls): - # Start SQL thread + """ + Start SQL thread + """ sqlLookup = sqlThread() sqlLookup.daemon = True sqlLookup.start() sql_ready.wait() + @classmethod + def setUp(cls): + """ + Drop all tables before each test case start + """ + # print("in setup start") + # tables = list(sqlQuery("select name from sqlite_master where type is 'table'")) + # print(tables) + # with SqlBulkExecute() as sql: + # for q in tables: + # print("table name : ", q) + # sql.execute("drop table if exists %s" % q) + # tables = list(sqlQuery("select name from sqlite_master where type is 'table'")) + # print(tables) + # print("in setup end") + + @classmethod + def tearDown(cls): + pass + @classmethod def tearDownClass(cls): + """ + Join the thread + """ sqlStoredProcedure('exit') for thread in threading.enumerate(): if thread.name == "SQL": thread.join() + def initialise_database(self, file): # pylint: disable=W0622, redefined-builtin + """ + Initialise DB + """ + with open(os.path.join(self.root_path, "tests/sql/{}.sql".format(file)), 'r') as sql_as_string: + sql_as_string = sql_as_string.read() + + sqlExecuteScript(sql_as_string) + + def version(self): + """ + Run SQL Scripts, Initialize DB with respect to versioning + and Upgrade DB schema for all versions + """ + def wrapper(*args): + """ + Run SQL and mocking DB for versions + """ + # import pdb; pdb.set_trace() + self = args[0] + func_name = func.__name__ + version = func_name.rsplit('_', 1)[-1] + + print("-------------=========") + print(version) + print("-------------=========") + + if int(version) == 8: + res = sqlQuery('''PRAGMA table_info('inbox');''') + print("""""""""""""""res""""""""""""""") + print(res) + + + # Update versions DB mocking + self.initialise_database("init_version_{}".format(version)) + + + + if int(version) == 9: + sqlThread().create_function() + + if int(version) == 8: + res = sqlQuery('''PRAGMA table_info('inbox');''') + print("""""""""""""""-----------res""""""""""""""") + print(res) + + + # Test versions + upgrade_db = UpgradeDB() + upgrade_db._upgrade_one_level_sql_statement(int(version)) # pylint: disable= W0212, protected-access + # upgrade_db.upgrade_to_latest(upgrade_db.cur, upgrade_db.conn) + # upgrade_db.upgrade_to_latest(upgrade_db.cur, upgrade_db.conn, int(version)) + return func(*args) # <-- use (self, ...) + func = self + return wrapper + def test_create_function(self): - """Check the result of enaddr function""" + """ + Test create_function and asserting the result + """ + + # call create function encoded_str = encodeAddress(4, 1, "21122112211221122112") - query = sqlQuery('SELECT enaddr(4, 1, "21122112211221122112")') - self.assertEqual( - query[0][-1], encoded_str, "test case fail for create_function") + # Initialise Database + self.initialise_database("create_function") + + sqlExecute('''INSERT INTO testhash (addressversion, hash) VALUES(4, "21122112211221122112")''') + # call function in query + + sqlExecute('''UPDATE testhash SET address=(enaddr(testhash.addressversion, 1, hash));''') + + # Assertion + query = sqlQuery('''select * from testhash;''') + self.assertEqual(query[0][-1], encoded_str, "test case fail for create_function") + sqlExecute('''DROP TABLE testhash''') + + @version + def test_sql_thread_version_2(self): + """ + Test with version 2 + """ + + # Assertion + res = sqlQuery(''' SELECT count(name) FROM sqlite_master WHERE type='table' AND name='inventory_backup' ''') + self.assertNotEqual(res[0][0], 1, "Table inventory_backup not deleted in versioning 2") + + @version + def test_sql_thread_version_3(self): + """ + Test with version 1 + Version 1 and 3 are same so will skip 3 + """ + + # Assertion after versioning + res = sqlQuery('''PRAGMA table_info('inventory');''') + result = list(filter_table_column(res, "tag")) + res = [tup for tup in res if any(i in tup for i in ["tag"])] + self.assertEqual(result, ['tag'], "Data not migrated for version 1") + self.assertEqual(res, [(5, 'tag', 'blob', 0, "''", 0)], "Data not migrated for version 1") + + @version + def test_sql_thread_version_4(self): + """ + Test with version 4 + """ + + # Assertion + res = sqlQuery('''select * from inventory where objecttype = 'pubkey';''') + self.assertNotEqual(len(res), 1, "Table inventory not deleted in versioning 4") + + @version + def test_sql_thread_version_5(self): + """ + Test with version 5 + """ + + # Assertion + res = sqlQuery(''' SELECT count(name) FROM sqlite_master WHERE type='table' AND name='knownnodes' ''') + + self.assertNotEqual(res[0][0], 1, "Table knownnodes not deleted in versioning 5") + res = sqlQuery(''' SELECT count(name) FROM sqlite_master + WHERE type='table' AND name='objectprocessorqueue'; ''') + self.assertNotEqual(len(res), 0, "Table objectprocessorqueue not created in versioning 5") + + @version + def test_sql_thread_version_6(self): + """ + Test with version 6 + """ + + # Assertion + + inventory = sqlQuery('''PRAGMA table_info('inventory');''') + inventory = list(filter_table_column(inventory, "expirestime")) + self.assertEqual(inventory, ['expirestime'], "Data not migrated for version 6") + + objectprocessorqueue = sqlQuery('''PRAGMA table_info('inventory');''') + objectprocessorqueue = list(filter_table_column(objectprocessorqueue, "objecttype")) + self.assertEqual(objectprocessorqueue, ['objecttype'], "Data not migrated for version 6") + + @version + def test_sql_thread_version_7(self): + """ + Test with version 7 + """ + + # Assertion + pubkeys = sqlQuery('''SELECT * FROM pubkeys ''') + self.assertEqual(pubkeys, [], "Data not migrated for version 7") + + inventory = sqlQuery('''SELECT * FROM inventory ''') + self.assertEqual(inventory, [], "Data not migrated for version 7") + + sent = sqlQuery('''SELECT status FROM sent ''') + self.assertEqual(sent, [('msgqueued',), ('msgqueued',)], "Data not migrated for version 7") + + @version + def test_sql_thread_version_8(self): + """ + Test with version 8 + """ + + # Assertion + res = sqlQuery('''PRAGMA table_info('inbox');''') + result = list(filter_table_column(res, "sighash")) + self.assertEqual(result, ['sighash'], "Data not migrated for version 8") + + # @version + # def test_sql_thread_version_9(self): + # """ + # Test with version 9 + # """ + # + # # Assertion + # res = sqlQuery(''' SELECT count(name) FROM sqlite_master WHERE type='table' AND name='pubkeys_backup' ''') + # self.assertNotEqual(res[0][0], 1, "Table pubkeys_backup not deleted") + # + # res = sqlQuery('''PRAGMA table_info('pubkeys');''') + # # res = res.fetchall() + # result = list(filter_table_column(res, "address")) + # self.assertEqual(result, ['address'], "Data not migrated for version 9") + + # @version + # def test_sql_thread_version_10(self): + # """ + # Test with version 10 + # """ + # + # # Assertion + # res = sqlQuery(''' SELECT count(name) FROM sqlite_master WHERE type='table' AND name='old_addressbook' ''') + # self.assertNotEqual(res[0][0], 1, "Table old_addressbook not deleted") + # self.assertEqual(len(res), 1, "Table old_addressbook not deleted") + # + # res = sqlQuery('''PRAGMA table_info('addressbook');''') + # result = list(filter_table_column(res, "address")) + # self.assertEqual(result, ['address'], "Data not migrated for version 10")