From ce0f99a8c2dade37f80b308b3e3b541c82d9d229 Mon Sep 17 00:00:00 2001 From: Muzahid Date: Fri, 26 Feb 2021 20:18:52 +0530 Subject: [PATCH] refactor the sqlthread test cases and fix py3 issue --- src/class_sqlThread.py | 55 +++++++++- src/helper_sql.py | 7 +- src/tests/sql/init_version_2.sql | 20 ++++ src/tests/sql/init_version_4.sql | 27 +++++ src/tests/sql/init_version_5.sql | 12 +++ src/tests/sql/init_version_6.sql | 27 +++++ src/tests/sql/init_version_7.sql | 69 +++++++++++++ src/tests/test_sqlthread.py | 168 ++++++++++++++++++++----------- 8 files changed, 321 insertions(+), 64 deletions(-) create mode 100644 src/tests/sql/init_version_2.sql create mode 100644 src/tests/sql/init_version_4.sql create mode 100644 src/tests/sql/init_version_5.sql create mode 100644 src/tests/sql/init_version_6.sql create mode 100644 src/tests/sql/init_version_7.sql diff --git a/src/class_sqlThread.py b/src/class_sqlThread.py index db6db07b..f3311d93 100644 --- a/src/class_sqlThread.py +++ b/src/class_sqlThread.py @@ -444,6 +444,56 @@ class sqlThread(threading.Thread, UpgradeDB): # Bitmessage users or modify the SQLite database? Add it right # above this line! + self.add_new_option() + + # try: + # testpayload = '\x00\x00' + # t = ('1234', 1, testpayload, '12345678', 'no') + # self.cur.execute('''INSERT INTO pubkeys VALUES(?,?,?,?,?)''', t) + # self.conn.commit() + # self.cur.execute( + # '''SELECT transmitdata FROM pubkeys WHERE address='1234' ''') + # queryreturn = self.cur.fetchall() + # for row in queryreturn: + # transmitdata, = row + # self.cur.execute('''DELETE FROM pubkeys WHERE address='1234' ''') + # self.conn.commit() + # if transmitdata == '': + # logger.fatal( + # 'Problem: The version of SQLite you have cannot store Null values.' + # ' Please download and install the latest revision of your version of Python' + # ' (for example, the latest Python 2.7 revision) and try again.\n') + # logger.fatal( + # 'PyBitmessage will now exit very abruptly.' + # ' You may now see threading errors related to this abrupt exit' + # ' but the problem you need to solve is related to SQLite.\n\n') + # os._exit(0) + # except Exception as err: + # if str(err) == 'database or disk is full': + # logger.fatal( + # '(While null value test) Alert: Your disk or data storage volume is full.' + # ' sqlThread will now exit.') + # queues.UISignalQueue.put(( + # 'alert', ( + # tr._translate( + # "MainWindow", + # "Disk full"), + # tr._translate( + # "MainWindow", + # 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), + # True))) + # os._exit(0) + # else: + # logger.error(err) + + # Let us check to see the last time we vaccumed the messages.dat file. + # If it has been more than a month let's do it now. + + self.check_vaccumed() + + + def add_new_option(self): + print("start func -=-=-=-=-=-=-=-=") try: testpayload = '\x00\x00' t = ('1234', 1, testpayload, '12345678', 'no') @@ -484,8 +534,9 @@ class sqlThread(threading.Thread, UpgradeDB): else: logger.error(err) - # Let us check to see the last time we vaccumed the messages.dat file. - # If it has been more than a month let's do it now. + + def check_vaccumed(self): + print(" chec vaccumed start") item = '''SELECT value FROM settings WHERE key='lastvacuumtime';''' parameters = '' self.cur.execute(item, parameters) diff --git a/src/helper_sql.py b/src/helper_sql.py index 043bccf2..85a6b32a 100644 --- a/src/helper_sql.py +++ b/src/helper_sql.py @@ -16,7 +16,12 @@ SQLite objects can only be used from one thread. or isn't thread-safe. """ -import Queue +# import Queue +try: + import queue as Queue #python2 +except ImportError: + import Queue #python3 + import threading sqlSubmitQueue = Queue.Queue() diff --git a/src/tests/sql/init_version_2.sql b/src/tests/sql/init_version_2.sql new file mode 100644 index 00000000..0e1abac8 --- /dev/null +++ b/src/tests/sql/init_version_2.sql @@ -0,0 +1,20 @@ +-- +-- Table structure for table `inventory` +-- + +CREATE TABLE IF NOT EXISTS `inventory` ( + `hash` blob NOT NULL, + `objecttype` int DEFAULT NULL, + `streamnumber` int NOT NULL, + `receivedtime` int NOT NULL, + `payload` blob DEFAULT NULL, + `integer` integer NOT NULL, + -- `tag` blob DEFAULT NULL, + UNIQUE(hash) ON CONFLICT REPLACE +) ; + +-- +-- Dumping data for table `inventory` +-- + +INSERT INTO `inventory` VALUES ('hash', 1, 1,1, 1,'test'); diff --git a/src/tests/sql/init_version_4.sql b/src/tests/sql/init_version_4.sql new file mode 100644 index 00000000..9ea168a1 --- /dev/null +++ b/src/tests/sql/init_version_4.sql @@ -0,0 +1,27 @@ +-- +-- Table structure for table `inventory` +-- + +CREATE TABLE IF NOT EXISTS `inventory` ( + `hash` blob NOT NULL, + `objecttype` text DEFAULT NULL, + `streamnumber` int NOT NULL, + `payload` blob DEFAULT NULL, + `integer` integer NOT NULL, + UNIQUE(hash) ON CONFLICT REPLACE +) ; + +-- +-- Dumping data for table `inventory` +-- + +INSERT INTO `inventory` VALUES ('hash', "pubkey", 1, 1,'test'); + +-- +-- Table structure for table `pubkeys` +-- + +CREATE TABLE IF NOT EXISTS `pubkeys` ( + `objecttype` int, + UNIQUE(objecttype) ON CONFLICT REPLACE +) ; \ No newline at end of file diff --git a/src/tests/sql/init_version_5.sql b/src/tests/sql/init_version_5.sql new file mode 100644 index 00000000..ee71c2aa --- /dev/null +++ b/src/tests/sql/init_version_5.sql @@ -0,0 +1,12 @@ +-- +-- Table structure for table `knownnodes` +-- + +CREATE TABLE IF NOT EXISTS `knownnodes` ( + `hash` blob NOT NULL, + `objecttype` int DEFAULT NULL, + `streamnumber` int NOT NULL, + `payload` blob DEFAULT NULL, + `integer` integer NOT NULL, + UNIQUE(hash) ON CONFLICT REPLACE +) ; diff --git a/src/tests/sql/init_version_6.sql b/src/tests/sql/init_version_6.sql new file mode 100644 index 00000000..ce029333 --- /dev/null +++ b/src/tests/sql/init_version_6.sql @@ -0,0 +1,27 @@ +-- +-- Table structure for table `inventory` +-- + +CREATE TABLE IF NOT EXISTS `inventory` ( + `hash` blob NOT NULL, + `objecttype` int DEFAULT NULL, + `streamnumber` int NOT NULL, + `payload` blob DEFAULT NULL, + `integer` integer NOT NULL, + UNIQUE(hash) ON CONFLICT REPLACE +) ; + +-- +-- Dumping data for table `inventory` +-- + +INSERT INTO `inventory` VALUES ('hash', 1, 1, 1,'test'); + +-- +-- Table structure for table `objectprocessorqueue` +-- + +CREATE TABLE IF NOT EXISTS `objectprocessorqueue` ( + `objecttype` int, + UNIQUE(objecttype) ON CONFLICT REPLACE +) ; \ No newline at end of file diff --git a/src/tests/sql/init_version_7.sql b/src/tests/sql/init_version_7.sql new file mode 100644 index 00000000..8d941fe3 --- /dev/null +++ b/src/tests/sql/init_version_7.sql @@ -0,0 +1,69 @@ +-- +-- Table structure for table `inventory` +-- + +CREATE TABLE IF NOT EXISTS `inventory` ( + `hash` blob NOT NULL, + `objecttype` int DEFAULT NULL, + `streamnumber` int NOT NULL, + `payload` blob DEFAULT NULL, + `integer` integer NOT NULL, + -- `tag` blob DEFAULT NULL, + UNIQUE(hash) ON CONFLICT REPLACE +) ; + +-- +-- Dumping data for table `inventory` +-- + +INSERT INTO `inventory` VALUES ('hash', 1, 1, 1,'test'); + +-- +-- Table structure for table `pubkeys` +-- + +CREATE TABLE IF NOT EXISTS `pubkeys` ( + `hash` text, + `addressversion` int, + `transmitdata` blob, + `time` int, + `usedpersonally` text, + UNIQUE(hash) ON CONFLICT REPLACE +) ; + + +-- +-- Dumping data for table `pubkeys` +-- + +INSERT INTO `pubkeys` VALUES ('hash','1','1','1','test'); + +-- +-- Table structure for table `sent` +-- + +CREATE TABLE IF NOT EXISTS `sent` ( + `msgid` blob DEFAULT NULL, + `toaddress` text DEFAULT NULL, + `toripe` blob DEFAULT NULL, + `fromaddress` text DEFAULT NULL, + `subject` text DEFAULT NULL, + `message` text DEFAULT NULL, + `ackdata` blob DEFAULT NULL, + `senttime` integer DEFAULT NULL, + `lastactiontime` integer DEFAULT NULL, + `sleeptill` integer DEFAULT NULL, + `status` text DEFAULT NULL, + `retrynumber` integer DEFAULT NULL, + `folder` text DEFAULT NULL, + `encodingtype` int DEFAULT NULL, + `ttl` int DEFAULT NULL +) ; + +-- +-- Dumping data for table `sent` +-- + +INSERT INTO `sent` VALUES +('msgid','toaddress','toripe','fromaddress','subject','message','ackdata','senttime','lastactiontime','sleeptill','doingmsgpow','retrynumber','folder','encodingtype','ttl'), +('msgid','toaddress','toripe','fromaddress','subject','message','ackdata','senttime','lastactiontime','sleeptill','badkey','retrynumber','folder','encodingtype','ttl'); \ No newline at end of file diff --git a/src/tests/test_sqlthread.py b/src/tests/test_sqlthread.py index 649cdd40..1e92bb51 100644 --- a/src/tests/test_sqlthread.py +++ b/src/tests/test_sqlthread.py @@ -4,22 +4,10 @@ import os import unittest -import shutil # used for moving the messages.dat file import sqlite3 import sys -import threading -import time -# print sys.path -# import state -# import sql -# from sql import init_version_1.sql from ..state import appdata from ..helper_sql import sqlStoredProcedure -# import threading -# from hashlib import sha256 -# from ..helper_sql import sqlQuery -# from ..version import softwareVersion -# from common import cleanup from ..class_sqlThread import (sqlThread, UpgradeDB) @@ -53,16 +41,11 @@ class TestSqlThread(unittest.TestCase): # Stop sql thread sqlStoredProcedure('exit') - def db_connection(self): - conn = sqlite3.connect(appdata + 'messages.dat') - conn.text_factory = str - return conn.cursor() - def normalize_version(self, file): try: root_path = os.path.dirname(os.path.dirname(__file__)) sql_file_path = os.path.join(root_path, 'tests/sql/') - sql_file_path = os.path.join(sql_file_path, "init_version_"+file+".sql") + sql_file_path = os.path.join(sql_file_path, "init_version_{}.sql".format(file)) sql_file = open(sql_file_path) sql_as_string = sql_file.read() self.cur.executescript(sql_as_string) @@ -81,91 +64,154 @@ class TestSqlThread(unittest.TestCase): if y == column: yield y + def versioning(func): + def wrapper(*args): + self = args[0] + func_name = func.__name__ + version = func_name.rsplit('_', 1)[-1] + + # Update versions DB mocking + self.normalize_version(version) + + # Test versions + upgrade_db = UpgradeDB() + upgrade_db.cur = self.cur + getattr(upgrade_db, "upgrade_schema_data_{}".format(version))() + ret = func(*args) + return ret # <-- use (self, ...) + return wrapper + + def change_state(self): + self.normalize_version("1") + + @versioning def test_sql_thread_version_1(self): """ Test with version 1 """ - # normalise Db for version 1 - self.normalize_version("1") - - # Test versioning - upgrade_db = UpgradeDB() - upgrade_db.cur = self.cur - upgrade_db.upgrade_schema_data_1() - # Assertion after versioning res = self.cur.execute('''PRAGMA table_info('inventory');''') res = res.fetchall() result = list(self.filter_table_column(res, "tag")) res = [tup for tup in res if any(i in tup for i in ["tag"])] - self.assertEqual(result, ['tag'] , "Data not migrated for version 1") - self.assertEqual(res, [(5, 'tag', 'blob', 0, "''", 0)] , "Data not migrated for version 1") - + self.assertEqual(result, ['tag'], "Data not migrated for version 1") + self.assertEqual(res, [(5, 'tag', 'blob', 0, "''", 0)], "Data not migrated for version 1") + @versioning def test_sql_thread_version_10(self): """ Test with version 10 """ - # Update version 10 - self.normalize_version("10") - - # Test versioning - upgrade_db = UpgradeDB() - upgrade_db.cur = self.cur - upgrade_db.upgrade_schema_data_10() - # Assertion self.cur.execute(''' SELECT count(name) FROM sqlite_master WHERE type='table' AND name='old_addressbook' ''') - self.assertNotEqual(self.cur.fetchone(), 1 , "Table old_addressbook not deleted") + self.assertNotEqual(self.cur.fetchone(), 1, "Table old_addressbook not deleted") res = self.cur.execute('''PRAGMA table_info('addressbook');''') res = res.fetchall() result = list(self.filter_table_column(res, "address")) - self.assertEqual(result, ['address'] , "Data not migrated for version 10") - + self.assertEqual(result, ['address'], "Data not migrated for version 10") + @versioning def test_sql_thread_version_9(self): """ Test with version 9 """ - # Update version 9 - self.normalize_version("9") - - # Test versioning - upgrade_db = UpgradeDB() - upgrade_db.cur = self.cur - upgrade_db.upgrade_schema_data_9() - # Assertion self.cur.execute(''' SELECT count(name) FROM sqlite_master WHERE type='table' AND name='pubkeys_backup' ''') - self.assertNotEqual(self.cur.fetchone(), 1 , "Table pubkeys_backup not deleted") + self.assertNotEqual(self.cur.fetchone(), 1, "Table pubkeys_backup not deleted") res = self.cur.execute('''PRAGMA table_info('pubkeys');''') res = res.fetchall() result = list(self.filter_table_column(res, "address")) - self.assertEqual(result, ['address'] , "Data not migrated for version 9") - + self.assertEqual(result, ['address'], "Data not migrated for version 9") + @versioning def test_sql_thread_version_8(self): """ Test with version 8 """ - # Update version 8 - self.normalize_version("8") - - # Test versioning - upgrade_db = UpgradeDB() - upgrade_db.cur = self.cur - upgrade_db.upgrade_schema_data_8() - - # # Assertion + # Assertion res = self.cur.execute('''PRAGMA table_info('inbox');''') res = res.fetchall() result = list(self.filter_table_column(res, "sighash")) - self.assertEqual(result, ['sighash'] , "Data not migrated for version 8") + self.assertEqual(result, ['sighash'], "Data not migrated for version 8") + @versioning + def test_sql_thread_version_7(self): + """ + Test with version 7 + """ + # Assertion + pubkeys = self.cur.execute('''SELECT * FROM pubkeys ''') + pubkeys = pubkeys.fetchall() + self.assertEqual(pubkeys, [], "Data not migrated for version 7") + + inventory = self.cur.execute('''SELECT * FROM inventory ''') + inventory = inventory.fetchall() + self.assertEqual(inventory, [], "Data not migrated for version 7") + + sent = self.cur.execute('''SELECT status FROM sent ''') + sent = sent.fetchall() + self.assertEqual(sent, [('msgqueued',), ('msgqueued',)], "Data not migrated for version 7") + + @versioning + def test_sql_thread_version_6(self): + """ + Test with version 6 + """ + + # Assertion + + inventory = self.cur.execute('''PRAGMA table_info('inventory');''') + inventory = inventory.fetchall() + inventory = list(self.filter_table_column(inventory, "expirestime")) + self.assertEqual(inventory, ['expirestime'], "Data not migrated for version 6") + + objectprocessorqueue = self.cur.execute('''PRAGMA table_info('inventory');''') + objectprocessorqueue = objectprocessorqueue.fetchall() + objectprocessorqueue = list(self.filter_table_column(objectprocessorqueue, "objecttype")) + self.assertEqual(objectprocessorqueue, ['objecttype'], "Data not migrated for version 6") + + @versioning + def test_sql_thread_version_5(self): + """ + Test with version 5 + """ + + # Assertion + self.cur.execute(''' SELECT count(name) FROM sqlite_master WHERE type='table' AND name='knownnodes' ''') + self.assertNotEqual(self.cur.fetchone(), 1, "Table knownnodes not deleted in versioning 5") + self.cur.execute( + ''' SELECT count(name) FROM sqlite_master WHERE type='table' AND name='objectprocessorqueue'; ''') + self.assertNotEqual(self.cur.fetchone(), 0, "Table objectprocessorqueue not created in versioning 5") + + @versioning + def test_sql_thread_version_4(self): + """ + Test with version 4 + """ + + # Assertion + self.cur.execute('''select * from inventory where objecttype = 'pubkey';''') + self.assertNotEqual(self.cur.fetchone(), 1, "Table knownnodes not deleted in versioning 5") + + def test_sql_thread_version_3(self): + """ + Test with version 3 and 1 both are same + """ + pass + + @versioning + def test_sql_thread_version_2(self): + """ + Test with version 2 + """ + + # Assertion + self.cur.execute(''' SELECT count(name) FROM sqlite_master WHERE type='table' AND name='inventory_backup' ''') + self.assertNotEqual(self.cur.fetchone(), 1, "Table inventory_backup not deleted in versioning 5")