refactor the sqlthread test cases and fix py3 issue

src/class_sqlThread.py

@@ -444,6 +444,56 @@ class sqlThread(threading.Thread, UpgradeDB):
         # Bitmessage users or modify the SQLite database? Add it right
         # above this line!
 
+        self.add_new_option()
+
+        # try:
+        #     testpayload = '\x00\x00'
+        #     t = ('1234', 1, testpayload, '12345678', 'no')
+        #     self.cur.execute('''INSERT INTO pubkeys VALUES(?,?,?,?,?)''', t)
+        #     self.conn.commit()
+        #     self.cur.execute(
+        #         '''SELECT transmitdata FROM pubkeys WHERE address='1234' ''')
+        #     queryreturn = self.cur.fetchall()
+        #     for row in queryreturn:
+        #         transmitdata, = row
+        #     self.cur.execute('''DELETE FROM pubkeys WHERE address='1234' ''')
+        #     self.conn.commit()
+        #     if transmitdata == '':
+        #         logger.fatal(
+        #             'Problem: The version of SQLite you have cannot store Null values.'
+        #             ' Please download and install the latest revision of your version of Python'
+        #             ' (for example, the latest Python 2.7 revision) and try again.\n')
+        #         logger.fatal(
+        #             'PyBitmessage will now exit very abruptly.'
+        #             ' You may now see threading errors related to this abrupt exit'
+        #             ' but the problem you need to solve is related to SQLite.\n\n')
+        #         os._exit(0)
+        # except Exception as err:
+        #     if str(err) == 'database or disk is full':
+        #         logger.fatal(
+        #             '(While null value test) Alert: Your disk or data storage volume is full.'
+        #             ' sqlThread will now exit.')
+        #         queues.UISignalQueue.put((
+        #             'alert', (
+        #                 tr._translate(
+        #                     "MainWindow",
+        #                     "Disk full"),
+        #                 tr._translate(
+        #                     "MainWindow",
+        #                     'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'),
+        #                 True)))
+        #         os._exit(0)
+        #     else:
+        #         logger.error(err)
+
+        # Let us check to see the last time we vaccumed the messages.dat file.
+        # If it has been more than a month let's do it now.
+
+        self.check_vaccumed()
+
+
+    def add_new_option(self):
+        print("start func -=-=-=-=-=-=-=-=")
         try:
             testpayload = '\x00\x00'
             t = ('1234', 1, testpayload, '12345678', 'no')
@@ -484,8 +534,9 @@ class sqlThread(threading.Thread, UpgradeDB):
             else:
                 logger.error(err)
 
-        # Let us check to see the last time we vaccumed the messages.dat file.
+
-        # If it has been more than a month let's do it now.
+    def check_vaccumed(self):
+        print(" chec vaccumed start")
         item = '''SELECT value FROM settings WHERE key='lastvacuumtime';'''
         parameters = ''
         self.cur.execute(item, parameters)

src/helper_sql.py

@@ -16,7 +16,12 @@ SQLite objects can only be used from one thread.
    or isn't thread-safe.
 """
 
-import Queue
+# import Queue
+try:
+    import queue as Queue #python2
+except ImportError:
+    import Queue #python3
+
 import threading
 
 sqlSubmitQueue = Queue.Queue()

src/tests/sql/init_version_2.sql

@@ -0,0 +1,20 @@
+--
+-- Table structure for table `inventory`
+--
+
+CREATE TABLE IF NOT EXISTS `inventory` (
+  `hash` blob NOT NULL,
+  `objecttype` int DEFAULT NULL,
+  `streamnumber` int NOT NULL,
+  `receivedtime` int NOT NULL,
+  `payload` blob DEFAULT NULL,
+  `integer` integer NOT NULL,
+  -- `tag` blob DEFAULT NULL,
+  UNIQUE(hash) ON CONFLICT REPLACE
+) ;
+
+--
+-- Dumping data for table `inventory`
+--
+
+INSERT INTO `inventory` VALUES ('hash', 1, 1,1, 1,'test');

src/tests/sql/init_version_4.sql

@@ -0,0 +1,27 @@
+--
+-- Table structure for table `inventory`
+--
+
+CREATE TABLE IF NOT EXISTS `inventory` (
+  `hash` blob NOT NULL,
+  `objecttype` text DEFAULT NULL,
+  `streamnumber` int NOT NULL,
+  `payload` blob DEFAULT NULL,
+  `integer` integer NOT NULL,
+  UNIQUE(hash) ON CONFLICT REPLACE
+) ;
+
+--
+-- Dumping data for table `inventory`
+--
+
+INSERT INTO `inventory` VALUES ('hash', "pubkey", 1, 1,'test');
+
+--
+-- Table structure for table `pubkeys`
+--
+
+CREATE TABLE IF NOT EXISTS `pubkeys` (
+  `objecttype` int,
+  UNIQUE(objecttype) ON CONFLICT REPLACE
+) ;

src/tests/sql/init_version_5.sql

@@ -0,0 +1,12 @@
+--
+-- Table structure for table `knownnodes`
+--
+
+CREATE TABLE IF NOT EXISTS `knownnodes` (
+  `hash` blob NOT NULL,
+  `objecttype` int DEFAULT NULL,
+  `streamnumber` int NOT NULL,
+  `payload` blob DEFAULT NULL,
+  `integer` integer NOT NULL,
+  UNIQUE(hash) ON CONFLICT REPLACE
+) ;

src/tests/sql/init_version_6.sql

@@ -0,0 +1,27 @@
+--
+-- Table structure for table `inventory`
+--
+
+CREATE TABLE IF NOT EXISTS `inventory` (
+  `hash` blob NOT NULL,
+  `objecttype` int DEFAULT NULL,
+  `streamnumber` int NOT NULL,
+  `payload` blob DEFAULT NULL,
+  `integer` integer NOT NULL,
+  UNIQUE(hash) ON CONFLICT REPLACE
+) ;
+
+--
+-- Dumping data for table `inventory`
+--
+
+INSERT INTO `inventory` VALUES ('hash', 1, 1, 1,'test');
+
+--
+-- Table structure for table `objectprocessorqueue`
+--
+
+CREATE TABLE IF NOT EXISTS `objectprocessorqueue` (
+  `objecttype` int,
+  UNIQUE(objecttype) ON CONFLICT REPLACE
+) ;

src/tests/sql/init_version_7.sql

@@ -0,0 +1,69 @@
+--
+-- Table structure for table `inventory`
+--
+
+CREATE TABLE IF NOT EXISTS `inventory` (
+  `hash` blob NOT NULL,
+  `objecttype` int DEFAULT NULL,
+  `streamnumber` int NOT NULL,
+  `payload` blob DEFAULT NULL,
+  `integer` integer NOT NULL,
+  -- `tag` blob DEFAULT NULL,
+  UNIQUE(hash) ON CONFLICT REPLACE
+) ;
+
+--
+-- Dumping data for table `inventory`
+--
+
+INSERT INTO `inventory` VALUES ('hash', 1, 1, 1,'test');
+
+--
+-- Table structure for table `pubkeys`
+--
+
+CREATE TABLE IF NOT EXISTS `pubkeys` (
+  `hash` text,
+  `addressversion` int,
+  `transmitdata` blob,
+  `time` int,
+  `usedpersonally` text,
+  UNIQUE(hash) ON CONFLICT REPLACE
+) ;
+
+
+--
+-- Dumping data for table `pubkeys`
+--
+
+INSERT INTO `pubkeys` VALUES ('hash','1','1','1','test');
+
+--
+-- Table structure for table `sent`
+--
+
+CREATE TABLE IF NOT EXISTS `sent` (
+  `msgid` blob DEFAULT NULL,
+  `toaddress` text DEFAULT NULL,
+  `toripe` blob DEFAULT NULL,
+  `fromaddress` text DEFAULT NULL,
+  `subject` text DEFAULT NULL,
+  `message` text DEFAULT NULL,
+  `ackdata` blob DEFAULT NULL,
+  `senttime` integer DEFAULT NULL,
+  `lastactiontime` integer DEFAULT NULL,
+  `sleeptill` integer DEFAULT NULL,
+  `status` text DEFAULT NULL,
+  `retrynumber` integer DEFAULT NULL,
+  `folder` text DEFAULT NULL,
+  `encodingtype` int DEFAULT NULL,
+  `ttl` int DEFAULT NULL
+) ;
+
+--
+-- Dumping data for table `sent`
+--
+
+INSERT INTO `sent` VALUES 
+('msgid','toaddress','toripe','fromaddress','subject','message','ackdata','senttime','lastactiontime','sleeptill','doingmsgpow','retrynumber','folder','encodingtype','ttl'),
+('msgid','toaddress','toripe','fromaddress','subject','message','ackdata','senttime','lastactiontime','sleeptill','badkey','retrynumber','folder','encodingtype','ttl');

src/tests/test_sqlthread.py

@@ -4,22 +4,10 @@
 
 import os
 import unittest
-import shutil  # used for moving the messages.dat file
 import sqlite3
 import sys
-import threading
-import time
-# print sys.path
-# import state
-# import sql
-# from sql import init_version_1.sql
 from ..state import appdata
 from ..helper_sql import sqlStoredProcedure
-# import threading
-# from hashlib import sha256
-# from ..helper_sql import sqlQuery
-# from ..version import softwareVersion
-# from common import cleanup
 from ..class_sqlThread import (sqlThread, UpgradeDB)
 
 
@@ -53,16 +41,11 @@ class TestSqlThread(unittest.TestCase):
         # Stop sql thread
         sqlStoredProcedure('exit')
 
-    def db_connection(self):
-        conn = sqlite3.connect(appdata + 'messages.dat')
-        conn.text_factory = str
-        return conn.cursor()
-
     def normalize_version(self, file):
         try:
             root_path = os.path.dirname(os.path.dirname(__file__))
             sql_file_path = os.path.join(root_path, 'tests/sql/')
-            sql_file_path = os.path.join(sql_file_path, "init_version_"+file+".sql")
+            sql_file_path = os.path.join(sql_file_path, "init_version_{}.sql".format(file))
             sql_file = open(sql_file_path)
             sql_as_string = sql_file.read()
             self.cur.executescript(sql_as_string)
@@ -81,91 +64,154 @@ class TestSqlThread(unittest.TestCase):
                 if y == column:
                     yield y
 
+    def versioning(func):
+        def wrapper(*args):
+            self = args[0]
+            func_name = func.__name__
+            version = func_name.rsplit('_', 1)[-1]
+
+            # Update versions DB mocking
+            self.normalize_version(version)
+
+            # Test versions
+            upgrade_db = UpgradeDB()
+            upgrade_db.cur = self.cur
+            getattr(upgrade_db, "upgrade_schema_data_{}".format(version))()
+            ret = func(*args)
+            return ret  # <-- use (self, ...)
+        return wrapper
+
+    def change_state(self):
+        self.normalize_version("1")
+
+    @versioning
     def test_sql_thread_version_1(self):
         """
             Test with version 1
         """
 
-        # normalise Db for version 1
-        self.normalize_version("1")
-
-        # Test versioning
-        upgrade_db = UpgradeDB()
-        upgrade_db.cur = self.cur
-        upgrade_db.upgrade_schema_data_1()
-
         # Assertion after versioning
         res = self.cur.execute('''PRAGMA table_info('inventory');''')
         res = res.fetchall()
         result = list(self.filter_table_column(res, "tag"))
         res = [tup for tup in res if any(i in tup for i in ["tag"])]
-        self.assertEqual(result, ['tag'] , "Data not migrated for version 1")
+        self.assertEqual(result, ['tag'], "Data not migrated for version 1")
-        self.assertEqual(res, [(5, 'tag', 'blob', 0, "''", 0)] , "Data not migrated for version 1")
+        self.assertEqual(res, [(5, 'tag', 'blob', 0, "''", 0)], "Data not migrated for version 1")
-
 
+    @versioning
     def test_sql_thread_version_10(self):
         """
             Test with version 10
         """
 
-        # Update version 10
-        self.normalize_version("10")
-
-        # Test versioning
-        upgrade_db = UpgradeDB()
-        upgrade_db.cur = self.cur
-        upgrade_db.upgrade_schema_data_10()
-
         # Assertion
         self.cur.execute(''' SELECT count(name) FROM sqlite_master WHERE type='table' AND name='old_addressbook' ''')
-        self.assertNotEqual(self.cur.fetchone(), 1 , "Table old_addressbook not deleted")
+        self.assertNotEqual(self.cur.fetchone(), 1, "Table old_addressbook not deleted")
 
         res = self.cur.execute('''PRAGMA table_info('addressbook');''')
         res = res.fetchall()
         result = list(self.filter_table_column(res, "address"))
-        self.assertEqual(result, ['address'] , "Data not migrated for version 10")
+        self.assertEqual(result, ['address'], "Data not migrated for version 10")
-
 
+    @versioning
     def test_sql_thread_version_9(self):
         """
             Test with version 9
         """
 
-        # Update version 9
-        self.normalize_version("9")
-
-        # Test versioning
-        upgrade_db = UpgradeDB()
-        upgrade_db.cur = self.cur
-        upgrade_db.upgrade_schema_data_9()
-
         # Assertion
         self.cur.execute(''' SELECT count(name) FROM sqlite_master WHERE type='table' AND name='pubkeys_backup' ''')
-        self.assertNotEqual(self.cur.fetchone(), 1 , "Table pubkeys_backup not deleted")
+        self.assertNotEqual(self.cur.fetchone(), 1, "Table pubkeys_backup not deleted")
 
         res = self.cur.execute('''PRAGMA table_info('pubkeys');''')
         res = res.fetchall()
         result = list(self.filter_table_column(res, "address"))
-        self.assertEqual(result, ['address'] , "Data not migrated for version 9")
+        self.assertEqual(result, ['address'], "Data not migrated for version 9")
-
 
+    @versioning
     def test_sql_thread_version_8(self):
         """
             Test with version 8
         """
 
-        # Update version 8
+        # Assertion
-        self.normalize_version("8")
-
-        # Test versioning
-        upgrade_db = UpgradeDB()
-        upgrade_db.cur = self.cur
-        upgrade_db.upgrade_schema_data_8()
-
-        # # Assertion
         res = self.cur.execute('''PRAGMA table_info('inbox');''')
         res = res.fetchall()
         result = list(self.filter_table_column(res, "sighash"))
-        self.assertEqual(result, ['sighash'] , "Data not migrated for version 8")
+        self.assertEqual(result, ['sighash'], "Data not migrated for version 8")
 
+    @versioning
+    def test_sql_thread_version_7(self):
+        """
+            Test with version 7
+        """
 
+        # Assertion
+        pubkeys = self.cur.execute('''SELECT * FROM pubkeys ''')
+        pubkeys = pubkeys.fetchall()
+        self.assertEqual(pubkeys, [], "Data not migrated for version 7")
+
+        inventory = self.cur.execute('''SELECT * FROM inventory ''')
+        inventory = inventory.fetchall()
+        self.assertEqual(inventory, [], "Data not migrated for version 7")
+
+        sent = self.cur.execute('''SELECT status FROM sent ''')
+        sent = sent.fetchall()
+        self.assertEqual(sent, [('msgqueued',), ('msgqueued',)], "Data not migrated for version 7")
+
+    @versioning
+    def test_sql_thread_version_6(self):
+        """
+            Test with version 6
+        """
+
+        # Assertion
+
+        inventory = self.cur.execute('''PRAGMA table_info('inventory');''')
+        inventory = inventory.fetchall()
+        inventory = list(self.filter_table_column(inventory, "expirestime"))
+        self.assertEqual(inventory, ['expirestime'], "Data not migrated for version 6")
+
+        objectprocessorqueue = self.cur.execute('''PRAGMA table_info('inventory');''')
+        objectprocessorqueue = objectprocessorqueue.fetchall()
+        objectprocessorqueue = list(self.filter_table_column(objectprocessorqueue, "objecttype"))
+        self.assertEqual(objectprocessorqueue, ['objecttype'], "Data not migrated for version 6")
+
+    @versioning
+    def test_sql_thread_version_5(self):
+        """
+            Test with version 5
+        """
+
+        # Assertion
+        self.cur.execute(''' SELECT count(name) FROM sqlite_master WHERE type='table' AND name='knownnodes' ''')
+        self.assertNotEqual(self.cur.fetchone(), 1, "Table knownnodes not deleted in versioning 5")
+        self.cur.execute(
+            ''' SELECT count(name) FROM sqlite_master WHERE type='table' AND name='objectprocessorqueue'; ''')
+        self.assertNotEqual(self.cur.fetchone(), 0, "Table objectprocessorqueue not created in versioning 5")
+
+    @versioning
+    def test_sql_thread_version_4(self):
+        """
+            Test with version 4
+        """
+
+        # Assertion
+        self.cur.execute('''select * from inventory where objecttype = 'pubkey';''')
+        self.assertNotEqual(self.cur.fetchone(), 1, "Table knownnodes not deleted in versioning 5")
+
+    def test_sql_thread_version_3(self):
+        """
+            Test with version 3 and 1 both are same
+        """
+        pass
+
+    @versioning
+    def test_sql_thread_version_2(self):
+        """
+            Test with version 2
+        """
+
+        # Assertion
+        self.cur.execute(''' SELECT count(name) FROM sqlite_master WHERE type='table' AND name='inventory_backup' ''')
+        self.assertNotEqual(self.cur.fetchone(), 1, "Table inventory_backup not deleted in versioning 5")