refactor the sqlthread test cases and fix py3 issue

This commit is contained in:
Muzahid 2021-02-26 20:18:52 +05:30
parent 17d40ef43c
commit ce0f99a8c2
Signed by untrusted user: cis-muzahid
GPG Key ID: 1DC85E7D3AB613EA
8 changed files with 321 additions and 64 deletions

View File

@ -444,6 +444,56 @@ class sqlThread(threading.Thread, UpgradeDB):
# Bitmessage users or modify the SQLite database? Add it right # Bitmessage users or modify the SQLite database? Add it right
# above this line! # above this line!
self.add_new_option()
# try:
# testpayload = '\x00\x00'
# t = ('1234', 1, testpayload, '12345678', 'no')
# self.cur.execute('''INSERT INTO pubkeys VALUES(?,?,?,?,?)''', t)
# self.conn.commit()
# self.cur.execute(
# '''SELECT transmitdata FROM pubkeys WHERE address='1234' ''')
# queryreturn = self.cur.fetchall()
# for row in queryreturn:
# transmitdata, = row
# self.cur.execute('''DELETE FROM pubkeys WHERE address='1234' ''')
# self.conn.commit()
# if transmitdata == '':
# logger.fatal(
# 'Problem: The version of SQLite you have cannot store Null values.'
# ' Please download and install the latest revision of your version of Python'
# ' (for example, the latest Python 2.7 revision) and try again.\n')
# logger.fatal(
# 'PyBitmessage will now exit very abruptly.'
# ' You may now see threading errors related to this abrupt exit'
# ' but the problem you need to solve is related to SQLite.\n\n')
# os._exit(0)
# except Exception as err:
# if str(err) == 'database or disk is full':
# logger.fatal(
# '(While null value test) Alert: Your disk or data storage volume is full.'
# ' sqlThread will now exit.')
# queues.UISignalQueue.put((
# 'alert', (
# tr._translate(
# "MainWindow",
# "Disk full"),
# tr._translate(
# "MainWindow",
# 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'),
# True)))
# os._exit(0)
# else:
# logger.error(err)
# Let us check to see the last time we vaccumed the messages.dat file.
# If it has been more than a month let's do it now.
self.check_vaccumed()
def add_new_option(self):
print("start func -=-=-=-=-=-=-=-=")
try: try:
testpayload = '\x00\x00' testpayload = '\x00\x00'
t = ('1234', 1, testpayload, '12345678', 'no') t = ('1234', 1, testpayload, '12345678', 'no')
@ -484,8 +534,9 @@ class sqlThread(threading.Thread, UpgradeDB):
else: else:
logger.error(err) logger.error(err)
# Let us check to see the last time we vaccumed the messages.dat file.
# If it has been more than a month let's do it now. def check_vaccumed(self):
print(" chec vaccumed start")
item = '''SELECT value FROM settings WHERE key='lastvacuumtime';''' item = '''SELECT value FROM settings WHERE key='lastvacuumtime';'''
parameters = '' parameters = ''
self.cur.execute(item, parameters) self.cur.execute(item, parameters)

View File

@ -16,7 +16,12 @@ SQLite objects can only be used from one thread.
or isn't thread-safe. or isn't thread-safe.
""" """
import Queue # import Queue
try:
import queue as Queue #python2
except ImportError:
import Queue #python3
import threading import threading
sqlSubmitQueue = Queue.Queue() sqlSubmitQueue = Queue.Queue()

View File

@ -0,0 +1,20 @@
--
-- Table structure for table `inventory`
--
CREATE TABLE IF NOT EXISTS `inventory` (
`hash` blob NOT NULL,
`objecttype` int DEFAULT NULL,
`streamnumber` int NOT NULL,
`receivedtime` int NOT NULL,
`payload` blob DEFAULT NULL,
`integer` integer NOT NULL,
-- `tag` blob DEFAULT NULL,
UNIQUE(hash) ON CONFLICT REPLACE
) ;
--
-- Dumping data for table `inventory`
--
INSERT INTO `inventory` VALUES ('hash', 1, 1,1, 1,'test');

View File

@ -0,0 +1,27 @@
--
-- Table structure for table `inventory`
--
CREATE TABLE IF NOT EXISTS `inventory` (
`hash` blob NOT NULL,
`objecttype` text DEFAULT NULL,
`streamnumber` int NOT NULL,
`payload` blob DEFAULT NULL,
`integer` integer NOT NULL,
UNIQUE(hash) ON CONFLICT REPLACE
) ;
--
-- Dumping data for table `inventory`
--
INSERT INTO `inventory` VALUES ('hash', "pubkey", 1, 1,'test');
--
-- Table structure for table `pubkeys`
--
CREATE TABLE IF NOT EXISTS `pubkeys` (
`objecttype` int,
UNIQUE(objecttype) ON CONFLICT REPLACE
) ;

View File

@ -0,0 +1,12 @@
--
-- Table structure for table `knownnodes`
--
CREATE TABLE IF NOT EXISTS `knownnodes` (
`hash` blob NOT NULL,
`objecttype` int DEFAULT NULL,
`streamnumber` int NOT NULL,
`payload` blob DEFAULT NULL,
`integer` integer NOT NULL,
UNIQUE(hash) ON CONFLICT REPLACE
) ;

View File

@ -0,0 +1,27 @@
--
-- Table structure for table `inventory`
--
CREATE TABLE IF NOT EXISTS `inventory` (
`hash` blob NOT NULL,
`objecttype` int DEFAULT NULL,
`streamnumber` int NOT NULL,
`payload` blob DEFAULT NULL,
`integer` integer NOT NULL,
UNIQUE(hash) ON CONFLICT REPLACE
) ;
--
-- Dumping data for table `inventory`
--
INSERT INTO `inventory` VALUES ('hash', 1, 1, 1,'test');
--
-- Table structure for table `objectprocessorqueue`
--
CREATE TABLE IF NOT EXISTS `objectprocessorqueue` (
`objecttype` int,
UNIQUE(objecttype) ON CONFLICT REPLACE
) ;

View File

@ -0,0 +1,69 @@
--
-- Table structure for table `inventory`
--
CREATE TABLE IF NOT EXISTS `inventory` (
`hash` blob NOT NULL,
`objecttype` int DEFAULT NULL,
`streamnumber` int NOT NULL,
`payload` blob DEFAULT NULL,
`integer` integer NOT NULL,
-- `tag` blob DEFAULT NULL,
UNIQUE(hash) ON CONFLICT REPLACE
) ;
--
-- Dumping data for table `inventory`
--
INSERT INTO `inventory` VALUES ('hash', 1, 1, 1,'test');
--
-- Table structure for table `pubkeys`
--
CREATE TABLE IF NOT EXISTS `pubkeys` (
`hash` text,
`addressversion` int,
`transmitdata` blob,
`time` int,
`usedpersonally` text,
UNIQUE(hash) ON CONFLICT REPLACE
) ;
--
-- Dumping data for table `pubkeys`
--
INSERT INTO `pubkeys` VALUES ('hash','1','1','1','test');
--
-- Table structure for table `sent`
--
CREATE TABLE IF NOT EXISTS `sent` (
`msgid` blob DEFAULT NULL,
`toaddress` text DEFAULT NULL,
`toripe` blob DEFAULT NULL,
`fromaddress` text DEFAULT NULL,
`subject` text DEFAULT NULL,
`message` text DEFAULT NULL,
`ackdata` blob DEFAULT NULL,
`senttime` integer DEFAULT NULL,
`lastactiontime` integer DEFAULT NULL,
`sleeptill` integer DEFAULT NULL,
`status` text DEFAULT NULL,
`retrynumber` integer DEFAULT NULL,
`folder` text DEFAULT NULL,
`encodingtype` int DEFAULT NULL,
`ttl` int DEFAULT NULL
) ;
--
-- Dumping data for table `sent`
--
INSERT INTO `sent` VALUES
('msgid','toaddress','toripe','fromaddress','subject','message','ackdata','senttime','lastactiontime','sleeptill','doingmsgpow','retrynumber','folder','encodingtype','ttl'),
('msgid','toaddress','toripe','fromaddress','subject','message','ackdata','senttime','lastactiontime','sleeptill','badkey','retrynumber','folder','encodingtype','ttl');

View File

@ -4,22 +4,10 @@
import os import os
import unittest import unittest
import shutil # used for moving the messages.dat file
import sqlite3 import sqlite3
import sys import sys
import threading
import time
# print sys.path
# import state
# import sql
# from sql import init_version_1.sql
from ..state import appdata from ..state import appdata
from ..helper_sql import sqlStoredProcedure from ..helper_sql import sqlStoredProcedure
# import threading
# from hashlib import sha256
# from ..helper_sql import sqlQuery
# from ..version import softwareVersion
# from common import cleanup
from ..class_sqlThread import (sqlThread, UpgradeDB) from ..class_sqlThread import (sqlThread, UpgradeDB)
@ -53,16 +41,11 @@ class TestSqlThread(unittest.TestCase):
# Stop sql thread # Stop sql thread
sqlStoredProcedure('exit') sqlStoredProcedure('exit')
def db_connection(self):
conn = sqlite3.connect(appdata + 'messages.dat')
conn.text_factory = str
return conn.cursor()
def normalize_version(self, file): def normalize_version(self, file):
try: try:
root_path = os.path.dirname(os.path.dirname(__file__)) root_path = os.path.dirname(os.path.dirname(__file__))
sql_file_path = os.path.join(root_path, 'tests/sql/') sql_file_path = os.path.join(root_path, 'tests/sql/')
sql_file_path = os.path.join(sql_file_path, "init_version_"+file+".sql") sql_file_path = os.path.join(sql_file_path, "init_version_{}.sql".format(file))
sql_file = open(sql_file_path) sql_file = open(sql_file_path)
sql_as_string = sql_file.read() sql_as_string = sql_file.read()
self.cur.executescript(sql_as_string) self.cur.executescript(sql_as_string)
@ -81,91 +64,154 @@ class TestSqlThread(unittest.TestCase):
if y == column: if y == column:
yield y yield y
def versioning(func):
def wrapper(*args):
self = args[0]
func_name = func.__name__
version = func_name.rsplit('_', 1)[-1]
# Update versions DB mocking
self.normalize_version(version)
# Test versions
upgrade_db = UpgradeDB()
upgrade_db.cur = self.cur
getattr(upgrade_db, "upgrade_schema_data_{}".format(version))()
ret = func(*args)
return ret # <-- use (self, ...)
return wrapper
def change_state(self):
self.normalize_version("1")
@versioning
def test_sql_thread_version_1(self): def test_sql_thread_version_1(self):
""" """
Test with version 1 Test with version 1
""" """
# normalise Db for version 1
self.normalize_version("1")
# Test versioning
upgrade_db = UpgradeDB()
upgrade_db.cur = self.cur
upgrade_db.upgrade_schema_data_1()
# Assertion after versioning # Assertion after versioning
res = self.cur.execute('''PRAGMA table_info('inventory');''') res = self.cur.execute('''PRAGMA table_info('inventory');''')
res = res.fetchall() res = res.fetchall()
result = list(self.filter_table_column(res, "tag")) result = list(self.filter_table_column(res, "tag"))
res = [tup for tup in res if any(i in tup for i in ["tag"])] res = [tup for tup in res if any(i in tup for i in ["tag"])]
self.assertEqual(result, ['tag'] , "Data not migrated for version 1") self.assertEqual(result, ['tag'], "Data not migrated for version 1")
self.assertEqual(res, [(5, 'tag', 'blob', 0, "''", 0)] , "Data not migrated for version 1") self.assertEqual(res, [(5, 'tag', 'blob', 0, "''", 0)], "Data not migrated for version 1")
@versioning
def test_sql_thread_version_10(self): def test_sql_thread_version_10(self):
""" """
Test with version 10 Test with version 10
""" """
# Update version 10
self.normalize_version("10")
# Test versioning
upgrade_db = UpgradeDB()
upgrade_db.cur = self.cur
upgrade_db.upgrade_schema_data_10()
# Assertion # Assertion
self.cur.execute(''' SELECT count(name) FROM sqlite_master WHERE type='table' AND name='old_addressbook' ''') self.cur.execute(''' SELECT count(name) FROM sqlite_master WHERE type='table' AND name='old_addressbook' ''')
self.assertNotEqual(self.cur.fetchone(), 1 , "Table old_addressbook not deleted") self.assertNotEqual(self.cur.fetchone(), 1, "Table old_addressbook not deleted")
res = self.cur.execute('''PRAGMA table_info('addressbook');''') res = self.cur.execute('''PRAGMA table_info('addressbook');''')
res = res.fetchall() res = res.fetchall()
result = list(self.filter_table_column(res, "address")) result = list(self.filter_table_column(res, "address"))
self.assertEqual(result, ['address'] , "Data not migrated for version 10") self.assertEqual(result, ['address'], "Data not migrated for version 10")
@versioning
def test_sql_thread_version_9(self): def test_sql_thread_version_9(self):
""" """
Test with version 9 Test with version 9
""" """
# Update version 9
self.normalize_version("9")
# Test versioning
upgrade_db = UpgradeDB()
upgrade_db.cur = self.cur
upgrade_db.upgrade_schema_data_9()
# Assertion # Assertion
self.cur.execute(''' SELECT count(name) FROM sqlite_master WHERE type='table' AND name='pubkeys_backup' ''') self.cur.execute(''' SELECT count(name) FROM sqlite_master WHERE type='table' AND name='pubkeys_backup' ''')
self.assertNotEqual(self.cur.fetchone(), 1 , "Table pubkeys_backup not deleted") self.assertNotEqual(self.cur.fetchone(), 1, "Table pubkeys_backup not deleted")
res = self.cur.execute('''PRAGMA table_info('pubkeys');''') res = self.cur.execute('''PRAGMA table_info('pubkeys');''')
res = res.fetchall() res = res.fetchall()
result = list(self.filter_table_column(res, "address")) result = list(self.filter_table_column(res, "address"))
self.assertEqual(result, ['address'] , "Data not migrated for version 9") self.assertEqual(result, ['address'], "Data not migrated for version 9")
@versioning
def test_sql_thread_version_8(self): def test_sql_thread_version_8(self):
""" """
Test with version 8 Test with version 8
""" """
# Update version 8 # Assertion
self.normalize_version("8")
# Test versioning
upgrade_db = UpgradeDB()
upgrade_db.cur = self.cur
upgrade_db.upgrade_schema_data_8()
# # Assertion
res = self.cur.execute('''PRAGMA table_info('inbox');''') res = self.cur.execute('''PRAGMA table_info('inbox');''')
res = res.fetchall() res = res.fetchall()
result = list(self.filter_table_column(res, "sighash")) result = list(self.filter_table_column(res, "sighash"))
self.assertEqual(result, ['sighash'] , "Data not migrated for version 8") self.assertEqual(result, ['sighash'], "Data not migrated for version 8")
@versioning
def test_sql_thread_version_7(self):
"""
Test with version 7
"""
# Assertion
pubkeys = self.cur.execute('''SELECT * FROM pubkeys ''')
pubkeys = pubkeys.fetchall()
self.assertEqual(pubkeys, [], "Data not migrated for version 7")
inventory = self.cur.execute('''SELECT * FROM inventory ''')
inventory = inventory.fetchall()
self.assertEqual(inventory, [], "Data not migrated for version 7")
sent = self.cur.execute('''SELECT status FROM sent ''')
sent = sent.fetchall()
self.assertEqual(sent, [('msgqueued',), ('msgqueued',)], "Data not migrated for version 7")
@versioning
def test_sql_thread_version_6(self):
"""
Test with version 6
"""
# Assertion
inventory = self.cur.execute('''PRAGMA table_info('inventory');''')
inventory = inventory.fetchall()
inventory = list(self.filter_table_column(inventory, "expirestime"))
self.assertEqual(inventory, ['expirestime'], "Data not migrated for version 6")
objectprocessorqueue = self.cur.execute('''PRAGMA table_info('inventory');''')
objectprocessorqueue = objectprocessorqueue.fetchall()
objectprocessorqueue = list(self.filter_table_column(objectprocessorqueue, "objecttype"))
self.assertEqual(objectprocessorqueue, ['objecttype'], "Data not migrated for version 6")
@versioning
def test_sql_thread_version_5(self):
"""
Test with version 5
"""
# Assertion
self.cur.execute(''' SELECT count(name) FROM sqlite_master WHERE type='table' AND name='knownnodes' ''')
self.assertNotEqual(self.cur.fetchone(), 1, "Table knownnodes not deleted in versioning 5")
self.cur.execute(
''' SELECT count(name) FROM sqlite_master WHERE type='table' AND name='objectprocessorqueue'; ''')
self.assertNotEqual(self.cur.fetchone(), 0, "Table objectprocessorqueue not created in versioning 5")
@versioning
def test_sql_thread_version_4(self):
"""
Test with version 4
"""
# Assertion
self.cur.execute('''select * from inventory where objecttype = 'pubkey';''')
self.assertNotEqual(self.cur.fetchone(), 1, "Table knownnodes not deleted in versioning 5")
def test_sql_thread_version_3(self):
"""
Test with version 3 and 1 both are same
"""
pass
@versioning
def test_sql_thread_version_2(self):
"""
Test with version 2
"""
# Assertion
self.cur.execute(''' SELECT count(name) FROM sqlite_master WHERE type='table' AND name='inventory_backup' ''')
self.assertNotEqual(self.cur.fetchone(), 1, "Table inventory_backup not deleted in versioning 5")