From 04bd5a0264603798cf5472f6bc54be88305e7fe7 Mon Sep 17 00:00:00 2001 From: Muzahid Date: Thu, 4 Mar 2021 19:45:41 +0530 Subject: [PATCH 1/5] Add sqlite functions while versioning --- src/class_sqlThread.py | 30 ++++++++----- src/helper_sql.py | 18 +++++++- src/tests/sql/create_function.sql | 11 +++++ src/tests/test_sqlthread.py | 73 +++++++++++++++++++++++++++++++ 4 files changed, 120 insertions(+), 12 deletions(-) create mode 100644 src/tests/sql/create_function.sql create mode 100644 src/tests/test_sqlthread.py diff --git a/src/class_sqlThread.py b/src/class_sqlThread.py index 84188408..86460e7b 100644 --- a/src/class_sqlThread.py +++ b/src/class_sqlThread.py @@ -8,7 +8,6 @@ import sqlite3 import sys import threading import time - import helper_sql import helper_startup import paths @@ -18,6 +17,7 @@ import tr from bmconfigparser import BMConfigParser from debug import logger # pylint: disable=attribute-defined-outside-init,protected-access +from addresses import encodeAddress class sqlThread(threading.Thread): @@ -35,6 +35,9 @@ class sqlThread(threading.Thread): self.cur.execute('PRAGMA secure_delete = true') + # call create_function for encode address + self.create_function() + try: self.cur.execute( '''CREATE TABLE inbox (msgid blob, toaddress text, fromaddress text, subject text,''' @@ -325,6 +328,7 @@ class sqlThread(threading.Thread): # We'll also need a `sleeptill` field and a `ttl` field. Also we # can combine the pubkeyretrynumber and msgretrynumber into one. + item = '''SELECT value FROM settings WHERE key='version';''' parameters = '' self.cur.execute(item, parameters) @@ -358,16 +362,11 @@ class sqlThread(threading.Thread): logger.debug('In messages.dat database, adding address field to the pubkeys table.') # We're going to have to calculate the address for each row in the pubkeys # table. Then we can take out the hash field. - self.cur.execute('''ALTER TABLE pubkeys ADD address text DEFAULT '' ''') - self.cur.execute('''SELECT hash, addressversion FROM pubkeys''') - queryResult = self.cur.fetchall() - from addresses import encodeAddress - for row in queryResult: - addressHash, addressVersion = row - address = encodeAddress(addressVersion, 1, hash) - item = '''UPDATE pubkeys SET address=? WHERE hash=?;''' - parameters = (address, addressHash) - self.cur.execute(item, parameters) + self.cur.execute('''ALTER TABLE pubkeys ADD address text DEFAULT '' ;''') + + # replica for loop to update hashed address + self.cur.execute('''UPDATE pubkeys SET address=(enaddr(pubkeys.addressversion, 1, hash)) WHERE hash=pubkeys.hash; ''') + # Now we can remove the hash field from the pubkeys table. self.cur.execute( '''CREATE TEMPORARY TABLE pubkeys_backup''' @@ -622,3 +621,12 @@ class sqlThread(threading.Thread): helper_sql.sqlReturnQueue.put((self.cur.fetchall(), rowcount)) # helper_sql.sqlSubmitQueue.task_done() + + def create_function(self): + # create_function + try: + self.conn.create_function("enaddr", 3, func=encodeAddress, deterministic=True) + except (TypeError, sqlite3.NotSupportedError) as err: + logger.debug( + "Got error while pass deterministic in sqlite create function {}, Passing 3 params".format(err)) + self.conn.create_function("enaddr", 3, encodeAddress) diff --git a/src/helper_sql.py b/src/helper_sql.py index 043bccf2..26398848 100644 --- a/src/helper_sql.py +++ b/src/helper_sql.py @@ -16,9 +16,16 @@ SQLite objects can only be used from one thread. or isn't thread-safe. """ -import Queue + +# import Queue +try: + import queue as Queue #python3 +except ImportError: + import Queue #python2 + import threading + sqlSubmitQueue = Queue.Queue() """the queue for SQL""" sqlReturnQueue = Queue.Queue() @@ -105,6 +112,15 @@ def sqlExecute(sql_statement, *args): return rowcount +def sqlExecuteScript(sql_statement): + """Execute SQL script statement""" + + statements = sql_statement.split(";") + with SqlBulkExecute() as sql: + for q in statements: + sql.execute("{}".format(q)) + + def sqlStoredProcedure(procName): """Schedule procName to be run""" assert sql_available diff --git a/src/tests/sql/create_function.sql b/src/tests/sql/create_function.sql new file mode 100644 index 00000000..b66d1aa1 --- /dev/null +++ b/src/tests/sql/create_function.sql @@ -0,0 +1,11 @@ +CREATE TABLE `testhash` ( + `addressversion` int DEFAULT NULL, + `hash` blob DEFAULT NULL, + `address` text DEFAULT NULL, + UNIQUE(address) ON CONFLICT IGNORE +); + + + +INSERT INTO testhash (addressversion, hash) VALUES(4, "21122112211221122112"); + diff --git a/src/tests/test_sqlthread.py b/src/tests/test_sqlthread.py new file mode 100644 index 00000000..d51c6623 --- /dev/null +++ b/src/tests/test_sqlthread.py @@ -0,0 +1,73 @@ +""" + Test for sqlThread +""" + +import os +import unittest +from ..helper_sql import sqlStoredProcedure, sql_ready, sqlExecute, SqlBulkExecute, sqlQuery, sqlExecuteScript +from ..class_sqlThread import (sqlThread) +from ..addresses import encodeAddress +from .common import skip_python3 + + +skip_python3() + + +class TestSqlThread(unittest.TestCase): + """ + Test case for SQLThread + """ + + # query file path + root_path = os.path.dirname(os.path.dirname(__file__)) + + @classmethod + def setUpClass(cls): + # Start SQL thread + sqlLookup = sqlThread() + sqlLookup.daemon = False + sqlLookup.start() + sql_ready.wait() + + @classmethod + def setUp(cls): + tables = list(sqlQuery("select name from sqlite_master where type is 'table'")) + with SqlBulkExecute() as sql: + for q in tables: + sql.execute("drop table if exists %s" % q) + + @classmethod + def tearDown(cls): + pass + + @classmethod + def tearDownClass(cls): + # Stop sql thread + sqlStoredProcedure('exit') + + def initialise_database(self, file): + """ + Initialise DB + """ + + sql_as_string = open(os.path.join(self.root_path, "tests/sql/{}.sql".format(file))).read() + sqlExecuteScript(sql_as_string) + + + def test_create_function(self): + # call create function + + encoded_str = encodeAddress(4, 1, "21122112211221122112") + + # Initialise Database + self.initialise_database("create_function") + + sqlExecute('''INSERT INTO testhash (addressversion, hash) VALUES(4, "21122112211221122112")''') + # call function in query + + sqlExecute('''UPDATE testhash SET address=(enaddr(testhash.addressversion, 1, hash)) WHERE hash=testhash.hash''') + + # Assertion + query = sqlQuery('''select * from testhash;''') + self.assertEqual(query[0][-1], encoded_str, "test case fail for create_function") + sqlExecute('''DROP TABLE testhash''') -- 2.45.1 From 06cab993d9a7ec78d0fc0ea116314cc0ae2d3653 Mon Sep 17 00:00:00 2001 From: Muzahid Date: Tue, 16 Mar 2021 15:18:38 +0530 Subject: [PATCH 2/5] Fix configparser import error --- src/bmconfigparser.py | 76 ++++++++++++++++++++--------- src/tests/test_config.py | 33 ------------- src/tests/test_config_functional.py | 39 +++++++++++++++ 3 files changed, 93 insertions(+), 55 deletions(-) create mode 100644 src/tests/test_config_functional.py diff --git a/src/bmconfigparser.py b/src/bmconfigparser.py index ff43fd7c..be7b3e87 100644 --- a/src/bmconfigparser.py +++ b/src/bmconfigparser.py @@ -2,13 +2,22 @@ BMConfigParser class definition and default configuration settings """ -import ConfigParser +import sys +if sys.version_info[0] == 3: + # python 3 + import configparser as ConfigParser + SafeConfigParser = ConfigParser.ConfigParser +else: + # python 2 + import ConfigParser + SafeConfigParser = ConfigParser.SafeConfigParser + +import state +from singleton import Singleton import os import shutil from datetime import datetime -import state -from singleton import Singleton BMConfigDefaults = { "bitmessagesettings": { @@ -43,7 +52,8 @@ BMConfigDefaults = { @Singleton -class BMConfigParser(ConfigParser.SafeConfigParser): +class BMConfigParser(SafeConfigParser): + """ Singleton class inherited from :class:`ConfigParser.SafeConfigParser` with additional methods specific to bitmessage config. @@ -60,26 +70,47 @@ class BMConfigParser(ConfigParser.SafeConfigParser): raise ValueError("Invalid value %s" % value) return ConfigParser.ConfigParser.set(self, section, option, value) - def get(self, section, option, raw=False, variables=None): - # pylint: disable=arguments-differ - try: - if section == "bitmessagesettings" and option == "timeformat": + def get(self, section, option, raw=False, vars=None): + if sys.version_info[0] == 3: + # pylint: disable=arguments-differ + try: + if section == "bitmessagesettings" and option == "timeformat": + return ConfigParser.ConfigParser.get( + self, section, option) + try: + return self._temp[section][option] + except KeyError: + pass return ConfigParser.ConfigParser.get( - self, section, option, raw, variables) + self, section, option) + except ConfigParser.InterpolationError: + return ConfigParser.ConfigParser.get( + self, section, option) + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError) as e: + try: + return BMConfigDefaults[section][option] + except (KeyError, ValueError, AttributeError): + raise e + else: + # pylint: disable=arguments-differ try: - return self._temp[section][option] - except KeyError: - pass - return ConfigParser.ConfigParser.get( - self, section, option, True, variables) - except ConfigParser.InterpolationError: - return ConfigParser.ConfigParser.get( - self, section, option, True, variables) - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError) as e: - try: - return BMConfigDefaults[section][option] - except (KeyError, ValueError, AttributeError): - raise e + if section == "bitmessagesettings" and option == "timeformat": + return ConfigParser.ConfigParser.get( + self, section, option, raw, vars) + try: + return self._temp[section][option] + except KeyError: + pass + return ConfigParser.ConfigParser.get( + self, section, option, True, vars) + except ConfigParser.InterpolationError: + return ConfigParser.ConfigParser.get( + self, section, option, True, vars) + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError) as e: + try: + return BMConfigDefaults[section][option] + except (KeyError, ValueError, AttributeError): + raise e def setTemp(self, section, option, value=None): """Temporary set option to value, not saving.""" @@ -191,3 +222,4 @@ class BMConfigParser(ConfigParser.SafeConfigParser): if value < 0 or value > 8: return False return True + diff --git a/src/tests/test_config.py b/src/tests/test_config.py index a3b90a4c..d44ec738 100644 --- a/src/tests/test_config.py +++ b/src/tests/test_config.py @@ -2,11 +2,7 @@ Various tests for config """ -import os import unittest -import tempfile - -from .test_process import TestProcessProto from pybitmessage.bmconfigparser import BMConfigParser @@ -38,32 +34,3 @@ class TestConfig(unittest.TestCase): BMConfigParser().safeGetInt('nonexistent', 'nonexistent'), 0) self.assertEqual( BMConfigParser().safeGetInt('nonexistent', 'nonexistent', 42), 42) - - -class TestProcessConfig(TestProcessProto): - """A test case for keys.dat""" - home = tempfile.mkdtemp() - - def test_config_defaults(self): - """Test settings in the generated config""" - self._stop_process() - self._kill_process() - config = BMConfigParser() - config.read(os.path.join(self.home, 'keys.dat')) - - self.assertEqual(config.safeGetInt( - 'bitmessagesettings', 'settingsversion'), 10) - self.assertEqual(config.safeGetInt( - 'bitmessagesettings', 'port'), 8444) - # don't connect - self.assertTrue(config.safeGetBoolean( - 'bitmessagesettings', 'dontconnect')) - # API disabled - self.assertFalse(config.safeGetBoolean( - 'bitmessagesettings', 'apienabled')) - - # extralowdifficulty is false - self.assertEqual(config.safeGetInt( - 'bitmessagesettings', 'defaultnoncetrialsperbyte'), 1000) - self.assertEqual(config.safeGetInt( - 'bitmessagesettings', 'defaultpayloadlengthextrabytes'), 1000) diff --git a/src/tests/test_config_functional.py b/src/tests/test_config_functional.py new file mode 100644 index 00000000..3b973911 --- /dev/null +++ b/src/tests/test_config_functional.py @@ -0,0 +1,39 @@ +""" +Various tests for config +""" + +import os +import unittest +import tempfile + +from .test_process import TestProcessProto +from pybitmessage.bmconfigparser import BMConfigParser + + +class TestProcessConfig(TestProcessProto): + """A test case for keys.dat""" + home = tempfile.mkdtemp() + + def test_config_defaults(self): + """Test settings in the generated config""" + self._stop_process() + self._kill_process() + config = BMConfigParser() + config.read(os.path.join(self.home, 'keys.dat')) + + self.assertEqual(config.safeGetInt( + 'bitmessagesettings', 'settingsversion'), 10) + self.assertEqual(config.safeGetInt( + 'bitmessagesettings', 'port'), 8444) + # don't connect + self.assertTrue(config.safeGetBoolean( + 'bitmessagesettings', 'dontconnect')) + # API disabled + self.assertFalse(config.safeGetBoolean( + 'bitmessagesettings', 'apienabled')) + + # extralowdifficulty is false + self.assertEqual(config.safeGetInt( + 'bitmessagesettings', 'defaultnoncetrialsperbyte'), 1000) + self.assertEqual(config.safeGetInt( + 'bitmessagesettings', 'defaultpayloadlengthextrabytes'), 1000) -- 2.45.1 From 1b8dc18ef640e85e390f086f12af8eba117992f8 Mon Sep 17 00:00:00 2001 From: Muzahid Date: Mon, 5 Apr 2021 18:28:26 +0530 Subject: [PATCH 3/5] remove functions imports --- src/tests/test_config_functional.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/tests/test_config_functional.py b/src/tests/test_config_functional.py index 3b973911..1492dbdc 100644 --- a/src/tests/test_config_functional.py +++ b/src/tests/test_config_functional.py @@ -3,10 +3,9 @@ Various tests for config """ import os -import unittest import tempfile -from .test_process import TestProcessProto +from test_process import TestProcessProto from pybitmessage.bmconfigparser import BMConfigParser @@ -16,8 +15,6 @@ class TestProcessConfig(TestProcessProto): def test_config_defaults(self): """Test settings in the generated config""" - self._stop_process() - self._kill_process() config = BMConfigParser() config.read(os.path.join(self.home, 'keys.dat')) -- 2.45.1 From f075d27fae05fb001d92f11659b7e5300b17c933 Mon Sep 17 00:00:00 2001 From: Muzahid Date: Mon, 5 Apr 2021 19:08:10 +0530 Subject: [PATCH 4/5] add dot in importing --- src/tests/test_config_functional.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/test_config_functional.py b/src/tests/test_config_functional.py index 1492dbdc..4195efd6 100644 --- a/src/tests/test_config_functional.py +++ b/src/tests/test_config_functional.py @@ -5,7 +5,7 @@ Various tests for config import os import tempfile -from test_process import TestProcessProto +from .test_process import TestProcessProto from pybitmessage.bmconfigparser import BMConfigParser -- 2.45.1 From f5fba7d1a83b1f64334c3962a289d166f79fcdfe Mon Sep 17 00:00:00 2001 From: Muzahid Date: Tue, 6 Apr 2021 13:04:44 +0530 Subject: [PATCH 5/5] update changes and rename file name --- .../{test_config_functional.py => test_config_process.py} | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) rename src/tests/{test_config_functional.py => test_config_process.py} (95%) diff --git a/src/tests/test_config_functional.py b/src/tests/test_config_process.py similarity index 95% rename from src/tests/test_config_functional.py rename to src/tests/test_config_process.py index 4195efd6..f3cf19f2 100644 --- a/src/tests/test_config_functional.py +++ b/src/tests/test_config_process.py @@ -4,18 +4,20 @@ Various tests for config import os import tempfile - -from .test_process import TestProcessProto from pybitmessage.bmconfigparser import BMConfigParser +from .test_process import TestProcessProto class TestProcessConfig(TestProcessProto): """A test case for keys.dat""" home = tempfile.mkdtemp() + def test_config_defaults(self): """Test settings in the generated config""" config = BMConfigParser() + self._stop_process() + self._kill_process() config.read(os.path.join(self.home, 'keys.dat')) self.assertEqual(config.safeGetInt( -- 2.45.1