From 70dcb944c8bb9b9eb5dc1dfd219c6438b90c057a Mon Sep 17 00:00:00 2001 From: Muzahid Date: Tue, 18 Jan 2022 19:59:17 +0530 Subject: [PATCH] add sql scripts files for sqlthread, change in sql import, updated code for sql file loading and fixed some linting --- packages/pyinstaller/bitmessagemain.spec | 23 ++++-- setup.py | 2 +- src/build_osx.py | 1 + src/sql/config_setting_ver_2.sql | 1 + src/sql/config_setting_ver_3.sql | 5 ++ src/sql/init_version_10.sql | 15 ++++ src/sql/init_version_2.sql | 29 +++++++ src/sql/init_version_3.sql | 5 ++ src/sql/init_version_4.sql | 17 ++++ src/sql/init_version_5.sql | 12 +++ src/sql/init_version_6.sql | 25 ++++++ src/sql/init_version_7.sql | 11 +++ src/sql/init_version_8.sql | 7 ++ src/sql/init_version_9.sql | 74 +++++++++++++++++ src/sql/initialize_schema.sql | 100 +++++++++++++++++++++++ src/sql/upg_sc_if_old_ver_1.sql | 30 +++++++ src/sql/upg_sc_if_old_ver_2.sql | 7 ++ 17 files changed, 355 insertions(+), 9 deletions(-) create mode 100644 src/sql/config_setting_ver_2.sql create mode 100644 src/sql/config_setting_ver_3.sql create mode 100644 src/sql/init_version_10.sql create mode 100644 src/sql/init_version_2.sql create mode 100644 src/sql/init_version_3.sql create mode 100644 src/sql/init_version_4.sql create mode 100644 src/sql/init_version_5.sql create mode 100644 src/sql/init_version_6.sql create mode 100644 src/sql/init_version_7.sql create mode 100644 src/sql/init_version_8.sql create mode 100644 src/sql/init_version_9.sql create mode 100644 src/sql/initialize_schema.sql create mode 100644 src/sql/upg_sc_if_old_ver_1.sql create mode 100644 src/sql/upg_sc_if_old_ver_2.sql diff --git a/packages/pyinstaller/bitmessagemain.spec b/packages/pyinstaller/bitmessagemain.spec index 8d38ec09..2ee65035 100644 --- a/packages/pyinstaller/bitmessagemain.spec +++ b/packages/pyinstaller/bitmessagemain.spec @@ -6,6 +6,7 @@ import time from PyInstaller.utils.hooks import copy_metadata + site_root = os.path.abspath(HOMEPATH) spec_root = os.path.abspath(SPECPATH) arch = 32 if ctypes.sizeof(ctypes.c_voidp) == 4 else 64 @@ -22,13 +23,13 @@ os.chdir(srcPath) snapshot = False -hookspath=os.path.join(spec_root, 'hooks') +hookspath = os.path.join(spec_root, 'hooks') a = Analysis( [os.path.join(srcPath, 'bitmessagemain.py')], - datas = [ + datas=[ (os.path.join(spec_root[:-20], 'pybitmessage.egg-info') + '/*', - 'pybitmessage.egg-info') + 'pybitmessage.egg-info') ] + copy_metadata('msgpack-python') + copy_metadata('qrcode') + copy_metadata('six') + copy_metadata('stem'), pathex=[outPath], @@ -72,6 +73,13 @@ a.datas += [ for file_ in os.listdir(dir_append) if file_.endswith('.ui') ] +sql_dir = os.path.join(srcPath, 'sql') + +a.datas += [ + ('sql', os.path.join(sql_dir, file_), 'DATA') + for file_ in os.listdir(sql_dir) if file_.endswith('.sql') +] + # append the translations directory a.datas += addTranslations() a.datas += [('default.ini', os.path.join(srcPath, 'default.ini'), 'DATA')] @@ -88,16 +96,15 @@ a.binaries += [ ('libeay32.dll', os.path.join(openSSLPath, 'libeay32.dll'), 'BINARY'), (os.path.join('bitmsghash', 'bitmsghash%i.dll' % arch), os.path.join(srcPath, 'bitmsghash', 'bitmsghash%i.dll' % arch), - 'BINARY'), + 'BINARY'), (os.path.join('bitmsghash', 'bitmsghash.cl'), - os.path.join(srcPath, 'bitmsghash', 'bitmsghash.cl'), 'BINARY'), + os.path.join(srcPath, 'bitmsghash', 'bitmsghash.cl'), 'BINARY'), (os.path.join('sslkeys', 'cert.pem'), - os.path.join(srcPath, 'sslkeys', 'cert.pem'), 'BINARY'), + os.path.join(srcPath, 'sslkeys', 'cert.pem'), 'BINARY'), (os.path.join('sslkeys', 'key.pem'), - os.path.join(srcPath, 'sslkeys', 'key.pem'), 'BINARY') + os.path.join(srcPath, 'sslkeys', 'key.pem'), 'BINARY') ] - from version import softwareVersion today = time.strftime("%Y%m%d") diff --git a/setup.py b/setup.py index fea2e58a..171526c3 100644 --- a/setup.py +++ b/setup.py @@ -136,7 +136,7 @@ if __name__ == "__main__": packages=packages, package_data={'': [ 'bitmessageqt/*.ui', 'bitmsghash/*.cl', 'sslkeys/*.pem', - 'translations/*.ts', 'translations/*.qm', 'default.ini', + 'translations/*.ts', 'translations/*.qm', 'default.ini', 'sql/*.sql', 'images/*.png', 'images/*.ico', 'images/*.icns' ]}, data_files=data_files, diff --git a/src/build_osx.py b/src/build_osx.py index 3fcefbfb..d83e9b9b 100644 --- a/src/build_osx.py +++ b/src/build_osx.py @@ -10,6 +10,7 @@ mainscript = ["bitmessagemain.py"] DATA_FILES = [ ('', ['sslkeys', 'images', 'default.ini']), + ('sql', glob('sql/*.sql')), ('bitmsghash', ['bitmsghash/bitmsghash.cl', 'bitmsghash/bitmsghash.so']), ('translations', glob('translations/*.qm')), ('ui', glob('bitmessageqt/*.ui')), diff --git a/src/sql/config_setting_ver_2.sql b/src/sql/config_setting_ver_2.sql new file mode 100644 index 00000000..087d297a --- /dev/null +++ b/src/sql/config_setting_ver_2.sql @@ -0,0 +1 @@ +ALTER TABLE pubkeys ADD usedpersonally text DEFAULT 'no'; diff --git a/src/sql/config_setting_ver_3.sql b/src/sql/config_setting_ver_3.sql new file mode 100644 index 00000000..4bdcccc8 --- /dev/null +++ b/src/sql/config_setting_ver_3.sql @@ -0,0 +1,5 @@ +ALTER TABLE inbox ADD encodingtype int DEFAULT '2'; + +ALTER TABLE inbox ADD read bool DEFAULT '1'; + +ALTER TABLE sent ADD encodingtype int DEFAULT '2'; diff --git a/src/sql/init_version_10.sql b/src/sql/init_version_10.sql new file mode 100644 index 00000000..8bd8b0b3 --- /dev/null +++ b/src/sql/init_version_10.sql @@ -0,0 +1,15 @@ +-- -- +-- -- Update the address colunm to unique in addressbook table +-- -- + +ALTER TABLE addressbook RENAME TO old_addressbook; + +CREATE TABLE `addressbook` ( + `label` text , + `address` text , + UNIQUE(address) ON CONFLICT IGNORE +) ; + +INSERT INTO addressbook SELECT label, address FROM old_addressbook; + +DROP TABLE old_addressbook; diff --git a/src/sql/init_version_2.sql b/src/sql/init_version_2.sql new file mode 100644 index 00000000..ea42df4c --- /dev/null +++ b/src/sql/init_version_2.sql @@ -0,0 +1,29 @@ +-- +-- Let's get rid of the first20bytesofencryptedmessage field in the inventory table. +-- + +CREATE TEMP TABLE `inventory_backup` ( + `hash` blob , + `objecttype` text , + `streamnumber` int , + `payload` blob , + `receivedtime` int , + UNIQUE(hash) ON CONFLICT REPLACE +) ; + +INSERT INTO `inventory_backup` SELECT hash, objecttype, streamnumber, payload, receivedtime FROM inventory; + +DROP TABLE inventory; + +CREATE TABLE `inventory` ( + `hash` blob , + `objecttype` text , + `streamnumber` int , + `payload` blob , + `receivedtime` int , + UNIQUE(hash) ON CONFLICT REPLACE +) ; + +INSERT INTO inventory SELECT hash, objecttype, streamnumber, payload, receivedtime FROM inventory_backup; + +DROP TABLE inventory_backup; diff --git a/src/sql/init_version_3.sql b/src/sql/init_version_3.sql new file mode 100644 index 00000000..9de784a5 --- /dev/null +++ b/src/sql/init_version_3.sql @@ -0,0 +1,5 @@ +-- +-- Add a new column to the inventory table to store tags. +-- + +ALTER TABLE inventory ADD tag blob DEFAULT ''; diff --git a/src/sql/init_version_4.sql b/src/sql/init_version_4.sql new file mode 100644 index 00000000..d2fd393d --- /dev/null +++ b/src/sql/init_version_4.sql @@ -0,0 +1,17 @@ + -- + -- Add a new column to the pubkeys table to store the address version. + -- We're going to trash all of our pubkeys and let them be redownloaded. + -- + +DROP TABLE pubkeys; + +CREATE TABLE `pubkeys` ( + `hash` blob , + `addressversion` int , + `transmitdata` blob , + `time` int , + `usedpersonally` text , + UNIQUE(hash, addressversion) ON CONFLICT REPLACE +) ; + +DELETE FROM inventory WHERE objecttype = 'pubkey'; diff --git a/src/sql/init_version_5.sql b/src/sql/init_version_5.sql new file mode 100644 index 00000000..a13fa8cf --- /dev/null +++ b/src/sql/init_version_5.sql @@ -0,0 +1,12 @@ + -- + -- Add a new table: objectprocessorqueue with which to hold objects + -- that have yet to be processed if the user shuts down Bitmessage. + -- + +DROP TABLE knownnodes; + +CREATE TABLE `objectprocessorqueue` ( + `objecttype` text, + `data` blob, + UNIQUE(objecttype, data) ON CONFLICT REPLACE +) ; diff --git a/src/sql/init_version_6.sql b/src/sql/init_version_6.sql new file mode 100644 index 00000000..b9a03669 --- /dev/null +++ b/src/sql/init_version_6.sql @@ -0,0 +1,25 @@ +-- +-- changes related to protocol v3 +-- In table inventory and objectprocessorqueue, objecttype is now +-- an integer (it was a human-friendly string previously) +-- + +DROP TABLE inventory; + +CREATE TABLE `inventory` ( + `hash` blob, + `objecttype` int, + `streamnumber` int, + `payload` blob, + `expirestime` integer, + `tag` blob, + UNIQUE(hash) ON CONFLICT REPLACE +) ; + +DROP TABLE objectprocessorqueue; + +CREATE TABLE `objectprocessorqueue` ( + `objecttype` int, + `data` blob, + UNIQUE(objecttype, data) ON CONFLICT REPLACE +) ; diff --git a/src/sql/init_version_7.sql b/src/sql/init_version_7.sql new file mode 100644 index 00000000..a2f6f6e3 --- /dev/null +++ b/src/sql/init_version_7.sql @@ -0,0 +1,11 @@ +-- +-- The format of data stored in the pubkeys table has changed. Let's +-- clear it, and the pubkeys from inventory, so that they'll +-- be re-downloaded. +-- + +DELETE FROM inventory WHERE objecttype = 1; + +DELETE FROM pubkeys; + +UPDATE sent SET status='msgqueued' WHERE status='doingmsgpow' or status='badkey'; diff --git a/src/sql/init_version_8.sql b/src/sql/init_version_8.sql new file mode 100644 index 00000000..0c1813d3 --- /dev/null +++ b/src/sql/init_version_8.sql @@ -0,0 +1,7 @@ +-- +-- Add a new column to the inbox table to store the hash of +-- the message signature. We'll use this as temporary message UUID +-- in order to detect duplicates. +-- + +ALTER TABLE inbox ADD sighash blob DEFAULT ''; diff --git a/src/sql/init_version_9.sql b/src/sql/init_version_9.sql new file mode 100644 index 00000000..bc8296b9 --- /dev/null +++ b/src/sql/init_version_9.sql @@ -0,0 +1,74 @@ +CREATE TEMPORARY TABLE `sent_backup` ( + `msgid` blob, + `toaddress` text, + `toripe` blob, + `fromaddress` text, + `subject` text, + `message` text, + `ackdata` blob, + `lastactiontime` integer, + `status` text, + `retrynumber` integer, + `folder` text, + `encodingtype` int +) ; + +INSERT INTO sent_backup SELECT msgid, toaddress, toripe, fromaddress, subject, message, ackdata, lastactiontime, status, 0, folder, encodingtype FROM sent; + +DROP TABLE sent; + +CREATE TABLE `sent` ( + `msgid` blob, + `toaddress` text, + `toripe` blob, + `fromaddress` text, + `subject` text, + `message` text, + `ackdata` blob, + `senttime` integer, + `lastactiontime` integer, + `sleeptill` int, + `status` text, + `retrynumber` integer, + `folder` text, + `encodingtype` int, + `ttl` int +) ; + +INSERT INTO sent SELECT msgid, toaddress, toripe, fromaddress, subject, message, ackdata, lastactiontime, lastactiontime, 0, status, 0, folder, encodingtype, 216000 FROM sent_backup; + +DROP TABLE sent_backup; + +ALTER TABLE pubkeys ADD address text DEFAULT '' ; + +-- +-- replica for loop to update hashed address +-- + +UPDATE pubkeys SET address=(enaddr(pubkeys.addressversion, 1, hash)); + +CREATE TEMPORARY TABLE `pubkeys_backup` ( + `address` text, + `addressversion` int, + `transmitdata` blob, + `time` int, + `usedpersonally` text, + UNIQUE(address) ON CONFLICT REPLACE +) ; + +INSERT INTO pubkeys_backup SELECT address, addressversion, transmitdata, `time`, usedpersonally FROM pubkeys; + +DROP TABLE pubkeys; + +CREATE TABLE `pubkeys` ( + `address` text, + `addressversion` int, + `transmitdata` blob, + `time` int, + `usedpersonally` text, + UNIQUE(address) ON CONFLICT REPLACE +) ; + +INSERT INTO pubkeys SELECT address, addressversion, transmitdata, `time`, usedpersonally FROM pubkeys_backup; + +DROP TABLE pubkeys_backup; diff --git a/src/sql/initialize_schema.sql b/src/sql/initialize_schema.sql new file mode 100644 index 00000000..8413aa0a --- /dev/null +++ b/src/sql/initialize_schema.sql @@ -0,0 +1,100 @@ +CREATE TABLE `inbox` ( + `msgid` blob, + `toaddress` text, + `fromaddress` text, + `subject` text, + `received` text, + `message` text, + `folder` text, + `encodingtype` int, + `read` bool, + `sighash` blob, +UNIQUE(msgid) ON CONFLICT REPLACE +) ; + +CREATE TABLE `sent` ( + `msgid` blob, + `toaddress` text, + `toripe` blob, + `fromaddress` text, + `subject` text, + `message` text, + `ackdata` blob, + `senttime` integer, + `lastactiontime` integer, + `sleeptill` integer, + `status` text, + `retrynumber` integer, + `folder` text, + `encodingtype` int, + `ttl` int +) ; + + +CREATE TABLE `subscriptions` ( + `label` text, + `address` text, + `enabled` bool +) ; + + +CREATE TABLE `addressbook` ( + `label` text, + `address` text, + UNIQUE(address) ON CONFLICT IGNORE +) ; + + + CREATE TABLE `blacklist` ( + `label` text, + `address` text, + `enabled` bool + ) ; + + + CREATE TABLE `whitelist` ( + `label` text, + `address` text, + `enabled` bool + ) ; + + +CREATE TABLE `pubkeys` ( + `address` text, + `addressversion` int, + `transmitdata` blob, + `time` int, + `usedpersonally` text, + UNIQUE(address) ON CONFLICT REPLACE +) ; + + +CREATE TABLE `inventory` ( + `hash` blob, + `objecttype` int, + `streamnumber` int, + `payload` blob, + `expirestime` integer, + `tag` blob, + UNIQUE(hash) ON CONFLICT REPLACE +) ; + + +INSERT INTO subscriptions VALUES ('Bitmessage new releases/announcements', 'BM-GtovgYdgs7qXPkoYaRgrLFuFKz1SFpsw', 1); + + + CREATE TABLE `settings` ( + `key` blob, + `value` blob, + UNIQUE(key) ON CONFLICT REPLACE + ) ; + +INSERT INTO settings VALUES('version','11'); + +INSERT INTO settings VALUES('lastvacuumtime', CAST(strftime('%s', 'now') AS STR) ); + +CREATE TABLE `objectprocessorqueue` ( + `objecttype` int, + `data` blob, + UNIQUE(objecttype, data) ON CONFLICT REPLACE +) ; diff --git a/src/sql/upg_sc_if_old_ver_1.sql b/src/sql/upg_sc_if_old_ver_1.sql new file mode 100644 index 00000000..18a5ecfc --- /dev/null +++ b/src/sql/upg_sc_if_old_ver_1.sql @@ -0,0 +1,30 @@ +CREATE TEMPORARY TABLE `pubkeys_backup` ( + `hash` blob, + `transmitdata` blob, + `time` int, + `usedpersonally` text, + UNIQUE(hash) ON CONFLICT REPLACE +) ; + +INSERT INTO `pubkeys_backup` SELECT hash, transmitdata, `time`, usedpersonally FROM `pubkeys`; + +DROP TABLE `pubkeys` + +CREATE TABLE `pubkeys` ( + `hash` blob, + `transmitdata` blob, + `time` int, + `usedpersonally` text, + UNIQUE(hash) ON CONFLICT REPLACE +) ; + + +INSERT INTO `pubkeys` SELECT hash, transmitdata, `time`, usedpersonally FROM `pubkeys_backup`; + +DROP TABLE `pubkeys_backup`; + +DELETE FROM inventory WHERE objecttype = 'pubkey'; + +DELETE FROM subscriptions WHERE address='BM-BbkPSZbzPwpVcYZpU4yHwf9ZPEapN5Zx' + +INSERT INTO subscriptions VALUES('Bitmessage new releases/announcements','BM-GtovgYdgs7qXPkoYaRgrLFuFKz1SFpsw',1) diff --git a/src/sql/upg_sc_if_old_ver_2.sql b/src/sql/upg_sc_if_old_ver_2.sql new file mode 100644 index 00000000..1fde0098 --- /dev/null +++ b/src/sql/upg_sc_if_old_ver_2.sql @@ -0,0 +1,7 @@ +UPDATE `sent` SET status='doingmsgpow' WHERE status='doingpow'; + +UPDATE `sent` SET status='msgsent' WHERE status='sentmessage'; + +UPDATE `sent` SET status='doingpubkeypow' WHERE status='findingpubkey'; + +UPDATE `sent` SET status='broadcastqueued' WHERE status='broadcastpending';