add sql scripts files for sqlthread, change in sql import, updated code for sql file loading and fixed some linting

packages/pyinstaller/bitmessagemain.spec

@@ -6,6 +6,7 @@ import time
 
 from PyInstaller.utils.hooks import copy_metadata
 
+
 site_root = os.path.abspath(HOMEPATH)
 spec_root = os.path.abspath(SPECPATH)
 arch = 32 if ctypes.sizeof(ctypes.c_voidp) == 4 else 64
@@ -22,11 +23,11 @@ os.chdir(srcPath)
 
 snapshot = False
 
-hookspath=os.path.join(spec_root, 'hooks')
+hookspath = os.path.join(spec_root, 'hooks')
 
 a = Analysis(
     [os.path.join(srcPath, 'bitmessagemain.py')],
-    datas = [
+    datas=[
         (os.path.join(spec_root[:-20], 'pybitmessage.egg-info') + '/*',
             'pybitmessage.egg-info')
     ] + copy_metadata('msgpack-python') + copy_metadata('qrcode')
@@ -72,6 +73,13 @@ a.datas += [
     for file_ in os.listdir(dir_append) if file_.endswith('.ui')
 ]
 
+sql_dir = os.path.join(srcPath, 'sql')
+
+a.datas += [
+    ('sql', os.path.join(sql_dir, file_), 'DATA')
+    for file_ in os.listdir(sql_dir) if file_.endswith('.sql')
+]
+
 # append the translations directory
 a.datas += addTranslations()
 a.datas += [('default.ini', os.path.join(srcPath, 'default.ini'), 'DATA')]
@@ -97,7 +105,6 @@ a.binaries += [
         os.path.join(srcPath, 'sslkeys', 'key.pem'), 'BINARY')
 ]
 
-
 from version import softwareVersion
 
 today = time.strftime("%Y%m%d")

setup.py

@@ -136,7 +136,7 @@ if __name__ == "__main__":
         packages=packages,
         package_data={'': [
             'bitmessageqt/*.ui', 'bitmsghash/*.cl', 'sslkeys/*.pem',
-            'translations/*.ts', 'translations/*.qm', 'default.ini',
+            'translations/*.ts', 'translations/*.qm', 'default.ini', 'sql/*.sql',
             'images/*.png', 'images/*.ico', 'images/*.icns'
         ]},
         data_files=data_files,

src/build_osx.py

@@ -10,6 +10,7 @@ mainscript = ["bitmessagemain.py"]
 
 DATA_FILES = [
     ('', ['sslkeys', 'images', 'default.ini']),
+    ('sql', glob('sql/*.sql')),
     ('bitmsghash', ['bitmsghash/bitmsghash.cl', 'bitmsghash/bitmsghash.so']),
     ('translations', glob('translations/*.qm')),
     ('ui', glob('bitmessageqt/*.ui')),

src/sql/config_setting_ver_2.sql

@@ -0,0 +1 @@
+ALTER TABLE pubkeys ADD usedpersonally text DEFAULT 'no';

src/sql/config_setting_ver_3.sql

@@ -0,0 +1,5 @@
+ALTER TABLE inbox ADD encodingtype int DEFAULT '2';
+
+ALTER TABLE inbox ADD read bool DEFAULT '1';
+
+ALTER TABLE sent ADD encodingtype int DEFAULT '2';

src/sql/init_version_10.sql

@@ -0,0 +1,15 @@
+-- --
+-- --  Update the address colunm to unique in addressbook table
+-- --
+
+ALTER TABLE addressbook RENAME TO old_addressbook;
+
+CREATE TABLE `addressbook` (
+    `label` text ,
+    `address` text ,
+    UNIQUE(address) ON CONFLICT IGNORE
+) ;
+
+INSERT INTO addressbook SELECT label, address FROM old_addressbook;
+
+DROP TABLE old_addressbook;

src/sql/init_version_2.sql

@@ -0,0 +1,29 @@
+--
+-- Let's get rid of the first20bytesofencryptedmessage field in the inventory table.
+--
+
+CREATE TEMP TABLE `inventory_backup` (
+    `hash` blob ,
+    `objecttype` text ,
+    `streamnumber` int ,
+    `payload` blob ,
+    `receivedtime` int ,
+    UNIQUE(hash) ON CONFLICT REPLACE
+) ;
+
+INSERT INTO `inventory_backup` SELECT  hash, objecttype, streamnumber, payload, receivedtime FROM inventory;
+
+DROP TABLE inventory;
+
+CREATE TABLE `inventory` (
+    `hash` blob ,
+    `objecttype` text ,
+    `streamnumber` int ,
+    `payload` blob ,
+    `receivedtime` int ,
+    UNIQUE(hash) ON CONFLICT REPLACE
+) ;
+
+INSERT INTO inventory SELECT hash, objecttype, streamnumber, payload, receivedtime FROM inventory_backup;
+
+DROP TABLE inventory_backup;

src/sql/init_version_3.sql

@@ -0,0 +1,5 @@
+--
+-- Add a new column to the inventory table to store tags.
+--
+
+ALTER TABLE inventory ADD tag blob DEFAULT '';

src/sql/init_version_4.sql

@@ -0,0 +1,17 @@
+ --
+ -- Add a new column to the pubkeys table to store the address version.
+ -- We're going to trash all of our pubkeys and let them be redownloaded.
+ --
+
+DROP TABLE pubkeys;
+
+CREATE TABLE `pubkeys` (
+    `hash` blob ,
+    `addressversion` int ,
+    `transmitdata` blob ,
+    `time` int ,
+    `usedpersonally` text ,
+    UNIQUE(hash, addressversion) ON CONFLICT REPLACE
+) ;
+
+DELETE FROM inventory WHERE objecttype = 'pubkey';

src/sql/init_version_5.sql

@@ -0,0 +1,12 @@
+ --
+ -- Add a new table: objectprocessorqueue with which to hold objects
+ -- that have yet to be processed if the user shuts down Bitmessage.
+ --
+
+DROP TABLE knownnodes;
+
+CREATE TABLE `objectprocessorqueue` (
+    `objecttype` text,
+    `data` blob,
+    UNIQUE(objecttype, data) ON CONFLICT REPLACE
+) ;

src/sql/init_version_6.sql

@@ -0,0 +1,25 @@
+--
+-- changes related to protocol v3
+-- In table inventory and objectprocessorqueue, objecttype is now
+-- an integer (it was a human-friendly string previously)
+--
+
+DROP TABLE inventory;
+
+CREATE TABLE `inventory` (
+    `hash` blob,
+    `objecttype` int,
+    `streamnumber` int,
+    `payload` blob,
+    `expirestime` integer,
+    `tag` blob,
+    UNIQUE(hash) ON CONFLICT REPLACE
+) ;
+
+DROP TABLE objectprocessorqueue;
+
+CREATE TABLE `objectprocessorqueue` (
+    `objecttype` int,
+    `data` blob,
+    UNIQUE(objecttype, data) ON CONFLICT REPLACE
+) ;

src/sql/init_version_7.sql

@@ -0,0 +1,11 @@
+--
+-- The format of data stored in the pubkeys table has changed. Let's
+-- clear it, and the pubkeys from inventory, so that they'll
+-- be re-downloaded.
+--
+
+DELETE FROM inventory WHERE objecttype = 1;
+
+DELETE FROM pubkeys;
+
+UPDATE sent SET status='msgqueued' WHERE status='doingmsgpow' or status='badkey';

src/sql/init_version_8.sql

@@ -0,0 +1,7 @@
+--
+-- Add a new column to the inbox table to store the hash of
+-- the message signature. We'll use this as temporary message UUID
+-- in order to detect duplicates.
+--
+
+ALTER TABLE inbox ADD sighash blob DEFAULT '';

src/sql/init_version_9.sql

@@ -0,0 +1,74 @@
+CREATE TEMPORARY TABLE `sent_backup` (
+    `msgid` blob,
+    `toaddress` text,
+    `toripe` blob,
+    `fromaddress` text,
+    `subject` text,
+    `message` text,
+    `ackdata` blob,
+    `lastactiontime` integer,
+    `status` text,
+    `retrynumber` integer,
+    `folder` text,
+    `encodingtype` int
+) ;
+
+INSERT INTO sent_backup SELECT msgid, toaddress, toripe, fromaddress, subject, message, ackdata, lastactiontime, status, 0, folder, encodingtype FROM sent;
+
+DROP TABLE sent;
+
+CREATE TABLE `sent` (
+    `msgid` blob,
+    `toaddress` text,
+    `toripe` blob,
+    `fromaddress` text,
+    `subject` text,
+    `message` text,
+    `ackdata` blob,
+    `senttime` integer,
+    `lastactiontime` integer,
+    `sleeptill` int,
+    `status` text,
+    `retrynumber` integer,
+    `folder` text,
+    `encodingtype` int,
+    `ttl` int
+) ;
+
+INSERT INTO sent SELECT msgid, toaddress, toripe, fromaddress, subject, message, ackdata, lastactiontime, lastactiontime, 0, status, 0, folder, encodingtype, 216000 FROM sent_backup;
+
+DROP TABLE sent_backup;
+
+ALTER TABLE pubkeys ADD address text DEFAULT '' ;
+
+--
+-- replica for loop to update hashed address
+--
+
+UPDATE pubkeys SET address=(enaddr(pubkeys.addressversion, 1, hash));
+
+CREATE TEMPORARY TABLE `pubkeys_backup` (
+    `address` text,
+    `addressversion` int,
+    `transmitdata` blob,
+    `time` int,
+    `usedpersonally` text,
+    UNIQUE(address) ON CONFLICT REPLACE
+) ;
+
+INSERT INTO pubkeys_backup SELECT address, addressversion, transmitdata, `time`, usedpersonally FROM pubkeys;
+
+DROP TABLE pubkeys;
+
+CREATE TABLE `pubkeys` (
+    `address` text,
+    `addressversion` int,
+    `transmitdata` blob,
+    `time` int,
+    `usedpersonally` text,
+    UNIQUE(address) ON CONFLICT REPLACE
+) ;
+
+INSERT INTO pubkeys SELECT address, addressversion, transmitdata, `time`, usedpersonally FROM pubkeys_backup;
+
+DROP TABLE pubkeys_backup;

src/sql/initialize_schema.sql

@@ -0,0 +1,100 @@
+CREATE TABLE `inbox` (
+   `msgid` blob,
+   `toaddress` text,
+   `fromaddress` text,
+   `subject` text,
+   `received` text,
+   `message` text,
+   `folder` text,
+   `encodingtype` int,
+   `read` bool,
+   `sighash` blob,
+UNIQUE(msgid) ON CONFLICT REPLACE
+) ;
+
+CREATE TABLE `sent` (
+   `msgid` blob,
+   `toaddress` text,
+   `toripe` blob,
+   `fromaddress` text,
+   `subject` text,
+   `message` text,
+   `ackdata` blob,
+   `senttime` integer,
+   `lastactiontime` integer,
+   `sleeptill` integer,
+   `status` text,
+   `retrynumber` integer,
+   `folder` text,
+   `encodingtype` int,
+   `ttl` int
+) ;
+
+
+CREATE TABLE `subscriptions` (
+   `label` text,
+   `address` text,
+   `enabled` bool
+) ;
+
+
+CREATE TABLE `addressbook` (
+   `label` text,
+   `address` text,
+   UNIQUE(address) ON CONFLICT IGNORE
+) ;
+
+
+ CREATE TABLE `blacklist` (
+    `label` text,
+    `address` text,
+    `enabled` bool
+ ) ;
+
+
+ CREATE TABLE `whitelist` (
+    `label` text,
+    `address` text,
+    `enabled` bool
+ ) ;
+
+
+CREATE TABLE `pubkeys` (
+    `address` text,
+    `addressversion` int,
+    `transmitdata` blob,
+    `time` int,
+    `usedpersonally` text,
+    UNIQUE(address) ON CONFLICT REPLACE
+) ;
+
+
+CREATE TABLE `inventory` (
+    `hash` blob,
+    `objecttype` int,
+    `streamnumber` int,
+    `payload` blob,
+    `expirestime` integer,
+    `tag` blob,
+    UNIQUE(hash) ON CONFLICT REPLACE
+) ;
+
+
+INSERT INTO subscriptions VALUES ('Bitmessage new releases/announcements', 'BM-GtovgYdgs7qXPkoYaRgrLFuFKz1SFpsw', 1);
+
+
+ CREATE TABLE `settings` (
+    `key` blob,
+    `value` blob,
+    UNIQUE(key) ON CONFLICT REPLACE
+ ) ;
+
+INSERT INTO settings VALUES('version','11');
+
+INSERT INTO settings VALUES('lastvacuumtime', CAST(strftime('%s', 'now') AS STR) );
+
+CREATE TABLE `objectprocessorqueue` (
+    `objecttype` int,
+    `data` blob,
+    UNIQUE(objecttype, data) ON CONFLICT REPLACE
+) ;

src/sql/upg_sc_if_old_ver_1.sql

@@ -0,0 +1,30 @@
+CREATE TEMPORARY TABLE `pubkeys_backup` (
+    `hash` blob,
+    `transmitdata` blob,
+    `time` int,
+    `usedpersonally` text,
+    UNIQUE(hash) ON CONFLICT REPLACE
+) ;
+
+INSERT INTO `pubkeys_backup` SELECT hash, transmitdata, `time`, usedpersonally FROM `pubkeys`;
+
+DROP TABLE `pubkeys`
+
+CREATE TABLE `pubkeys` (
+    `hash` blob,
+    `transmitdata` blob,
+    `time` int,
+    `usedpersonally` text,
+    UNIQUE(hash) ON CONFLICT REPLACE
+) ;
+
+
+INSERT INTO `pubkeys` SELECT hash, transmitdata, `time`, usedpersonally FROM `pubkeys_backup`;
+
+DROP TABLE `pubkeys_backup`;
+
+DELETE FROM inventory WHERE objecttype = 'pubkey';
+
+DELETE FROM subscriptions WHERE address='BM-BbkPSZbzPwpVcYZpU4yHwf9ZPEapN5Zx'
+
+INSERT INTO subscriptions VALUES('Bitmessage new releases/announcements','BM-GtovgYdgs7qXPkoYaRgrLFuFKz1SFpsw',1)

src/sql/upg_sc_if_old_ver_2.sql

@@ -0,0 +1,7 @@
+UPDATE `sent` SET status='doingmsgpow' WHERE status='doingpow';
+
+UPDATE `sent` SET status='msgsent' WHERE status='sentmessage';
+
+UPDATE `sent` SET status='doingpubkeypow' WHERE status='findingpubkey';
+
+UPDATE `sent` SET status='broadcastqueued' WHERE status='broadcastpending';