Added flatpak build manifests with split dependencies

Closes: #1696

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "packages/flatpak/shared-modules"]
+	path = packages/flatpak/shared-modules
+	url = https://github.com/flathub/shared-modules.git

packages/flatpak/org.bitmessage.BaseApp.json

@@ -0,0 +1,57 @@
+{
+ "id": "org.bitmessage.BaseApp",
+ "branch": "19.08",
+ "runtime": "org.freedesktop.Platform",
+ "sdk": "org.freedesktop.Sdk",
+ "runtime-version": "19.08",
+ "separate-locales": false,
+ "modules": [
+  "shared-modules/python2.7/python-2.7.json",
+  "shared-modules/qt4/qt4-4.8.7-minimal.json",
+  {
+    "name": "python-sip",
+    "sources": [
+      {
+        "type": "archive",
+        "url": "https://www.riverbankcomputing.com/static/Downloads/sip/4.19.25/sip-4.19.25.tar.gz",
+        "sha256": "b39d93e937647807bac23579edbff25fe46d16213f708370072574ab1f1b4211"
+      }
+    ],
+    "buildsystem": "simple",
+    "build-commands": [
+      "python configure.py --sip-module PyQt4.sip --no-dist-info",
+      "make",
+      "make install"
+    ]
+  },
+  {
+    "name": "python-qt4",
+    "sources": [
+      {
+        "type": "archive",
+        "url": "http://sourceforge.net/projects/pyqt/files/PyQt4/PyQt-4.12.3/PyQt4_gpl_x11-4.12.3.tar.gz",
+        "sha256": "a00f5abef240a7b5852b7924fa5fdf5174569525dc076cd368a566619e56d472"
+      }
+    ],
+    "buildsystem": "simple",
+    "build-commands": [
+      "python configure.py -w --confirm-license",
+      "make",
+      "make install"
+    ]
+  },
+  {
+    "name" : "PyBitmessage-dependencies",
+    "buildsystem" : "simple",
+    "build-options": {
+      "build-args": [
+        "--share=network"
+      ]
+    },
+    "build-commands": [
+      "pip --version",
+      "pip install setuptools msgpack"
+    ]
+  }
+ ]
+}

packages/flatpak/org.bitmessage.PyBitmessage.json

@@ -0,0 +1,48 @@
+{
+    "app-id": "org.bitmessage.PyBitmessage",
+    "runtime": "org.freedesktop.Platform",
+    "runtime-version": "19.08",
+    "branch": "stable",
+    "sdk": "org.freedesktop.Sdk",
+    "base": "org.bitmessage.BaseApp",
+    "command": "pybitmessage",
+    "base-version":"stable",
+    "finish-args" : [
+      "--share=network",
+      "--socket=x11",
+      "--share=ipc",
+      "--filesystem=xdg-config/PyBitmessage:create"
+    ],
+    "modules": [
+      {
+        "name" : "PyBitmessage",
+        "buildsystem" : "simple",
+        "build-options": {
+          "build-args": [
+            "--share=network"
+          ]
+        },
+        "build-commands": [
+          "python --version",
+          "pwd",
+          "ls",
+          "python checkdeps.py",
+          "python setup.py install --prefix=/app --exec-prefix=/app",
+          "sed -i 's~/usr/bin/~/app/bin/~' /app/bin/pybitmessage",
+          "cat /app/bin/pybitmessage",
+          "mv /app/share/applications/pybitmessage.desktop /app/share/applications/org.bitmessage.PyBitmessage.desktop",
+          "sed -i 's~Icon=pybitmessage~Icon=org.bitmessage.PyBitmessage~' /app/share/applications/org.bitmessage.PyBitmessage.desktop",
+          "mv /app/share/icons/hicolor/scalable/apps/pybitmessage.svg /app/share/icons/hicolor/scalable/apps/org.bitmessage.PyBitmessage.svg",
+          "mv /app/share/icons/hicolor/24x24/apps/pybitmessage.png /app/share/icons/hicolor/24x24/apps/org.bitmessage.PyBitmessage.png",
+          "which pybitmessage"
+        ],
+        "sources" : [
+          {
+            "type" : "dir",
+            "path" : "../../"
+          }
+        ]
+      }
+    ]
+  }
+  

packages/flatpak/shared-modules

@@ -0,0 +1 @@
+Subproject commit fd4d38328ccb078b88ad4a891807e593ae8de806

src/api.py

@@ -73,7 +73,8 @@ from SimpleXMLRPCServer import SimpleXMLRPCRequestHandler, SimpleXMLRPCServer
 from struct import pack
 
 import defaults
-import helper_db
+import helper_inbox
+import helper_sent
 import network.stats
 import proofofwork
 import queues
@@ -586,9 +587,14 @@ class BMRPCDispatcher(object):
         label = self._decode(label, "base64")
         address = addBMIfNotPresent(address)
         self._verifyAddress(address)
-        if not helper_db.put_addressbook(label, address):
+        # TODO: add unique together constraint in the table
+        queryreturn = sqlQuery(
+            "SELECT address FROM addressbook WHERE address=?", address)
+        if queryreturn != []:
             raise APIError(
                 16, 'You already have this address in your address book.')
+
+        sqlExecute("INSERT INTO addressbook VALUES(?,?)", label, address)
         queues.UISignalQueue.put(('rerenderMessagelistFromLabels', ''))
         queues.UISignalQueue.put(('rerenderMessagelistToLabels', ''))
         queues.UISignalQueue.put(('rerenderAddressBook', ''))
@@ -1050,23 +1056,23 @@ class BMRPCDispatcher(object):
         """
         msgid = self._decode(msgid, "hex")
         # Trash if in inbox table
-        helper_db.put_trash(msgid)
+        helper_inbox.trash(msgid)
         # Trash if in sent table
-        helper_db.put_trash(msgid, sent=True)
+        sqlExecute("UPDATE sent SET folder='trash' WHERE msgid=?", msgid)
         return 'Trashed message (assuming message existed).'
 
     @command('trashInboxMessage')
     def HandleTrashInboxMessage(self, msgid):
         """Trash inbox message by msgid (encoded in hex)."""
         msgid = self._decode(msgid, "hex")
-        helper_db.put_trash(msgid)
+        helper_inbox.trash(msgid)
         return 'Trashed inbox message (assuming message existed).'
 
     @command('trashSentMessage')
     def HandleTrashSentMessage(self, msgid):
         """Trash sent message by msgid (encoded in hex)."""
         msgid = self._decode(msgid, "hex")
-        helper_db.put_trash(msgid, sent=True)
+        sqlExecute('''UPDATE sent SET folder='trash' WHERE msgid=?''', msgid)
         return 'Trashed sent message (assuming message existed).'
 
     @command('sendMessage')
@@ -1104,15 +1110,18 @@ class BMRPCDispatcher(object):
         if not fromAddressEnabled:
             raise APIError(14, 'Your fromAddress is disabled. Cannot send.')
 
-        stealthLevel = BMConfigParser().safeGetInt(
-            'bitmessagesettings', 'ackstealthlevel')
-        ackdata = genAckPayload(streamNumber, stealthLevel)
+        ackdata = helper_sent.insert(
+            toAddress=toAddress, fromAddress=fromAddress,
+            subject=subject, message=message, encoding=encodingType, ttl=TTL)
 
-        helper_db.put_sent(
-            toAddress, fromAddress, subject, message, ackdata,
-            'msgqueued', encodingType, ttl=TTL)
+        toLabel = ''
+        queryreturn = sqlQuery(
+            "SELECT label FROM addressbook WHERE address=?", toAddress)
+        try:
+            toLabel, = queryreturn[0][0]
+        except IndexError:
+            pass
 
-        toLabel = helper_db.get_label(toAddress) or ''
         queues.UISignalQueue.put(('displayNewSentMessage', (
             toAddress, toLabel, fromAddress, subject, message, ackdata)))
         queues.workerQueue.put(('sendmessage', toAddress))
@@ -1142,15 +1151,15 @@ class BMRPCDispatcher(object):
             self.config.getboolean(fromAddress, 'enabled')
         except BaseException:
             raise APIError(
-                13, 'could not find your fromAddress in the keys.dat file.')
-        streamNumber = decodeAddress(fromAddress)[2]
-        ackdata = genAckPayload(streamNumber, 0)
-        toAddress = toLabel = str_broadcast_subscribers
+                13, 'Could not find your fromAddress in the keys.dat file.')
+        toAddress = str_broadcast_subscribers
 
-        helper_db.put_sent(
-            toAddress, fromAddress, subject, message, ackdata,
-            'broadcastqueued', encodingType, ttl=TTL)
+        ackdata = helper_sent.insert(
+            fromAddress=fromAddress, subject=subject,
+            message=message, status='broadcastqueued',
+            encoding=encodingType)
 
+        toLabel = str_broadcast_subscribers
         queues.UISignalQueue.put(('displayNewSentMessage', (
             toAddress, toLabel, fromAddress, subject, message, ackdata)))
         queues.workerQueue.put(('sendbroadcast', ''))
@@ -1188,8 +1197,15 @@ class BMRPCDispatcher(object):
             except UnicodeDecodeError:
                 raise APIError(17, 'Label is not valid UTF-8 data.')
         self._verifyAddress(address)
-        if not helper_db.put_subscriptions(label, address):
+        address = addBMIfNotPresent(address)
+        # First we must check to see if the address is already in the
+        # subscriptions list.
+        queryreturn = sqlQuery(
+            "SELECT * FROM subscriptions WHERE address=?", address)
+        if queryreturn:
             raise APIError(16, 'You are already subscribed to that address.')
+        sqlExecute(
+            "INSERT INTO subscriptions VALUES (?,?,?)", label, address, True)
         shared.reloadBroadcastSendersForWhichImWatching()
         queues.UISignalQueue.put(('rerenderMessagelistFromLabels', ''))
         queues.UISignalQueue.put(('rerenderSubscriptions', ''))

src/bitmessagemain.py

@@ -321,9 +321,10 @@ class Main(object):
                 receiveQueueThread = ReceiveQueueThread(i)
                 receiveQueueThread.daemon = True
                 receiveQueueThread.start()
-            announceThread = AnnounceThread()
-            announceThread.daemon = True
-            announceThread.start()
+            if config.safeGetBoolean('bitmessagesettings', 'udp'):
+                state.announceThread = AnnounceThread()
+                state.announceThread.daemon = True
+                state.announceThread.start()
             state.invThread = InvThread()
             state.invThread.daemon = True
             state.invThread.start()

src/bitmessageqt/__init__.py

@@ -35,10 +35,9 @@ from foldertree import (
     MessageList_TimeWidget)
 import settingsmixin
 import support
-from helper_ackPayload import genAckPayload
-from helper_sql import (
-    sqlQuery, sqlExecute, sqlExecuteChunked, sqlStoredProcedure)
-import helper_db
+from helper_sql import sqlQuery, sqlExecute, sqlExecuteChunked, sqlStoredProcedure
+import helper_addressbook
+import helper_search
 import l10n
 from utils import str_broadcast_subscribers, avatarize
 from account import (
@@ -55,6 +54,7 @@ from statusbar import BMStatusBar
 import sound
 # This is needed for tray icon
 import bitmessage_icons_rc  # noqa:F401 pylint: disable=unused-import
+import helper_sent
 
 try:
     from plugins.plugin import get_plugin, get_plugins
@@ -1214,7 +1214,7 @@ class MyForm(settingsmixin.SMainWindow):
             tableWidget.setColumnHidden(1, bool(account))
             xAddress = 'fromaddress'
 
-        queryreturn = helper_db.search_sql(
+        queryreturn = helper_search.search_sql(
             xAddress, account, "sent", where, what, False)
 
         for row in queryreturn:
@@ -1254,7 +1254,7 @@ class MyForm(settingsmixin.SMainWindow):
             tableWidget.setColumnHidden(0, False)
             tableWidget.setColumnHidden(1, False)
 
-        queryreturn = helper_db.search_sql(
+        queryreturn = helper_search.search_sql(
             xAddress, account, folder, where, what, unreadOnly)
 
         for row in queryreturn:
@@ -1937,7 +1937,8 @@ class MyForm(settingsmixin.SMainWindow):
 
         newRows = {}
         # subscriptions
-        for row in helper_db.get_subscriptions():
+        queryreturn = sqlQuery('SELECT label, address FROM subscriptions WHERE enabled = 1')
+        for row in queryreturn:
             label, address = row
             newRows[address] = [label, AccountMixin.SUBSCRIPTION]
         # chans
@@ -1947,7 +1948,8 @@ class MyForm(settingsmixin.SMainWindow):
             if (account.type == AccountMixin.CHAN and BMConfigParser().safeGetBoolean(address, 'enabled')):
                 newRows[address] = [account.getLabel(), AccountMixin.CHAN]
         # normal accounts
-        for row in helper_db.get_addressbook():
+        queryreturn = sqlQuery('SELECT * FROM addressbook')
+        for row in queryreturn:
             label, address = row
             newRows[address] = [label, AccountMixin.NORMAL]
 
@@ -2164,19 +2166,18 @@ class MyForm(settingsmixin.SMainWindow):
                                 " send the message but it won\'t send until"
                                 " you connect.")
                             )
-                        stealthLevel = BMConfigParser().safeGetInt(
-                            'bitmessagesettings', 'ackstealthlevel')
-                        ackdata = genAckPayload(streamNumber, stealthLevel)
-                        helper_db.put_sent(
-                            toAddress, fromAddress, subject, message, ackdata,
-                            'msgqueued', encoding, ripe
-                        )
-
-                        toLabel = helper_db.get_label(toAddress) or ''
+                        ackdata = helper_sent.insert(
+                            toAddress=toAddress, fromAddress=fromAddress,
+                            subject=subject, message=message, encoding=encoding)
+                        toLabel = ''
+                        queryreturn = sqlQuery('''select label from addressbook where address=?''',
+                                               toAddress)
+                        if queryreturn != []:
+                            for row in queryreturn:
+                                toLabel, = row
 
                         self.displayNewSentMessage(
-                            toAddress, toLabel, fromAddress, subject,
-                            message, ackdata)
+                            toAddress, toLabel, fromAddress, subject, message, ackdata)
                         queues.workerQueue.put(('sendmessage', toAddress))
 
                         self.click_pushButtonClear()
@@ -2201,14 +2202,15 @@ class MyForm(settingsmixin.SMainWindow):
                 # We don't actually need the ackdata for acknowledgement since
                 # this is a broadcast message, but we can use it to update the
                 # user interface when the POW is done generating.
-                streamNumber = decodeAddress(fromAddress)[2]
-                ackdata = genAckPayload(streamNumber, 0)
-                toAddress = toLabel = str_broadcast_subscribers
+                toAddress = str_broadcast_subscribers
 
-                helper_db.put_sent(
-                    toAddress, fromAddress, subject, message, ackdata,
-                    'broadcastqueued', encoding
-                )
+                # msgid. We don't know what this will be until the POW is done.
+                ackdata = helper_sent.insert(
+                    fromAddress=fromAddress,
+                    subject=subject, message=message,
+                    status='broadcastqueued', encoding=encoding)
+
+                toLabel = str_broadcast_subscribers
 
                 self.displayNewSentMessage(
                     toAddress, toLabel, fromAddress, subject, message, ackdata)
@@ -2336,7 +2338,7 @@ class MyForm(settingsmixin.SMainWindow):
                 self.ui.treeWidgetChans
             ) and self.getCurrentAccount(treeWidget) != toAddress:
                 continue
-            elif not helper_db.check_match(
+            elif not helper_search.check_match(
                 toAddress, fromAddress, subject, message,
                 self.getCurrentSearchOption(tab),
                 self.getCurrentSearchLine(tab)
@@ -2366,7 +2368,7 @@ class MyForm(settingsmixin.SMainWindow):
             tab += 1
             if tab == 1:
                 tab = 2
-            if not helper_db.check_match(
+            if not helper_search.check_match(
                 toAddress, fromAddress, subject, message,
                 self.getCurrentSearchOption(tab),
                 self.getCurrentSearchLine(tab)
@@ -2425,7 +2427,10 @@ class MyForm(settingsmixin.SMainWindow):
         except AttributeError:
             return
 
-        if not helper_db.put_addressbook(label, address):
+        # First we must check to see if the address is already in the
+        # address book. The user cannot add it again or else it will
+        # cause problems when updating and deleting the entry.
+        if shared.isAddressInMyAddressBook(address):
             self.updateStatusBar(_translate(
                 "MainWindow",
                 "Error: You cannot add the same address to your"
@@ -2434,25 +2439,30 @@ class MyForm(settingsmixin.SMainWindow):
             ))
             return
 
-        self.rerenderMessagelistFromLabels()
-        self.rerenderMessagelistToLabels()
-        self.rerenderAddressBook()
-
-    def addSubscription(self, address, label):
-        if not helper_db.put_subscriptions(label, address):
+        if helper_addressbook.insert(label=label, address=address):
+            self.rerenderMessagelistFromLabels()
+            self.rerenderMessagelistToLabels()
+            self.rerenderAddressBook()
+        else:
             self.updateStatusBar(_translate(
                 "MainWindow",
-                "Error: You cannot add the same address to your"
-                " subscriptions twice. Perhaps rename the existing one"
-                " if you want."
+                "Error: You cannot add your own address in the address book."
             ))
-            return
 
+    def addSubscription(self, address, label):
+        # This should be handled outside of this function, for error displaying
+        # and such, but it must also be checked here.
+        if shared.isAddressInMySubscriptionsList(address):
+            return
+        # Add to database (perhaps this should be separated from the MyForm class)
+        sqlExecute(
+            '''INSERT INTO subscriptions VALUES (?,?,?)''',
+            label, address, True
+        )
         self.rerenderMessagelistFromLabels()
         shared.reloadBroadcastSendersForWhichImWatching()
         self.rerenderAddressBook()
         self.rerenderTabTreeSubscriptions()
-        return True
 
     def click_pushButtonAddSubscription(self):
         dialog = dialogs.NewSubscriptionDialog(self)
@@ -2462,9 +2472,19 @@ class MyForm(settingsmixin.SMainWindow):
         except AttributeError:
             return
 
-        if not self.addSubscription(address, label):
+        # We must check to see if the address is already in the
+        # subscriptions list. The user cannot add it again or else it
+        # will cause problems when updating and deleting the entry.
+        if shared.isAddressInMySubscriptionsList(address):
+            self.updateStatusBar(_translate(
+                "MainWindow",
+                "Error: You cannot add the same address to your"
+                " subscriptions twice. Perhaps rename the existing one"
+                " if you want."
+            ))
             return
 
+        self.addSubscription(address, label)
         # Now, if the user wants to display old broadcasts, let's get
         # them out of the inventory and put them
         # to the objectProcessorQueue to be processed
@@ -2795,7 +2815,11 @@ class MyForm(settingsmixin.SMainWindow):
         textEdit = self.getCurrentMessageTextedit()
         if not msgid:
             return
-        messageText = helper_db.get_message(msgid)
+        queryreturn = sqlQuery(
+            '''select message from inbox where msgid=?''', msgid)
+        if queryreturn != []:
+            for row in queryreturn:
+                messageText, = row
 
         lines = messageText.split('\n')
         totalLines = len(lines)
@@ -3028,20 +3052,24 @@ class MyForm(settingsmixin.SMainWindow):
             currentInboxRow, 1).data(QtCore.Qt.UserRole)
         recipientAddress = tableWidget.item(
             currentInboxRow, 0).data(QtCore.Qt.UserRole)
-        label = "\"" + tableWidget.item(currentInboxRow, 2).subject + \
-            "\" in " + BMConfigParser().get(recipientAddress, "label")
-        if not helper_db.put_blacklist(label, addressAtCurrentInboxRow):
+        # Let's make sure that it isn't already in the address book
+        queryreturn = sqlQuery('''select * from blacklist where address=?''',
+                               addressAtCurrentInboxRow)
+        if queryreturn == []:
+            label = "\"" + tableWidget.item(currentInboxRow, 2).subject + "\" in " + BMConfigParser().get(recipientAddress, "label")
+            sqlExecute('''INSERT INTO blacklist VALUES (?,?, ?)''',
+                       label,
+                       addressAtCurrentInboxRow, True)
+            self.ui.blackwhitelist.rerenderBlackWhiteList()
+            self.updateStatusBar(_translate(
+                "MainWindow",
+                "Entry added to the blacklist. Edit the label to your liking.")
+            )
+        else:
             self.updateStatusBar(_translate(
                 "MainWindow",
                 "Error: You cannot add the same address to your blacklist"
                 " twice. Try renaming the existing one if you want."))
-            return
-
-        self.ui.blackwhitelist.rerenderBlackWhiteList()
-        self.updateStatusBar(_translate(
-            "MainWindow",
-            "Entry added to the blacklist. Edit the label to your liking.")
-        )
 
     def deleteRowFromMessagelist(
         self, row=None, inventoryHash=None, ackData=None, messageLists=None
@@ -3140,7 +3168,11 @@ class MyForm(settingsmixin.SMainWindow):
 
         # Retrieve the message data out of the SQL database
         msgid = tableWidget.item(currentInboxRow, 3).data()
-        message = helper_db.get_message(msgid)
+        queryreturn = sqlQuery(
+            '''select message from inbox where msgid=?''', msgid)
+        if queryreturn != []:
+            for row in queryreturn:
+                message, = row
 
         defaultFilename = "".join(x for x in subjectAtCurrentInboxRow if x.isalnum()) + '.txt'
         filename = QtGui.QFileDialog.getSaveFileName(self, _translate("MainWindow","Save As..."), defaultFilename, "Text files (*.txt);;All files (*.*)")
@@ -3248,6 +3280,15 @@ class MyForm(settingsmixin.SMainWindow):
 
     def on_action_AddressBookSubscribe(self):
         for item in self.getAddressbookSelectedItems():
+            # Then subscribe to it...
+            # provided it's not already in the address book
+            if shared.isAddressInMySubscriptionsList(item.address):
+                self.updateStatusBar(_translate(
+                    "MainWindow",
+                    "Error: You cannot add the same address to your"
+                    " subscriptions twice. Perhaps rename the existing"
+                    " one if you want."))
+                continue
             self.addSubscription(item.address, item.label)
             self.ui.tabWidget.setCurrentIndex(
                 self.ui.tabWidget.indexOf(self.ui.subscriptions)

src/bitmessageqt/account.py

@@ -4,30 +4,43 @@ account.py
 ==========
 
 Account related functions.
+
 """
 
+from __future__ import absolute_import
+
 import inspect
 import re
 import sys
+import time
+
+from PyQt4 import QtGui
 
-import helper_db
 import queues
 from addresses import decodeAddress
 from bmconfigparser import BMConfigParser
 from helper_ackPayload import genAckPayload
-from helper_sql import sqlQuery
-from foldertree import AccountMixin
-from utils import str_broadcast_subscribers
-from tr import _translate
+from helper_sql import sqlQuery, sqlExecute
+from .foldertree import AccountMixin
+from .utils import str_broadcast_subscribers
 
 
 def getSortedAccounts():
     """Get a sorted list of configSections"""
+
     configSections = BMConfigParser().addresses()
     configSections.sort(
         cmp=lambda x, y: cmp(
-            unicode(BMConfigParser().get(x, 'label'), 'utf-8').lower(),
-            unicode(BMConfigParser().get(y, 'label'), 'utf-8').lower()))
+            unicode(
+                BMConfigParser().get(
+                    x,
+                    'label'),
+                'utf-8').lower(),
+            unicode(
+                BMConfigParser().get(
+                    y,
+                    'label'),
+                'utf-8').lower()))
     return configSections
 
 
@@ -94,27 +107,29 @@ def accountClass(address):
 
 class AccountColor(AccountMixin):  # pylint: disable=too-few-public-methods
     """Set the type of account"""
+
     def __init__(self, address, address_type=None):
         self.isEnabled = True
         self.address = address
-        if address_type:
-            self.type = address_type
-            return
-        # AccountMixin.setType()
-        if self.address is None:
-            self.type = self.ALL
-        elif BMConfigParser().safeGetBoolean(self.address, 'chan'):
-            self.type = self.CHAN
-        elif BMConfigParser().safeGetBoolean(self.address, 'mailinglist'):
-            self.type = self.MAILINGLIST
-        elif helper_db.get_label(self.address, "subscriptions"):
-            self.type = AccountMixin.SUBSCRIPTION
+        if address_type is None:
+            if address is None:
+                self.type = AccountMixin.ALL
+            elif BMConfigParser().safeGetBoolean(self.address, 'mailinglist'):
+                self.type = AccountMixin.MAILINGLIST
+            elif BMConfigParser().safeGetBoolean(self.address, 'chan'):
+                self.type = AccountMixin.CHAN
+            elif sqlQuery(
+                    '''select label from subscriptions where address=?''', self.address):
+                self.type = AccountMixin.SUBSCRIPTION
+            else:
+                self.type = AccountMixin.NORMAL
         else:
-            self.type = self.NORMAL
+            self.type = address_type
 
 
 class BMAccount(object):
     """Encapsulate a Bitmessage account"""
+
     def __init__(self, address=None):
         self.address = address
         self.type = AccountMixin.NORMAL
@@ -125,19 +140,29 @@ class BMAccount(object):
                 self.type = AccountMixin.MAILINGLIST
         elif self.address == str_broadcast_subscribers:
             self.type = AccountMixin.BROADCAST
-        elif helper_db.get_label(self.address, "subscriptions"):
-            self.type = AccountMixin.SUBSCRIPTION
+        else:
+            queryreturn = sqlQuery(
+                '''select label from subscriptions where address=?''', self.address)
+            if queryreturn:
+                self.type = AccountMixin.SUBSCRIPTION
 
     def getLabel(self, address=None):
         """Get a label for this bitmessage account"""
         if address is None:
             address = self.address
-        if BMConfigParser().has_section(address):
-            return BMConfigParser().get(address, 'label')
-        return (
-            helper_db.get_label(address) or
-            helper_db.get_label(address, "subscriptions") or address
-        )
+        label = BMConfigParser().safeGet(address, 'label', address)
+        queryreturn = sqlQuery(
+            '''select label from addressbook where address=?''', address)
+        if queryreturn != []:
+            for row in queryreturn:
+                label, = row
+        else:
+            queryreturn = sqlQuery(
+                '''select label from subscriptions where address=?''', address)
+            if queryreturn != []:
+                for row in queryreturn:
+                    label, = row
+        return label
 
     def parseMessage(self, toAddress, fromAddress, subject, message):
         """Set metadata and address labels on self"""
@@ -161,7 +186,9 @@ class NoAccount(BMAccount):
         self.type = AccountMixin.NORMAL
 
     def getLabel(self, address=None):
-        return address or self.address
+        if address is None:
+            address = self.address
+        return address
 
 
 class SubscriptionAccount(BMAccount):
@@ -186,17 +213,31 @@ class GatewayAccount(BMAccount):
 
     def send(self):
         """Override the send method for gateway accounts"""
-        streamNumber, ripe = decodeAddress(self.toAddress)[2:]
-        stealthLevel = BMConfigParser().safeGetInt(
-            'bitmessagesettings', 'ackstealthlevel')
+
+        # pylint: disable=unused-variable
+        status, addressVersionNumber, streamNumber, ripe = decodeAddress(self.toAddress)
+        stealthLevel = BMConfigParser().safeGetInt('bitmessagesettings', 'ackstealthlevel')
         ackdata = genAckPayload(streamNumber, stealthLevel)
-        helper_db.put_sent(
-            self.toAddress, self.fromAddress, self.subject, self.message,
-            ackdata, 'msgqueued', 2, ripe,
-            min(
-                BMConfigParser().getint('bitmessagesettings', 'ttl'),
-                86400 * 2)  # not necessary to have a TTL higher than 2 days
+        sqlExecute(
+            '''INSERT INTO sent VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)''',
+            '',
+            self.toAddress,
+            ripe,
+            self.fromAddress,
+            self.subject,
+            self.message,
+            ackdata,
+            int(time.time()),  # sentTime (this will never change)
+            int(time.time()),  # lastActionTime
+            0,  # sleepTill time. This will get set when the POW gets done.
+            'msgqueued',
+            0,  # retryNumber
+            'sent',  # folder
+            2,  # encodingtype
+            # not necessary to have a TTL higher than 2 days
+            min(BMConfigParser().getint('bitmessagesettings', 'ttl'), 86400 * 2)
         )
+
         queues.workerQueue.put(('sendmessage', self.toAddress))
 
 
@@ -251,7 +292,9 @@ class MailchuckAccount(GatewayAccount):
 
         self.toAddress = self.registrationAddress
         self.subject = "config"
-        self.message = _translate("Mailchuck", """# You can use this to configure your email gateway account
+        self.message = QtGui.QApplication.translate(
+            "Mailchuck",
+            """# You can use this to configure your email gateway account
 # Uncomment the setting you want to use
 # Here are the options:
 #

src/bitmessageqt/blacklist.py

@@ -1,11 +1,10 @@
 from PyQt4 import QtCore, QtGui
 
-import helper_db
 import widgets
 from addresses import addBMIfNotPresent
 from bmconfigparser import BMConfigParser
 from dialogs import AddAddressDialog
-from helper_sql import sqlExecute
+from helper_sql import sqlExecute, sqlQuery
 from queues import UISignalQueue
 from retranslateui import RetranslateMixin
 from tr import _translate
@@ -62,14 +61,16 @@ class Blacklist(QtGui.QWidget, RetranslateMixin):
                     _translate("MainWindow", "Address is valid."):
                 address = addBMIfNotPresent(str(
                     self.NewBlacklistDialogInstance.lineEditAddress.text()))
-                label = str(self.NewBlacklistDialogInstance.lineEditLabel.text().toUtf8())
-                if helper_db.put_addresslist(
-                    label, address,
-                    group='blacklist'
-                    if BMConfigParser().get(
-                        'bitmessagesettings', 'blackwhitelist') == 'black'
-                    else 'whitelist'
-                ):
+                # First we must check to see if the address is already in the
+                # address book. The user cannot add it again or else it will
+                # cause problems when updating and deleting the entry.
+                t = (address,)
+                if BMConfigParser().get('bitmessagesettings', 'blackwhitelist') == 'black':
+                    sql = '''select * from blacklist where address=?'''
+                else:
+                    sql = '''select * from whitelist where address=?'''
+                queryreturn = sqlQuery(sql,*t)
+                if queryreturn == []:
                     self.tableWidgetBlacklist.setSortingEnabled(False)
                     self.tableWidgetBlacklist.insertRow(0)
                     newItem = QtGui.QTableWidgetItem(unicode(
@@ -81,6 +82,12 @@ class Blacklist(QtGui.QWidget, RetranslateMixin):
                         QtCore.Qt.ItemIsSelectable | QtCore.Qt.ItemIsEnabled)
                     self.tableWidgetBlacklist.setItem(0, 1, newItem)
                     self.tableWidgetBlacklist.setSortingEnabled(True)
+                    t = (str(self.NewBlacklistDialogInstance.lineEditLabel.text().toUtf8()), address, True)
+                    if BMConfigParser().get('bitmessagesettings', 'blackwhitelist') == 'black':
+                        sql = '''INSERT INTO blacklist VALUES (?,?,?)'''
+                    else:
+                        sql = '''INSERT INTO whitelist VALUES (?,?,?)'''
+                    sqlExecute(sql, *t)
                 else:
                     UISignalQueue.put((
                         'updateStatusBar',
@@ -151,18 +158,19 @@ class Blacklist(QtGui.QWidget, RetranslateMixin):
 
     def rerenderBlackWhiteList(self):
         tabs = self.parent().parent()
-        list_type = BMConfigParser().get(
-            'bitmessagesettings', 'blackwhitelist')
-        tabs.setTabText(
-            tabs.indexOf(self),
-            _translate('blacklist', 'Blacklist') if list_type == 'black'
-            else _translate('blacklist', 'Whitelist'))
-
+        if BMConfigParser().get('bitmessagesettings', 'blackwhitelist') == 'black':
+            tabs.setTabText(tabs.indexOf(self), _translate('blacklist', 'Blacklist'))
+        else:
+            tabs.setTabText(tabs.indexOf(self), _translate('blacklist', 'Whitelist'))
         self.tableWidgetBlacklist.setRowCount(0)
+        listType = BMConfigParser().get('bitmessagesettings', 'blackwhitelist')
+        if listType == 'black':
+            queryreturn = sqlQuery('''SELECT label, address, enabled FROM blacklist''')
+        else:
+            queryreturn = sqlQuery('''SELECT label, address, enabled FROM whitelist''')
         self.tableWidgetBlacklist.setSortingEnabled(False)
-        for label, address, enabled in helper_db.get_addresslist(
-            group='blacklist' if list_type == 'black' else 'whiteslist'
-        ):
+        for row in queryreturn:
+            label, address, enabled = row
             self.tableWidgetBlacklist.insertRow(0)
             newItem = QtGui.QTableWidgetItem(unicode(label, 'utf-8'))
             if not enabled:

src/bitmessageqt/foldertree.py

@@ -8,8 +8,8 @@ from cgi import escape
 
 from PyQt4 import QtCore, QtGui
 
-import helper_db
 from bmconfigparser import BMConfigParser
+from helper_sql import sqlExecute, sqlQuery
 from settingsmixin import SettingsMixin
 from tr import _translate
 from utils import avatarize
@@ -110,13 +110,15 @@ class AccountMixin(object):
             self.type = self.CHAN
         elif BMConfigParser().safeGetBoolean(self.address, 'mailinglist'):
             self.type = self.MAILINGLIST
-        elif helper_db.get_label(self.address, "subscriptions"):
+        elif sqlQuery(
+                '''select label from subscriptions where address=?''', self.address):
             self.type = AccountMixin.SUBSCRIPTION
         else:
             self.type = self.NORMAL
 
     def defaultLabel(self):
         """Default label (in case no label is set manually)"""
+        queryreturn = None
         retval = None
         if self.type in (
                 AccountMixin.NORMAL,
@@ -125,9 +127,16 @@ class AccountMixin(object):
                 retval = unicode(
                     BMConfigParser().get(self.address, 'label'), 'utf-8')
             except Exception:
-                retval = helper_db.get_label(self.address)
+                queryreturn = sqlQuery(
+                    '''select label from addressbook where address=?''', self.address)
         elif self.type == AccountMixin.SUBSCRIPTION:
-            retval = helper_db.get_label(self.address, "subscriptions")
+            queryreturn = sqlQuery(
+                '''select label from subscriptions where address=?''', self.address)
+        if queryreturn is not None:
+            if queryreturn != []:
+                for row in queryreturn:
+                    retval, = row
+                    retval = unicode(retval, 'utf-8')
         elif self.address is None or self.type == AccountMixin.ALL:
             return unicode(
                 str(_translate("MainWindow", "All accounts")), 'utf-8')
@@ -297,8 +306,13 @@ class Ui_SubscriptionWidget(Ui_AddressWidget):
             parent, pos, address, unreadCount, enabled)
 
     def _getLabel(self):
-        return helper_db.get_label(self.address, "subscriptions") \
-            or self.address
+        queryreturn = sqlQuery(
+            '''select label from subscriptions where address=?''', self.address)
+        if queryreturn != []:
+            for row in queryreturn:
+                retval, = row
+            return unicode(retval, 'utf-8', 'ignore')
+        return unicode(self.address, 'utf-8')
 
     def setType(self):
         """Set account type"""
@@ -313,7 +327,9 @@ class Ui_SubscriptionWidget(Ui_AddressWidget):
                     value.toString().toUtf8()).decode('utf-8', 'ignore')
             else:
                 label = unicode(value, 'utf-8', 'ignore')
-            helper_db.set_label(self.address, label, "subscriptions")
+            sqlExecute(
+                '''UPDATE subscriptions SET label=? WHERE address=?''',
+                label, self.address)
         return super(Ui_SubscriptionWidget, self).setData(column, role, value)
 
 
@@ -386,6 +402,7 @@ class MessageList_AddressWidget(BMAddressWidget):
         if label is not None:
             return
         newLabel = self.address
+        queryreturn = None
         if self.type in (
                 AccountMixin.NORMAL,
                 AccountMixin.CHAN, AccountMixin.MAILINGLIST):
@@ -394,9 +411,14 @@ class MessageList_AddressWidget(BMAddressWidget):
                     BMConfigParser().get(self.address, 'label'),
                     'utf-8', 'ignore')
             except:
-                newLabel = helper_db.get_label(self.address)
+                queryreturn = sqlQuery(
+                    '''select label from addressbook where address=?''', self.address)
         elif self.type == AccountMixin.SUBSCRIPTION:
-            newLabel = helper_db.get_label(self.address, "subscriptions")
+            queryreturn = sqlQuery(
+                '''select label from subscriptions where address=?''', self.address)
+        if queryreturn:
+            for row in queryreturn:
+                newLabel = unicode(row[0], 'utf-8', 'ignore')
 
         self.label = newLabel
 
@@ -503,9 +525,9 @@ class Ui_AddressBookWidgetItem(BMAddressWidget):
                     BMConfigParser().set(self.address, 'label', self.label)
                     BMConfigParser().save()
                 except:
-                    helper_db.set_label(self.address, self.label)
+                    sqlExecute('''UPDATE addressbook set label=? WHERE address=?''', self.label, self.address)
             elif self.type == AccountMixin.SUBSCRIPTION:
-                helper_db.set_label(self.address, self.label, "subscriptions")
+                sqlExecute('''UPDATE subscriptions set label=? WHERE address=?''', self.label, self.address)
             else:
                 pass
         return super(Ui_AddressBookWidgetItem, self).setData(role, value)

src/bitmessageqt/settings.py

@@ -19,7 +19,7 @@ import widgets
 from bmconfigparser import BMConfigParser
 from helper_sql import sqlExecute, sqlStoredProcedure
 from helper_startup import start_proxyconfig
-from network import knownnodes
+from network import knownnodes, AnnounceThread
 from network.asyncore_pollchoose import set_rates
 from tr import _translate
 
@@ -138,6 +138,8 @@ class SettingsDialog(QtGui.QDialog):
             config.get('bitmessagesettings', 'port')))
         self.checkBoxUPnP.setChecked(
             config.safeGetBoolean('bitmessagesettings', 'upnp'))
+        self.checkBoxUDP.setChecked(
+            config.safeGetBoolean('bitmessagesettings', 'udp'))
         self.checkBoxAuthentication.setChecked(
             config.getboolean('bitmessagesettings', 'socksauthentication'))
         self.checkBoxSocksListen.setChecked(
@@ -326,7 +328,8 @@ class SettingsDialog(QtGui.QDialog):
                 self.lineEditTCPPort.text()):
             self.config.set(
                 'bitmessagesettings', 'port', str(self.lineEditTCPPort.text()))
-            if not self.config.safeGetBoolean('bitmessagesettings', 'dontconnect'):
+            if not self.config.safeGetBoolean(
+                    'bitmessagesettings', 'dontconnect'):
                 self.net_restart_needed = True
 
         if self.checkBoxUPnP.isChecked() != self.config.safeGetBoolean(
@@ -339,11 +342,26 @@ class SettingsDialog(QtGui.QDialog):
                 upnpThread = upnp.uPnPThread()
                 upnpThread.start()
 
+        udp_enabled = self.checkBoxUDP.isChecked()
+        if udp_enabled != self.config.safeGetBoolean(
+                'bitmessagesettings', 'udp'):
+            self.config.set('bitmessagesettings', 'udp', str(udp_enabled))
+            if udp_enabled:
+                announceThread = AnnounceThread()
+                announceThread.daemon = True
+                announceThread.start()
+            else:
+                try:
+                    state.announceThread.stopThread()
+                except AttributeError:
+                    pass
+
         proxytype_index = self.comboBoxProxyType.currentIndex()
         if proxytype_index == 0:
             if self._proxy_type and state.statusIconColor != 'red':
                 self.net_restart_needed = True
-        elif state.statusIconColor == 'red' and self.config.safeGetBoolean('bitmessagesettings', 'dontconnect'):
+        elif state.statusIconColor == 'red' and self.config.safeGetBoolean(
+                'bitmessagesettings', 'dontconnect'):
             self.net_restart_needed = False
         elif self.comboBoxProxyType.currentText() != self._proxy_type:
             self.net_restart_needed = True
@@ -369,8 +387,11 @@ class SettingsDialog(QtGui.QDialog):
             self.lineEditSocksPassword.text()))
         self.config.set('bitmessagesettings', 'sockslisten', str(
             self.checkBoxSocksListen.isChecked()))
-        if self.checkBoxOnionOnly.isChecked() \
-                and not self.config.safeGetBoolean('bitmessagesettings', 'onionservicesonly'):
+        if (
+            self.checkBoxOnionOnly.isChecked()
+            and not self.config.safeGetBoolean(
+                'bitmessagesettings', 'onionservicesonly')
+        ):
             self.net_restart_needed = True
         self.config.set('bitmessagesettings', 'onionservicesonly', str(
             self.checkBoxOnionOnly.isChecked()))
@@ -432,8 +453,8 @@ class SettingsDialog(QtGui.QDialog):
         acceptableDifficultyChanged = False
 
         if (
-                float(self.lineEditMaxAcceptableTotalDifficulty.text()) >= 1
-                or float(self.lineEditMaxAcceptableTotalDifficulty.text()) == 0
+            float(self.lineEditMaxAcceptableTotalDifficulty.text()) >= 1
+            or float(self.lineEditMaxAcceptableTotalDifficulty.text()) == 0
         ):
             if self.config.get(
                     'bitmessagesettings', 'maxacceptablenoncetrialsperbyte'
@@ -449,8 +470,8 @@ class SettingsDialog(QtGui.QDialog):
                         * defaults.networkDefaultProofOfWorkNonceTrialsPerByte))
                 )
         if (
-                float(self.lineEditMaxAcceptableSmallMessageDifficulty.text()) >= 1
-                or float(self.lineEditMaxAcceptableSmallMessageDifficulty.text()) == 0
+            float(self.lineEditMaxAcceptableSmallMessageDifficulty.text()) >= 1
+            or float(self.lineEditMaxAcceptableSmallMessageDifficulty.text()) == 0
         ):
             if self.config.get(
                     'bitmessagesettings', 'maxacceptablepayloadlengthextrabytes'
@@ -541,8 +562,8 @@ class SettingsDialog(QtGui.QDialog):
         self.parent.updateStartOnLogon()
 
         if (
-                state.appdata != paths.lookupExeFolder()
-                and self.checkBoxPortableMode.isChecked()
+            state.appdata != paths.lookupExeFolder()
+            and self.checkBoxPortableMode.isChecked()
         ):
             # If we are NOT using portable mode now but the user selected
             # that we should...
@@ -564,8 +585,8 @@ class SettingsDialog(QtGui.QDialog):
                 pass
 
         if (
-                state.appdata == paths.lookupExeFolder()
-                and not self.checkBoxPortableMode.isChecked()
+            state.appdata == paths.lookupExeFolder()
+            and not self.checkBoxPortableMode.isChecked()
         ):
             # If we ARE using portable mode now but the user selected
             # that we shouldn't...

src/bitmessageqt/settings.ui

@@ -231,7 +231,7 @@
          </layout>
         </widget>
        </item>
-       <item row="2" column="0">
+       <item row="3" column="0">
         <widget class="QGroupBox" name="groupBox_3">
          <property name="title">
           <string>Bandwidth limit</string>
@@ -322,7 +322,7 @@
          </layout>
         </widget>
        </item>
-       <item row="1" column="0">
+       <item row="2" column="0">
         <widget class="QGroupBox" name="groupBox_2">
          <property name="title">
           <string>Proxy server / Tor</string>
@@ -432,7 +432,14 @@
          </layout>
         </widget>
        </item>
-       <item row="3" column="0">
+       <item row="1" column="0">
+        <widget class="QCheckBox" name="checkBoxUDP">
+         <property name="text">
+          <string>Announce self by UDP</string>
+         </property>
+        </widget>
+       </item>
+       <item row="4" column="0">
         <spacer name="verticalSpacer">
          <property name="orientation">
           <enum>Qt::Vertical</enum>

src/bitmessageqt/support.py

@@ -10,7 +10,6 @@ from PyQt4 import QtCore
 
 import account
 import defaults
-import helper_db
 import network.stats
 import paths
 import proofofwork
@@ -18,7 +17,7 @@ import queues
 import state
 from bmconfigparser import BMConfigParser
 from foldertree import AccountMixin
-from helper_sql import sqlExecute
+from helper_sql import sqlExecute, sqlQuery
 from l10n import getTranslationLanguage
 from openclpow import openclEnabled
 from pyelliptic.openssl import OpenSSL
@@ -68,8 +67,12 @@ Connected hosts: {}
 
 
 def checkAddressBook(myapp):
-    sqlExecute('DELETE FROM addressbook WHERE address=?', OLD_SUPPORT_ADDRESS)
-    if helper_db.put_addressbook(SUPPORT_LABEL.toUtf8(), SUPPORT_ADDRESS):
+    sqlExecute('DELETE from addressbook WHERE address=?', OLD_SUPPORT_ADDRESS)
+    queryreturn = sqlQuery('SELECT * FROM addressbook WHERE address=?', SUPPORT_ADDRESS)
+    if queryreturn == []:
+        sqlExecute(
+            'INSERT INTO addressbook VALUES (?,?)',
+            SUPPORT_LABEL.toUtf8(), SUPPORT_ADDRESS)
         myapp.rerenderAddressBook()
 
 

src/bitmessageqt/tests/__init__.py

@@ -2,6 +2,10 @@
 
 from addressbook import TestAddressbook
 from main import TestMain, TestUISignaler
+from settings import TestSettings
 from support import TestSupport
 
-__all__ = ["TestAddressbook", "TestMain", "TestSupport", "TestUISignaler"]
+__all__ = [
+    "TestAddressbook", "TestMain", "TestSettings", "TestSupport",
+    "TestUISignaler"
+]

src/bitmessageqt/tests/settings.py

@@ -0,0 +1,34 @@
+import threading
+import time
+
+from main import TestBase
+from bmconfigparser import BMConfigParser
+from bitmessageqt import settings
+
+
+class TestSettings(TestBase):
+    """A test case for the "Settings" dialog"""
+    def setUp(self):
+        super(TestSettings, self).setUp()
+        self.dialog = settings.SettingsDialog(self.window)
+
+    def test_udp(self):
+        """Test the effect of checkBoxUDP"""
+        udp_setting = BMConfigParser().safeGetBoolean(
+            'bitmessagesettings', 'udp')
+        self.assertEqual(udp_setting, self.dialog.checkBoxUDP.isChecked())
+        self.dialog.checkBoxUDP.setChecked(not udp_setting)
+        self.dialog.accept()
+        self.assertEqual(
+            not udp_setting,
+            BMConfigParser().safeGetBoolean('bitmessagesettings', 'udp'))
+        time.sleep(5)
+        for thread in threading.enumerate():
+            if thread.name == 'Announcer':  # find Announcer thread
+                if udp_setting:
+                    self.fail(
+                        'Announcer thread is running while udp set to False')
+                break
+        else:
+            if not udp_setting:
+                self.fail('No Announcer thread found while udp set to True')

src/bmconfigparser.py

@@ -2,13 +2,22 @@
 BMConfigParser class definition and default configuration settings
 """
 
-import ConfigParser
+import sys
+if sys.version_info[0] == 3:
+    # python 3
+    import configparser as ConfigParser
+    SafeConfigParser = ConfigParser.ConfigParser
+else:
+    # python 2
+    import ConfigParser
+    SafeConfigParser = ConfigParser.SafeConfigParser
+
+import state
+from singleton import Singleton
 import os
 import shutil
 from datetime import datetime
 
-import state
-from singleton import Singleton
 
 BMConfigDefaults = {
     "bitmessagesettings": {
@@ -19,30 +28,32 @@ BMConfigDefaults = {
         "maxtotalconnections": 200,
         "maxuploadrate": 0,
         "apiinterface": "127.0.0.1",
-        "apiport": 8442
+        "apiport": 8442,
+        "udp": "True"
     },
     "threads": {
         "receive": 3,
     },
     "network": {
-        "bind": '',
+        "bind": "",
         "dandelion": 90,
     },
     "inventory": {
         "storage": "sqlite",
-        "acceptmismatch": False,
+        "acceptmismatch": "False",
     },
     "knownnodes": {
         "maxnodes": 20000,
     },
     "zlib": {
-        'maxsize': 1048576
+        "maxsize": 1048576
     }
 }
 
 
 @Singleton
-class BMConfigParser(ConfigParser.SafeConfigParser):
+class BMConfigParser(SafeConfigParser):
+
     """
     Singleton class inherited from :class:`ConfigParser.SafeConfigParser`
     with additional methods specific to bitmessage config.
@@ -59,26 +70,47 @@ class BMConfigParser(ConfigParser.SafeConfigParser):
             raise ValueError("Invalid value %s" % value)
         return ConfigParser.ConfigParser.set(self, section, option, value)
 
-    def get(self, section, option, raw=False, variables=None):
-        # pylint: disable=arguments-differ
-        try:
-            if section == "bitmessagesettings" and option == "timeformat":
+    def get(self, section, option, raw=False, vars=None):
+        if sys.version_info[0] == 3:
+            # pylint: disable=arguments-differ
+           try:
+                if section == "bitmessagesettings" and option == "timeformat":
+                    return ConfigParser.ConfigParser.get(
+                        self, section, option)
+                try:
+                    return self._temp[section][option]
+                except KeyError:
+                    pass
                 return ConfigParser.ConfigParser.get(
-                    self, section, option, raw, variables)
+                    self, section, option)
+           except ConfigParser.InterpolationError:
+                return ConfigParser.ConfigParser.get(
+                    self, section, option)
+           except (ConfigParser.NoSectionError, ConfigParser.NoOptionError) as e:
+                try:
+                    return BMConfigDefaults[section][option]
+                except (KeyError, ValueError, AttributeError):
+                    raise e
+        else:
+            # pylint: disable=arguments-differ
             try:
-                return self._temp[section][option]
-            except KeyError:
-                pass
-            return ConfigParser.ConfigParser.get(
-                self, section, option, True, variables)
-        except ConfigParser.InterpolationError:
-            return ConfigParser.ConfigParser.get(
-                self, section, option, True, variables)
-        except (ConfigParser.NoSectionError, ConfigParser.NoOptionError) as e:
-            try:
-                return BMConfigDefaults[section][option]
-            except (KeyError, ValueError, AttributeError):
-                raise e
+                if section == "bitmessagesettings" and option == "timeformat":
+                    return ConfigParser.ConfigParser.get(
+                        self, section, option, raw, vars)
+                try:
+                    return self._temp[section][option]
+                except KeyError:
+                    pass
+                return ConfigParser.ConfigParser.get(
+                    self, section, option, True, vars)
+            except ConfigParser.InterpolationError:
+                return ConfigParser.ConfigParser.get(
+                    self, section, option, True, vars)
+            except (ConfigParser.NoSectionError, ConfigParser.NoOptionError) as e:
+                try:
+                    return BMConfigDefaults[section][option]
+                except (KeyError, ValueError, AttributeError):
+                    raise e
 
     def setTemp(self, section, option, value=None):
         """Temporary set option to value, not saving."""
@@ -190,3 +222,4 @@ class BMConfigParser(ConfigParser.SafeConfigParser):
         if value < 0 or value > 8:
             return False
         return True
+

src/class_objectProcessor.py

@@ -10,10 +10,12 @@ import random
 import threading
 import time
 from binascii import hexlify
+from subprocess import call  # nosec
 
 import helper_bitcoin
+import helper_inbox
 import helper_msgcoding
-import helper_db
+import helper_sent
 import highlevelcrypto
 import l10n
 import protocol
@@ -27,10 +29,10 @@ from addresses import (
 )
 from bmconfigparser import BMConfigParser
 from fallback import RIPEMD160Hash
-from helper_ackPayload import genAckPayload
 from helper_sql import sql_ready, SqlBulkExecute, sqlExecute, sqlQuery
 from network import bmproto, knownnodes
 from network.node import Peer
+# pylint: disable=too-many-locals, too-many-return-statements, too-many-branches, too-many-statements
 
 logger = logging.getLogger('default')
 
@@ -341,7 +343,24 @@ class objectProcessor(threading.Thread):
                 )
 
             address = encodeAddress(addressVersion, streamNumber, ripe)
-            helper_db.put_pubkey(address, addressVersion, dataToStore)
+
+            queryreturn = sqlQuery(
+                "SELECT usedpersonally FROM pubkeys WHERE address=?"
+                " AND usedpersonally='yes'", address)
+            # if this pubkey is already in our database and if we have
+            # used it personally:
+            if queryreturn != []:
+                logger.info(
+                    'We HAVE used this pubkey personally. Updating time.')
+                t = (address, addressVersion, dataToStore,
+                     int(time.time()), 'yes')
+            else:
+                logger.info(
+                    'We have NOT used this pubkey personally. Inserting'
+                    ' in database.')
+                t = (address, addressVersion, dataToStore,
+                     int(time.time()), 'no')
+            sqlExecute('''INSERT INTO pubkeys VALUES (?,?,?,?,?)''', *t)
             self.possibleNewPubkey(address)
         if addressVersion == 3:
             if len(data) < 170:  # sanity check.
@@ -389,7 +408,23 @@ class objectProcessor(threading.Thread):
                 )
 
             address = encodeAddress(addressVersion, streamNumber, ripe)
-            helper_db.put_pubkey(address, addressVersion, dataToStore)
+            queryreturn = sqlQuery(
+                "SELECT usedpersonally FROM pubkeys WHERE address=?"
+                " AND usedpersonally='yes'", address)
+            # if this pubkey is already in our database and if we have
+            # used it personally:
+            if queryreturn != []:
+                logger.info(
+                    'We HAVE used this pubkey personally. Updating time.')
+                t = (address, addressVersion, dataToStore,
+                     int(time.time()), 'yes')
+            else:
+                logger.info(
+                    'We have NOT used this pubkey personally. Inserting'
+                    ' in database.')
+                t = (address, addressVersion, dataToStore,
+                     int(time.time()), 'no')
+            sqlExecute('''INSERT INTO pubkeys VALUES (?,?,?,?,?)''', *t)
             self.possibleNewPubkey(address)
 
         if addressVersion == 4:
@@ -588,10 +623,13 @@ class objectProcessor(threading.Thread):
 
         # Let's store the public key in case we want to reply to this
         # person.
-        helper_db.put_pubkey(
-            fromAddress, sendersAddressVersionNumber,
-            decryptedData[:endOfThePublicKeyPosition], True
-        )
+        sqlExecute(
+            '''INSERT INTO pubkeys VALUES (?,?,?,?,?)''',
+            fromAddress,
+            sendersAddressVersionNumber,
+            decryptedData[:endOfThePublicKeyPosition],
+            int(time.time()),
+            'yes')
 
         # Check to see whether we happen to be awaiting this
         # pubkey in order to send a message. If we are, it will do the POW
@@ -655,56 +693,78 @@ class objectProcessor(threading.Thread):
         subject = decodedMessage.subject
         body = decodedMessage.body
 
-        if helper_db.put_inbox(
-            toAddress, fromAddress, subject, body, inventoryHash, sigHash,
-            encoding=messageEncodingType
-        ) is False:
-            # logger.info('This msg is already in our inbox. Ignoring it.')
-            return
+        # Let us make sure that we haven't already received this message
+        if helper_inbox.isMessageAlreadyInInbox(sigHash):
+            logger.info('This msg is already in our inbox. Ignoring it.')
+            blockMessage = True
+        if not blockMessage:
+            if messageEncodingType != 0:
+                t = (inventoryHash, toAddress, fromAddress, subject,
+                     int(time.time()), body, 'inbox', messageEncodingType,
+                     0, sigHash)
+                helper_inbox.insert(t)
 
-        # Let us now check and see whether our receiving address is
-        # behaving as a mailing list
-        if BMConfigParser().safeGetBoolean(toAddress, 'mailinglist'):
-            mailingListName = BMConfigParser().safeGet(
-                toAddress, 'mailinglistname', '')
+                queues.UISignalQueue.put(('displayNewInboxMessage', (
+                    inventoryHash, toAddress, fromAddress, subject, body)))
 
-            # Let us send out this message as a broadcast
-            subject = self.addMailingListNameToSubject(
-                subject, mailingListName)
-            # Let us now send this message out as a broadcast
-            message = time.strftime("%a, %Y-%m-%d %H:%M:%S UTC", time.gmtime(
-            )) + '   Message ostensibly from ' + fromAddress + ':\n\n' + body
-            # The fromAddress for the broadcast that we are about to
-            # send is the toAddress (my address) for the msg message
-            # we are currently processing.
-            fromAddress = toAddress
-            # We don't actually need the ackdata for acknowledgement
-            # since this is a broadcast message but we can use it to
-            # update the user interface when the POW is done generating.
-            streamNumber = decodeAddress(fromAddress)[2]
+            # If we are behaving as an API then we might need to run an
+            # outside command to let some program know that a new message
+            # has arrived.
+            if BMConfigParser().safeGetBoolean(
+                    'bitmessagesettings', 'apienabled'):
+                try:
+                    apiNotifyPath = BMConfigParser().get(
+                        'bitmessagesettings', 'apinotifypath')
+                except:
+                    apiNotifyPath = ''
+                if apiNotifyPath != '':
+                    call([apiNotifyPath, "newMessage"])
 
-            ackdata = genAckPayload(streamNumber, 0)
-            toAddress = toLabel = '[Broadcast subscribers]'
+            # Let us now check and see whether our receiving address is
+            # behaving as a mailing list
+            if BMConfigParser().safeGetBoolean(toAddress, 'mailinglist') \
+                    and messageEncodingType != 0:
+                try:
+                    mailingListName = BMConfigParser().get(
+                        toAddress, 'mailinglistname')
+                except:
+                    mailingListName = ''
+                # Let us send out this message as a broadcast
+                subject = self.addMailingListNameToSubject(
+                    subject, mailingListName)
+                # Let us now send this message out as a broadcast
+                message = time.strftime(
+                    "%a, %Y-%m-%d %H:%M:%S UTC", time.gmtime()
+                ) + '   Message ostensibly from ' + fromAddress \
+                    + ':\n\n' + body
+                # The fromAddress for the broadcast that we are about to
+                # send is the toAddress (my address) for the msg message
+                # we are currently processing.
+                fromAddress = toAddress
+                # We don't actually need the ackdata for acknowledgement
+                # since this is a broadcast message but we can use it to
+                # update the user interface when the POW is done generating.
+                toAddress = '[Broadcast subscribers]'
 
-            # We really should have a discussion about how to
-            # set the TTL for mailing list broadcasts. This is obviously
-            # hard-coded.
-            TTL = 2 * 7 * 24 * 60 * 60  # 2 weeks
-            helper_db.put_sent(
-                toAddress, fromAddress, subject, message, ackdata,
-                'broadcastqueued', messageEncodingType, ttl=TTL
-            )
+                ackdata = helper_sent.insert(
+                    fromAddress=fromAddress,
+                    status='broadcastqueued',
+                    subject=subject,
+                    message=message,
+                    encoding=messageEncodingType)
 
-            queues.UISignalQueue.put((
-                'displayNewSentMessage',
-                (toAddress, toLabel, fromAddress, subject, message, ackdata)
-            ))
-            queues.workerQueue.put(('sendbroadcast', ''))
+                queues.UISignalQueue.put((
+                    'displayNewSentMessage', (
+                        toAddress, '[Broadcast subscribers]', fromAddress,
+                        subject, message, ackdata)
+                ))
+                queues.workerQueue.put(('sendbroadcast', ''))
 
         # Don't send ACK if invalid, blacklisted senders, invisible
         # messages, disabled or chan
         if (
             self.ackDataHasAValidHeader(ackData) and not blockMessage and
+            messageEncodingType != 0 and
             not BMConfigParser().safeGetBoolean(toAddress, 'dontsendack') and
             not BMConfigParser().safeGetBoolean(toAddress, 'chan')
         ):
@@ -915,11 +975,12 @@ class objectProcessor(threading.Thread):
         logger.info('fromAddress: %s', fromAddress)
 
         # Let's store the public key in case we want to reply to this person.
-        helper_db.put_pubkey(
-            fromAddress, sendersAddressVersion,
-            decryptedData[:endOfPubkeyPosition],
-            True
-        )
+        sqlExecute('''INSERT INTO pubkeys VALUES (?,?,?,?,?)''',
+                   fromAddress,
+                   sendersAddressVersion,
+                   decryptedData[:endOfPubkeyPosition],
+                   int(time.time()),
+                   'yes')
 
         # Check to see whether we happen to be awaiting this
         # pubkey in order to send a message. If we are, it will do the POW
@@ -939,12 +1000,27 @@ class objectProcessor(threading.Thread):
         body = decodedMessage.body
 
         toAddress = '[Broadcast subscribers]'
-        if helper_db.put_inbox(
-            toAddress, fromAddress, subject, body, inventoryHash, sigHash,
-            encoding=messageEncodingType
-        ) is False:
-            # logger.info('This broadcast is already in inbox. Ignoring it.')
+        if helper_inbox.isMessageAlreadyInInbox(sigHash):
+            logger.info('This broadcast is already in our inbox. Ignoring it.')
             return
+        t = (inventoryHash, toAddress, fromAddress, subject, int(
+            time.time()), body, 'inbox', messageEncodingType, 0, sigHash)
+        helper_inbox.insert(t)
+
+        queues.UISignalQueue.put(('displayNewInboxMessage', (
+            inventoryHash, toAddress, fromAddress, subject, body)))
+
+        # If we are behaving as an API then we might need to run an
+        # outside command to let some program know that a new message
+        # has arrived.
+        if BMConfigParser().safeGetBoolean('bitmessagesettings', 'apienabled'):
+            try:
+                apiNotifyPath = BMConfigParser().get(
+                    'bitmessagesettings', 'apinotifypath')
+            except:
+                apiNotifyPath = ''
+            if apiNotifyPath != '':
+                call([apiNotifyPath, "newBroadcast"])
 
         # Display timing data
         logger.info(

src/class_singleWorker.py

@@ -10,9 +10,10 @@ import hashlib
 import time
 from binascii import hexlify, unhexlify
 from struct import pack
+from subprocess import call  # nosec
 
 import defaults
-import helper_db
+import helper_inbox
 import helper_msgcoding
 import helper_random
 import helper_sql
@@ -1311,11 +1312,25 @@ class singleWorker(StoppableThread):
                 # Used to detect and ignore duplicate messages in our inbox
                 sigHash = hashlib.sha512(hashlib.sha512(
                     signature).digest()).digest()[32:]
+                t = (inventoryHash, toaddress, fromaddress, subject, int(
+                    time.time()), message, 'inbox', encoding, 0, sigHash)
+                helper_inbox.insert(t)
 
-                helper_db.put_inbox(
-                    toaddress, fromaddress, subject, message, inventoryHash, sigHash,
-                    encoding=encoding
-                )
+                queues.UISignalQueue.put(('displayNewInboxMessage', (
+                    inventoryHash, toaddress, fromaddress, subject, message)))
+
+                # If we are behaving as an API then we might need to run an
+                # outside command to let some program know that a new message
+                # has arrived.
+                if BMConfigParser().safeGetBoolean(
+                        'bitmessagesettings', 'apienabled'):
+                    try:
+                        apiNotifyPath = BMConfigParser().get(
+                            'bitmessagesettings', 'apinotifypath')
+                    except:
+                        apiNotifyPath = ''
+                    if apiNotifyPath != '':
+                        call([apiNotifyPath, "newMessage"])
 
     def requestPubKey(self, toAddress):
         """Send a getpubkey object"""

src/helper_db.py

@@ -1,208 +0,0 @@
-import subprocess  # nosec
-import time
-
-import queues
-from bmconfigparser import BMConfigParser
-from helper_search import search_sql, check_match
-from helper_sql import sqlExecute, sqlQuery
-
-
-__all__ = ["search_sql", "check_match"]
-
-_groups = ("blacklist", "whitelist", "subscriptions", "addressbook")
-_groups_enable = ("blacklist", "whitelist", "subscriptions")
-
-
-# + genAckPayload
-def put_sent(
-        to_address, from_address, subject, message, ackdata,
-        status, encoding,
-        ripe='', ttl=0, msgid='', sent_time=None, last_action_time=None,
-        sleep_till_time=0, retrynumber=0):
-    """Put message into Sent table"""
-    # We don't know msgid until the POW is done.
-    # sleep_till_time will get set when the POW gets done
-    if not sent_time:
-        sent_time = time.time()
-    if not last_action_time:
-        last_action_time = sent_time
-    if not ttl:
-        ttl = BMConfigParser().getint('bitmessagesettings', 'ttl')
-    sqlExecute(
-        "INSERT INTO sent VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
-        msgid, to_address, ripe, from_address, subject, message, ackdata,
-        int(sent_time), int(last_action_time), sleep_till_time, status, retrynumber,
-        'sent', encoding, ttl
-    )
-
-
-def _in_inbox_already(sighash):
-    return sqlQuery(
-        "SELECT COUNT(*) FROM inbox WHERE sighash=?", sighash
-    )[0][0] != 0
-
-
-def put_inbox(
-        to_address, from_address, subject, message, msgid, sighash,
-        encoding=0, received_time=None, broadcast=False):
-    """Put message into Inbox table"""
-    if not received_time:
-        received_time = time.time()
-    if encoding == 0 or _in_inbox_already(sighash):
-        return False
-
-    sqlExecute(
-        "INSERT INTO inbox VALUES (?,?,?,?,?,?,?,?,?,?)",
-        msgid, to_address, from_address, subject, int(received_time), message,
-        'inbox', encoding, 0, sighash
-    )
-
-    queues.UISignalQueue.put((
-        'displayNewInboxMessage',
-        (msgid, to_address, from_address, subject, message)
-    ))
-
-    # If we are behaving as an API then we might need to run an
-    # outside command to let some program know that a new message
-    # has arrived.
-    if BMConfigParser().safeGetBoolean('bitmessagesettings', 'apienabled'):
-        apinotify_path = BMConfigParser().safeGet(
-            'bitmessagesettings', 'apinotifypath')
-        if apinotify_path:
-            subprocess.call([
-                apinotify_path,
-                "newBroadcast" if broadcast else "newMessage"])
-
-
-def put_trash(msgid, sent=False):
-    """Put inbox message (or sent if sent=True) into trash by msgid"""
-    sqlExecute(
-        "UPDATE %s SET folder='trash' WHERE msgid=?" %
-        ('sent' if sent else 'inbox'), msgid)
-    if not sent:
-        queues.UISignalQueue.put(('removeInboxRowByMsgid', msgid))
-
-
-def put_pubkey(address, address_version, data, used_personally=None):
-    """Put pubkey into Pubkeys table"""
-    if used_personally is None:
-        if sqlQuery(
-            "SELECT * FROM pubkeys WHERE address=? AND usedpersonally='yes'",
-            address
-        ) == []:
-            used_personally = False
-        else:
-            sqlExecute(
-                "UPDATE pubkeys SET time=? WHERE address=?",
-                time.time(), address
-            )
-            return
-    sqlExecute(
-        "INSERT INTO pubkeys VALUES (?,?,?,?,?)",
-        address, address_version, data, time.time(),
-        'yes' if used_personally else 'no'
-    )
-
-
-def _in_group_already(address, group="addressbook"):
-    if group not in _groups:
-        return True
-    # elif group in _groups_enable:
-    #     try:
-    #         return sqlQuery(
-    #             "SELECT enabled FROM %s WHERE address=?" % group, address
-    #         )[-1][0]
-    #     except IndexError:
-    #         return
-    else:
-        return sqlQuery(
-            "SELECT * FROM %s WHERE address=?" % group, address)
-
-
-def put_addresslist(label, address, group="blacklist", enabled=True):
-    """Put address into address list (e.g. blacklist, subscriptions...)"""
-    # We must check to see if the address is already in the
-    # subscriptions list. The user cannot add it again or else it
-    # will cause problems when updating and deleting the entry.
-    # FIXME: address should be primary key in this case
-    if _in_group_already(address, group):
-        return False
-    sqlExecute(
-        "INSERT INTO %s VALUES (?,?,?)" % group, label, address, enabled)
-    return True
-
-
-def put_blacklist(label, address):
-    """Put address into blacklist"""
-    return put_addresslist(label, address, "blacklist")
-
-
-def put_subscriptions(label, address, enabled=True):
-    """Put address into subscriptions"""
-    return put_addresslist(label, address, "subscriptions", enabled)
-
-
-def put_addressbook(label, address):
-    """Put address into Addressbook"""
-    # First we must check to see if the address is already in the
-    # address book. The user cannot add it again or else it will
-    # cause problems when updating and deleting the entry.
-    if _in_group_already(address):
-        return False
-    sqlExecute("INSERT INTO addressbook VALUES (?,?)", label, address)
-    return True
-
-
-def get_subscriptions():
-    """Generator for Subscriptions"""
-    queryreturn = sqlQuery(
-        "SELECT label, address FROM subscriptions WHERE enabled=1")
-    for row in queryreturn:
-        yield row
-
-
-def get_addressbook():
-    """Generator for Addressbook"""
-    queryreturn = sqlQuery("SELECT * FROM addressbook")
-    for row in queryreturn:
-        yield row
-
-
-def get_addresslist(group="blacklist"):
-    """Generator for address list given by group arg"""
-    if group not in _groups:
-        return
-    queryreturn = sqlQuery("SELECT * FROM %s" % group)
-    for row in queryreturn:
-        yield row
-
-
-def get_label(address, group="addressbook"):
-    """
-    Get address label from address list given by group arg
-    (default is addressbook)
-    """
-    if group not in _groups:
-        return
-    queryreturn = sqlQuery(
-        "SELECT label FROM %s WHERE address=?" % group, address)
-    try:
-        return unicode(queryreturn[-1][0], 'utf-8')
-    except IndexError:
-        pass
-
-
-def set_label(address, label, group="addressbook"):
-    """Set address label in the address list given by group arg"""
-    if group not in _groups:
-        return
-    sqlExecute("UPDATE %s set label=? WHERE address=?" % group, label, address)
-
-
-def get_message(msgid):
-    """Get inbox message by msgid"""
-    queryreturn = sqlQuery("SELECT message FROM inbox WHERE msgid=?", msgid)
-    try:
-        return queryreturn[-1][0]
-    except IndexError:
-        return ''

src/helper_inbox.py

@@ -0,0 +1,25 @@
+"""Helper Inbox performs inbox messages related operations"""
+
+import queues
+from helper_sql import sqlExecute, sqlQuery
+
+
+def insert(t):
+    """Perform an insert into the "inbox" table"""
+    sqlExecute('''INSERT INTO inbox VALUES (?,?,?,?,?,?,?,?,?,?)''', *t)
+    # shouldn't emit changedInboxUnread and displayNewInboxMessage
+    # at the same time
+    # queues.UISignalQueue.put(('changedInboxUnread', None))
+
+
+def trash(msgid):
+    """Mark a message in the `inbox` as `trash`"""
+    sqlExecute('''UPDATE inbox SET folder='trash' WHERE msgid=?''', msgid)
+    queues.UISignalQueue.put(('removeInboxRowByMsgid', msgid))
+
+
+def isMessageAlreadyInInbox(sigHash):
+    """Check for previous instances of this message"""
+    queryReturn = sqlQuery(
+        '''SELECT COUNT(*) FROM inbox WHERE sighash=?''', sigHash)
+    return queryReturn[0][0] != 0

src/helper_sent.py

@@ -0,0 +1,48 @@
+"""
+Insert values into sent table
+"""
+
+import time
+import uuid
+from addresses import decodeAddress
+from bmconfigparser import BMConfigParser
+from helper_ackPayload import genAckPayload
+from helper_sql import sqlExecute
+
+
+# pylint: disable=too-many-arguments
+def insert(msgid=None, toAddress='[Broadcast subscribers]', fromAddress=None, subject=None,
+           message=None, status='msgqueued', ripe=None, ackdata=None, sentTime=None,
+           lastActionTime=None, sleeptill=0, retryNumber=0, encoding=2, ttl=None, folder='sent'):
+    """Perform an insert into the `sent` table"""
+    # pylint: disable=unused-variable
+    # pylint: disable-msg=too-many-locals
+
+    valid_addr = True
+    if not ripe or not ackdata:
+        addr = fromAddress if toAddress == '[Broadcast subscribers]' else toAddress
+        new_status, addressVersionNumber, streamNumber, new_ripe = decodeAddress(addr)
+        valid_addr = True if new_status == 'success' else False
+        if not ripe:
+            ripe = new_ripe
+
+        if not ackdata:
+            stealthLevel = BMConfigParser().safeGetInt(
+                'bitmessagesettings', 'ackstealthlevel')
+            new_ackdata = genAckPayload(streamNumber, stealthLevel)
+            ackdata = new_ackdata
+    if valid_addr:
+        msgid = msgid if msgid else uuid.uuid4().bytes
+        sentTime = sentTime if sentTime else int(time.time())  # sentTime (this doesn't change)
+        lastActionTime = lastActionTime if lastActionTime else int(time.time())
+
+        ttl = ttl if ttl else BMConfigParser().getint('bitmessagesettings', 'ttl')
+
+        t = (msgid, toAddress, ripe, fromAddress, subject, message, ackdata,
+             sentTime, lastActionTime, sleeptill, status, retryNumber, folder,
+             encoding, ttl)
+
+        sqlExecute('''INSERT INTO sent VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)''', *t)
+        return ackdata
+    else:
+        return None

src/network/announcethread.py

@@ -7,7 +7,6 @@ import state
 from bmconfigparser import BMConfigParser
 from network.assemble import assemble_addr
 from network.connectionpool import BMConnectionPool
-from network.udp import UDPSocket
 from node import Peer
 from threads import StoppableThread
 
@@ -15,12 +14,13 @@ from threads import StoppableThread
 class AnnounceThread(StoppableThread):
     """A thread to manage regular announcing of this node"""
     name = "Announcer"
+    announceInterval = 60
 
     def run(self):
         lastSelfAnnounced = 0
         while not self._stopped and state.shutdown == 0:
             processed = 0
-            if lastSelfAnnounced < time.time() - UDPSocket.announceInterval:
+            if lastSelfAnnounced < time.time() - self.announceInterval:
                 self.announceSelf()
                 lastSelfAnnounced = time.time()
             if processed == 0:

src/network/udp.py

@@ -8,6 +8,7 @@ import time
 import protocol
 import state
 from bmproto import BMProto
+from constants import MAX_TIME_OFFSET
 from node import Peer
 from objectracker import ObjectTracker
 from queues import receiveDataQueue
@@ -18,7 +19,6 @@ logger = logging.getLogger('default')
 class UDPSocket(BMProto):  # pylint: disable=too-many-instance-attributes
     """Bitmessage protocol over UDP (class)"""
     port = 8444
-    announceInterval = 60
 
     def __init__(self, host=None, sock=None, announcing=False):
         # pylint: disable=bad-super-call
@@ -82,8 +82,8 @@ class UDPSocket(BMProto):  # pylint: disable=too-many-instance-attributes
             decodedIP = protocol.checkIPAddress(str(ip))
             if stream not in state.streamsInWhichIAmParticipating:
                 continue
-            if (seenTime < time.time() - self.maxTimeOffset
-                    or seenTime > time.time() + self.maxTimeOffset):
+            if (seenTime < time.time() - MAX_TIME_OFFSET
+                    or seenTime > time.time() + MAX_TIME_OFFSET):
                 continue
             if decodedIP is False:
                 # if the address isn't local, interpret it as
@@ -94,9 +94,8 @@ class UDPSocket(BMProto):  # pylint: disable=too-many-instance-attributes
         logger.debug(
             "received peer discovery from %s:%i (port %i):",
             self.destination.host, self.destination.port, remoteport)
-        if self.local:
-            state.discoveredPeers[Peer(self.destination.host, remoteport)] = \
-                time.time()
+        state.discoveredPeers[Peer(self.destination.host, remoteport)] = \
+            time.time()
         return True
 
     def bm_command_portcheck(self):
@@ -125,9 +124,9 @@ class UDPSocket(BMProto):  # pylint: disable=too-many-instance-attributes
 
     def handle_read(self):
         try:
-            (recdata, addr) = self.socket.recvfrom(self._buf_len)
-        except socket.error as e:
-            logger.error("socket error: %s", e)
+            recdata, addr = self.socket.recvfrom(self._buf_len)
+        except socket.error:
+            logger.error("socket error on recvfrom:", exc_info=True)
             return
 
         self.destination = Peer(*addr)
@@ -143,7 +142,7 @@ class UDPSocket(BMProto):  # pylint: disable=too-many-instance-attributes
         try:
             retval = self.socket.sendto(
                 self.write_buf, ('<broadcast>', self.port))
-        except socket.error as e:
-            logger.error("socket error on sendto: %s", e)
+        except socket.error:
+            logger.error("socket error on sendto:", exc_info=True)
             retval = len(self.write_buf)
         self.slice_write_buf(retval)

src/protocol.py

@@ -14,7 +14,6 @@ from binascii import hexlify
 from struct import Struct, pack, unpack
 
 import defaults
-import helper_db
 import highlevelcrypto
 import state
 from addresses import (
@@ -22,6 +21,7 @@ from addresses import (
 from bmconfigparser import BMConfigParser
 from debug import logger
 from fallback import RIPEMD160Hash
+from helper_sql import sqlExecute
 from version import softwareVersion
 
 # Service flags
@@ -503,7 +503,8 @@ def decryptAndCheckPubkeyPayload(data, address):
             hexlify(publicSigningKey), hexlify(publicEncryptionKey)
         )
 
-        helper_db.put_pubkey(address, addressVersion, storedData, True)
+        t = (address, addressVersion, storedData, int(time.time()), 'yes')
+        sqlExecute('''INSERT INTO pubkeys VALUES (?,?,?,?,?)''', *t)
         return 'successful'
     except varintDecodeError:
         logger.info(

src/tests/core.py

@@ -11,6 +11,7 @@ import shutil
 import socket
 import string
 import sys
+import threading
 import time
 import unittest
 
@@ -61,6 +62,13 @@ class TestCore(unittest.TestCase):
     """Test case, which runs in main pybitmessage thread"""
     addr = 'BM-2cVvkzJuQDsQHLqxRXc6HZGPLZnkBLzEZY'
 
+    def tearDown(self):
+        """Reset possible unexpected settings after test"""
+        knownnodes.addKnownNode(1, Peer('127.0.0.1', 8444), is_self=True)
+        BMConfigParser().remove_option('bitmessagesettings', 'dontconnect')
+        BMConfigParser().remove_option('bitmessagesettings', 'onionservicesonly')
+        BMConfigParser().set('bitmessagesettings', 'socksproxytype', 'none')
+
     def test_msgcoding(self):
         """test encoding and decoding (originally from helper_msgcoding)"""
         msg_data = {
@@ -270,6 +278,36 @@ class TestCore(unittest.TestCase):
                     return
         self.fail('Failed to connect to at least 3 nodes within 360 sec')
 
+    def test_udp(self):
+        """check default udp setting and presence of Announcer thread"""
+        self.assertTrue(
+            BMConfigParser().safeGetBoolean('bitmessagesettings', 'udp'))
+        for thread in threading.enumerate():
+            if thread.name == 'Announcer':  # find Announcer thread
+                break
+        else:
+            return self.fail('No Announcer thread found')
+
+        for _ in range(20):  # wait for UDP socket
+            for sock in BMConnectionPool().udpSockets.values():
+                thread.announceSelf()
+                break
+            else:
+                time.sleep(1)
+                continue
+            break
+        else:
+            self.fail('UDP socket is not started')
+
+        for _ in range(20):
+            if state.discoveredPeers:
+                peer = state.discoveredPeers.keys()[0]
+                self.assertEqual(peer.port, 8444)
+                break
+            time.sleep(1)
+        else:
+            self.fail('No self in discovered peers')
+
     @staticmethod
     def _decode_msg(data, pattern):
         proto = BMProto()
@@ -366,10 +404,6 @@ def run():
         qt_tests = loader.loadTestsFromModule(bitmessageqt.tests)
         suite.addTests(qt_tests)
 
-    import sql
-    sql_tests = loader.loadTestsFromModule(sql)
-    suite.addTest(sql_tests)
-
     def keep_exc(ex_cls, exc, tb):  # pylint: disable=unused-argument
         """Own exception hook for test cases"""
         excQueue.put(('tests', exc))

src/tests/sql.py

@@ -1,33 +0,0 @@
-"""
-SQL related core tests
-"""
-
-import unittest
-
-import helper_db
-
-
-class TestSQL(unittest.TestCase):
-    """A test case for common SQL-related functions"""
-    test_address_1 = 'BM-2cVRo3WuKL5vDCKq2XJi1yR366nkHgHfcL'
-    test_address_2 = 'BM-2cVJn1BusL1j4UezuYkvpqNPCr7kCebeLw'
-
-    def test_subscriptions(self):
-        """Put address into subscriptions and check that it was added"""
-        self.assertTrue(
-            helper_db.put_subscriptions('pass_1', self.test_address_1))
-        self.assertFalse(
-            helper_db.put_subscriptions('pass_2', self.test_address_1))
-
-        helper_db.put_subscriptions('pass_3', self.test_address_2, False)
-
-        for label, addr in helper_db.get_subscriptions():
-            if addr == self.test_address_1:
-                self.assertEqual(label, 'pass_1')
-                break
-        else:
-            self.fail('Could not find added address in subscriptions')
-
-        for label, addr in helper_db.get_subscriptions():
-            if addr == self.test_address_2:
-                self.fail('Got disabled subscription')

src/tests/test_config.py

@@ -2,11 +2,7 @@
 Various tests for config
 """
 
-import os
 import unittest
-import tempfile
-
-from .test_process import TestProcessProto
 from pybitmessage.bmconfigparser import BMConfigParser
 
 
@@ -38,32 +34,3 @@ class TestConfig(unittest.TestCase):
             BMConfigParser().safeGetInt('nonexistent', 'nonexistent'), 0)
         self.assertEqual(
             BMConfigParser().safeGetInt('nonexistent', 'nonexistent', 42), 42)
-
-
-class TestProcessConfig(TestProcessProto):
-    """A test case for keys.dat"""
-    home = tempfile.mkdtemp()
-
-    def test_config_defaults(self):
-        """Test settings in the generated config"""
-        self._stop_process()
-        self._kill_process()
-        config = BMConfigParser()
-        config.read(os.path.join(self.home, 'keys.dat'))
-
-        self.assertEqual(config.safeGetInt(
-            'bitmessagesettings', 'settingsversion'), 10)
-        self.assertEqual(config.safeGetInt(
-            'bitmessagesettings', 'port'), 8444)
-        # don't connect
-        self.assertTrue(config.safeGetBoolean(
-            'bitmessagesettings', 'dontconnect'))
-        # API disabled
-        self.assertFalse(config.safeGetBoolean(
-            'bitmessagesettings', 'apienabled'))
-
-        # extralowdifficulty is false
-        self.assertEqual(config.safeGetInt(
-            'bitmessagesettings', 'defaultnoncetrialsperbyte'), 1000)
-        self.assertEqual(config.safeGetInt(
-            'bitmessagesettings', 'defaultpayloadlengthextrabytes'), 1000)

src/tests/test_config_process.py

@@ -0,0 +1,38 @@
+"""
+Various tests for config
+"""
+
+import os
+import tempfile
+from pybitmessage.bmconfigparser import BMConfigParser
+from .test_process import TestProcessProto
+
+
+class TestProcessConfig(TestProcessProto):
+    """A test case for keys.dat"""
+    home = tempfile.mkdtemp()
+
+
+    def test_config_defaults(self):
+        """Test settings in the generated config"""
+        config = BMConfigParser()
+        self._stop_process()
+        self._kill_process()
+        config.read(os.path.join(self.home, 'keys.dat'))
+
+        self.assertEqual(config.safeGetInt(
+            'bitmessagesettings', 'settingsversion'), 10)
+        self.assertEqual(config.safeGetInt(
+            'bitmessagesettings', 'port'), 8444)
+        # don't connect
+        self.assertTrue(config.safeGetBoolean(
+            'bitmessagesettings', 'dontconnect'))
+        # API disabled
+        self.assertFalse(config.safeGetBoolean(
+            'bitmessagesettings', 'apienabled'))
+
+        # extralowdifficulty is false
+        self.assertEqual(config.safeGetInt(
+            'bitmessagesettings', 'defaultnoncetrialsperbyte'), 1000)
+        self.assertEqual(config.safeGetInt(
+            'bitmessagesettings', 'defaultpayloadlengthextrabytes'), 1000)