Test helper_db

src/api.py

@@ -73,8 +73,7 @@ from SimpleXMLRPCServer import SimpleXMLRPCRequestHandler, SimpleXMLRPCServer
 from struct import pack
 
 import defaults
-import helper_inbox
-import helper_sent
+import helper_db
 import network.stats
 import proofofwork
 import queues
@@ -587,14 +586,9 @@ class BMRPCDispatcher(object):
         label = self._decode(label, "base64")
         address = addBMIfNotPresent(address)
         self._verifyAddress(address)
-        # TODO: add unique together constraint in the table
-        queryreturn = sqlQuery(
-            "SELECT address FROM addressbook WHERE address=?", address)
-        if queryreturn != []:
+        if not helper_db.put_addressbook(label, address):
             raise APIError(
                 16, 'You already have this address in your address book.')
-
-        sqlExecute("INSERT INTO addressbook VALUES(?,?)", label, address)
         queues.UISignalQueue.put(('rerenderMessagelistFromLabels', ''))
         queues.UISignalQueue.put(('rerenderMessagelistToLabels', ''))
         queues.UISignalQueue.put(('rerenderAddressBook', ''))
@@ -1056,23 +1050,23 @@ class BMRPCDispatcher(object):
         """
         msgid = self._decode(msgid, "hex")
         # Trash if in inbox table
-        helper_inbox.trash(msgid)
+        helper_db.put_trash(msgid)
         # Trash if in sent table
-        sqlExecute("UPDATE sent SET folder='trash' WHERE msgid=?", msgid)
+        helper_db.put_trash(msgid, sent=True)
         return 'Trashed message (assuming message existed).'
 
     @command('trashInboxMessage')
     def HandleTrashInboxMessage(self, msgid):
         """Trash inbox message by msgid (encoded in hex)."""
         msgid = self._decode(msgid, "hex")
-        helper_inbox.trash(msgid)
+        helper_db.put_trash(msgid)
         return 'Trashed inbox message (assuming message existed).'
 
     @command('trashSentMessage')
     def HandleTrashSentMessage(self, msgid):
         """Trash sent message by msgid (encoded in hex)."""
         msgid = self._decode(msgid, "hex")
-        sqlExecute('''UPDATE sent SET folder='trash' WHERE msgid=?''', msgid)
+        helper_db.put_trash(msgid, sent=True)
         return 'Trashed sent message (assuming message existed).'
 
     @command('sendMessage')
@@ -1110,18 +1104,15 @@ class BMRPCDispatcher(object):
         if not fromAddressEnabled:
             raise APIError(14, 'Your fromAddress is disabled. Cannot send.')
 
-        ackdata = helper_sent.insert(
-            toAddress=toAddress, fromAddress=fromAddress,
-            subject=subject, message=message, encoding=encodingType, ttl=TTL)
+        stealthLevel = BMConfigParser().safeGetInt(
+            'bitmessagesettings', 'ackstealthlevel')
+        ackdata = genAckPayload(streamNumber, stealthLevel)
 
-        toLabel = ''
-        queryreturn = sqlQuery(
-            "SELECT label FROM addressbook WHERE address=?", toAddress)
-        try:
-            toLabel, = queryreturn[0][0]
-        except IndexError:
-            pass
+        helper_db.put_sent(
+            toAddress, fromAddress, subject, message, ackdata,
+            'msgqueued', encodingType, ttl=TTL)
 
+        toLabel = helper_db.get_label(toAddress) or ''
         queues.UISignalQueue.put(('displayNewSentMessage', (
             toAddress, toLabel, fromAddress, subject, message, ackdata)))
         queues.workerQueue.put(('sendmessage', toAddress))
@@ -1151,15 +1142,15 @@ class BMRPCDispatcher(object):
             self.config.getboolean(fromAddress, 'enabled')
         except BaseException:
             raise APIError(
-                13, 'Could not find your fromAddress in the keys.dat file.')
-        toAddress = str_broadcast_subscribers
+                13, 'could not find your fromAddress in the keys.dat file.')
+        streamNumber = decodeAddress(fromAddress)[2]
+        ackdata = genAckPayload(streamNumber, 0)
+        toAddress = toLabel = str_broadcast_subscribers
 
-        ackdata = helper_sent.insert(
-            fromAddress=fromAddress, subject=subject,
-            message=message, status='broadcastqueued',
-            encoding=encodingType)
+        helper_db.put_sent(
+            toAddress, fromAddress, subject, message, ackdata,
+            'broadcastqueued', encodingType, ttl=TTL)
 
-        toLabel = str_broadcast_subscribers
         queues.UISignalQueue.put(('displayNewSentMessage', (
             toAddress, toLabel, fromAddress, subject, message, ackdata)))
         queues.workerQueue.put(('sendbroadcast', ''))
@@ -1197,15 +1188,8 @@ class BMRPCDispatcher(object):
             except UnicodeDecodeError:
                 raise APIError(17, 'Label is not valid UTF-8 data.')
         self._verifyAddress(address)
-        address = addBMIfNotPresent(address)
-        # First we must check to see if the address is already in the
-        # subscriptions list.
-        queryreturn = sqlQuery(
-            "SELECT * FROM subscriptions WHERE address=?", address)
-        if queryreturn:
+        if not helper_db.put_subscriptions(label, address):
             raise APIError(16, 'You are already subscribed to that address.')
-        sqlExecute(
-            "INSERT INTO subscriptions VALUES (?,?,?)", label, address, True)
         shared.reloadBroadcastSendersForWhichImWatching()
         queues.UISignalQueue.put(('rerenderMessagelistFromLabels', ''))
         queues.UISignalQueue.put(('rerenderSubscriptions', ''))

src/bitmessageqt/__init__.py

@@ -35,9 +35,10 @@ from foldertree import (
     MessageList_TimeWidget)
 import settingsmixin
 import support
-from helper_sql import sqlQuery, sqlExecute, sqlExecuteChunked, sqlStoredProcedure
-import helper_addressbook
-import helper_search
+from helper_ackPayload import genAckPayload
+from helper_sql import (
+    sqlQuery, sqlExecute, sqlExecuteChunked, sqlStoredProcedure)
+import helper_db
 import l10n
 from utils import str_broadcast_subscribers, avatarize
 from account import (
@@ -54,7 +55,6 @@ from statusbar import BMStatusBar
 import sound
 # This is needed for tray icon
 import bitmessage_icons_rc  # noqa:F401 pylint: disable=unused-import
-import helper_sent
 
 try:
     from plugins.plugin import get_plugin, get_plugins
@@ -1214,7 +1214,7 @@ class MyForm(settingsmixin.SMainWindow):
             tableWidget.setColumnHidden(1, bool(account))
             xAddress = 'fromaddress'
 
-        queryreturn = helper_search.search_sql(
+        queryreturn = helper_db.search_sql(
             xAddress, account, "sent", where, what, False)
 
         for row in queryreturn:
@@ -1254,7 +1254,7 @@ class MyForm(settingsmixin.SMainWindow):
             tableWidget.setColumnHidden(0, False)
             tableWidget.setColumnHidden(1, False)
 
-        queryreturn = helper_search.search_sql(
+        queryreturn = helper_db.search_sql(
             xAddress, account, folder, where, what, unreadOnly)
 
         for row in queryreturn:
@@ -1937,8 +1937,7 @@ class MyForm(settingsmixin.SMainWindow):
 
         newRows = {}
         # subscriptions
-        queryreturn = sqlQuery('SELECT label, address FROM subscriptions WHERE enabled = 1')
-        for row in queryreturn:
+        for row in helper_db.get_subscriptions():
             label, address = row
             newRows[address] = [label, AccountMixin.SUBSCRIPTION]
         # chans
@@ -1948,8 +1947,7 @@ class MyForm(settingsmixin.SMainWindow):
             if (account.type == AccountMixin.CHAN and BMConfigParser().safeGetBoolean(address, 'enabled')):
                 newRows[address] = [account.getLabel(), AccountMixin.CHAN]
         # normal accounts
-        queryreturn = sqlQuery('SELECT * FROM addressbook')
-        for row in queryreturn:
+        for row in helper_db.get_addressbook():
             label, address = row
             newRows[address] = [label, AccountMixin.NORMAL]
 
@@ -2166,18 +2164,19 @@ class MyForm(settingsmixin.SMainWindow):
                                 " send the message but it won\'t send until"
                                 " you connect.")
                             )
-                        ackdata = helper_sent.insert(
-                            toAddress=toAddress, fromAddress=fromAddress,
-                            subject=subject, message=message, encoding=encoding)
-                        toLabel = ''
-                        queryreturn = sqlQuery('''select label from addressbook where address=?''',
-                                               toAddress)
-                        if queryreturn != []:
-                            for row in queryreturn:
-                                toLabel, = row
+                        stealthLevel = BMConfigParser().safeGetInt(
+                            'bitmessagesettings', 'ackstealthlevel')
+                        ackdata = genAckPayload(streamNumber, stealthLevel)
+                        helper_db.put_sent(
+                            toAddress, fromAddress, subject, message, ackdata,
+                            'msgqueued', encoding, ripe
+                        )
+
+                        toLabel = helper_db.get_label(toAddress) or ''
 
                         self.displayNewSentMessage(
-                            toAddress, toLabel, fromAddress, subject, message, ackdata)
+                            toAddress, toLabel, fromAddress, subject,
+                            message, ackdata)
                         queues.workerQueue.put(('sendmessage', toAddress))
 
                         self.click_pushButtonClear()
@@ -2202,15 +2201,14 @@ class MyForm(settingsmixin.SMainWindow):
                 # We don't actually need the ackdata for acknowledgement since
                 # this is a broadcast message, but we can use it to update the
                 # user interface when the POW is done generating.
-                toAddress = str_broadcast_subscribers
+                streamNumber = decodeAddress(fromAddress)[2]
+                ackdata = genAckPayload(streamNumber, 0)
+                toAddress = toLabel = str_broadcast_subscribers
 
-                # msgid. We don't know what this will be until the POW is done.
-                ackdata = helper_sent.insert(
-                    fromAddress=fromAddress,
-                    subject=subject, message=message,
-                    status='broadcastqueued', encoding=encoding)
-
-                toLabel = str_broadcast_subscribers
+                helper_db.put_sent(
+                    toAddress, fromAddress, subject, message, ackdata,
+                    'broadcastqueued', encoding
+                )
 
                 self.displayNewSentMessage(
                     toAddress, toLabel, fromAddress, subject, message, ackdata)
@@ -2338,7 +2336,7 @@ class MyForm(settingsmixin.SMainWindow):
                 self.ui.treeWidgetChans
             ) and self.getCurrentAccount(treeWidget) != toAddress:
                 continue
-            elif not helper_search.check_match(
+            elif not helper_db.check_match(
                 toAddress, fromAddress, subject, message,
                 self.getCurrentSearchOption(tab),
                 self.getCurrentSearchLine(tab)
@@ -2368,7 +2366,7 @@ class MyForm(settingsmixin.SMainWindow):
             tab += 1
             if tab == 1:
                 tab = 2
-            if not helper_search.check_match(
+            if not helper_db.check_match(
                 toAddress, fromAddress, subject, message,
                 self.getCurrentSearchOption(tab),
                 self.getCurrentSearchLine(tab)
@@ -2427,10 +2425,7 @@ class MyForm(settingsmixin.SMainWindow):
         except AttributeError:
             return
 
-        # First we must check to see if the address is already in the
-        # address book. The user cannot add it again or else it will
-        # cause problems when updating and deleting the entry.
-        if shared.isAddressInMyAddressBook(address):
+        if not helper_db.put_addressbook(label, address):
             self.updateStatusBar(_translate(
                 "MainWindow",
                 "Error: You cannot add the same address to your"
@@ -2439,30 +2434,25 @@ class MyForm(settingsmixin.SMainWindow):
             ))
             return
 
-        if helper_addressbook.insert(label=label, address=address):
-            self.rerenderMessagelistFromLabels()
-            self.rerenderMessagelistToLabels()
-            self.rerenderAddressBook()
-        else:
+        self.rerenderMessagelistFromLabels()
+        self.rerenderMessagelistToLabels()
+        self.rerenderAddressBook()
+
+    def addSubscription(self, address, label):
+        if not helper_db.put_subscriptions(label, address):
             self.updateStatusBar(_translate(
                 "MainWindow",
-                "Error: You cannot add your own address in the address book."
+                "Error: You cannot add the same address to your"
+                " subscriptions twice. Perhaps rename the existing one"
+                " if you want."
             ))
-
-    def addSubscription(self, address, label):
-        # This should be handled outside of this function, for error displaying
-        # and such, but it must also be checked here.
-        if shared.isAddressInMySubscriptionsList(address):
             return
-        # Add to database (perhaps this should be separated from the MyForm class)
-        sqlExecute(
-            '''INSERT INTO subscriptions VALUES (?,?,?)''',
-            label, address, True
-        )
+
         self.rerenderMessagelistFromLabels()
         shared.reloadBroadcastSendersForWhichImWatching()
         self.rerenderAddressBook()
         self.rerenderTabTreeSubscriptions()
+        return True
 
     def click_pushButtonAddSubscription(self):
         dialog = dialogs.NewSubscriptionDialog(self)
@@ -2472,19 +2462,9 @@ class MyForm(settingsmixin.SMainWindow):
         except AttributeError:
             return
 
-        # We must check to see if the address is already in the
-        # subscriptions list. The user cannot add it again or else it
-        # will cause problems when updating and deleting the entry.
-        if shared.isAddressInMySubscriptionsList(address):
-            self.updateStatusBar(_translate(
-                "MainWindow",
-                "Error: You cannot add the same address to your"
-                " subscriptions twice. Perhaps rename the existing one"
-                " if you want."
-            ))
+        if not self.addSubscription(address, label):
             return
 
-        self.addSubscription(address, label)
         # Now, if the user wants to display old broadcasts, let's get
         # them out of the inventory and put them
         # to the objectProcessorQueue to be processed
@@ -2815,11 +2795,7 @@ class MyForm(settingsmixin.SMainWindow):
         textEdit = self.getCurrentMessageTextedit()
         if not msgid:
             return
-        queryreturn = sqlQuery(
-            '''select message from inbox where msgid=?''', msgid)
-        if queryreturn != []:
-            for row in queryreturn:
-                messageText, = row
+        messageText = helper_db.get_message(msgid)
 
         lines = messageText.split('\n')
         totalLines = len(lines)
@@ -3052,24 +3028,20 @@ class MyForm(settingsmixin.SMainWindow):
             currentInboxRow, 1).data(QtCore.Qt.UserRole)
         recipientAddress = tableWidget.item(
             currentInboxRow, 0).data(QtCore.Qt.UserRole)
-        # Let's make sure that it isn't already in the address book
-        queryreturn = sqlQuery('''select * from blacklist where address=?''',
-                               addressAtCurrentInboxRow)
-        if queryreturn == []:
-            label = "\"" + tableWidget.item(currentInboxRow, 2).subject + "\" in " + BMConfigParser().get(recipientAddress, "label")
-            sqlExecute('''INSERT INTO blacklist VALUES (?,?, ?)''',
-                       label,
-                       addressAtCurrentInboxRow, True)
-            self.ui.blackwhitelist.rerenderBlackWhiteList()
-            self.updateStatusBar(_translate(
-                "MainWindow",
-                "Entry added to the blacklist. Edit the label to your liking.")
-            )
-        else:
+        label = "\"" + tableWidget.item(currentInboxRow, 2).subject + \
+            "\" in " + BMConfigParser().get(recipientAddress, "label")
+        if not helper_db.put_blacklist(label, addressAtCurrentInboxRow):
             self.updateStatusBar(_translate(
                 "MainWindow",
                 "Error: You cannot add the same address to your blacklist"
                 " twice. Try renaming the existing one if you want."))
+            return
+
+        self.ui.blackwhitelist.rerenderBlackWhiteList()
+        self.updateStatusBar(_translate(
+            "MainWindow",
+            "Entry added to the blacklist. Edit the label to your liking.")
+        )
 
     def deleteRowFromMessagelist(
         self, row=None, inventoryHash=None, ackData=None, messageLists=None
@@ -3168,11 +3140,7 @@ class MyForm(settingsmixin.SMainWindow):
 
         # Retrieve the message data out of the SQL database
         msgid = tableWidget.item(currentInboxRow, 3).data()
-        queryreturn = sqlQuery(
-            '''select message from inbox where msgid=?''', msgid)
-        if queryreturn != []:
-            for row in queryreturn:
-                message, = row
+        message = helper_db.get_message(msgid)
 
         defaultFilename = "".join(x for x in subjectAtCurrentInboxRow if x.isalnum()) + '.txt'
         filename = QtGui.QFileDialog.getSaveFileName(self, _translate("MainWindow","Save As..."), defaultFilename, "Text files (*.txt);;All files (*.*)")
@@ -3280,15 +3248,6 @@ class MyForm(settingsmixin.SMainWindow):
 
     def on_action_AddressBookSubscribe(self):
         for item in self.getAddressbookSelectedItems():
-            # Then subscribe to it...
-            # provided it's not already in the address book
-            if shared.isAddressInMySubscriptionsList(item.address):
-                self.updateStatusBar(_translate(
-                    "MainWindow",
-                    "Error: You cannot add the same address to your"
-                    " subscriptions twice. Perhaps rename the existing"
-                    " one if you want."))
-                continue
             self.addSubscription(item.address, item.label)
             self.ui.tabWidget.setCurrentIndex(
                 self.ui.tabWidget.indexOf(self.ui.subscriptions)

src/bitmessageqt/account.py

@@ -4,43 +4,30 @@ account.py
 ==========
 
 Account related functions.
-
 """
 
-from __future__ import absolute_import
-
 import inspect
 import re
 import sys
-import time
-
-from PyQt4 import QtGui
 
+import helper_db
 import queues
 from addresses import decodeAddress
 from bmconfigparser import BMConfigParser
 from helper_ackPayload import genAckPayload
-from helper_sql import sqlQuery, sqlExecute
-from .foldertree import AccountMixin
-from .utils import str_broadcast_subscribers
+from helper_sql import sqlQuery
+from foldertree import AccountMixin
+from utils import str_broadcast_subscribers
+from tr import _translate
 
 
 def getSortedAccounts():
     """Get a sorted list of configSections"""
-
     configSections = BMConfigParser().addresses()
     configSections.sort(
         cmp=lambda x, y: cmp(
-            unicode(
-                BMConfigParser().get(
-                    x,
-                    'label'),
-                'utf-8').lower(),
-            unicode(
-                BMConfigParser().get(
-                    y,
-                    'label'),
-                'utf-8').lower()))
+            unicode(BMConfigParser().get(x, 'label'), 'utf-8').lower(),
+            unicode(BMConfigParser().get(y, 'label'), 'utf-8').lower()))
     return configSections
 
 
@@ -107,29 +94,27 @@ def accountClass(address):
 
 class AccountColor(AccountMixin):  # pylint: disable=too-few-public-methods
     """Set the type of account"""
-
     def __init__(self, address, address_type=None):
         self.isEnabled = True
         self.address = address
-        if address_type is None:
-            if address is None:
-                self.type = AccountMixin.ALL
-            elif BMConfigParser().safeGetBoolean(self.address, 'mailinglist'):
-                self.type = AccountMixin.MAILINGLIST
-            elif BMConfigParser().safeGetBoolean(self.address, 'chan'):
-                self.type = AccountMixin.CHAN
-            elif sqlQuery(
-                    '''select label from subscriptions where address=?''', self.address):
-                self.type = AccountMixin.SUBSCRIPTION
-            else:
-                self.type = AccountMixin.NORMAL
-        else:
+        if address_type:
             self.type = address_type
+            return
+        # AccountMixin.setType()
+        if self.address is None:
+            self.type = self.ALL
+        elif BMConfigParser().safeGetBoolean(self.address, 'chan'):
+            self.type = self.CHAN
+        elif BMConfigParser().safeGetBoolean(self.address, 'mailinglist'):
+            self.type = self.MAILINGLIST
+        elif helper_db.get_label(self.address, "subscriptions"):
+            self.type = AccountMixin.SUBSCRIPTION
+        else:
+            self.type = self.NORMAL
 
 
 class BMAccount(object):
     """Encapsulate a Bitmessage account"""
-
     def __init__(self, address=None):
         self.address = address
         self.type = AccountMixin.NORMAL
@@ -140,29 +125,19 @@ class BMAccount(object):
                 self.type = AccountMixin.MAILINGLIST
         elif self.address == str_broadcast_subscribers:
             self.type = AccountMixin.BROADCAST
-        else:
-            queryreturn = sqlQuery(
-                '''select label from subscriptions where address=?''', self.address)
-            if queryreturn:
-                self.type = AccountMixin.SUBSCRIPTION
+        elif helper_db.get_label(self.address, "subscriptions"):
+            self.type = AccountMixin.SUBSCRIPTION
 
     def getLabel(self, address=None):
         """Get a label for this bitmessage account"""
         if address is None:
             address = self.address
-        label = BMConfigParser().safeGet(address, 'label', address)
-        queryreturn = sqlQuery(
-            '''select label from addressbook where address=?''', address)
-        if queryreturn != []:
-            for row in queryreturn:
-                label, = row
-        else:
-            queryreturn = sqlQuery(
-                '''select label from subscriptions where address=?''', address)
-            if queryreturn != []:
-                for row in queryreturn:
-                    label, = row
-        return label
+        if BMConfigParser().has_section(address):
+            return BMConfigParser().get(address, 'label')
+        return (
+            helper_db.get_label(address) or
+            helper_db.get_label(address, "subscriptions") or address
+        )
 
     def parseMessage(self, toAddress, fromAddress, subject, message):
         """Set metadata and address labels on self"""
@@ -186,9 +161,7 @@ class NoAccount(BMAccount):
         self.type = AccountMixin.NORMAL
 
     def getLabel(self, address=None):
-        if address is None:
-            address = self.address
-        return address
+        return address or self.address
 
 
 class SubscriptionAccount(BMAccount):
@@ -213,31 +186,17 @@ class GatewayAccount(BMAccount):
 
     def send(self):
         """Override the send method for gateway accounts"""
-
-        # pylint: disable=unused-variable
-        status, addressVersionNumber, streamNumber, ripe = decodeAddress(self.toAddress)
-        stealthLevel = BMConfigParser().safeGetInt('bitmessagesettings', 'ackstealthlevel')
+        streamNumber, ripe = decodeAddress(self.toAddress)[2:]
+        stealthLevel = BMConfigParser().safeGetInt(
+            'bitmessagesettings', 'ackstealthlevel')
         ackdata = genAckPayload(streamNumber, stealthLevel)
-        sqlExecute(
-            '''INSERT INTO sent VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)''',
-            '',
-            self.toAddress,
-            ripe,
-            self.fromAddress,
-            self.subject,
-            self.message,
-            ackdata,
-            int(time.time()),  # sentTime (this will never change)
-            int(time.time()),  # lastActionTime
-            0,  # sleepTill time. This will get set when the POW gets done.
-            'msgqueued',
-            0,  # retryNumber
-            'sent',  # folder
-            2,  # encodingtype
-            # not necessary to have a TTL higher than 2 days
-            min(BMConfigParser().getint('bitmessagesettings', 'ttl'), 86400 * 2)
+        helper_db.put_sent(
+            self.toAddress, self.fromAddress, self.subject, self.message,
+            ackdata, 'msgqueued', 2, ripe,
+            min(
+                BMConfigParser().getint('bitmessagesettings', 'ttl'),
+                86400 * 2)  # not necessary to have a TTL higher than 2 days
         )
-
         queues.workerQueue.put(('sendmessage', self.toAddress))
 
 
@@ -292,9 +251,7 @@ class MailchuckAccount(GatewayAccount):
 
         self.toAddress = self.registrationAddress
         self.subject = "config"
-        self.message = QtGui.QApplication.translate(
-            "Mailchuck",
-            """# You can use this to configure your email gateway account
+        self.message = _translate("Mailchuck", """# You can use this to configure your email gateway account
 # Uncomment the setting you want to use
 # Here are the options:
 #

src/bitmessageqt/blacklist.py

@@ -1,10 +1,11 @@
 from PyQt4 import QtCore, QtGui
 
+import helper_db
 import widgets
 from addresses import addBMIfNotPresent
 from bmconfigparser import BMConfigParser
 from dialogs import AddAddressDialog
-from helper_sql import sqlExecute, sqlQuery
+from helper_sql import sqlExecute
 from queues import UISignalQueue
 from retranslateui import RetranslateMixin
 from tr import _translate
@@ -61,16 +62,14 @@ class Blacklist(QtGui.QWidget, RetranslateMixin):
                     _translate("MainWindow", "Address is valid."):
                 address = addBMIfNotPresent(str(
                     self.NewBlacklistDialogInstance.lineEditAddress.text()))
-                # First we must check to see if the address is already in the
-                # address book. The user cannot add it again or else it will
-                # cause problems when updating and deleting the entry.
-                t = (address,)
-                if BMConfigParser().get('bitmessagesettings', 'blackwhitelist') == 'black':
-                    sql = '''select * from blacklist where address=?'''
-                else:
-                    sql = '''select * from whitelist where address=?'''
-                queryreturn = sqlQuery(sql,*t)
-                if queryreturn == []:
+                label = str(self.NewBlacklistDialogInstance.lineEditLabel.text().toUtf8())
+                if helper_db.put_addresslist(
+                    label, address,
+                    group='blacklist'
+                    if BMConfigParser().get(
+                        'bitmessagesettings', 'blackwhitelist') == 'black'
+                    else 'whitelist'
+                ):
                     self.tableWidgetBlacklist.setSortingEnabled(False)
                     self.tableWidgetBlacklist.insertRow(0)
                     newItem = QtGui.QTableWidgetItem(unicode(
@@ -82,12 +81,6 @@ class Blacklist(QtGui.QWidget, RetranslateMixin):
                         QtCore.Qt.ItemIsSelectable | QtCore.Qt.ItemIsEnabled)
                     self.tableWidgetBlacklist.setItem(0, 1, newItem)
                     self.tableWidgetBlacklist.setSortingEnabled(True)
-                    t = (str(self.NewBlacklistDialogInstance.lineEditLabel.text().toUtf8()), address, True)
-                    if BMConfigParser().get('bitmessagesettings', 'blackwhitelist') == 'black':
-                        sql = '''INSERT INTO blacklist VALUES (?,?,?)'''
-                    else:
-                        sql = '''INSERT INTO whitelist VALUES (?,?,?)'''
-                    sqlExecute(sql, *t)
                 else:
                     UISignalQueue.put((
                         'updateStatusBar',
@@ -158,19 +151,18 @@ class Blacklist(QtGui.QWidget, RetranslateMixin):
 
     def rerenderBlackWhiteList(self):
         tabs = self.parent().parent()
-        if BMConfigParser().get('bitmessagesettings', 'blackwhitelist') == 'black':
-            tabs.setTabText(tabs.indexOf(self), _translate('blacklist', 'Blacklist'))
-        else:
-            tabs.setTabText(tabs.indexOf(self), _translate('blacklist', 'Whitelist'))
+        list_type = BMConfigParser().get(
+            'bitmessagesettings', 'blackwhitelist')
+        tabs.setTabText(
+            tabs.indexOf(self),
+            _translate('blacklist', 'Blacklist') if list_type == 'black'
+            else _translate('blacklist', 'Whitelist'))
+
         self.tableWidgetBlacklist.setRowCount(0)
-        listType = BMConfigParser().get('bitmessagesettings', 'blackwhitelist')
-        if listType == 'black':
-            queryreturn = sqlQuery('''SELECT label, address, enabled FROM blacklist''')
-        else:
-            queryreturn = sqlQuery('''SELECT label, address, enabled FROM whitelist''')
         self.tableWidgetBlacklist.setSortingEnabled(False)
-        for row in queryreturn:
-            label, address, enabled = row
+        for label, address, enabled in helper_db.get_addresslist(
+            group='blacklist' if list_type == 'black' else 'whiteslist'
+        ):
             self.tableWidgetBlacklist.insertRow(0)
             newItem = QtGui.QTableWidgetItem(unicode(label, 'utf-8'))
             if not enabled:

src/bitmessageqt/foldertree.py

@@ -8,8 +8,8 @@ from cgi import escape
 
 from PyQt4 import QtCore, QtGui
 
+import helper_db
 from bmconfigparser import BMConfigParser
-from helper_sql import sqlExecute, sqlQuery
 from settingsmixin import SettingsMixin
 from tr import _translate
 from utils import avatarize
@@ -110,15 +110,13 @@ class AccountMixin(object):
             self.type = self.CHAN
         elif BMConfigParser().safeGetBoolean(self.address, 'mailinglist'):
             self.type = self.MAILINGLIST
-        elif sqlQuery(
-                '''select label from subscriptions where address=?''', self.address):
+        elif helper_db.get_label(self.address, "subscriptions"):
             self.type = AccountMixin.SUBSCRIPTION
         else:
             self.type = self.NORMAL
 
     def defaultLabel(self):
         """Default label (in case no label is set manually)"""
-        queryreturn = None
         retval = None
         if self.type in (
                 AccountMixin.NORMAL,
@@ -127,16 +125,9 @@ class AccountMixin(object):
                 retval = unicode(
                     BMConfigParser().get(self.address, 'label'), 'utf-8')
             except Exception:
-                queryreturn = sqlQuery(
-                    '''select label from addressbook where address=?''', self.address)
+                retval = helper_db.get_label(self.address)
         elif self.type == AccountMixin.SUBSCRIPTION:
-            queryreturn = sqlQuery(
-                '''select label from subscriptions where address=?''', self.address)
-        if queryreturn is not None:
-            if queryreturn != []:
-                for row in queryreturn:
-                    retval, = row
-                    retval = unicode(retval, 'utf-8')
+            retval = helper_db.get_label(self.address, "subscriptions")
         elif self.address is None or self.type == AccountMixin.ALL:
             return unicode(
                 str(_translate("MainWindow", "All accounts")), 'utf-8')
@@ -306,13 +297,8 @@ class Ui_SubscriptionWidget(Ui_AddressWidget):
             parent, pos, address, unreadCount, enabled)
 
     def _getLabel(self):
-        queryreturn = sqlQuery(
-            '''select label from subscriptions where address=?''', self.address)
-        if queryreturn != []:
-            for row in queryreturn:
-                retval, = row
-            return unicode(retval, 'utf-8', 'ignore')
-        return unicode(self.address, 'utf-8')
+        return helper_db.get_label(self.address, "subscriptions") \
+            or self.address
 
     def setType(self):
         """Set account type"""
@@ -327,9 +313,7 @@ class Ui_SubscriptionWidget(Ui_AddressWidget):
                     value.toString().toUtf8()).decode('utf-8', 'ignore')
             else:
                 label = unicode(value, 'utf-8', 'ignore')
-            sqlExecute(
-                '''UPDATE subscriptions SET label=? WHERE address=?''',
-                label, self.address)
+            helper_db.set_label(self.address, label, "subscriptions")
         return super(Ui_SubscriptionWidget, self).setData(column, role, value)
 
 
@@ -402,7 +386,6 @@ class MessageList_AddressWidget(BMAddressWidget):
         if label is not None:
             return
         newLabel = self.address
-        queryreturn = None
         if self.type in (
                 AccountMixin.NORMAL,
                 AccountMixin.CHAN, AccountMixin.MAILINGLIST):
@@ -411,14 +394,9 @@ class MessageList_AddressWidget(BMAddressWidget):
                     BMConfigParser().get(self.address, 'label'),
                     'utf-8', 'ignore')
             except:
-                queryreturn = sqlQuery(
-                    '''select label from addressbook where address=?''', self.address)
+                newLabel = helper_db.get_label(self.address)
         elif self.type == AccountMixin.SUBSCRIPTION:
-            queryreturn = sqlQuery(
-                '''select label from subscriptions where address=?''', self.address)
-        if queryreturn:
-            for row in queryreturn:
-                newLabel = unicode(row[0], 'utf-8', 'ignore')
+            newLabel = helper_db.get_label(self.address, "subscriptions")
 
         self.label = newLabel
 
@@ -525,9 +503,9 @@ class Ui_AddressBookWidgetItem(BMAddressWidget):
                     BMConfigParser().set(self.address, 'label', self.label)
                     BMConfigParser().save()
                 except:
-                    sqlExecute('''UPDATE addressbook set label=? WHERE address=?''', self.label, self.address)
+                    helper_db.set_label(self.address, self.label)
             elif self.type == AccountMixin.SUBSCRIPTION:
-                sqlExecute('''UPDATE subscriptions set label=? WHERE address=?''', self.label, self.address)
+                helper_db.set_label(self.address, self.label, "subscriptions")
             else:
                 pass
         return super(Ui_AddressBookWidgetItem, self).setData(role, value)

src/bitmessageqt/support.py

@@ -10,6 +10,7 @@ from PyQt4 import QtCore
 
 import account
 import defaults
+import helper_db
 import network.stats
 import paths
 import proofofwork
@@ -17,7 +18,7 @@ import queues
 import state
 from bmconfigparser import BMConfigParser
 from foldertree import AccountMixin
-from helper_sql import sqlExecute, sqlQuery
+from helper_sql import sqlExecute
 from l10n import getTranslationLanguage
 from openclpow import openclEnabled
 from pyelliptic.openssl import OpenSSL
@@ -67,12 +68,8 @@ Connected hosts: {}
 
 
 def checkAddressBook(myapp):
-    sqlExecute('DELETE from addressbook WHERE address=?', OLD_SUPPORT_ADDRESS)
-    queryreturn = sqlQuery('SELECT * FROM addressbook WHERE address=?', SUPPORT_ADDRESS)
-    if queryreturn == []:
-        sqlExecute(
-            'INSERT INTO addressbook VALUES (?,?)',
-            SUPPORT_LABEL.toUtf8(), SUPPORT_ADDRESS)
+    sqlExecute('DELETE FROM addressbook WHERE address=?', OLD_SUPPORT_ADDRESS)
+    if helper_db.put_addressbook(SUPPORT_LABEL.toUtf8(), SUPPORT_ADDRESS):
         myapp.rerenderAddressBook()
 
 

src/class_objectProcessor.py

@@ -10,12 +10,10 @@ import random
 import threading
 import time
 from binascii import hexlify
-from subprocess import call  # nosec
 
 import helper_bitcoin
-import helper_inbox
 import helper_msgcoding
-import helper_sent
+import helper_db
 import highlevelcrypto
 import l10n
 import protocol
@@ -29,10 +27,10 @@ from addresses import (
 )
 from bmconfigparser import BMConfigParser
 from fallback import RIPEMD160Hash
+from helper_ackPayload import genAckPayload
 from helper_sql import sql_ready, SqlBulkExecute, sqlExecute, sqlQuery
 from network import bmproto, knownnodes
 from network.node import Peer
-# pylint: disable=too-many-locals, too-many-return-statements, too-many-branches, too-many-statements
 
 logger = logging.getLogger('default')
 
@@ -343,24 +341,7 @@ class objectProcessor(threading.Thread):
                 )
 
             address = encodeAddress(addressVersion, streamNumber, ripe)
-
-            queryreturn = sqlQuery(
-                "SELECT usedpersonally FROM pubkeys WHERE address=?"
-                " AND usedpersonally='yes'", address)
-            # if this pubkey is already in our database and if we have
-            # used it personally:
-            if queryreturn != []:
-                logger.info(
-                    'We HAVE used this pubkey personally. Updating time.')
-                t = (address, addressVersion, dataToStore,
-                     int(time.time()), 'yes')
-            else:
-                logger.info(
-                    'We have NOT used this pubkey personally. Inserting'
-                    ' in database.')
-                t = (address, addressVersion, dataToStore,
-                     int(time.time()), 'no')
-            sqlExecute('''INSERT INTO pubkeys VALUES (?,?,?,?,?)''', *t)
+            helper_db.put_pubkey(address, addressVersion, dataToStore)
             self.possibleNewPubkey(address)
         if addressVersion == 3:
             if len(data) < 170:  # sanity check.
@@ -408,23 +389,7 @@ class objectProcessor(threading.Thread):
                 )
 
             address = encodeAddress(addressVersion, streamNumber, ripe)
-            queryreturn = sqlQuery(
-                "SELECT usedpersonally FROM pubkeys WHERE address=?"
-                " AND usedpersonally='yes'", address)
-            # if this pubkey is already in our database and if we have
-            # used it personally:
-            if queryreturn != []:
-                logger.info(
-                    'We HAVE used this pubkey personally. Updating time.')
-                t = (address, addressVersion, dataToStore,
-                     int(time.time()), 'yes')
-            else:
-                logger.info(
-                    'We have NOT used this pubkey personally. Inserting'
-                    ' in database.')
-                t = (address, addressVersion, dataToStore,
-                     int(time.time()), 'no')
-            sqlExecute('''INSERT INTO pubkeys VALUES (?,?,?,?,?)''', *t)
+            helper_db.put_pubkey(address, addressVersion, dataToStore)
             self.possibleNewPubkey(address)
 
         if addressVersion == 4:
@@ -623,13 +588,10 @@ class objectProcessor(threading.Thread):
 
         # Let's store the public key in case we want to reply to this
         # person.
-        sqlExecute(
-            '''INSERT INTO pubkeys VALUES (?,?,?,?,?)''',
-            fromAddress,
-            sendersAddressVersionNumber,
-            decryptedData[:endOfThePublicKeyPosition],
-            int(time.time()),
-            'yes')
+        helper_db.put_pubkey(
+            fromAddress, sendersAddressVersionNumber,
+            decryptedData[:endOfThePublicKeyPosition], True
+        )
 
         # Check to see whether we happen to be awaiting this
         # pubkey in order to send a message. If we are, it will do the POW
@@ -693,78 +655,56 @@ class objectProcessor(threading.Thread):
         subject = decodedMessage.subject
         body = decodedMessage.body
 
-        # Let us make sure that we haven't already received this message
-        if helper_inbox.isMessageAlreadyInInbox(sigHash):
-            logger.info('This msg is already in our inbox. Ignoring it.')
-            blockMessage = True
-        if not blockMessage:
-            if messageEncodingType != 0:
-                t = (inventoryHash, toAddress, fromAddress, subject,
-                     int(time.time()), body, 'inbox', messageEncodingType,
-                     0, sigHash)
-                helper_inbox.insert(t)
-
-                queues.UISignalQueue.put(('displayNewInboxMessage', (
-                    inventoryHash, toAddress, fromAddress, subject, body)))
-
-            # If we are behaving as an API then we might need to run an
-            # outside command to let some program know that a new message
-            # has arrived.
-            if BMConfigParser().safeGetBoolean(
-                    'bitmessagesettings', 'apienabled'):
-                try:
-                    apiNotifyPath = BMConfigParser().get(
-                        'bitmessagesettings', 'apinotifypath')
-                except:
-                    apiNotifyPath = ''
-                if apiNotifyPath != '':
-                    call([apiNotifyPath, "newMessage"])
-
-            # Let us now check and see whether our receiving address is
-            # behaving as a mailing list
-            if BMConfigParser().safeGetBoolean(toAddress, 'mailinglist') \
-                    and messageEncodingType != 0:
-                try:
-                    mailingListName = BMConfigParser().get(
-                        toAddress, 'mailinglistname')
-                except:
-                    mailingListName = ''
-                # Let us send out this message as a broadcast
-                subject = self.addMailingListNameToSubject(
-                    subject, mailingListName)
-                # Let us now send this message out as a broadcast
-                message = time.strftime(
-                    "%a, %Y-%m-%d %H:%M:%S UTC", time.gmtime()
-                ) + '   Message ostensibly from ' + fromAddress \
-                    + ':\n\n' + body
-                # The fromAddress for the broadcast that we are about to
-                # send is the toAddress (my address) for the msg message
-                # we are currently processing.
-                fromAddress = toAddress
-                # We don't actually need the ackdata for acknowledgement
-                # since this is a broadcast message but we can use it to
-                # update the user interface when the POW is done generating.
-                toAddress = '[Broadcast subscribers]'
-
-                ackdata = helper_sent.insert(
-                    fromAddress=fromAddress,
-                    status='broadcastqueued',
-                    subject=subject,
-                    message=message,
-                    encoding=messageEncodingType)
-
-                queues.UISignalQueue.put((
-                    'displayNewSentMessage', (
-                        toAddress, '[Broadcast subscribers]', fromAddress,
-                        subject, message, ackdata)
-                ))
-                queues.workerQueue.put(('sendbroadcast', ''))
+        if helper_db.put_inbox(
+            toAddress, fromAddress, subject, body, inventoryHash, sigHash,
+            encoding=messageEncodingType
+        ) is False:
+            # logger.info('This msg is already in our inbox. Ignoring it.')
+            return
+
+        # Let us now check and see whether our receiving address is
+        # behaving as a mailing list
+        if BMConfigParser().safeGetBoolean(toAddress, 'mailinglist'):
+            mailingListName = BMConfigParser().safeGet(
+                toAddress, 'mailinglistname', '')
+
+            # Let us send out this message as a broadcast
+            subject = self.addMailingListNameToSubject(
+                subject, mailingListName)
+            # Let us now send this message out as a broadcast
+            message = time.strftime("%a, %Y-%m-%d %H:%M:%S UTC", time.gmtime(
+            )) + '   Message ostensibly from ' + fromAddress + ':\n\n' + body
+            # The fromAddress for the broadcast that we are about to
+            # send is the toAddress (my address) for the msg message
+            # we are currently processing.
+            fromAddress = toAddress
+            # We don't actually need the ackdata for acknowledgement
+            # since this is a broadcast message but we can use it to
+            # update the user interface when the POW is done generating.
+            streamNumber = decodeAddress(fromAddress)[2]
+
+            ackdata = genAckPayload(streamNumber, 0)
+            toAddress = toLabel = '[Broadcast subscribers]'
+
+            # We really should have a discussion about how to
+            # set the TTL for mailing list broadcasts. This is obviously
+            # hard-coded.
+            TTL = 2 * 7 * 24 * 60 * 60  # 2 weeks
+            helper_db.put_sent(
+                toAddress, fromAddress, subject, message, ackdata,
+                'broadcastqueued', messageEncodingType, ttl=TTL
+            )
+
+            queues.UISignalQueue.put((
+                'displayNewSentMessage',
+                (toAddress, toLabel, fromAddress, subject, message, ackdata)
+            ))
+            queues.workerQueue.put(('sendbroadcast', ''))
 
         # Don't send ACK if invalid, blacklisted senders, invisible
         # messages, disabled or chan
         if (
             self.ackDataHasAValidHeader(ackData) and not blockMessage and
-            messageEncodingType != 0 and
             not BMConfigParser().safeGetBoolean(toAddress, 'dontsendack') and
             not BMConfigParser().safeGetBoolean(toAddress, 'chan')
         ):
@@ -975,12 +915,11 @@ class objectProcessor(threading.Thread):
         logger.info('fromAddress: %s', fromAddress)
 
         # Let's store the public key in case we want to reply to this person.
-        sqlExecute('''INSERT INTO pubkeys VALUES (?,?,?,?,?)''',
-                   fromAddress,
-                   sendersAddressVersion,
-                   decryptedData[:endOfPubkeyPosition],
-                   int(time.time()),
-                   'yes')
+        helper_db.put_pubkey(
+            fromAddress, sendersAddressVersion,
+            decryptedData[:endOfPubkeyPosition],
+            True
+        )
 
         # Check to see whether we happen to be awaiting this
         # pubkey in order to send a message. If we are, it will do the POW
@@ -1000,27 +939,12 @@ class objectProcessor(threading.Thread):
         body = decodedMessage.body
 
         toAddress = '[Broadcast subscribers]'
-        if helper_inbox.isMessageAlreadyInInbox(sigHash):
-            logger.info('This broadcast is already in our inbox. Ignoring it.')
+        if helper_db.put_inbox(
+            toAddress, fromAddress, subject, body, inventoryHash, sigHash,
+            encoding=messageEncodingType
+        ) is False:
+            # logger.info('This broadcast is already in inbox. Ignoring it.')
             return
-        t = (inventoryHash, toAddress, fromAddress, subject, int(
-            time.time()), body, 'inbox', messageEncodingType, 0, sigHash)
-        helper_inbox.insert(t)
-
-        queues.UISignalQueue.put(('displayNewInboxMessage', (
-            inventoryHash, toAddress, fromAddress, subject, body)))
-
-        # If we are behaving as an API then we might need to run an
-        # outside command to let some program know that a new message
-        # has arrived.
-        if BMConfigParser().safeGetBoolean('bitmessagesettings', 'apienabled'):
-            try:
-                apiNotifyPath = BMConfigParser().get(
-                    'bitmessagesettings', 'apinotifypath')
-            except:
-                apiNotifyPath = ''
-            if apiNotifyPath != '':
-                call([apiNotifyPath, "newBroadcast"])
 
         # Display timing data
         logger.info(

src/class_singleWorker.py

@@ -10,10 +10,9 @@ import hashlib
 import time
 from binascii import hexlify, unhexlify
 from struct import pack
-from subprocess import call  # nosec
 
 import defaults
-import helper_inbox
+import helper_db
 import helper_msgcoding
 import helper_random
 import helper_sql
@@ -1312,25 +1311,11 @@ class singleWorker(StoppableThread):
                 # Used to detect and ignore duplicate messages in our inbox
                 sigHash = hashlib.sha512(hashlib.sha512(
                     signature).digest()).digest()[32:]
-                t = (inventoryHash, toaddress, fromaddress, subject, int(
-                    time.time()), message, 'inbox', encoding, 0, sigHash)
-                helper_inbox.insert(t)
-
-                queues.UISignalQueue.put(('displayNewInboxMessage', (
-                    inventoryHash, toaddress, fromaddress, subject, message)))
-
-                # If we are behaving as an API then we might need to run an
-                # outside command to let some program know that a new message
-                # has arrived.
-                if BMConfigParser().safeGetBoolean(
-                        'bitmessagesettings', 'apienabled'):
-                    try:
-                        apiNotifyPath = BMConfigParser().get(
-                            'bitmessagesettings', 'apinotifypath')
-                    except:
-                        apiNotifyPath = ''
-                    if apiNotifyPath != '':
-                        call([apiNotifyPath, "newMessage"])
+
+                helper_db.put_inbox(
+                    toaddress, fromaddress, subject, message, inventoryHash, sigHash,
+                    encoding=encoding
+                )
 
     def requestPubKey(self, toAddress):
         """Send a getpubkey object"""

src/helper_db.py

@@ -0,0 +1,208 @@
+import subprocess  # nosec
+import time
+
+import queues
+from bmconfigparser import BMConfigParser
+from helper_search import search_sql, check_match
+from helper_sql import sqlExecute, sqlQuery
+
+
+__all__ = ["search_sql", "check_match"]
+
+_groups = ("blacklist", "whitelist", "subscriptions", "addressbook")
+_groups_enable = ("blacklist", "whitelist", "subscriptions")
+
+
+# + genAckPayload
+def put_sent(
+        to_address, from_address, subject, message, ackdata,
+        status, encoding,
+        ripe='', ttl=0, msgid='', sent_time=None, last_action_time=None,
+        sleep_till_time=0, retrynumber=0):
+    """Put message into Sent table"""
+    # We don't know msgid until the POW is done.
+    # sleep_till_time will get set when the POW gets done
+    if not sent_time:
+        sent_time = time.time()
+    if not last_action_time:
+        last_action_time = sent_time
+    if not ttl:
+        ttl = BMConfigParser().getint('bitmessagesettings', 'ttl')
+    sqlExecute(
+        "INSERT INTO sent VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
+        msgid, to_address, ripe, from_address, subject, message, ackdata,
+        int(sent_time), int(last_action_time), sleep_till_time, status, retrynumber,
+        'sent', encoding, ttl
+    )
+
+
+def _in_inbox_already(sighash):
+    return sqlQuery(
+        "SELECT COUNT(*) FROM inbox WHERE sighash=?", sighash
+    )[0][0] != 0
+
+
+def put_inbox(
+        to_address, from_address, subject, message, msgid, sighash,
+        encoding=0, received_time=None, broadcast=False):
+    """Put message into Inbox table"""
+    if not received_time:
+        received_time = time.time()
+    if encoding == 0 or _in_inbox_already(sighash):
+        return False
+
+    sqlExecute(
+        "INSERT INTO inbox VALUES (?,?,?,?,?,?,?,?,?,?)",
+        msgid, to_address, from_address, subject, int(received_time), message,
+        'inbox', encoding, 0, sighash
+    )
+
+    queues.UISignalQueue.put((
+        'displayNewInboxMessage',
+        (msgid, to_address, from_address, subject, message)
+    ))
+
+    # If we are behaving as an API then we might need to run an
+    # outside command to let some program know that a new message
+    # has arrived.
+    if BMConfigParser().safeGetBoolean('bitmessagesettings', 'apienabled'):
+        apinotify_path = BMConfigParser().safeGet(
+            'bitmessagesettings', 'apinotifypath')
+        if apinotify_path:
+            subprocess.call([
+                apinotify_path,
+                "newBroadcast" if broadcast else "newMessage"])
+
+
+def put_trash(msgid, sent=False):
+    """Put inbox message (or sent if sent=True) into trash by msgid"""
+    sqlExecute(
+        "UPDATE %s SET folder='trash' WHERE msgid=?" %
+        ('sent' if sent else 'inbox'), msgid)