From cc2fa62b90d8e609220d15a324a836bbb670cb99 Mon Sep 17 00:00:00 2001 From: Dmitri Bogomolov <4glitch@gmail.com> Date: Mon, 12 Mar 2018 12:55:42 +0200 Subject: [PATCH] Moved most of frequently used SQL-queries to helper_db module. --- src/bitmessageqt/__init__.py | 149 ++++++++++--------------- src/bitmessageqt/account.py | 121 +++++++-------------- src/bitmessageqt/foldertree.py | 44 ++------ src/bitmessageqt/support.py | 11 +- src/class_objectProcessor.py | 192 ++++++++++----------------------- src/class_singleWorker.py | 25 +---- src/helper_db.py | 180 +++++++++++++++++++++++++++++++ src/helper_sent.py | 48 --------- src/protocol.py | 5 +- 9 files changed, 353 insertions(+), 422 deletions(-) create mode 100644 src/helper_db.py delete mode 100644 src/helper_sent.py diff --git a/src/bitmessageqt/__init__.py b/src/bitmessageqt/__init__.py index eb140fb9..7cab9a43 100644 --- a/src/bitmessageqt/__init__.py +++ b/src/bitmessageqt/__init__.py @@ -35,9 +35,10 @@ from foldertree import ( MessageList_TimeWidget) import settingsmixin import support -from helper_sql import sqlQuery, sqlExecute, sqlExecuteChunked, sqlStoredProcedure -import helper_addressbook -import helper_search +from helper_ackPayload import genAckPayload +from helper_sql import ( + sqlQuery, sqlExecute, sqlExecuteChunked, sqlStoredProcedure) +import helper_db import l10n from utils import str_broadcast_subscribers, avatarize from account import ( @@ -54,7 +55,6 @@ from statusbar import BMStatusBar import sound # This is needed for tray icon import bitmessage_icons_rc # noqa:F401 pylint: disable=unused-import -import helper_sent try: from plugins.plugin import get_plugin, get_plugins @@ -1214,7 +1214,7 @@ class MyForm(settingsmixin.SMainWindow): tableWidget.setColumnHidden(1, bool(account)) xAddress = 'fromaddress' - queryreturn = helper_search.search_sql( + queryreturn = helper_db.search_sql( xAddress, account, "sent", where, what, False) for row in queryreturn: @@ -1254,7 +1254,7 @@ class MyForm(settingsmixin.SMainWindow): tableWidget.setColumnHidden(0, False) tableWidget.setColumnHidden(1, False) - queryreturn = helper_search.search_sql( + queryreturn = helper_db.search_sql( xAddress, account, folder, where, what, unreadOnly) for row in queryreturn: @@ -1937,8 +1937,7 @@ class MyForm(settingsmixin.SMainWindow): newRows = {} # subscriptions - queryreturn = sqlQuery('SELECT label, address FROM subscriptions WHERE enabled = 1') - for row in queryreturn: + for row in helper_db.get_subscriptions(): label, address = row newRows[address] = [label, AccountMixin.SUBSCRIPTION] # chans @@ -1948,8 +1947,7 @@ class MyForm(settingsmixin.SMainWindow): if (account.type == AccountMixin.CHAN and BMConfigParser().safeGetBoolean(address, 'enabled')): newRows[address] = [account.getLabel(), AccountMixin.CHAN] # normal accounts - queryreturn = sqlQuery('SELECT * FROM addressbook') - for row in queryreturn: + for row in helper_db.get_addressbook(): label, address = row newRows[address] = [label, AccountMixin.NORMAL] @@ -2166,18 +2164,19 @@ class MyForm(settingsmixin.SMainWindow): " send the message but it won\'t send until" " you connect.") ) - ackdata = helper_sent.insert( - toAddress=toAddress, fromAddress=fromAddress, - subject=subject, message=message, encoding=encoding) - toLabel = '' - queryreturn = sqlQuery('''select label from addressbook where address=?''', - toAddress) - if queryreturn != []: - for row in queryreturn: - toLabel, = row + stealthLevel = BMConfigParser().safeGetInt( + 'bitmessagesettings', 'ackstealthlevel') + ackdata = genAckPayload(streamNumber, stealthLevel) + helper_db.put_sent( + toAddress, fromAddress, subject, message, ackdata, + 'msgqueued', encoding, ripe + ) + + toLabel = helper_db.get_label(toAddress) or '' self.displayNewSentMessage( - toAddress, toLabel, fromAddress, subject, message, ackdata) + toAddress, toLabel, fromAddress, subject, + message, ackdata) queues.workerQueue.put(('sendmessage', toAddress)) self.click_pushButtonClear() @@ -2202,15 +2201,14 @@ class MyForm(settingsmixin.SMainWindow): # We don't actually need the ackdata for acknowledgement since # this is a broadcast message, but we can use it to update the # user interface when the POW is done generating. - toAddress = str_broadcast_subscribers + streamNumber = decodeAddress(fromAddress)[2] + ackdata = genAckPayload(streamNumber, 0) + toAddress = toLabel = str_broadcast_subscribers - # msgid. We don't know what this will be until the POW is done. - ackdata = helper_sent.insert( - fromAddress=fromAddress, - subject=subject, message=message, - status='broadcastqueued', encoding=encoding) - - toLabel = str_broadcast_subscribers + helper_db.put_sent( + toAddress, fromAddress, subject, message, ackdata, + 'broadcastqueued', encoding + ) self.displayNewSentMessage( toAddress, toLabel, fromAddress, subject, message, ackdata) @@ -2338,7 +2336,7 @@ class MyForm(settingsmixin.SMainWindow): self.ui.treeWidgetChans ) and self.getCurrentAccount(treeWidget) != toAddress: continue - elif not helper_search.check_match( + elif not helper_db.check_match( toAddress, fromAddress, subject, message, self.getCurrentSearchOption(tab), self.getCurrentSearchLine(tab) @@ -2368,7 +2366,7 @@ class MyForm(settingsmixin.SMainWindow): tab += 1 if tab == 1: tab = 2 - if not helper_search.check_match( + if not helper_db.check_match( toAddress, fromAddress, subject, message, self.getCurrentSearchOption(tab), self.getCurrentSearchLine(tab) @@ -2427,10 +2425,7 @@ class MyForm(settingsmixin.SMainWindow): except AttributeError: return - # First we must check to see if the address is already in the - # address book. The user cannot add it again or else it will - # cause problems when updating and deleting the entry. - if shared.isAddressInMyAddressBook(address): + if not helper_db.put_addressbook(label, address): self.updateStatusBar(_translate( "MainWindow", "Error: You cannot add the same address to your" @@ -2439,30 +2434,25 @@ class MyForm(settingsmixin.SMainWindow): )) return - if helper_addressbook.insert(label=label, address=address): - self.rerenderMessagelistFromLabels() - self.rerenderMessagelistToLabels() - self.rerenderAddressBook() - else: - self.updateStatusBar(_translate( - "MainWindow", - "Error: You cannot add your own address in the address book." - )) + self.rerenderMessagelistFromLabels() + self.rerenderMessagelistToLabels() + self.rerenderAddressBook() def addSubscription(self, address, label): - # This should be handled outside of this function, for error displaying - # and such, but it must also be checked here. - if shared.isAddressInMySubscriptionsList(address): + if not helper_db.put_subscriptions(label, address): + self.updateStatusBar(_translate( + "MainWindow", + "Error: You cannot add the same address to your" + " subscriptions twice. Perhaps rename the existing one" + " if you want." + )) return - # Add to database (perhaps this should be separated from the MyForm class) - sqlExecute( - '''INSERT INTO subscriptions VALUES (?,?,?)''', - label, address, True - ) + self.rerenderMessagelistFromLabels() shared.reloadBroadcastSendersForWhichImWatching() self.rerenderAddressBook() self.rerenderTabTreeSubscriptions() + return True def click_pushButtonAddSubscription(self): dialog = dialogs.NewSubscriptionDialog(self) @@ -2472,19 +2462,9 @@ class MyForm(settingsmixin.SMainWindow): except AttributeError: return - # We must check to see if the address is already in the - # subscriptions list. The user cannot add it again or else it - # will cause problems when updating and deleting the entry. - if shared.isAddressInMySubscriptionsList(address): - self.updateStatusBar(_translate( - "MainWindow", - "Error: You cannot add the same address to your" - " subscriptions twice. Perhaps rename the existing one" - " if you want." - )) + if not self.addSubscription(address, label): return - self.addSubscription(address, label) # Now, if the user wants to display old broadcasts, let's get # them out of the inventory and put them # to the objectProcessorQueue to be processed @@ -2815,11 +2795,7 @@ class MyForm(settingsmixin.SMainWindow): textEdit = self.getCurrentMessageTextedit() if not msgid: return - queryreturn = sqlQuery( - '''select message from inbox where msgid=?''', msgid) - if queryreturn != []: - for row in queryreturn: - messageText, = row + messageText = helper_db.get_message(msgid) lines = messageText.split('\n') totalLines = len(lines) @@ -3052,24 +3028,20 @@ class MyForm(settingsmixin.SMainWindow): currentInboxRow, 1).data(QtCore.Qt.UserRole) recipientAddress = tableWidget.item( currentInboxRow, 0).data(QtCore.Qt.UserRole) - # Let's make sure that it isn't already in the address book - queryreturn = sqlQuery('''select * from blacklist where address=?''', - addressAtCurrentInboxRow) - if queryreturn == []: - label = "\"" + tableWidget.item(currentInboxRow, 2).subject + "\" in " + BMConfigParser().get(recipientAddress, "label") - sqlExecute('''INSERT INTO blacklist VALUES (?,?, ?)''', - label, - addressAtCurrentInboxRow, True) - self.ui.blackwhitelist.rerenderBlackWhiteList() - self.updateStatusBar(_translate( - "MainWindow", - "Entry added to the blacklist. Edit the label to your liking.") - ) - else: + label = "\"" + tableWidget.item(currentInboxRow, 2).subject + \ + "\" in " + BMConfigParser().get(recipientAddress, "label") + if not helper_db.put_blacklist(label, addressAtCurrentInboxRow): self.updateStatusBar(_translate( "MainWindow", "Error: You cannot add the same address to your blacklist" " twice. Try renaming the existing one if you want.")) + return + + self.ui.blackwhitelist.rerenderBlackWhiteList() + self.updateStatusBar(_translate( + "MainWindow", + "Entry added to the blacklist. Edit the label to your liking.") + ) def deleteRowFromMessagelist( self, row=None, inventoryHash=None, ackData=None, messageLists=None @@ -3168,11 +3140,7 @@ class MyForm(settingsmixin.SMainWindow): # Retrieve the message data out of the SQL database msgid = tableWidget.item(currentInboxRow, 3).data() - queryreturn = sqlQuery( - '''select message from inbox where msgid=?''', msgid) - if queryreturn != []: - for row in queryreturn: - message, = row + message = helper_db.get_message(msgid) defaultFilename = "".join(x for x in subjectAtCurrentInboxRow if x.isalnum()) + '.txt' filename = QtGui.QFileDialog.getSaveFileName(self, _translate("MainWindow","Save As..."), defaultFilename, "Text files (*.txt);;All files (*.*)") @@ -3280,15 +3248,6 @@ class MyForm(settingsmixin.SMainWindow): def on_action_AddressBookSubscribe(self): for item in self.getAddressbookSelectedItems(): - # Then subscribe to it... - # provided it's not already in the address book - if shared.isAddressInMySubscriptionsList(item.address): - self.updateStatusBar(_translate( - "MainWindow", - "Error: You cannot add the same address to your" - " subscriptions twice. Perhaps rename the existing" - " one if you want.")) - continue self.addSubscription(item.address, item.label) self.ui.tabWidget.setCurrentIndex( self.ui.tabWidget.indexOf(self.ui.subscriptions) diff --git a/src/bitmessageqt/account.py b/src/bitmessageqt/account.py index 50ea3548..374d1e91 100644 --- a/src/bitmessageqt/account.py +++ b/src/bitmessageqt/account.py @@ -4,43 +4,30 @@ account.py ========== Account related functions. - """ -from __future__ import absolute_import - import inspect import re import sys -import time - -from PyQt4 import QtGui +import helper_db import queues from addresses import decodeAddress from bmconfigparser import BMConfigParser from helper_ackPayload import genAckPayload -from helper_sql import sqlQuery, sqlExecute -from .foldertree import AccountMixin -from .utils import str_broadcast_subscribers +from helper_sql import sqlQuery +from foldertree import AccountMixin +from utils import str_broadcast_subscribers +from tr import _translate def getSortedAccounts(): """Get a sorted list of configSections""" - configSections = BMConfigParser().addresses() configSections.sort( cmp=lambda x, y: cmp( - unicode( - BMConfigParser().get( - x, - 'label'), - 'utf-8').lower(), - unicode( - BMConfigParser().get( - y, - 'label'), - 'utf-8').lower())) + unicode(BMConfigParser().get(x, 'label'), 'utf-8').lower(), + unicode(BMConfigParser().get(y, 'label'), 'utf-8').lower())) return configSections @@ -107,29 +94,27 @@ def accountClass(address): class AccountColor(AccountMixin): # pylint: disable=too-few-public-methods """Set the type of account""" - def __init__(self, address, address_type=None): self.isEnabled = True self.address = address - if address_type is None: - if address is None: - self.type = AccountMixin.ALL - elif BMConfigParser().safeGetBoolean(self.address, 'mailinglist'): - self.type = AccountMixin.MAILINGLIST - elif BMConfigParser().safeGetBoolean(self.address, 'chan'): - self.type = AccountMixin.CHAN - elif sqlQuery( - '''select label from subscriptions where address=?''', self.address): - self.type = AccountMixin.SUBSCRIPTION - else: - self.type = AccountMixin.NORMAL - else: + if address_type: self.type = address_type + return + # AccountMixin.setType() + if self.address is None: + self.type = self.ALL + elif BMConfigParser().safeGetBoolean(self.address, 'chan'): + self.type = self.CHAN + elif BMConfigParser().safeGetBoolean(self.address, 'mailinglist'): + self.type = self.MAILINGLIST + elif helper_db.get_label(self.address, "subscriptions"): + self.type = AccountMixin.SUBSCRIPTION + else: + self.type = self.NORMAL class BMAccount(object): """Encapsulate a Bitmessage account""" - def __init__(self, address=None): self.address = address self.type = AccountMixin.NORMAL @@ -140,29 +125,19 @@ class BMAccount(object): self.type = AccountMixin.MAILINGLIST elif self.address == str_broadcast_subscribers: self.type = AccountMixin.BROADCAST - else: - queryreturn = sqlQuery( - '''select label from subscriptions where address=?''', self.address) - if queryreturn: - self.type = AccountMixin.SUBSCRIPTION + elif helper_db.get_label(self.address, "subscriptions"): + self.type = AccountMixin.SUBSCRIPTION def getLabel(self, address=None): """Get a label for this bitmessage account""" if address is None: address = self.address - label = BMConfigParser().safeGet(address, 'label', address) - queryreturn = sqlQuery( - '''select label from addressbook where address=?''', address) - if queryreturn != []: - for row in queryreturn: - label, = row - else: - queryreturn = sqlQuery( - '''select label from subscriptions where address=?''', address) - if queryreturn != []: - for row in queryreturn: - label, = row - return label + if BMConfigParser().has_section(address): + return BMConfigParser().get(address, 'label') + return ( + helper_db.get_label(address) or + helper_db.get_label(address, "subscriptions") or address + ) def parseMessage(self, toAddress, fromAddress, subject, message): """Set metadata and address labels on self""" @@ -186,9 +161,7 @@ class NoAccount(BMAccount): self.type = AccountMixin.NORMAL def getLabel(self, address=None): - if address is None: - address = self.address - return address + return address or self.address class SubscriptionAccount(BMAccount): @@ -213,31 +186,17 @@ class GatewayAccount(BMAccount): def send(self): """Override the send method for gateway accounts""" - - # pylint: disable=unused-variable - status, addressVersionNumber, streamNumber, ripe = decodeAddress(self.toAddress) - stealthLevel = BMConfigParser().safeGetInt('bitmessagesettings', 'ackstealthlevel') + streamNumber, ripe = decodeAddress(self.toAddress)[2:] + stealthLevel = BMConfigParser().safeGetInt( + 'bitmessagesettings', 'ackstealthlevel') ackdata = genAckPayload(streamNumber, stealthLevel) - sqlExecute( - '''INSERT INTO sent VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)''', - '', - self.toAddress, - ripe, - self.fromAddress, - self.subject, - self.message, - ackdata, - int(time.time()), # sentTime (this will never change) - int(time.time()), # lastActionTime - 0, # sleepTill time. This will get set when the POW gets done. - 'msgqueued', - 0, # retryNumber - 'sent', # folder - 2, # encodingtype - # not necessary to have a TTL higher than 2 days - min(BMConfigParser().getint('bitmessagesettings', 'ttl'), 86400 * 2) + helper_db.put_sent( + self.toAddress, self.fromAddress, self.subject, self.message, + ackdata, 'msgqueued', 2, ripe, + min( + BMConfigParser().getint('bitmessagesettings', 'ttl'), + 86400 * 2) # not necessary to have a TTL higher than 2 days ) - queues.workerQueue.put(('sendmessage', self.toAddress)) @@ -292,9 +251,7 @@ class MailchuckAccount(GatewayAccount): self.toAddress = self.registrationAddress self.subject = "config" - self.message = QtGui.QApplication.translate( - "Mailchuck", - """# You can use this to configure your email gateway account + self.message = _translate("Mailchuck", """# You can use this to configure your email gateway account # Uncomment the setting you want to use # Here are the options: # diff --git a/src/bitmessageqt/foldertree.py b/src/bitmessageqt/foldertree.py index e6d64427..78af13f8 100644 --- a/src/bitmessageqt/foldertree.py +++ b/src/bitmessageqt/foldertree.py @@ -8,8 +8,8 @@ from cgi import escape from PyQt4 import QtCore, QtGui +import helper_db from bmconfigparser import BMConfigParser -from helper_sql import sqlExecute, sqlQuery from settingsmixin import SettingsMixin from tr import _translate from utils import avatarize @@ -110,15 +110,13 @@ class AccountMixin(object): self.type = self.CHAN elif BMConfigParser().safeGetBoolean(self.address, 'mailinglist'): self.type = self.MAILINGLIST - elif sqlQuery( - '''select label from subscriptions where address=?''', self.address): + elif helper_db.get_label(self.address, "subscriptions"): self.type = AccountMixin.SUBSCRIPTION else: self.type = self.NORMAL def defaultLabel(self): """Default label (in case no label is set manually)""" - queryreturn = None retval = None if self.type in ( AccountMixin.NORMAL, @@ -127,16 +125,9 @@ class AccountMixin(object): retval = unicode( BMConfigParser().get(self.address, 'label'), 'utf-8') except Exception: - queryreturn = sqlQuery( - '''select label from addressbook where address=?''', self.address) + retval = helper_db.get_label(self.address) elif self.type == AccountMixin.SUBSCRIPTION: - queryreturn = sqlQuery( - '''select label from subscriptions where address=?''', self.address) - if queryreturn is not None: - if queryreturn != []: - for row in queryreturn: - retval, = row - retval = unicode(retval, 'utf-8') + retval = helper_db.get_label(self.address, "subscriptions") elif self.address is None or self.type == AccountMixin.ALL: return unicode( str(_translate("MainWindow", "All accounts")), 'utf-8') @@ -306,13 +297,8 @@ class Ui_SubscriptionWidget(Ui_AddressWidget): parent, pos, address, unreadCount, enabled) def _getLabel(self): - queryreturn = sqlQuery( - '''select label from subscriptions where address=?''', self.address) - if queryreturn != []: - for row in queryreturn: - retval, = row - return unicode(retval, 'utf-8', 'ignore') - return unicode(self.address, 'utf-8') + return helper_db.get_label(self.address, "subscriptions") \ + or self.address def setType(self): """Set account type""" @@ -327,9 +313,7 @@ class Ui_SubscriptionWidget(Ui_AddressWidget): value.toString().toUtf8()).decode('utf-8', 'ignore') else: label = unicode(value, 'utf-8', 'ignore') - sqlExecute( - '''UPDATE subscriptions SET label=? WHERE address=?''', - label, self.address) + helper_db.set_label(self.address, label, "subscriptions") return super(Ui_SubscriptionWidget, self).setData(column, role, value) @@ -402,7 +386,6 @@ class MessageList_AddressWidget(BMAddressWidget): if label is not None: return newLabel = self.address - queryreturn = None if self.type in ( AccountMixin.NORMAL, AccountMixin.CHAN, AccountMixin.MAILINGLIST): @@ -411,14 +394,9 @@ class MessageList_AddressWidget(BMAddressWidget): BMConfigParser().get(self.address, 'label'), 'utf-8', 'ignore') except: - queryreturn = sqlQuery( - '''select label from addressbook where address=?''', self.address) + newLabel = helper_db.get_label(self.address) elif self.type == AccountMixin.SUBSCRIPTION: - queryreturn = sqlQuery( - '''select label from subscriptions where address=?''', self.address) - if queryreturn: - for row in queryreturn: - newLabel = unicode(row[0], 'utf-8', 'ignore') + newLabel = helper_db.get_label(self.address, "subscriptions") self.label = newLabel @@ -525,9 +503,9 @@ class Ui_AddressBookWidgetItem(BMAddressWidget): BMConfigParser().set(self.address, 'label', self.label) BMConfigParser().save() except: - sqlExecute('''UPDATE addressbook set label=? WHERE address=?''', self.label, self.address) + helper_db.set_label(self.address, self.label) elif self.type == AccountMixin.SUBSCRIPTION: - sqlExecute('''UPDATE subscriptions set label=? WHERE address=?''', self.label, self.address) + helper_db.set_label(self.address, self.label, "subscriptions") else: pass return super(Ui_AddressBookWidgetItem, self).setData(role, value) diff --git a/src/bitmessageqt/support.py b/src/bitmessageqt/support.py index ac02e2ca..31cfc0dc 100644 --- a/src/bitmessageqt/support.py +++ b/src/bitmessageqt/support.py @@ -10,6 +10,7 @@ from PyQt4 import QtCore import account import defaults +import helper_db import network.stats import paths import proofofwork @@ -17,7 +18,7 @@ import queues import state from bmconfigparser import BMConfigParser from foldertree import AccountMixin -from helper_sql import sqlExecute, sqlQuery +from helper_sql import sqlExecute from l10n import getTranslationLanguage from openclpow import openclEnabled from pyelliptic.openssl import OpenSSL @@ -67,12 +68,8 @@ Connected hosts: {} def checkAddressBook(myapp): - sqlExecute('DELETE from addressbook WHERE address=?', OLD_SUPPORT_ADDRESS) - queryreturn = sqlQuery('SELECT * FROM addressbook WHERE address=?', SUPPORT_ADDRESS) - if queryreturn == []: - sqlExecute( - 'INSERT INTO addressbook VALUES (?,?)', - SUPPORT_LABEL.toUtf8(), SUPPORT_ADDRESS) + sqlExecute('DELETE FROM addressbook WHERE address=?', OLD_SUPPORT_ADDRESS) + if helper_db.put_addressbook(SUPPORT_LABEL.toUtf8(), SUPPORT_ADDRESS): myapp.rerenderAddressBook() diff --git a/src/class_objectProcessor.py b/src/class_objectProcessor.py index 6fa31a47..f8dea48d 100644 --- a/src/class_objectProcessor.py +++ b/src/class_objectProcessor.py @@ -10,12 +10,10 @@ import random import threading import time from binascii import hexlify -from subprocess import call # nosec import helper_bitcoin -import helper_inbox import helper_msgcoding -import helper_sent +import helper_db import highlevelcrypto import l10n import protocol @@ -29,10 +27,10 @@ from addresses import ( ) from bmconfigparser import BMConfigParser from fallback import RIPEMD160Hash +from helper_ackPayload import genAckPayload from helper_sql import sql_ready, SqlBulkExecute, sqlExecute, sqlQuery from network import bmproto, knownnodes from network.node import Peer -# pylint: disable=too-many-locals, too-many-return-statements, too-many-branches, too-many-statements logger = logging.getLogger('default') @@ -343,24 +341,7 @@ class objectProcessor(threading.Thread): ) address = encodeAddress(addressVersion, streamNumber, ripe) - - queryreturn = sqlQuery( - "SELECT usedpersonally FROM pubkeys WHERE address=?" - " AND usedpersonally='yes'", address) - # if this pubkey is already in our database and if we have - # used it personally: - if queryreturn != []: - logger.info( - 'We HAVE used this pubkey personally. Updating time.') - t = (address, addressVersion, dataToStore, - int(time.time()), 'yes') - else: - logger.info( - 'We have NOT used this pubkey personally. Inserting' - ' in database.') - t = (address, addressVersion, dataToStore, - int(time.time()), 'no') - sqlExecute('''INSERT INTO pubkeys VALUES (?,?,?,?,?)''', *t) + helper_db.put_pubkey(address, addressVersion, dataToStore) self.possibleNewPubkey(address) if addressVersion == 3: if len(data) < 170: # sanity check. @@ -408,23 +389,7 @@ class objectProcessor(threading.Thread): ) address = encodeAddress(addressVersion, streamNumber, ripe) - queryreturn = sqlQuery( - "SELECT usedpersonally FROM pubkeys WHERE address=?" - " AND usedpersonally='yes'", address) - # if this pubkey is already in our database and if we have - # used it personally: - if queryreturn != []: - logger.info( - 'We HAVE used this pubkey personally. Updating time.') - t = (address, addressVersion, dataToStore, - int(time.time()), 'yes') - else: - logger.info( - 'We have NOT used this pubkey personally. Inserting' - ' in database.') - t = (address, addressVersion, dataToStore, - int(time.time()), 'no') - sqlExecute('''INSERT INTO pubkeys VALUES (?,?,?,?,?)''', *t) + helper_db.put_pubkey(address, addressVersion, dataToStore) self.possibleNewPubkey(address) if addressVersion == 4: @@ -623,13 +588,10 @@ class objectProcessor(threading.Thread): # Let's store the public key in case we want to reply to this # person. - sqlExecute( - '''INSERT INTO pubkeys VALUES (?,?,?,?,?)''', - fromAddress, - sendersAddressVersionNumber, - decryptedData[:endOfThePublicKeyPosition], - int(time.time()), - 'yes') + helper_db.put_pubkey( + fromAddress, sendersAddressVersionNumber, + decryptedData[:endOfThePublicKeyPosition], True + ) # Check to see whether we happen to be awaiting this # pubkey in order to send a message. If we are, it will do the POW @@ -693,78 +655,56 @@ class objectProcessor(threading.Thread): subject = decodedMessage.subject body = decodedMessage.body - # Let us make sure that we haven't already received this message - if helper_inbox.isMessageAlreadyInInbox(sigHash): - logger.info('This msg is already in our inbox. Ignoring it.') - blockMessage = True - if not blockMessage: - if messageEncodingType != 0: - t = (inventoryHash, toAddress, fromAddress, subject, - int(time.time()), body, 'inbox', messageEncodingType, - 0, sigHash) - helper_inbox.insert(t) + if helper_db.put_inbox( + toAddress, fromAddress, subject, body, inventoryHash, sigHash, + encoding=messageEncodingType + ) is False: + # logger.info('This msg is already in our inbox. Ignoring it.') + return - queues.UISignalQueue.put(('displayNewInboxMessage', ( - inventoryHash, toAddress, fromAddress, subject, body))) + # Let us now check and see whether our receiving address is + # behaving as a mailing list + if BMConfigParser().safeGetBoolean(toAddress, 'mailinglist'): + mailingListName = BMConfigParser().safeGet( + toAddress, 'mailinglistname', '') - # If we are behaving as an API then we might need to run an - # outside command to let some program know that a new message - # has arrived. - if BMConfigParser().safeGetBoolean( - 'bitmessagesettings', 'apienabled'): - try: - apiNotifyPath = BMConfigParser().get( - 'bitmessagesettings', 'apinotifypath') - except: - apiNotifyPath = '' - if apiNotifyPath != '': - call([apiNotifyPath, "newMessage"]) + # Let us send out this message as a broadcast + subject = self.addMailingListNameToSubject( + subject, mailingListName) + # Let us now send this message out as a broadcast + message = time.strftime("%a, %Y-%m-%d %H:%M:%S UTC", time.gmtime( + )) + ' Message ostensibly from ' + fromAddress + ':\n\n' + body + # The fromAddress for the broadcast that we are about to + # send is the toAddress (my address) for the msg message + # we are currently processing. + fromAddress = toAddress + # We don't actually need the ackdata for acknowledgement + # since this is a broadcast message but we can use it to + # update the user interface when the POW is done generating. + streamNumber = decodeAddress(fromAddress)[2] - # Let us now check and see whether our receiving address is - # behaving as a mailing list - if BMConfigParser().safeGetBoolean(toAddress, 'mailinglist') \ - and messageEncodingType != 0: - try: - mailingListName = BMConfigParser().get( - toAddress, 'mailinglistname') - except: - mailingListName = '' - # Let us send out this message as a broadcast - subject = self.addMailingListNameToSubject( - subject, mailingListName) - # Let us now send this message out as a broadcast - message = time.strftime( - "%a, %Y-%m-%d %H:%M:%S UTC", time.gmtime() - ) + ' Message ostensibly from ' + fromAddress \ - + ':\n\n' + body - # The fromAddress for the broadcast that we are about to - # send is the toAddress (my address) for the msg message - # we are currently processing. - fromAddress = toAddress - # We don't actually need the ackdata for acknowledgement - # since this is a broadcast message but we can use it to - # update the user interface when the POW is done generating. - toAddress = '[Broadcast subscribers]' + ackdata = genAckPayload(streamNumber, 0) + toAddress = toLabel = '[Broadcast subscribers]' - ackdata = helper_sent.insert( - fromAddress=fromAddress, - status='broadcastqueued', - subject=subject, - message=message, - encoding=messageEncodingType) + # We really should have a discussion about how to + # set the TTL for mailing list broadcasts. This is obviously + # hard-coded. + TTL = 2 * 7 * 24 * 60 * 60 # 2 weeks + helper_db.put_sent( + toAddress, fromAddress, subject, message, ackdata, + 'broadcastqueued', messageEncodingType, ttl=TTL + ) - queues.UISignalQueue.put(( - 'displayNewSentMessage', ( - toAddress, '[Broadcast subscribers]', fromAddress, - subject, message, ackdata) - )) - queues.workerQueue.put(('sendbroadcast', '')) + queues.UISignalQueue.put(( + 'displayNewSentMessage', + (toAddress, toLabel, fromAddress, subject, message, ackdata) + )) + queues.workerQueue.put(('sendbroadcast', '')) # Don't send ACK if invalid, blacklisted senders, invisible # messages, disabled or chan if ( self.ackDataHasAValidHeader(ackData) and not blockMessage and - messageEncodingType != 0 and not BMConfigParser().safeGetBoolean(toAddress, 'dontsendack') and not BMConfigParser().safeGetBoolean(toAddress, 'chan') ): @@ -975,12 +915,11 @@ class objectProcessor(threading.Thread): logger.info('fromAddress: %s', fromAddress) # Let's store the public key in case we want to reply to this person. - sqlExecute('''INSERT INTO pubkeys VALUES (?,?,?,?,?)''', - fromAddress, - sendersAddressVersion, - decryptedData[:endOfPubkeyPosition], - int(time.time()), - 'yes') + helper_db.put_pubkey( + fromAddress, sendersAddressVersion, + decryptedData[:endOfPubkeyPosition], + True + ) # Check to see whether we happen to be awaiting this # pubkey in order to send a message. If we are, it will do the POW @@ -1000,27 +939,12 @@ class objectProcessor(threading.Thread): body = decodedMessage.body toAddress = '[Broadcast subscribers]' - if helper_inbox.isMessageAlreadyInInbox(sigHash): - logger.info('This broadcast is already in our inbox. Ignoring it.') + if helper_db.put_inbox( + toAddress, fromAddress, subject, body, inventoryHash, sigHash, + encoding=messageEncodingType + ) is False: + # logger.info('This broadcast is already in inbox. Ignoring it.') return - t = (inventoryHash, toAddress, fromAddress, subject, int( - time.time()), body, 'inbox', messageEncodingType, 0, sigHash) - helper_inbox.insert(t) - - queues.UISignalQueue.put(('displayNewInboxMessage', ( - inventoryHash, toAddress, fromAddress, subject, body))) - - # If we are behaving as an API then we might need to run an - # outside command to let some program know that a new message - # has arrived. - if BMConfigParser().safeGetBoolean('bitmessagesettings', 'apienabled'): - try: - apiNotifyPath = BMConfigParser().get( - 'bitmessagesettings', 'apinotifypath') - except: - apiNotifyPath = '' - if apiNotifyPath != '': - call([apiNotifyPath, "newBroadcast"]) # Display timing data logger.info( diff --git a/src/class_singleWorker.py b/src/class_singleWorker.py index 012313ac..9551570b 100644 --- a/src/class_singleWorker.py +++ b/src/class_singleWorker.py @@ -10,10 +10,9 @@ import hashlib import time from binascii import hexlify, unhexlify from struct import pack -from subprocess import call # nosec import defaults -import helper_inbox +import helper_db import helper_msgcoding import helper_random import helper_sql @@ -1312,25 +1311,11 @@ class singleWorker(StoppableThread): # Used to detect and ignore duplicate messages in our inbox sigHash = hashlib.sha512(hashlib.sha512( signature).digest()).digest()[32:] - t = (inventoryHash, toaddress, fromaddress, subject, int( - time.time()), message, 'inbox', encoding, 0, sigHash) - helper_inbox.insert(t) - queues.UISignalQueue.put(('displayNewInboxMessage', ( - inventoryHash, toaddress, fromaddress, subject, message))) - - # If we are behaving as an API then we might need to run an - # outside command to let some program know that a new message - # has arrived. - if BMConfigParser().safeGetBoolean( - 'bitmessagesettings', 'apienabled'): - try: - apiNotifyPath = BMConfigParser().get( - 'bitmessagesettings', 'apinotifypath') - except: - apiNotifyPath = '' - if apiNotifyPath != '': - call([apiNotifyPath, "newMessage"]) + helper_db.put_inbox( + toaddress, fromaddress, subject, message, inventoryHash, sigHash, + encoding=encoding + ) def requestPubKey(self, toAddress): """Send a getpubkey object""" diff --git a/src/helper_db.py b/src/helper_db.py new file mode 100644 index 00000000..33190ff6 --- /dev/null +++ b/src/helper_db.py @@ -0,0 +1,180 @@ +import subprocess # nosec +import time + +import queues +from bmconfigparser import BMConfigParser +from helper_search import search_sql, check_match +from helper_sql import sqlExecute, sqlQuery + + +__all__ = ["search_sql", "check_match"] + + +# + genAckPayload +def put_sent( + to_address, from_address, subject, message, ackdata, + status, encoding, + ripe='', ttl=0, msgid='', sent_time=None, last_action_time=None, + sleep_till_time=0, retrynumber=0): + """Put message into Sent table""" + # We don't know msgid until the POW is done. + # sleep_till_time will get set when the POW gets done + if not sent_time: + sent_time = time.time() + if not last_action_time: + last_action_time = sent_time + if not ttl: + ttl = BMConfigParser().getint('bitmessagesettings', 'ttl') + sqlExecute( + "INSERT INTO sent VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + msgid, to_address, ripe, from_address, subject, message, ackdata, + int(sent_time), int(last_action_time), sleep_till_time, status, retrynumber, + 'sent', encoding, ttl + ) + + +def _in_inbox_already(sighash): + return sqlQuery( + "SELECT COUNT(*) FROM inbox WHERE sighash=?", sighash + )[0][0] != 0 + + +def put_inbox( + to_address, from_address, subject, message, msgid, sighash, + encoding=0, received_time=None, broadcast=False): + """Put message into Inbox table""" + if not received_time: + received_time = time.time() + if encoding == 0 or _in_inbox_already(sighash): + return False + + sqlExecute( + "INSERT INTO inbox VALUES (?,?,?,?,?,?,?,?,?,?)", + msgid, to_address, from_address, subject, int(received_time), message, + 'inbox', encoding, 0, sighash + ) + + queues.UISignalQueue.put(( + 'displayNewInboxMessage', + (msgid, to_address, from_address, subject, message) + )) + + # If we are behaving as an API then we might need to run an + # outside command to let some program know that a new message + # has arrived. + if BMConfigParser().safeGetBoolean('bitmessagesettings', 'apienabled'): + apinotify_path = BMConfigParser().safeGet( + 'bitmessagesettings', 'apinotifypath') + if apinotify_path: + subprocess.call([ + apinotify_path, + "newBroadcast" if broadcast else "newMessage"]) + + +def put_pubkey(address, address_version, data, used_personally=None): + """Put pubkey into Pubkeys table""" + if used_personally is None: + if sqlQuery( + "SELECT * FROM pubkeys WHERE address=? AND usedpersonally='yes'", + address + ) == []: + used_personally = False + else: + sqlExecute( + "UPDATE pubkeys SET time=? WHERE address=?", + time.time(), address + ) + return + sqlExecute( + "INSERT INTO pubkeys VALUES (?,?,?,?,?)", + address, address_version, data, time.time(), + 'yes' if used_personally else 'no' + ) + + +def _in_group_already(address, group="addressbook"): + return sqlQuery( + "SELECT enabled FROM %s WHERE address=?" % group, address) + + +def put_addresslist(label, address, group="blacklist", enabled=True): + """Put address into address list (e.g. blacklist, subscriptions...)""" + # We must check to see if the address is already in the + # subscriptions list. The user cannot add it again or else it + # will cause problems when updating and deleting the entry. + # FIXME: address should be primary key in this case + if _in_group_already(address, group): + return False + sqlExecute( + "INSERT INTO %s VALUES (?,?,?)" % group, label, address, enabled) + return True + + +def put_blacklist(label, address): + """Put address into blacklist""" + return put_addresslist(label, address, "blacklist") + + +def put_subscriptions(label, address, enabled=True): + """Put address into subscriptions""" + return put_addresslist(label, address, "subscriptions", enabled) + + +def put_addressbook(label, address): + """Put address into Addressbook""" + # First we must check to see if the address is already in the + # address book. The user cannot add it again or else it will + # cause problems when updating and deleting the entry. + if _in_group_already(address): + return False + sqlExecute("INSERT INTO addressbook VALUES (?,?)", label, address) + return True + + +def get_subscriptions(): + """Generator for Subscriptions""" + queryreturn = sqlQuery( + "SELECT label, address FROM subscriptions WHERE enabled=1") + for row in queryreturn: + yield row + + +def get_addressbook(): + """Generator for Addressbook""" + queryreturn = sqlQuery("SELECT * FROM addressbook") + for row in queryreturn: + yield row + + +def get_addresslist(group="blacklist"): + """Generator for address list given by group arg""" + queryreturn = sqlQuery("SELECT * FROM %s" % group) + for row in queryreturn: + yield row + + +def get_label(address, group="addressbook"): + """ + Get address label from address list given by group arg + (default is addressbook) + """ + queryreturn = sqlQuery( + "SELECT label FROM %s WHERE address=?" % group, address) + try: + return unicode(queryreturn[-1][0], 'utf-8') + except IndexError: + pass + + +def set_label(address, label, group="addressbook"): + """Set address label in the address list given by group arg""" + sqlExecute("UPDATE %s set label=? WHERE address=?" % group, label, address) + + +def get_message(msgid): + """Get inbox message by msgid""" + queryreturn = sqlQuery("SELECT message FROM inbox WHERE msgid=?", msgid) + try: + return queryreturn[-1][0] + except IndexError: + return '' diff --git a/src/helper_sent.py b/src/helper_sent.py deleted file mode 100644 index d83afce6..00000000 --- a/src/helper_sent.py +++ /dev/null @@ -1,48 +0,0 @@ -""" -Insert values into sent table -""" - -import time -import uuid -from addresses import decodeAddress -from bmconfigparser import BMConfigParser -from helper_ackPayload import genAckPayload -from helper_sql import sqlExecute - - -# pylint: disable=too-many-arguments -def insert(msgid=None, toAddress='[Broadcast subscribers]', fromAddress=None, subject=None, - message=None, status='msgqueued', ripe=None, ackdata=None, sentTime=None, - lastActionTime=None, sleeptill=0, retryNumber=0, encoding=2, ttl=None, folder='sent'): - """Perform an insert into the `sent` table""" - # pylint: disable=unused-variable - # pylint: disable-msg=too-many-locals - - valid_addr = True - if not ripe or not ackdata: - addr = fromAddress if toAddress == '[Broadcast subscribers]' else toAddress - new_status, addressVersionNumber, streamNumber, new_ripe = decodeAddress(addr) - valid_addr = True if new_status == 'success' else False - if not ripe: - ripe = new_ripe - - if not ackdata: - stealthLevel = BMConfigParser().safeGetInt( - 'bitmessagesettings', 'ackstealthlevel') - new_ackdata = genAckPayload(streamNumber, stealthLevel) - ackdata = new_ackdata - if valid_addr: - msgid = msgid if msgid else uuid.uuid4().bytes - sentTime = sentTime if sentTime else int(time.time()) # sentTime (this doesn't change) - lastActionTime = lastActionTime if lastActionTime else int(time.time()) - - ttl = ttl if ttl else BMConfigParser().getint('bitmessagesettings', 'ttl') - - t = (msgid, toAddress, ripe, fromAddress, subject, message, ackdata, - sentTime, lastActionTime, sleeptill, status, retryNumber, folder, - encoding, ttl) - - sqlExecute('''INSERT INTO sent VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)''', *t) - return ackdata - else: - return None diff --git a/src/protocol.py b/src/protocol.py index 4f2d0856..8d0bbb8e 100644 --- a/src/protocol.py +++ b/src/protocol.py @@ -14,6 +14,7 @@ from binascii import hexlify from struct import Struct, pack, unpack import defaults +import helper_db import highlevelcrypto import state from addresses import ( @@ -21,7 +22,6 @@ from addresses import ( from bmconfigparser import BMConfigParser from debug import logger from fallback import RIPEMD160Hash -from helper_sql import sqlExecute from version import softwareVersion # Service flags @@ -503,8 +503,7 @@ def decryptAndCheckPubkeyPayload(data, address): hexlify(publicSigningKey), hexlify(publicEncryptionKey) ) - t = (address, addressVersion, storedData, int(time.time()), 'yes') - sqlExecute('''INSERT INTO pubkeys VALUES (?,?,?,?,?)''', *t) + helper_db.put_pubkey(address, addressVersion, storedData, True) return 'successful' except varintDecodeError: logger.info(