From cc2fa62b90d8e609220d15a324a836bbb670cb99 Mon Sep 17 00:00:00 2001 From: Dmitri Bogomolov <4glitch@gmail.com> Date: Mon, 12 Mar 2018 12:55:42 +0200 Subject: [PATCH 1/7] Moved most of frequently used SQL-queries to helper_db module. --- src/bitmessageqt/__init__.py | 149 ++++++++++--------------- src/bitmessageqt/account.py | 121 +++++++-------------- src/bitmessageqt/foldertree.py | 44 ++------ src/bitmessageqt/support.py | 11 +- src/class_objectProcessor.py | 192 ++++++++++----------------------- src/class_singleWorker.py | 25 +---- src/helper_db.py | 180 +++++++++++++++++++++++++++++++ src/helper_sent.py | 48 --------- src/protocol.py | 5 +- 9 files changed, 353 insertions(+), 422 deletions(-) create mode 100644 src/helper_db.py delete mode 100644 src/helper_sent.py diff --git a/src/bitmessageqt/__init__.py b/src/bitmessageqt/__init__.py index eb140fb9..7cab9a43 100644 --- a/src/bitmessageqt/__init__.py +++ b/src/bitmessageqt/__init__.py @@ -35,9 +35,10 @@ from foldertree import ( MessageList_TimeWidget) import settingsmixin import support -from helper_sql import sqlQuery, sqlExecute, sqlExecuteChunked, sqlStoredProcedure -import helper_addressbook -import helper_search +from helper_ackPayload import genAckPayload +from helper_sql import ( + sqlQuery, sqlExecute, sqlExecuteChunked, sqlStoredProcedure) +import helper_db import l10n from utils import str_broadcast_subscribers, avatarize from account import ( @@ -54,7 +55,6 @@ from statusbar import BMStatusBar import sound # This is needed for tray icon import bitmessage_icons_rc # noqa:F401 pylint: disable=unused-import -import helper_sent try: from plugins.plugin import get_plugin, get_plugins @@ -1214,7 +1214,7 @@ class MyForm(settingsmixin.SMainWindow): tableWidget.setColumnHidden(1, bool(account)) xAddress = 'fromaddress' - queryreturn = helper_search.search_sql( + queryreturn = helper_db.search_sql( xAddress, account, "sent", where, what, False) for row in queryreturn: @@ -1254,7 +1254,7 @@ class MyForm(settingsmixin.SMainWindow): tableWidget.setColumnHidden(0, False) tableWidget.setColumnHidden(1, False) - queryreturn = helper_search.search_sql( + queryreturn = helper_db.search_sql( xAddress, account, folder, where, what, unreadOnly) for row in queryreturn: @@ -1937,8 +1937,7 @@ class MyForm(settingsmixin.SMainWindow): newRows = {} # subscriptions - queryreturn = sqlQuery('SELECT label, address FROM subscriptions WHERE enabled = 1') - for row in queryreturn: + for row in helper_db.get_subscriptions(): label, address = row newRows[address] = [label, AccountMixin.SUBSCRIPTION] # chans @@ -1948,8 +1947,7 @@ class MyForm(settingsmixin.SMainWindow): if (account.type == AccountMixin.CHAN and BMConfigParser().safeGetBoolean(address, 'enabled')): newRows[address] = [account.getLabel(), AccountMixin.CHAN] # normal accounts - queryreturn = sqlQuery('SELECT * FROM addressbook') - for row in queryreturn: + for row in helper_db.get_addressbook(): label, address = row newRows[address] = [label, AccountMixin.NORMAL] @@ -2166,18 +2164,19 @@ class MyForm(settingsmixin.SMainWindow): " send the message but it won\'t send until" " you connect.") ) - ackdata = helper_sent.insert( - toAddress=toAddress, fromAddress=fromAddress, - subject=subject, message=message, encoding=encoding) - toLabel = '' - queryreturn = sqlQuery('''select label from addressbook where address=?''', - toAddress) - if queryreturn != []: - for row in queryreturn: - toLabel, = row + stealthLevel = BMConfigParser().safeGetInt( + 'bitmessagesettings', 'ackstealthlevel') + ackdata = genAckPayload(streamNumber, stealthLevel) + helper_db.put_sent( + toAddress, fromAddress, subject, message, ackdata, + 'msgqueued', encoding, ripe + ) + + toLabel = helper_db.get_label(toAddress) or '' self.displayNewSentMessage( - toAddress, toLabel, fromAddress, subject, message, ackdata) + toAddress, toLabel, fromAddress, subject, + message, ackdata) queues.workerQueue.put(('sendmessage', toAddress)) self.click_pushButtonClear() @@ -2202,15 +2201,14 @@ class MyForm(settingsmixin.SMainWindow): # We don't actually need the ackdata for acknowledgement since # this is a broadcast message, but we can use it to update the # user interface when the POW is done generating. - toAddress = str_broadcast_subscribers + streamNumber = decodeAddress(fromAddress)[2] + ackdata = genAckPayload(streamNumber, 0) + toAddress = toLabel = str_broadcast_subscribers - # msgid. We don't know what this will be until the POW is done. - ackdata = helper_sent.insert( - fromAddress=fromAddress, - subject=subject, message=message, - status='broadcastqueued', encoding=encoding) - - toLabel = str_broadcast_subscribers + helper_db.put_sent( + toAddress, fromAddress, subject, message, ackdata, + 'broadcastqueued', encoding + ) self.displayNewSentMessage( toAddress, toLabel, fromAddress, subject, message, ackdata) @@ -2338,7 +2336,7 @@ class MyForm(settingsmixin.SMainWindow): self.ui.treeWidgetChans ) and self.getCurrentAccount(treeWidget) != toAddress: continue - elif not helper_search.check_match( + elif not helper_db.check_match( toAddress, fromAddress, subject, message, self.getCurrentSearchOption(tab), self.getCurrentSearchLine(tab) @@ -2368,7 +2366,7 @@ class MyForm(settingsmixin.SMainWindow): tab += 1 if tab == 1: tab = 2 - if not helper_search.check_match( + if not helper_db.check_match( toAddress, fromAddress, subject, message, self.getCurrentSearchOption(tab), self.getCurrentSearchLine(tab) @@ -2427,10 +2425,7 @@ class MyForm(settingsmixin.SMainWindow): except AttributeError: return - # First we must check to see if the address is already in the - # address book. The user cannot add it again or else it will - # cause problems when updating and deleting the entry. - if shared.isAddressInMyAddressBook(address): + if not helper_db.put_addressbook(label, address): self.updateStatusBar(_translate( "MainWindow", "Error: You cannot add the same address to your" @@ -2439,30 +2434,25 @@ class MyForm(settingsmixin.SMainWindow): )) return - if helper_addressbook.insert(label=label, address=address): - self.rerenderMessagelistFromLabels() - self.rerenderMessagelistToLabels() - self.rerenderAddressBook() - else: - self.updateStatusBar(_translate( - "MainWindow", - "Error: You cannot add your own address in the address book." - )) + self.rerenderMessagelistFromLabels() + self.rerenderMessagelistToLabels() + self.rerenderAddressBook() def addSubscription(self, address, label): - # This should be handled outside of this function, for error displaying - # and such, but it must also be checked here. - if shared.isAddressInMySubscriptionsList(address): + if not helper_db.put_subscriptions(label, address): + self.updateStatusBar(_translate( + "MainWindow", + "Error: You cannot add the same address to your" + " subscriptions twice. Perhaps rename the existing one" + " if you want." + )) return - # Add to database (perhaps this should be separated from the MyForm class) - sqlExecute( - '''INSERT INTO subscriptions VALUES (?,?,?)''', - label, address, True - ) + self.rerenderMessagelistFromLabels() shared.reloadBroadcastSendersForWhichImWatching() self.rerenderAddressBook() self.rerenderTabTreeSubscriptions() + return True def click_pushButtonAddSubscription(self): dialog = dialogs.NewSubscriptionDialog(self) @@ -2472,19 +2462,9 @@ class MyForm(settingsmixin.SMainWindow): except AttributeError: return - # We must check to see if the address is already in the - # subscriptions list. The user cannot add it again or else it - # will cause problems when updating and deleting the entry. - if shared.isAddressInMySubscriptionsList(address): - self.updateStatusBar(_translate( - "MainWindow", - "Error: You cannot add the same address to your" - " subscriptions twice. Perhaps rename the existing one" - " if you want." - )) + if not self.addSubscription(address, label): return - self.addSubscription(address, label) # Now, if the user wants to display old broadcasts, let's get # them out of the inventory and put them # to the objectProcessorQueue to be processed @@ -2815,11 +2795,7 @@ class MyForm(settingsmixin.SMainWindow): textEdit = self.getCurrentMessageTextedit() if not msgid: return - queryreturn = sqlQuery( - '''select message from inbox where msgid=?''', msgid) - if queryreturn != []: - for row in queryreturn: - messageText, = row + messageText = helper_db.get_message(msgid) lines = messageText.split('\n') totalLines = len(lines) @@ -3052,24 +3028,20 @@ class MyForm(settingsmixin.SMainWindow): currentInboxRow, 1).data(QtCore.Qt.UserRole) recipientAddress = tableWidget.item( currentInboxRow, 0).data(QtCore.Qt.UserRole) - # Let's make sure that it isn't already in the address book - queryreturn = sqlQuery('''select * from blacklist where address=?''', - addressAtCurrentInboxRow) - if queryreturn == []: - label = "\"" + tableWidget.item(currentInboxRow, 2).subject + "\" in " + BMConfigParser().get(recipientAddress, "label") - sqlExecute('''INSERT INTO blacklist VALUES (?,?, ?)''', - label, - addressAtCurrentInboxRow, True) - self.ui.blackwhitelist.rerenderBlackWhiteList() - self.updateStatusBar(_translate( - "MainWindow", - "Entry added to the blacklist. Edit the label to your liking.") - ) - else: + label = "\"" + tableWidget.item(currentInboxRow, 2).subject + \ + "\" in " + BMConfigParser().get(recipientAddress, "label") + if not helper_db.put_blacklist(label, addressAtCurrentInboxRow): self.updateStatusBar(_translate( "MainWindow", "Error: You cannot add the same address to your blacklist" " twice. Try renaming the existing one if you want.")) + return + + self.ui.blackwhitelist.rerenderBlackWhiteList() + self.updateStatusBar(_translate( + "MainWindow", + "Entry added to the blacklist. Edit the label to your liking.") + ) def deleteRowFromMessagelist( self, row=None, inventoryHash=None, ackData=None, messageLists=None @@ -3168,11 +3140,7 @@ class MyForm(settingsmixin.SMainWindow): # Retrieve the message data out of the SQL database msgid = tableWidget.item(currentInboxRow, 3).data() - queryreturn = sqlQuery( - '''select message from inbox where msgid=?''', msgid) - if queryreturn != []: - for row in queryreturn: - message, = row + message = helper_db.get_message(msgid) defaultFilename = "".join(x for x in subjectAtCurrentInboxRow if x.isalnum()) + '.txt' filename = QtGui.QFileDialog.getSaveFileName(self, _translate("MainWindow","Save As..."), defaultFilename, "Text files (*.txt);;All files (*.*)") @@ -3280,15 +3248,6 @@ class MyForm(settingsmixin.SMainWindow): def on_action_AddressBookSubscribe(self): for item in self.getAddressbookSelectedItems(): - # Then subscribe to it... - # provided it's not already in the address book - if shared.isAddressInMySubscriptionsList(item.address): - self.updateStatusBar(_translate( - "MainWindow", - "Error: You cannot add the same address to your" - " subscriptions twice. Perhaps rename the existing" - " one if you want.")) - continue self.addSubscription(item.address, item.label) self.ui.tabWidget.setCurrentIndex( self.ui.tabWidget.indexOf(self.ui.subscriptions) diff --git a/src/bitmessageqt/account.py b/src/bitmessageqt/account.py index 50ea3548..374d1e91 100644 --- a/src/bitmessageqt/account.py +++ b/src/bitmessageqt/account.py @@ -4,43 +4,30 @@ account.py ========== Account related functions. - """ -from __future__ import absolute_import - import inspect import re import sys -import time - -from PyQt4 import QtGui +import helper_db import queues from addresses import decodeAddress from bmconfigparser import BMConfigParser from helper_ackPayload import genAckPayload -from helper_sql import sqlQuery, sqlExecute -from .foldertree import AccountMixin -from .utils import str_broadcast_subscribers +from helper_sql import sqlQuery +from foldertree import AccountMixin +from utils import str_broadcast_subscribers +from tr import _translate def getSortedAccounts(): """Get a sorted list of configSections""" - configSections = BMConfigParser().addresses() configSections.sort( cmp=lambda x, y: cmp( - unicode( - BMConfigParser().get( - x, - 'label'), - 'utf-8').lower(), - unicode( - BMConfigParser().get( - y, - 'label'), - 'utf-8').lower())) + unicode(BMConfigParser().get(x, 'label'), 'utf-8').lower(), + unicode(BMConfigParser().get(y, 'label'), 'utf-8').lower())) return configSections @@ -107,29 +94,27 @@ def accountClass(address): class AccountColor(AccountMixin): # pylint: disable=too-few-public-methods """Set the type of account""" - def __init__(self, address, address_type=None): self.isEnabled = True self.address = address - if address_type is None: - if address is None: - self.type = AccountMixin.ALL - elif BMConfigParser().safeGetBoolean(self.address, 'mailinglist'): - self.type = AccountMixin.MAILINGLIST - elif BMConfigParser().safeGetBoolean(self.address, 'chan'): - self.type = AccountMixin.CHAN - elif sqlQuery( - '''select label from subscriptions where address=?''', self.address): - self.type = AccountMixin.SUBSCRIPTION - else: - self.type = AccountMixin.NORMAL - else: + if address_type: self.type = address_type + return + # AccountMixin.setType() + if self.address is None: + self.type = self.ALL + elif BMConfigParser().safeGetBoolean(self.address, 'chan'): + self.type = self.CHAN + elif BMConfigParser().safeGetBoolean(self.address, 'mailinglist'): + self.type = self.MAILINGLIST + elif helper_db.get_label(self.address, "subscriptions"): + self.type = AccountMixin.SUBSCRIPTION + else: + self.type = self.NORMAL class BMAccount(object): """Encapsulate a Bitmessage account""" - def __init__(self, address=None): self.address = address self.type = AccountMixin.NORMAL @@ -140,29 +125,19 @@ class BMAccount(object): self.type = AccountMixin.MAILINGLIST elif self.address == str_broadcast_subscribers: self.type = AccountMixin.BROADCAST - else: - queryreturn = sqlQuery( - '''select label from subscriptions where address=?''', self.address) - if queryreturn: - self.type = AccountMixin.SUBSCRIPTION + elif helper_db.get_label(self.address, "subscriptions"): + self.type = AccountMixin.SUBSCRIPTION def getLabel(self, address=None): """Get a label for this bitmessage account""" if address is None: address = self.address - label = BMConfigParser().safeGet(address, 'label', address) - queryreturn = sqlQuery( - '''select label from addressbook where address=?''', address) - if queryreturn != []: - for row in queryreturn: - label, = row - else: - queryreturn = sqlQuery( - '''select label from subscriptions where address=?''', address) - if queryreturn != []: - for row in queryreturn: - label, = row - return label + if BMConfigParser().has_section(address): + return BMConfigParser().get(address, 'label') + return ( + helper_db.get_label(address) or + helper_db.get_label(address, "subscriptions") or address + ) def parseMessage(self, toAddress, fromAddress, subject, message): """Set metadata and address labels on self""" @@ -186,9 +161,7 @@ class NoAccount(BMAccount): self.type = AccountMixin.NORMAL def getLabel(self, address=None): - if address is None: - address = self.address - return address + return address or self.address class SubscriptionAccount(BMAccount): @@ -213,31 +186,17 @@ class GatewayAccount(BMAccount): def send(self): """Override the send method for gateway accounts""" - - # pylint: disable=unused-variable - status, addressVersionNumber, streamNumber, ripe = decodeAddress(self.toAddress) - stealthLevel = BMConfigParser().safeGetInt('bitmessagesettings', 'ackstealthlevel') + streamNumber, ripe = decodeAddress(self.toAddress)[2:] + stealthLevel = BMConfigParser().safeGetInt( + 'bitmessagesettings', 'ackstealthlevel') ackdata = genAckPayload(streamNumber, stealthLevel) - sqlExecute( - '''INSERT INTO sent VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)''', - '', - self.toAddress, - ripe, - self.fromAddress, - self.subject, - self.message, - ackdata, - int(time.time()), # sentTime (this will never change) - int(time.time()), # lastActionTime - 0, # sleepTill time. This will get set when the POW gets done. - 'msgqueued', - 0, # retryNumber - 'sent', # folder - 2, # encodingtype - # not necessary to have a TTL higher than 2 days - min(BMConfigParser().getint('bitmessagesettings', 'ttl'), 86400 * 2) + helper_db.put_sent( + self.toAddress, self.fromAddress, self.subject, self.message, + ackdata, 'msgqueued', 2, ripe, + min( + BMConfigParser().getint('bitmessagesettings', 'ttl'), + 86400 * 2) # not necessary to have a TTL higher than 2 days ) - queues.workerQueue.put(('sendmessage', self.toAddress)) @@ -292,9 +251,7 @@ class MailchuckAccount(GatewayAccount): self.toAddress = self.registrationAddress self.subject = "config" - self.message = QtGui.QApplication.translate( - "Mailchuck", - """# You can use this to configure your email gateway account + self.message = _translate("Mailchuck", """# You can use this to configure your email gateway account # Uncomment the setting you want to use # Here are the options: # diff --git a/src/bitmessageqt/foldertree.py b/src/bitmessageqt/foldertree.py index e6d64427..78af13f8 100644 --- a/src/bitmessageqt/foldertree.py +++ b/src/bitmessageqt/foldertree.py @@ -8,8 +8,8 @@ from cgi import escape from PyQt4 import QtCore, QtGui +import helper_db from bmconfigparser import BMConfigParser -from helper_sql import sqlExecute, sqlQuery from settingsmixin import SettingsMixin from tr import _translate from utils import avatarize @@ -110,15 +110,13 @@ class AccountMixin(object): self.type = self.CHAN elif BMConfigParser().safeGetBoolean(self.address, 'mailinglist'): self.type = self.MAILINGLIST - elif sqlQuery( - '''select label from subscriptions where address=?''', self.address): + elif helper_db.get_label(self.address, "subscriptions"): self.type = AccountMixin.SUBSCRIPTION else: self.type = self.NORMAL def defaultLabel(self): """Default label (in case no label is set manually)""" - queryreturn = None retval = None if self.type in ( AccountMixin.NORMAL, @@ -127,16 +125,9 @@ class AccountMixin(object): retval = unicode( BMConfigParser().get(self.address, 'label'), 'utf-8') except Exception: - queryreturn = sqlQuery( - '''select label from addressbook where address=?''', self.address) + retval = helper_db.get_label(self.address) elif self.type == AccountMixin.SUBSCRIPTION: - queryreturn = sqlQuery( - '''select label from subscriptions where address=?''', self.address) - if queryreturn is not None: - if queryreturn != []: - for row in queryreturn: - retval, = row - retval = unicode(retval, 'utf-8') + retval = helper_db.get_label(self.address, "subscriptions") elif self.address is None or self.type == AccountMixin.ALL: return unicode( str(_translate("MainWindow", "All accounts")), 'utf-8') @@ -306,13 +297,8 @@ class Ui_SubscriptionWidget(Ui_AddressWidget): parent, pos, address, unreadCount, enabled) def _getLabel(self): - queryreturn = sqlQuery( - '''select label from subscriptions where address=?''', self.address) - if queryreturn != []: - for row in queryreturn: - retval, = row - return unicode(retval, 'utf-8', 'ignore') - return unicode(self.address, 'utf-8') + return helper_db.get_label(self.address, "subscriptions") \ + or self.address def setType(self): """Set account type""" @@ -327,9 +313,7 @@ class Ui_SubscriptionWidget(Ui_AddressWidget): value.toString().toUtf8()).decode('utf-8', 'ignore') else: label = unicode(value, 'utf-8', 'ignore') - sqlExecute( - '''UPDATE subscriptions SET label=? WHERE address=?''', - label, self.address) + helper_db.set_label(self.address, label, "subscriptions") return super(Ui_SubscriptionWidget, self).setData(column, role, value) @@ -402,7 +386,6 @@ class MessageList_AddressWidget(BMAddressWidget): if label is not None: return newLabel = self.address - queryreturn = None if self.type in ( AccountMixin.NORMAL, AccountMixin.CHAN, AccountMixin.MAILINGLIST): @@ -411,14 +394,9 @@ class MessageList_AddressWidget(BMAddressWidget): BMConfigParser().get(self.address, 'label'), 'utf-8', 'ignore') except: - queryreturn = sqlQuery( - '''select label from addressbook where address=?''', self.address) + newLabel = helper_db.get_label(self.address) elif self.type == AccountMixin.SUBSCRIPTION: - queryreturn = sqlQuery( - '''select label from subscriptions where address=?''', self.address) - if queryreturn: - for row in queryreturn: - newLabel = unicode(row[0], 'utf-8', 'ignore') + newLabel = helper_db.get_label(self.address, "subscriptions") self.label = newLabel @@ -525,9 +503,9 @@ class Ui_AddressBookWidgetItem(BMAddressWidget): BMConfigParser().set(self.address, 'label', self.label) BMConfigParser().save() except: - sqlExecute('''UPDATE addressbook set label=? WHERE address=?''', self.label, self.address) + helper_db.set_label(self.address, self.label) elif self.type == AccountMixin.SUBSCRIPTION: - sqlExecute('''UPDATE subscriptions set label=? WHERE address=?''', self.label, self.address) + helper_db.set_label(self.address, self.label, "subscriptions") else: pass return super(Ui_AddressBookWidgetItem, self).setData(role, value) diff --git a/src/bitmessageqt/support.py b/src/bitmessageqt/support.py index ac02e2ca..31cfc0dc 100644 --- a/src/bitmessageqt/support.py +++ b/src/bitmessageqt/support.py @@ -10,6 +10,7 @@ from PyQt4 import QtCore import account import defaults +import helper_db import network.stats import paths import proofofwork @@ -17,7 +18,7 @@ import queues import state from bmconfigparser import BMConfigParser from foldertree import AccountMixin -from helper_sql import sqlExecute, sqlQuery +from helper_sql import sqlExecute from l10n import getTranslationLanguage from openclpow import openclEnabled from pyelliptic.openssl import OpenSSL @@ -67,12 +68,8 @@ Connected hosts: {} def checkAddressBook(myapp): - sqlExecute('DELETE from addressbook WHERE address=?', OLD_SUPPORT_ADDRESS) - queryreturn = sqlQuery('SELECT * FROM addressbook WHERE address=?', SUPPORT_ADDRESS) - if queryreturn == []: - sqlExecute( - 'INSERT INTO addressbook VALUES (?,?)', - SUPPORT_LABEL.toUtf8(), SUPPORT_ADDRESS) + sqlExecute('DELETE FROM addressbook WHERE address=?', OLD_SUPPORT_ADDRESS) + if helper_db.put_addressbook(SUPPORT_LABEL.toUtf8(), SUPPORT_ADDRESS): myapp.rerenderAddressBook() diff --git a/src/class_objectProcessor.py b/src/class_objectProcessor.py index 6fa31a47..f8dea48d 100644 --- a/src/class_objectProcessor.py +++ b/src/class_objectProcessor.py @@ -10,12 +10,10 @@ import random import threading import time from binascii import hexlify -from subprocess import call # nosec import helper_bitcoin -import helper_inbox import helper_msgcoding -import helper_sent +import helper_db import highlevelcrypto import l10n import protocol @@ -29,10 +27,10 @@ from addresses import ( ) from bmconfigparser import BMConfigParser from fallback import RIPEMD160Hash +from helper_ackPayload import genAckPayload from helper_sql import sql_ready, SqlBulkExecute, sqlExecute, sqlQuery from network import bmproto, knownnodes from network.node import Peer -# pylint: disable=too-many-locals, too-many-return-statements, too-many-branches, too-many-statements logger = logging.getLogger('default') @@ -343,24 +341,7 @@ class objectProcessor(threading.Thread): ) address = encodeAddress(addressVersion, streamNumber, ripe) - - queryreturn = sqlQuery( - "SELECT usedpersonally FROM pubkeys WHERE address=?" - " AND usedpersonally='yes'", address) - # if this pubkey is already in our database and if we have - # used it personally: - if queryreturn != []: - logger.info( - 'We HAVE used this pubkey personally. Updating time.') - t = (address, addressVersion, dataToStore, - int(time.time()), 'yes') - else: - logger.info( - 'We have NOT used this pubkey personally. Inserting' - ' in database.') - t = (address, addressVersion, dataToStore, - int(time.time()), 'no') - sqlExecute('''INSERT INTO pubkeys VALUES (?,?,?,?,?)''', *t) + helper_db.put_pubkey(address, addressVersion, dataToStore) self.possibleNewPubkey(address) if addressVersion == 3: if len(data) < 170: # sanity check. @@ -408,23 +389,7 @@ class objectProcessor(threading.Thread): ) address = encodeAddress(addressVersion, streamNumber, ripe) - queryreturn = sqlQuery( - "SELECT usedpersonally FROM pubkeys WHERE address=?" - " AND usedpersonally='yes'", address) - # if this pubkey is already in our database and if we have - # used it personally: - if queryreturn != []: - logger.info( - 'We HAVE used this pubkey personally. Updating time.') - t = (address, addressVersion, dataToStore, - int(time.time()), 'yes') - else: - logger.info( - 'We have NOT used this pubkey personally. Inserting' - ' in database.') - t = (address, addressVersion, dataToStore, - int(time.time()), 'no') - sqlExecute('''INSERT INTO pubkeys VALUES (?,?,?,?,?)''', *t) + helper_db.put_pubkey(address, addressVersion, dataToStore) self.possibleNewPubkey(address) if addressVersion == 4: @@ -623,13 +588,10 @@ class objectProcessor(threading.Thread): # Let's store the public key in case we want to reply to this # person. - sqlExecute( - '''INSERT INTO pubkeys VALUES (?,?,?,?,?)''', - fromAddress, - sendersAddressVersionNumber, - decryptedData[:endOfThePublicKeyPosition], - int(time.time()), - 'yes') + helper_db.put_pubkey( + fromAddress, sendersAddressVersionNumber, + decryptedData[:endOfThePublicKeyPosition], True + ) # Check to see whether we happen to be awaiting this # pubkey in order to send a message. If we are, it will do the POW @@ -693,78 +655,56 @@ class objectProcessor(threading.Thread): subject = decodedMessage.subject body = decodedMessage.body - # Let us make sure that we haven't already received this message - if helper_inbox.isMessageAlreadyInInbox(sigHash): - logger.info('This msg is already in our inbox. Ignoring it.') - blockMessage = True - if not blockMessage: - if messageEncodingType != 0: - t = (inventoryHash, toAddress, fromAddress, subject, - int(time.time()), body, 'inbox', messageEncodingType, - 0, sigHash) - helper_inbox.insert(t) + if helper_db.put_inbox( + toAddress, fromAddress, subject, body, inventoryHash, sigHash, + encoding=messageEncodingType + ) is False: + # logger.info('This msg is already in our inbox. Ignoring it.') + return - queues.UISignalQueue.put(('displayNewInboxMessage', ( - inventoryHash, toAddress, fromAddress, subject, body))) + # Let us now check and see whether our receiving address is + # behaving as a mailing list + if BMConfigParser().safeGetBoolean(toAddress, 'mailinglist'): + mailingListName = BMConfigParser().safeGet( + toAddress, 'mailinglistname', '') - # If we are behaving as an API then we might need to run an - # outside command to let some program know that a new message - # has arrived. - if BMConfigParser().safeGetBoolean( - 'bitmessagesettings', 'apienabled'): - try: - apiNotifyPath = BMConfigParser().get( - 'bitmessagesettings', 'apinotifypath') - except: - apiNotifyPath = '' - if apiNotifyPath != '': - call([apiNotifyPath, "newMessage"]) + # Let us send out this message as a broadcast + subject = self.addMailingListNameToSubject( + subject, mailingListName) + # Let us now send this message out as a broadcast + message = time.strftime("%a, %Y-%m-%d %H:%M:%S UTC", time.gmtime( + )) + ' Message ostensibly from ' + fromAddress + ':\n\n' + body + # The fromAddress for the broadcast that we are about to + # send is the toAddress (my address) for the msg message + # we are currently processing. + fromAddress = toAddress + # We don't actually need the ackdata for acknowledgement + # since this is a broadcast message but we can use it to + # update the user interface when the POW is done generating. + streamNumber = decodeAddress(fromAddress)[2] - # Let us now check and see whether our receiving address is - # behaving as a mailing list - if BMConfigParser().safeGetBoolean(toAddress, 'mailinglist') \ - and messageEncodingType != 0: - try: - mailingListName = BMConfigParser().get( - toAddress, 'mailinglistname') - except: - mailingListName = '' - # Let us send out this message as a broadcast - subject = self.addMailingListNameToSubject( - subject, mailingListName) - # Let us now send this message out as a broadcast - message = time.strftime( - "%a, %Y-%m-%d %H:%M:%S UTC", time.gmtime() - ) + ' Message ostensibly from ' + fromAddress \ - + ':\n\n' + body - # The fromAddress for the broadcast that we are about to - # send is the toAddress (my address) for the msg message - # we are currently processing. - fromAddress = toAddress - # We don't actually need the ackdata for acknowledgement - # since this is a broadcast message but we can use it to - # update the user interface when the POW is done generating. - toAddress = '[Broadcast subscribers]' + ackdata = genAckPayload(streamNumber, 0) + toAddress = toLabel = '[Broadcast subscribers]' - ackdata = helper_sent.insert( - fromAddress=fromAddress, - status='broadcastqueued', - subject=subject, - message=message, - encoding=messageEncodingType) + # We really should have a discussion about how to + # set the TTL for mailing list broadcasts. This is obviously + # hard-coded. + TTL = 2 * 7 * 24 * 60 * 60 # 2 weeks + helper_db.put_sent( + toAddress, fromAddress, subject, message, ackdata, + 'broadcastqueued', messageEncodingType, ttl=TTL + ) - queues.UISignalQueue.put(( - 'displayNewSentMessage', ( - toAddress, '[Broadcast subscribers]', fromAddress, - subject, message, ackdata) - )) - queues.workerQueue.put(('sendbroadcast', '')) + queues.UISignalQueue.put(( + 'displayNewSentMessage', + (toAddress, toLabel, fromAddress, subject, message, ackdata) + )) + queues.workerQueue.put(('sendbroadcast', '')) # Don't send ACK if invalid, blacklisted senders, invisible # messages, disabled or chan if ( self.ackDataHasAValidHeader(ackData) and not blockMessage and - messageEncodingType != 0 and not BMConfigParser().safeGetBoolean(toAddress, 'dontsendack') and not BMConfigParser().safeGetBoolean(toAddress, 'chan') ): @@ -975,12 +915,11 @@ class objectProcessor(threading.Thread): logger.info('fromAddress: %s', fromAddress) # Let's store the public key in case we want to reply to this person. - sqlExecute('''INSERT INTO pubkeys VALUES (?,?,?,?,?)''', - fromAddress, - sendersAddressVersion, - decryptedData[:endOfPubkeyPosition], - int(time.time()), - 'yes') + helper_db.put_pubkey( + fromAddress, sendersAddressVersion, + decryptedData[:endOfPubkeyPosition], + True + ) # Check to see whether we happen to be awaiting this # pubkey in order to send a message. If we are, it will do the POW @@ -1000,27 +939,12 @@ class objectProcessor(threading.Thread): body = decodedMessage.body toAddress = '[Broadcast subscribers]' - if helper_inbox.isMessageAlreadyInInbox(sigHash): - logger.info('This broadcast is already in our inbox. Ignoring it.') + if helper_db.put_inbox( + toAddress, fromAddress, subject, body, inventoryHash, sigHash, + encoding=messageEncodingType + ) is False: + # logger.info('This broadcast is already in inbox. Ignoring it.') return - t = (inventoryHash, toAddress, fromAddress, subject, int( - time.time()), body, 'inbox', messageEncodingType, 0, sigHash) - helper_inbox.insert(t) - - queues.UISignalQueue.put(('displayNewInboxMessage', ( - inventoryHash, toAddress, fromAddress, subject, body))) - - # If we are behaving as an API then we might need to run an - # outside command to let some program know that a new message - # has arrived. - if BMConfigParser().safeGetBoolean('bitmessagesettings', 'apienabled'): - try: - apiNotifyPath = BMConfigParser().get( - 'bitmessagesettings', 'apinotifypath') - except: - apiNotifyPath = '' - if apiNotifyPath != '': - call([apiNotifyPath, "newBroadcast"]) # Display timing data logger.info( diff --git a/src/class_singleWorker.py b/src/class_singleWorker.py index 012313ac..9551570b 100644 --- a/src/class_singleWorker.py +++ b/src/class_singleWorker.py @@ -10,10 +10,9 @@ import hashlib import time from binascii import hexlify, unhexlify from struct import pack -from subprocess import call # nosec import defaults -import helper_inbox +import helper_db import helper_msgcoding import helper_random import helper_sql @@ -1312,25 +1311,11 @@ class singleWorker(StoppableThread): # Used to detect and ignore duplicate messages in our inbox sigHash = hashlib.sha512(hashlib.sha512( signature).digest()).digest()[32:] - t = (inventoryHash, toaddress, fromaddress, subject, int( - time.time()), message, 'inbox', encoding, 0, sigHash) - helper_inbox.insert(t) - queues.UISignalQueue.put(('displayNewInboxMessage', ( - inventoryHash, toaddress, fromaddress, subject, message))) - - # If we are behaving as an API then we might need to run an - # outside command to let some program know that a new message - # has arrived. - if BMConfigParser().safeGetBoolean( - 'bitmessagesettings', 'apienabled'): - try: - apiNotifyPath = BMConfigParser().get( - 'bitmessagesettings', 'apinotifypath') - except: - apiNotifyPath = '' - if apiNotifyPath != '': - call([apiNotifyPath, "newMessage"]) + helper_db.put_inbox( + toaddress, fromaddress, subject, message, inventoryHash, sigHash, + encoding=encoding + ) def requestPubKey(self, toAddress): """Send a getpubkey object""" diff --git a/src/helper_db.py b/src/helper_db.py new file mode 100644 index 00000000..33190ff6 --- /dev/null +++ b/src/helper_db.py @@ -0,0 +1,180 @@ +import subprocess # nosec +import time + +import queues +from bmconfigparser import BMConfigParser +from helper_search import search_sql, check_match +from helper_sql import sqlExecute, sqlQuery + + +__all__ = ["search_sql", "check_match"] + + +# + genAckPayload +def put_sent( + to_address, from_address, subject, message, ackdata, + status, encoding, + ripe='', ttl=0, msgid='', sent_time=None, last_action_time=None, + sleep_till_time=0, retrynumber=0): + """Put message into Sent table""" + # We don't know msgid until the POW is done. + # sleep_till_time will get set when the POW gets done + if not sent_time: + sent_time = time.time() + if not last_action_time: + last_action_time = sent_time + if not ttl: + ttl = BMConfigParser().getint('bitmessagesettings', 'ttl') + sqlExecute( + "INSERT INTO sent VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + msgid, to_address, ripe, from_address, subject, message, ackdata, + int(sent_time), int(last_action_time), sleep_till_time, status, retrynumber, + 'sent', encoding, ttl + ) + + +def _in_inbox_already(sighash): + return sqlQuery( + "SELECT COUNT(*) FROM inbox WHERE sighash=?", sighash + )[0][0] != 0 + + +def put_inbox( + to_address, from_address, subject, message, msgid, sighash, + encoding=0, received_time=None, broadcast=False): + """Put message into Inbox table""" + if not received_time: + received_time = time.time() + if encoding == 0 or _in_inbox_already(sighash): + return False + + sqlExecute( + "INSERT INTO inbox VALUES (?,?,?,?,?,?,?,?,?,?)", + msgid, to_address, from_address, subject, int(received_time), message, + 'inbox', encoding, 0, sighash + ) + + queues.UISignalQueue.put(( + 'displayNewInboxMessage', + (msgid, to_address, from_address, subject, message) + )) + + # If we are behaving as an API then we might need to run an + # outside command to let some program know that a new message + # has arrived. + if BMConfigParser().safeGetBoolean('bitmessagesettings', 'apienabled'): + apinotify_path = BMConfigParser().safeGet( + 'bitmessagesettings', 'apinotifypath') + if apinotify_path: + subprocess.call([ + apinotify_path, + "newBroadcast" if broadcast else "newMessage"]) + + +def put_pubkey(address, address_version, data, used_personally=None): + """Put pubkey into Pubkeys table""" + if used_personally is None: + if sqlQuery( + "SELECT * FROM pubkeys WHERE address=? AND usedpersonally='yes'", + address + ) == []: + used_personally = False + else: + sqlExecute( + "UPDATE pubkeys SET time=? WHERE address=?", + time.time(), address + ) + return + sqlExecute( + "INSERT INTO pubkeys VALUES (?,?,?,?,?)", + address, address_version, data, time.time(), + 'yes' if used_personally else 'no' + ) + + +def _in_group_already(address, group="addressbook"): + return sqlQuery( + "SELECT enabled FROM %s WHERE address=?" % group, address) + + +def put_addresslist(label, address, group="blacklist", enabled=True): + """Put address into address list (e.g. blacklist, subscriptions...)""" + # We must check to see if the address is already in the + # subscriptions list. The user cannot add it again or else it + # will cause problems when updating and deleting the entry. + # FIXME: address should be primary key in this case + if _in_group_already(address, group): + return False + sqlExecute( + "INSERT INTO %s VALUES (?,?,?)" % group, label, address, enabled) + return True + + +def put_blacklist(label, address): + """Put address into blacklist""" + return put_addresslist(label, address, "blacklist") + + +def put_subscriptions(label, address, enabled=True): + """Put address into subscriptions""" + return put_addresslist(label, address, "subscriptions", enabled) + + +def put_addressbook(label, address): + """Put address into Addressbook""" + # First we must check to see if the address is already in the + # address book. The user cannot add it again or else it will + # cause problems when updating and deleting the entry. + if _in_group_already(address): + return False + sqlExecute("INSERT INTO addressbook VALUES (?,?)", label, address) + return True + + +def get_subscriptions(): + """Generator for Subscriptions""" + queryreturn = sqlQuery( + "SELECT label, address FROM subscriptions WHERE enabled=1") + for row in queryreturn: + yield row + + +def get_addressbook(): + """Generator for Addressbook""" + queryreturn = sqlQuery("SELECT * FROM addressbook") + for row in queryreturn: + yield row + + +def get_addresslist(group="blacklist"): + """Generator for address list given by group arg""" + queryreturn = sqlQuery("SELECT * FROM %s" % group) + for row in queryreturn: + yield row + + +def get_label(address, group="addressbook"): + """ + Get address label from address list given by group arg + (default is addressbook) + """ + queryreturn = sqlQuery( + "SELECT label FROM %s WHERE address=?" % group, address) + try: + return unicode(queryreturn[-1][0], 'utf-8') + except IndexError: + pass + + +def set_label(address, label, group="addressbook"): + """Set address label in the address list given by group arg""" + sqlExecute("UPDATE %s set label=? WHERE address=?" % group, label, address) + + +def get_message(msgid): + """Get inbox message by msgid""" + queryreturn = sqlQuery("SELECT message FROM inbox WHERE msgid=?", msgid) + try: + return queryreturn[-1][0] + except IndexError: + return '' diff --git a/src/helper_sent.py b/src/helper_sent.py deleted file mode 100644 index d83afce6..00000000 --- a/src/helper_sent.py +++ /dev/null @@ -1,48 +0,0 @@ -""" -Insert values into sent table -""" - -import time -import uuid -from addresses import decodeAddress -from bmconfigparser import BMConfigParser -from helper_ackPayload import genAckPayload -from helper_sql import sqlExecute - - -# pylint: disable=too-many-arguments -def insert(msgid=None, toAddress='[Broadcast subscribers]', fromAddress=None, subject=None, - message=None, status='msgqueued', ripe=None, ackdata=None, sentTime=None, - lastActionTime=None, sleeptill=0, retryNumber=0, encoding=2, ttl=None, folder='sent'): - """Perform an insert into the `sent` table""" - # pylint: disable=unused-variable - # pylint: disable-msg=too-many-locals - - valid_addr = True - if not ripe or not ackdata: - addr = fromAddress if toAddress == '[Broadcast subscribers]' else toAddress - new_status, addressVersionNumber, streamNumber, new_ripe = decodeAddress(addr) - valid_addr = True if new_status == 'success' else False - if not ripe: - ripe = new_ripe - - if not ackdata: - stealthLevel = BMConfigParser().safeGetInt( - 'bitmessagesettings', 'ackstealthlevel') - new_ackdata = genAckPayload(streamNumber, stealthLevel) - ackdata = new_ackdata - if valid_addr: - msgid = msgid if msgid else uuid.uuid4().bytes - sentTime = sentTime if sentTime else int(time.time()) # sentTime (this doesn't change) - lastActionTime = lastActionTime if lastActionTime else int(time.time()) - - ttl = ttl if ttl else BMConfigParser().getint('bitmessagesettings', 'ttl') - - t = (msgid, toAddress, ripe, fromAddress, subject, message, ackdata, - sentTime, lastActionTime, sleeptill, status, retryNumber, folder, - encoding, ttl) - - sqlExecute('''INSERT INTO sent VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)''', *t) - return ackdata - else: - return None diff --git a/src/protocol.py b/src/protocol.py index 4f2d0856..8d0bbb8e 100644 --- a/src/protocol.py +++ b/src/protocol.py @@ -14,6 +14,7 @@ from binascii import hexlify from struct import Struct, pack, unpack import defaults +import helper_db import highlevelcrypto import state from addresses import ( @@ -21,7 +22,6 @@ from addresses import ( from bmconfigparser import BMConfigParser from debug import logger from fallback import RIPEMD160Hash -from helper_sql import sqlExecute from version import softwareVersion # Service flags @@ -503,8 +503,7 @@ def decryptAndCheckPubkeyPayload(data, address): hexlify(publicSigningKey), hexlify(publicEncryptionKey) ) - t = (address, addressVersion, storedData, int(time.time()), 'yes') - sqlExecute('''INSERT INTO pubkeys VALUES (?,?,?,?,?)''', *t) + helper_db.put_pubkey(address, addressVersion, storedData, True) return 'successful' except varintDecodeError: logger.info( -- 2.45.1 From 9bca4faf9fcaebf99319ce3f19c58114b5a92147 Mon Sep 17 00:00:00 2001 From: Dmitri Bogomolov <4glitch@gmail.com> Date: Mon, 12 Mar 2018 18:28:32 +0200 Subject: [PATCH 2/7] Added additional checks against SQL-injection --- src/helper_db.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/helper_db.py b/src/helper_db.py index 33190ff6..7ff6d65d 100644 --- a/src/helper_db.py +++ b/src/helper_db.py @@ -9,6 +9,9 @@ from helper_sql import sqlExecute, sqlQuery __all__ = ["search_sql", "check_match"] +_groups = ("blacklist", "whitelist", "subscriptions", "addressbook") +_groups_enable = ("blacklist", "whitelist", "subscriptions") + # + genAckPayload def put_sent( @@ -93,8 +96,18 @@ def put_pubkey(address, address_version, data, used_personally=None): def _in_group_already(address, group="addressbook"): - return sqlQuery( - "SELECT enabled FROM %s WHERE address=?" % group, address) + if group not in _groups: + return True + # elif group in _groups_enable: + # try: + # return sqlQuery( + # "SELECT enabled FROM %s WHERE address=?" % group, address + # )[-1][0] + # except IndexError: + # return + else: + return sqlQuery( + "SELECT * FROM %s WHERE address=?" % group, address) def put_addresslist(label, address, group="blacklist", enabled=True): @@ -148,6 +161,8 @@ def get_addressbook(): def get_addresslist(group="blacklist"): """Generator for address list given by group arg""" + if group not in _groups: + return queryreturn = sqlQuery("SELECT * FROM %s" % group) for row in queryreturn: yield row @@ -158,6 +173,8 @@ def get_label(address, group="addressbook"): Get address label from address list given by group arg (default is addressbook) """ + if group not in _groups: + return queryreturn = sqlQuery( "SELECT label FROM %s WHERE address=?" % group, address) try: @@ -168,6 +185,8 @@ def get_label(address, group="addressbook"): def set_label(address, label, group="addressbook"): """Set address label in the address list given by group arg""" + if group not in _groups: + return sqlExecute("UPDATE %s set label=? WHERE address=?" % group, label, address) -- 2.45.1 From 7dd75eec21c923c0bb3d0c163e4dfb152796e32a Mon Sep 17 00:00:00 2001 From: Dmitri Bogomolov <4glitch@gmail.com> Date: Mon, 5 Aug 2019 17:59:55 +0300 Subject: [PATCH 3/7] Partially replaced queries in bitmessageqt.blacklist --- src/bitmessageqt/blacklist.py | 48 +++++++++++++++-------------------- 1 file changed, 20 insertions(+), 28 deletions(-) diff --git a/src/bitmessageqt/blacklist.py b/src/bitmessageqt/blacklist.py index 3897bddc..7b66d9d6 100644 --- a/src/bitmessageqt/blacklist.py +++ b/src/bitmessageqt/blacklist.py @@ -1,10 +1,11 @@ from PyQt4 import QtCore, QtGui +import helper_db import widgets from addresses import addBMIfNotPresent from bmconfigparser import BMConfigParser from dialogs import AddAddressDialog -from helper_sql import sqlExecute, sqlQuery +from helper_sql import sqlExecute from queues import UISignalQueue from retranslateui import RetranslateMixin from tr import _translate @@ -61,16 +62,14 @@ class Blacklist(QtGui.QWidget, RetranslateMixin): _translate("MainWindow", "Address is valid."): address = addBMIfNotPresent(str( self.NewBlacklistDialogInstance.lineEditAddress.text())) - # First we must check to see if the address is already in the - # address book. The user cannot add it again or else it will - # cause problems when updating and deleting the entry. - t = (address,) - if BMConfigParser().get('bitmessagesettings', 'blackwhitelist') == 'black': - sql = '''select * from blacklist where address=?''' - else: - sql = '''select * from whitelist where address=?''' - queryreturn = sqlQuery(sql,*t) - if queryreturn == []: + label = str(self.NewBlacklistDialogInstance.lineEditLabel.text().toUtf8()) + if helper_db.put_addresslist( + label, address, + group='blacklist' + if BMConfigParser().get( + 'bitmessagesettings', 'blackwhitelist') == 'black' + else 'whitelist' + ): self.tableWidgetBlacklist.setSortingEnabled(False) self.tableWidgetBlacklist.insertRow(0) newItem = QtGui.QTableWidgetItem(unicode( @@ -82,12 +81,6 @@ class Blacklist(QtGui.QWidget, RetranslateMixin): QtCore.Qt.ItemIsSelectable | QtCore.Qt.ItemIsEnabled) self.tableWidgetBlacklist.setItem(0, 1, newItem) self.tableWidgetBlacklist.setSortingEnabled(True) - t = (str(self.NewBlacklistDialogInstance.lineEditLabel.text().toUtf8()), address, True) - if BMConfigParser().get('bitmessagesettings', 'blackwhitelist') == 'black': - sql = '''INSERT INTO blacklist VALUES (?,?,?)''' - else: - sql = '''INSERT INTO whitelist VALUES (?,?,?)''' - sqlExecute(sql, *t) else: UISignalQueue.put(( 'updateStatusBar', @@ -158,19 +151,18 @@ class Blacklist(QtGui.QWidget, RetranslateMixin): def rerenderBlackWhiteList(self): tabs = self.parent().parent() - if BMConfigParser().get('bitmessagesettings', 'blackwhitelist') == 'black': - tabs.setTabText(tabs.indexOf(self), _translate('blacklist', 'Blacklist')) - else: - tabs.setTabText(tabs.indexOf(self), _translate('blacklist', 'Whitelist')) + list_type = BMConfigParser().get( + 'bitmessagesettings', 'blackwhitelist') + tabs.setTabText( + tabs.indexOf(self), + _translate('blacklist', 'Blacklist') if list_type == 'black' + else _translate('blacklist', 'Whitelist')) + self.tableWidgetBlacklist.setRowCount(0) - listType = BMConfigParser().get('bitmessagesettings', 'blackwhitelist') - if listType == 'black': - queryreturn = sqlQuery('''SELECT label, address, enabled FROM blacklist''') - else: - queryreturn = sqlQuery('''SELECT label, address, enabled FROM whitelist''') self.tableWidgetBlacklist.setSortingEnabled(False) - for row in queryreturn: - label, address, enabled = row + for label, address, enabled in helper_db.get_addresslist( + group='blacklist' if list_type == 'black' else 'whiteslist' + ): self.tableWidgetBlacklist.insertRow(0) newItem = QtGui.QTableWidgetItem(unicode(label, 'utf-8')) if not enabled: -- 2.45.1 From 93679483874c4905f5a887ddbc1e8261770a98d0 Mon Sep 17 00:00:00 2001 From: Dmitri Bogomolov <4glitch@gmail.com> Date: Thu, 22 Aug 2019 17:29:08 +0300 Subject: [PATCH 4/7] Replaced appropriate queries in api --- src/api.py | 49 +++++++++++++++++-------------------------------- 1 file changed, 17 insertions(+), 32 deletions(-) diff --git a/src/api.py b/src/api.py index 9c0c18a7..2b0aece6 100644 --- a/src/api.py +++ b/src/api.py @@ -73,8 +73,8 @@ from SimpleXMLRPCServer import SimpleXMLRPCRequestHandler, SimpleXMLRPCServer from struct import pack import defaults +import helper_db import helper_inbox -import helper_sent import network.stats import proofofwork import queues @@ -587,14 +587,9 @@ class BMRPCDispatcher(object): label = self._decode(label, "base64") address = addBMIfNotPresent(address) self._verifyAddress(address) - # TODO: add unique together constraint in the table - queryreturn = sqlQuery( - "SELECT address FROM addressbook WHERE address=?", address) - if queryreturn != []: + if not helper_db.put_addressbook(label, address): raise APIError( 16, 'You already have this address in your address book.') - - sqlExecute("INSERT INTO addressbook VALUES(?,?)", label, address) queues.UISignalQueue.put(('rerenderMessagelistFromLabels', '')) queues.UISignalQueue.put(('rerenderMessagelistToLabels', '')) queues.UISignalQueue.put(('rerenderAddressBook', '')) @@ -1110,18 +1105,15 @@ class BMRPCDispatcher(object): if not fromAddressEnabled: raise APIError(14, 'Your fromAddress is disabled. Cannot send.') - ackdata = helper_sent.insert( - toAddress=toAddress, fromAddress=fromAddress, - subject=subject, message=message, encoding=encodingType, ttl=TTL) + stealthLevel = BMConfigParser().safeGetInt( + 'bitmessagesettings', 'ackstealthlevel') + ackdata = genAckPayload(streamNumber, stealthLevel) - toLabel = '' - queryreturn = sqlQuery( - "SELECT label FROM addressbook WHERE address=?", toAddress) - try: - toLabel, = queryreturn[0][0] - except IndexError: - pass + helper_db.put_sent( + toAddress, fromAddress, subject, message, ackdata, + 'msgqueued', encodingType, ttl=TTL) + toLabel = helper_db.get_label(toAddress) or '' queues.UISignalQueue.put(('displayNewSentMessage', ( toAddress, toLabel, fromAddress, subject, message, ackdata))) queues.workerQueue.put(('sendmessage', toAddress)) @@ -1151,15 +1143,15 @@ class BMRPCDispatcher(object): self.config.getboolean(fromAddress, 'enabled') except BaseException: raise APIError( - 13, 'Could not find your fromAddress in the keys.dat file.') - toAddress = str_broadcast_subscribers + 13, 'could not find your fromAddress in the keys.dat file.') + streamNumber = decodeAddress(fromAddress)[2] + ackdata = genAckPayload(streamNumber, 0) + toAddress = toLabel = str_broadcast_subscribers - ackdata = helper_sent.insert( - fromAddress=fromAddress, subject=subject, - message=message, status='broadcastqueued', - encoding=encodingType) + helper_db.put_sent( + toAddress, fromAddress, subject, message, ackdata, + 'broadcastqueued', encodingType, ttl=TTL) - toLabel = str_broadcast_subscribers queues.UISignalQueue.put(('displayNewSentMessage', ( toAddress, toLabel, fromAddress, subject, message, ackdata))) queues.workerQueue.put(('sendbroadcast', '')) @@ -1197,15 +1189,8 @@ class BMRPCDispatcher(object): except UnicodeDecodeError: raise APIError(17, 'Label is not valid UTF-8 data.') self._verifyAddress(address) - address = addBMIfNotPresent(address) - # First we must check to see if the address is already in the - # subscriptions list. - queryreturn = sqlQuery( - "SELECT * FROM subscriptions WHERE address=?", address) - if queryreturn: + if not helper_db.put_subscriptions(label, address): raise APIError(16, 'You are already subscribed to that address.') - sqlExecute( - "INSERT INTO subscriptions VALUES (?,?,?)", label, address, True) shared.reloadBroadcastSendersForWhichImWatching() queues.UISignalQueue.put(('rerenderMessagelistFromLabels', '')) queues.UISignalQueue.put(('rerenderSubscriptions', '')) -- 2.45.1 From 1364116304ad6b3b4a4fd26ee7cb2370649e36d2 Mon Sep 17 00:00:00 2001 From: Dmitri Bogomolov <4glitch@gmail.com> Date: Fri, 27 Sep 2019 14:57:29 +0300 Subject: [PATCH 5/7] Added helper_db.put_trash and replaced helper_inbox in api --- src/api.py | 9 ++++----- src/helper_db.py | 9 +++++++++ 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/api.py b/src/api.py index 2b0aece6..9162d5a0 100644 --- a/src/api.py +++ b/src/api.py @@ -74,7 +74,6 @@ from struct import pack import defaults import helper_db -import helper_inbox import network.stats import proofofwork import queues @@ -1051,23 +1050,23 @@ class BMRPCDispatcher(object): """ msgid = self._decode(msgid, "hex") # Trash if in inbox table - helper_inbox.trash(msgid) + helper_db.put_trash(msgid) # Trash if in sent table - sqlExecute("UPDATE sent SET folder='trash' WHERE msgid=?", msgid) + helper_db.put_trash(msgid, sent=True) return 'Trashed message (assuming message existed).' @command('trashInboxMessage') def HandleTrashInboxMessage(self, msgid): """Trash inbox message by msgid (encoded in hex).""" msgid = self._decode(msgid, "hex") - helper_inbox.trash(msgid) + helper_db.put_trash(msgid) return 'Trashed inbox message (assuming message existed).' @command('trashSentMessage') def HandleTrashSentMessage(self, msgid): """Trash sent message by msgid (encoded in hex).""" msgid = self._decode(msgid, "hex") - sqlExecute('''UPDATE sent SET folder='trash' WHERE msgid=?''', msgid) + helper_db.put_trash(msgid, sent=True) return 'Trashed sent message (assuming message existed).' @command('sendMessage') diff --git a/src/helper_db.py b/src/helper_db.py index 7ff6d65d..e4b75c18 100644 --- a/src/helper_db.py +++ b/src/helper_db.py @@ -74,6 +74,15 @@ def put_inbox( "newBroadcast" if broadcast else "newMessage"]) +def put_trash(msgid, sent=False): + """Put inbox message (or sent if sent=True) into trash by msgid""" + sqlExecute( + "UPDATE %s SET folder='trash' WHERE msgid=?" % + ('sent' if sent else 'inbox'), msgid) + if not sent: + queues.UISignalQueue.put(('removeInboxRowByMsgid', msgid)) + + def put_pubkey(address, address_version, data, used_personally=None): """Put pubkey into Pubkeys table""" if used_personally is None: -- 2.45.1 From d4180b7ead858e4a7317ba0533b835a8f39aa0ec Mon Sep 17 00:00:00 2001 From: Dmitri Bogomolov <4glitch@gmail.com> Date: Fri, 27 Sep 2019 14:58:12 +0300 Subject: [PATCH 6/7] Removed helper_inbox --- src/helper_inbox.py | 25 ------------------------- 1 file changed, 25 deletions(-) delete mode 100644 src/helper_inbox.py diff --git a/src/helper_inbox.py b/src/helper_inbox.py deleted file mode 100644 index 654dd59d..00000000 --- a/src/helper_inbox.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Helper Inbox performs inbox messages related operations""" - -import queues -from helper_sql import sqlExecute, sqlQuery - - -def insert(t): - """Perform an insert into the "inbox" table""" - sqlExecute('''INSERT INTO inbox VALUES (?,?,?,?,?,?,?,?,?,?)''', *t) - # shouldn't emit changedInboxUnread and displayNewInboxMessage - # at the same time - # queues.UISignalQueue.put(('changedInboxUnread', None)) - - -def trash(msgid): - """Mark a message in the `inbox` as `trash`""" - sqlExecute('''UPDATE inbox SET folder='trash' WHERE msgid=?''', msgid) - queues.UISignalQueue.put(('removeInboxRowByMsgid', msgid)) - - -def isMessageAlreadyInInbox(sigHash): - """Check for previous instances of this message""" - queryReturn = sqlQuery( - '''SELECT COUNT(*) FROM inbox WHERE sighash=?''', sigHash) - return queryReturn[0][0] != 0 -- 2.45.1 From d71900220623e20bf5a3431cf7e30bf4c6d31946 Mon Sep 17 00:00:00 2001 From: Dmitri Bogomolov <4glitch@gmail.com> Date: Fri, 4 Oct 2019 23:56:56 +0300 Subject: [PATCH 7/7] Test helper_db --- src/tests/core.py | 4 ++++ src/tests/sql.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 src/tests/sql.py diff --git a/src/tests/core.py b/src/tests/core.py index 400f6315..51f0df9f 100644 --- a/src/tests/core.py +++ b/src/tests/core.py @@ -366,6 +366,10 @@ def run(): qt_tests = loader.loadTestsFromModule(bitmessageqt.tests) suite.addTests(qt_tests) + import sql + sql_tests = loader.loadTestsFromModule(sql) + suite.addTest(sql_tests) + def keep_exc(ex_cls, exc, tb): # pylint: disable=unused-argument """Own exception hook for test cases""" excQueue.put(('tests', exc)) diff --git a/src/tests/sql.py b/src/tests/sql.py new file mode 100644 index 00000000..c2f1d8f1 --- /dev/null +++ b/src/tests/sql.py @@ -0,0 +1,33 @@ +""" +SQL related core tests +""" + +import unittest + +import helper_db + + +class TestSQL(unittest.TestCase): + """A test case for common SQL-related functions""" + test_address_1 = 'BM-2cVRo3WuKL5vDCKq2XJi1yR366nkHgHfcL' + test_address_2 = 'BM-2cVJn1BusL1j4UezuYkvpqNPCr7kCebeLw' + + def test_subscriptions(self): + """Put address into subscriptions and check that it was added""" + self.assertTrue( + helper_db.put_subscriptions('pass_1', self.test_address_1)) + self.assertFalse( + helper_db.put_subscriptions('pass_2', self.test_address_1)) + + helper_db.put_subscriptions('pass_3', self.test_address_2, False) + + for label, addr in helper_db.get_subscriptions(): + if addr == self.test_address_1: + self.assertEqual(label, 'pass_1') + break + else: + self.fail('Could not find added address in subscriptions') + + for label, addr in helper_db.get_subscriptions(): + if addr == self.test_address_2: + self.fail('Got disabled subscription') -- 2.45.1