Merge pull request #532 from Atheros1/master

Don't store inbox & sent message data in memory
This commit is contained in:
Jonathan Warren 2013-10-19 15:42:00 -07:00
commit 1cf7d36d81
3 changed files with 83 additions and 71 deletions

View File

@ -560,7 +560,7 @@ class MyForm(QtGui.QMainWindow):
where = "toaddress || fromaddress || subject || message" where = "toaddress || fromaddress || subject || message"
sqlStatement = ''' sqlStatement = '''
SELECT toaddress, fromaddress, subject, message, status, ackdata, lastactiontime SELECT toaddress, fromaddress, subject, status, ackdata, lastactiontime
FROM sent WHERE folder="sent" AND %s LIKE ? FROM sent WHERE folder="sent" AND %s LIKE ?
ORDER BY lastactiontime ORDER BY lastactiontime
''' % (where,) ''' % (where,)
@ -570,9 +570,9 @@ class MyForm(QtGui.QMainWindow):
queryreturn = sqlQuery(sqlStatement, what) queryreturn = sqlQuery(sqlStatement, what)
for row in queryreturn: for row in queryreturn:
toAddress, fromAddress, subject, message, status, ackdata, lastactiontime = row toAddress, fromAddress, subject, status, ackdata, lastactiontime = row
subject = shared.fixPotentiallyInvalidUTF8Data(subject) subject = shared.fixPotentiallyInvalidUTF8Data(subject)
message = shared.fixPotentiallyInvalidUTF8Data(message) #message = shared.fixPotentiallyInvalidUTF8Data(message)
try: try:
fromLabel = shared.config.get(fromAddress, 'label') fromLabel = shared.config.get(fromAddress, 'label')
except: except:
@ -612,7 +612,7 @@ class MyForm(QtGui.QMainWindow):
self.ui.tableWidgetSent.setItem(0, 1, newItem) self.ui.tableWidgetSent.setItem(0, 1, newItem)
newItem = QtGui.QTableWidgetItem(unicode(subject, 'utf-8')) newItem = QtGui.QTableWidgetItem(unicode(subject, 'utf-8'))
newItem.setToolTip(unicode(subject, 'utf-8')) newItem.setToolTip(unicode(subject, 'utf-8'))
newItem.setData(Qt.UserRole, unicode(message, 'utf-8)')) #newItem.setData(Qt.UserRole, unicode(message, 'utf-8)')) # No longer hold the message in the table; we'll use a SQL query to display it as needed.
newItem.setFlags( newItem.setFlags(
QtCore.Qt.ItemIsSelectable | QtCore.Qt.ItemIsEnabled) QtCore.Qt.ItemIsSelectable | QtCore.Qt.ItemIsEnabled)
self.ui.tableWidgetSent.setItem(0, 2, newItem) self.ui.tableWidgetSent.setItem(0, 2, newItem)
@ -680,7 +680,7 @@ class MyForm(QtGui.QMainWindow):
where = "toaddress || fromaddress || subject || message" where = "toaddress || fromaddress || subject || message"
sqlStatement = ''' sqlStatement = '''
SELECT msgid, toaddress, fromaddress, subject, received, message, read SELECT msgid, toaddress, fromaddress, subject, received, read
FROM inbox WHERE folder="inbox" AND %s LIKE ? FROM inbox WHERE folder="inbox" AND %s LIKE ?
ORDER BY received ORDER BY received
''' % (where,) ''' % (where,)
@ -692,9 +692,9 @@ class MyForm(QtGui.QMainWindow):
font.setBold(True) font.setBold(True)
queryreturn = sqlQuery(sqlStatement, what) queryreturn = sqlQuery(sqlStatement, what)
for row in queryreturn: for row in queryreturn:
msgid, toAddress, fromAddress, subject, received, message, read = row msgid, toAddress, fromAddress, subject, received, read = row
subject = shared.fixPotentiallyInvalidUTF8Data(subject) subject = shared.fixPotentiallyInvalidUTF8Data(subject)
message = shared.fixPotentiallyInvalidUTF8Data(message) #message = shared.fixPotentiallyInvalidUTF8Data(message)
try: try:
if toAddress == self.str_broadcast_subscribers: if toAddress == self.str_broadcast_subscribers:
toLabel = self.str_broadcast_subscribers toLabel = self.str_broadcast_subscribers
@ -750,7 +750,7 @@ class MyForm(QtGui.QMainWindow):
self.ui.tableWidgetInbox.setItem(0, 1, newItem) self.ui.tableWidgetInbox.setItem(0, 1, newItem)
newItem = QtGui.QTableWidgetItem(unicode(subject, 'utf-8')) newItem = QtGui.QTableWidgetItem(unicode(subject, 'utf-8'))
newItem.setToolTip(unicode(subject, 'utf-8')) newItem.setToolTip(unicode(subject, 'utf-8'))
newItem.setData(Qt.UserRole, unicode(message, 'utf-8)')) #newItem.setData(Qt.UserRole, unicode(message, 'utf-8)')) # No longer hold the message in the table (and thus in memory); we'll use a SQL query when we need to display it.
newItem.setFlags( newItem.setFlags(
QtCore.Qt.ItemIsSelectable | QtCore.Qt.ItemIsEnabled) QtCore.Qt.ItemIsSelectable | QtCore.Qt.ItemIsEnabled)
if not read: if not read:
@ -1632,44 +1632,12 @@ class MyForm(QtGui.QMainWindow):
sqlExecute( sqlExecute(
'''INSERT INTO sent VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)''', *t) '''INSERT INTO sent VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)''', *t)
shared.workerQueue.put(('sendbroadcast', ''))
try:
fromLabel = shared.config.get(fromAddress, 'label')
except:
fromLabel = ''
if fromLabel == '':
fromLabel = fromAddress
toLabel = self.str_broadcast_subscribers toLabel = self.str_broadcast_subscribers
self.displayNewSentMessage(
toAddress, toLabel, fromAddress, subject, message, ackdata)
self.ui.tableWidgetSent.insertRow(0) shared.workerQueue.put(('sendbroadcast', ''))
newItem = QtGui.QTableWidgetItem(unicode(toLabel, 'utf-8'))
newItem.setData(Qt.UserRole, str(toAddress))
self.ui.tableWidgetSent.setItem(0, 0, newItem)
if fromLabel == '':
newItem = QtGui.QTableWidgetItem(
unicode(fromAddress, 'utf-8'))
else:
newItem = QtGui.QTableWidgetItem(
unicode(fromLabel, 'utf-8'))
newItem.setData(Qt.UserRole, str(fromAddress))
self.ui.tableWidgetSent.setItem(0, 1, newItem)
newItem = QtGui.QTableWidgetItem(unicode(subject, 'utf-8)'))
newItem.setData(Qt.UserRole, unicode(message, 'utf-8)'))
self.ui.tableWidgetSent.setItem(0, 2, newItem)
# newItem = QtGui.QTableWidgetItem('Doing work necessary to
# send broadcast...'+
# unicode(strftime(config.get('bitmessagesettings',
# 'timeformat'),localtime(int(time.time()))),'utf-8'))
newItem = myTableWidgetItem(_translate("MainWindow", "Work is queued."))
newItem.setData(Qt.UserRole, QByteArray(ackdata))
newItem.setData(33, int(time.time()))
self.ui.tableWidgetSent.setItem(0, 3, newItem)
self.ui.textEditSentMessage.setPlainText(
self.ui.tableWidgetSent.item(0, 2).data(Qt.UserRole).toPyObject())
self.ui.comboBoxSendFrom.setCurrentIndex(0) self.ui.comboBoxSendFrom.setCurrentIndex(0)
self.ui.labelFrom.setText('') self.ui.labelFrom.setText('')
@ -1765,7 +1733,7 @@ class MyForm(QtGui.QMainWindow):
self.ui.tableWidgetSent.setItem(0, 1, newItem) self.ui.tableWidgetSent.setItem(0, 1, newItem)
newItem = QtGui.QTableWidgetItem(unicode(subject, 'utf-8)')) newItem = QtGui.QTableWidgetItem(unicode(subject, 'utf-8)'))
newItem.setToolTip(unicode(subject, 'utf-8)')) newItem.setToolTip(unicode(subject, 'utf-8)'))
newItem.setData(Qt.UserRole, unicode(message, 'utf-8)')) #newItem.setData(Qt.UserRole, unicode(message, 'utf-8)')) # No longer hold the message in the table; we'll use a SQL query to display it as needed.
self.ui.tableWidgetSent.setItem(0, 2, newItem) self.ui.tableWidgetSent.setItem(0, 2, newItem)
# newItem = QtGui.QTableWidgetItem('Doing work necessary to send # newItem = QtGui.QTableWidgetItem('Doing work necessary to send
# broadcast...'+ # broadcast...'+
@ -1778,13 +1746,12 @@ class MyForm(QtGui.QMainWindow):
newItem.setData(Qt.UserRole, QByteArray(ackdata)) newItem.setData(Qt.UserRole, QByteArray(ackdata))
newItem.setData(33, int(time.time())) newItem.setData(33, int(time.time()))
self.ui.tableWidgetSent.setItem(0, 3, newItem) self.ui.tableWidgetSent.setItem(0, 3, newItem)
self.ui.textEditSentMessage.setPlainText( self.ui.textEditSentMessage.setPlainText(unicode(message, 'utf-8)'))
self.ui.tableWidgetSent.item(0, 2).data(Qt.UserRole).toPyObject())
self.ui.tableWidgetSent.setSortingEnabled(True) self.ui.tableWidgetSent.setSortingEnabled(True)
def displayNewInboxMessage(self, inventoryHash, toAddress, fromAddress, subject, message): def displayNewInboxMessage(self, inventoryHash, toAddress, fromAddress, subject, message):
subject = shared.fixPotentiallyInvalidUTF8Data(subject) subject = shared.fixPotentiallyInvalidUTF8Data(subject)
message = shared.fixPotentiallyInvalidUTF8Data(message) #message = shared.fixPotentiallyInvalidUTF8Data(message)
fromLabel = '' fromLabel = ''
queryreturn = sqlQuery( queryreturn = sqlQuery(
'''select label from addressbook where address=?''', fromAddress) '''select label from addressbook where address=?''', fromAddress)
@ -1838,7 +1805,7 @@ class MyForm(QtGui.QMainWindow):
self.ui.tableWidgetInbox.setItem(0, 1, newItem) self.ui.tableWidgetInbox.setItem(0, 1, newItem)
newItem = QtGui.QTableWidgetItem(unicode(subject, 'utf-8)')) newItem = QtGui.QTableWidgetItem(unicode(subject, 'utf-8)'))
newItem.setToolTip(unicode(subject, 'utf-8)')) newItem.setToolTip(unicode(subject, 'utf-8)'))
newItem.setData(Qt.UserRole, unicode(message, 'utf-8)')) #newItem.setData(Qt.UserRole, unicode(message, 'utf-8)')) # No longer hold the message in the table; we'll use a SQL query to display it as needed.
newItem.setFont(font) newItem.setFont(font)
self.ui.tableWidgetInbox.setItem(0, 2, newItem) self.ui.tableWidgetInbox.setItem(0, 2, newItem)
newItem = myTableWidgetItem(unicode(strftime(shared.config.get( newItem = myTableWidgetItem(unicode(strftime(shared.config.get(
@ -2255,10 +2222,18 @@ class MyForm(QtGui.QMainWindow):
def on_action_InboxMessageForceHtml(self): def on_action_InboxMessageForceHtml(self):
currentInboxRow = self.ui.tableWidgetInbox.currentRow() currentInboxRow = self.ui.tableWidgetInbox.currentRow()
lines = self.ui.tableWidgetInbox.item(
currentInboxRow, 2).data(Qt.UserRole).toPyObject().split('\n') msgid = str(self.ui.tableWidgetInbox.item(
currentInboxRow, 3).data(Qt.UserRole).toPyObject())
queryreturn = sqlQuery(
'''select message from inbox where msgid=?''', msgid)
if queryreturn != []:
for row in queryreturn:
messageAtCurrentInboxRow, = row
lines = messageAtCurrentInboxRow.split('\n')
for i in xrange(len(lines)): for i in xrange(len(lines)):
if lines[i].contains('Message ostensibly from '): if 'Message ostensibly from ' in lines[i]:
lines[i] = '<p style="font-size: 12px; color: grey;">%s</span></p>' % ( lines[i] = '<p style="font-size: 12px; color: grey;">%s</span></p>' % (
lines[i]) lines[i])
elif lines[i] == '------------------------------------------------------': elif lines[i] == '------------------------------------------------------':
@ -2292,6 +2267,13 @@ class MyForm(QtGui.QMainWindow):
currentInboxRow, 0).data(Qt.UserRole).toPyObject()) currentInboxRow, 0).data(Qt.UserRole).toPyObject())
fromAddressAtCurrentInboxRow = str(self.ui.tableWidgetInbox.item( fromAddressAtCurrentInboxRow = str(self.ui.tableWidgetInbox.item(
currentInboxRow, 1).data(Qt.UserRole).toPyObject()) currentInboxRow, 1).data(Qt.UserRole).toPyObject())
msgid = str(self.ui.tableWidgetInbox.item(
currentInboxRow, 3).data(Qt.UserRole).toPyObject())
queryreturn = sqlQuery(
'''select message from inbox where msgid=?''', msgid)
if queryreturn != []:
for row in queryreturn:
messageAtCurrentInboxRow, = row
if toAddressAtCurrentInboxRow == self.str_broadcast_subscribers: if toAddressAtCurrentInboxRow == self.str_broadcast_subscribers:
self.ui.labelFrom.setText('') self.ui.labelFrom.setText('')
elif not shared.config.has_section(toAddressAtCurrentInboxRow): elif not shared.config.has_section(toAddressAtCurrentInboxRow):
@ -2315,9 +2297,7 @@ class MyForm(QtGui.QMainWindow):
self.ui.lineEditTo.setText(str(toAddressAtCurrentInboxRow)) self.ui.lineEditTo.setText(str(toAddressAtCurrentInboxRow))
self.ui.comboBoxSendFrom.setCurrentIndex(0) self.ui.comboBoxSendFrom.setCurrentIndex(0)
# self.ui.comboBoxSendFrom.setEditText(str(self.ui.tableWidgetInbox.item(currentInboxRow,0).text)) self.ui.textEditMessage.setText('\n\n------------------------------------------------------\n' + unicode(messageAtCurrentInboxRow, 'utf-8)'))
self.ui.textEditMessage.setText('\n\n------------------------------------------------------\n' + self.ui.tableWidgetInbox.item(
currentInboxRow, 2).data(Qt.UserRole).toPyObject())
if self.ui.tableWidgetInbox.item(currentInboxRow, 2).text()[0:3] in ['Re:', 'RE:']: if self.ui.tableWidgetInbox.item(currentInboxRow, 2).text()[0:3] in ['Re:', 'RE:']:
self.ui.lineEditSubject.setText( self.ui.lineEditSubject.setText(
self.ui.tableWidgetInbox.item(currentInboxRow, 2).text()) self.ui.tableWidgetInbox.item(currentInboxRow, 2).text())
@ -2377,14 +2357,23 @@ class MyForm(QtGui.QMainWindow):
subjectAtCurrentInboxRow = str(self.ui.tableWidgetInbox.item(currentInboxRow,2).text()) subjectAtCurrentInboxRow = str(self.ui.tableWidgetInbox.item(currentInboxRow,2).text())
except: except:
subjectAtCurrentInboxRow = '' subjectAtCurrentInboxRow = ''
# Retrieve the message data out of the SQL database
msgid = str(self.ui.tableWidgetInbox.item(
currentInboxRow, 3).data(Qt.UserRole).toPyObject())
queryreturn = sqlQuery(
'''select message from inbox where msgid=?''', msgid)
if queryreturn != []:
for row in queryreturn:
message, = row
defaultFilename = "".join(x for x in subjectAtCurrentInboxRow if x.isalnum()) + '.txt' defaultFilename = "".join(x for x in subjectAtCurrentInboxRow if x.isalnum()) + '.txt'
data = self.ui.tableWidgetInbox.item(currentInboxRow,2).data(Qt.UserRole).toPyObject()
filename = QFileDialog.getSaveFileName(self, _translate("MainWindow","Save As..."), defaultFilename, "Text files (*.txt);;All files (*.*)") filename = QFileDialog.getSaveFileName(self, _translate("MainWindow","Save As..."), defaultFilename, "Text files (*.txt);;All files (*.*)")
if filename == '': if filename == '':
return return
try: try:
f = open(filename, 'w') f = open(filename, 'w')
f.write( self.ui.tableWidgetInbox.item(currentInboxRow,2).data(Qt.UserRole).toPyObject() ) f.write(message)
f.close() f.close()
except Exception, e: except Exception, e:
sys.stderr.write('Write error: '+ e) sys.stderr.write('Write error: '+ e)
@ -2716,21 +2705,27 @@ class MyForm(QtGui.QMainWindow):
fromAddress = str(self.ui.tableWidgetInbox.item( fromAddress = str(self.ui.tableWidgetInbox.item(
currentRow, 1).data(Qt.UserRole).toPyObject()) currentRow, 1).data(Qt.UserRole).toPyObject())
msgid = str(self.ui.tableWidgetInbox.item(
currentRow, 3).data(Qt.UserRole).toPyObject())
queryreturn = sqlQuery(
'''select message from inbox where msgid=?''', msgid)
if queryreturn != []:
for row in queryreturn:
message, = row
message = unicode(message, 'utf-8)')
# If we have received this message from either a broadcast address # If we have received this message from either a broadcast address
# or from someone in our address book, display as HTML # or from someone in our address book, display as HTML
if decodeAddress(fromAddress)[3] in shared.broadcastSendersForWhichImWatching or shared.isAddressInMyAddressBook(fromAddress): if decodeAddress(fromAddress)[3] in shared.broadcastSendersForWhichImWatching or shared.isAddressInMyAddressBook(fromAddress):
if len(self.ui.tableWidgetInbox.item(currentRow, 2).data(Qt.UserRole).toPyObject()) < 30000: if len(message) < 30000:
self.ui.textEditInboxMessage.setText(self.ui.tableWidgetInbox.item( self.ui.textEditInboxMessage.setText(message) # Only show the first 30K characters
currentRow, 2).data(Qt.UserRole).toPyObject()) # Only show the first 30K characters
else: else:
self.ui.textEditInboxMessage.setText(self.ui.tableWidgetInbox.item(currentRow, 2).data(Qt.UserRole).toPyObject()[ self.ui.textEditInboxMessage.setText(message[
:30000] + '\n\nDisplay of the remainder of the message truncated because it is too long.') # Only show the first 30K characters :30000] + '\n\nDisplay of the remainder of the message truncated because it is too long.') # Only show the first 30K characters
else: else:
if len(self.ui.tableWidgetInbox.item(currentRow, 2).data(Qt.UserRole).toPyObject()) < 30000: if len(message) < 30000:
self.ui.textEditInboxMessage.setPlainText(self.ui.tableWidgetInbox.item( self.ui.textEditInboxMessage.setPlainText(message) # Only show the first 30K characters
currentRow, 2).data(Qt.UserRole).toPyObject()) # Only show the first 30K characters
else: else:
self.ui.textEditInboxMessage.setPlainText(self.ui.tableWidgetInbox.item(currentRow, 2).data(Qt.UserRole).toPyObject()[ self.ui.textEditInboxMessage.setPlainText(message[
:30000] + '\n\nDisplay of the remainder of the message truncated because it is too long.') # Only show the first 30K characters :30000] + '\n\nDisplay of the remainder of the message truncated because it is too long.') # Only show the first 30K characters
self.ui.tableWidgetInbox.item(currentRow, 0).setFont(font) self.ui.tableWidgetInbox.item(currentRow, 0).setFont(font)
@ -2746,8 +2741,17 @@ class MyForm(QtGui.QMainWindow):
def tableWidgetSentItemClicked(self): def tableWidgetSentItemClicked(self):
currentRow = self.ui.tableWidgetSent.currentRow() currentRow = self.ui.tableWidgetSent.currentRow()
if currentRow >= 0: if currentRow >= 0:
self.ui.textEditSentMessage.setPlainText(self.ui.tableWidgetSent.item( ackdata = str(self.ui.tableWidgetSent.item(
currentRow, 2).data(Qt.UserRole).toPyObject()) currentRow, 3).data(Qt.UserRole).toPyObject())
queryreturn = sqlQuery(
'''select message from sent where ackdata=?''', ackdata)
if queryreturn != []:
for row in queryreturn:
message, = row
else:
message = "Error occurred: could not load message from disk."
message = unicode(message, 'utf-8)')
self.ui.textEditSentMessage.setPlainText(message)
def tableWidgetYourIdentitiesItemChanged(self): def tableWidgetYourIdentitiesItemChanged(self):
currentRow = self.ui.tableWidgetYourIdentities.currentRow() currentRow = self.ui.tableWidgetYourIdentities.currentRow()

View File

@ -268,7 +268,7 @@ class _OpenSSL:
self.EVP_MD_CTX_destroy.argtypes = [ctypes.c_void_p] self.EVP_MD_CTX_destroy.argtypes = [ctypes.c_void_p]
self.RAND_bytes = self._lib.RAND_bytes self.RAND_bytes = self._lib.RAND_bytes
self.RAND_bytes.restype = None self.RAND_bytes.restype = ctypes.c_int
self.RAND_bytes.argtypes = [ctypes.c_void_p, ctypes.c_int] self.RAND_bytes.argtypes = [ctypes.c_void_p, ctypes.c_int]
@ -394,7 +394,15 @@ class _OpenSSL:
OpenSSL random function OpenSSL random function
""" """
buffer = self.malloc(0, size) buffer = self.malloc(0, size)
self.RAND_bytes(buffer, size) # This pyelliptic library, by default, didn't check the return value of RAND_bytes. It is
# evidently possible that it returned an error and not-actually-random data. However, in
# tests on various operating systems, while generating hundreds of gigabytes of random
# strings of various sizes I could not get an error to occur. Also Bitcoin doesn't check
# the return value of RAND_bytes either.
# Fixed in Bitmessage version 0.4.2 (in source code on 2013-10-13)
while self.RAND_bytes(buffer, size) != 1:
import time
time.sleep(1)
return buffer.raw return buffer.raw
def malloc(self, data, size): def malloc(self, data, size):

View File

@ -279,6 +279,8 @@ def reloadBroadcastSendersForWhichImWatching():
def doCleanShutdown(): def doCleanShutdown():
global shutdown global shutdown
shutdown = 1 #Used to tell proof of work worker threads to exit. shutdown = 1 #Used to tell proof of work worker threads to exit.
broadcastToSendDataQueues((0, 'shutdown', 'all'))
knownNodesLock.acquire() knownNodesLock.acquire()
UISignalQueue.put(('updateStatusBar','Saving the knownNodes list of peers to disk...')) UISignalQueue.put(('updateStatusBar','Saving the knownNodes list of peers to disk...'))
output = open(appdata + 'knownnodes.dat', 'wb') output = open(appdata + 'knownnodes.dat', 'wb')
@ -290,8 +292,6 @@ def doCleanShutdown():
logger.info('Finished closing knownnodes.dat output file.') logger.info('Finished closing knownnodes.dat output file.')
UISignalQueue.put(('updateStatusBar','Done saving the knownNodes list of peers to disk.')) UISignalQueue.put(('updateStatusBar','Done saving the knownNodes list of peers to disk.'))
broadcastToSendDataQueues((0, 'shutdown', 'all'))
logger.info('Flushing inventory in memory out to disk...') logger.info('Flushing inventory in memory out to disk...')
UISignalQueue.put(( UISignalQueue.put((
'updateStatusBar', 'updateStatusBar',