Bitmessage/PyBitmessage-2024-12-01
Archived
2
0
Fork 0
Code Issues 168 Pull Requests 53 Actions Packages Projects Releases 5 Wiki Activity

Don't store inbox & sent message data in memory #532

Merged
Atheros1 merged 3 commits from master into master 2013-10-20 00:42:00 +02:00
Conversation 0 Commits 3 Files Changed 3 +83 -71
3 changed files with 83 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

138
src/bitmessageqt/__init__.py
View File

@ -560,7 +560,7 @@ class MyForm(QtGui.QMainWindow):
where = "toaddress || fromaddress || subject || message"
sqlStatement = '''
SELECT toaddress, fromaddress, subject, message, status, ackdata, lastactiontime
SELECT toaddress, fromaddress, subject, status, ackdata, lastactiontime
FROM sent WHERE folder="sent" AND %s LIKE ?
ORDER BY lastactiontime
''' % (where,)
@ -570,9 +570,9 @@ class MyForm(QtGui.QMainWindow):
queryreturn = sqlQuery(sqlStatement, what)
for row in queryreturn:
toAddress, fromAddress, subject, message, status, ackdata, lastactiontime = row
toAddress, fromAddress, subject, status, ackdata, lastactiontime = row
subject = shared.fixPotentiallyInvalidUTF8Data(subject)
message = shared.fixPotentiallyInvalidUTF8Data(message)
#message = shared.fixPotentiallyInvalidUTF8Data(message)
try:
fromLabel = shared.config.get(fromAddress, 'label')
except:
@ -612,7 +612,7 @@ class MyForm(QtGui.QMainWindow):
self.ui.tableWidgetSent.setItem(0, 1, newItem)
newItem = QtGui.QTableWidgetItem(unicode(subject, 'utf-8'))
newItem.setToolTip(unicode(subject, 'utf-8'))
newItem.setData(Qt.UserRole, unicode(message, 'utf-8)'))
#newItem.setData(Qt.UserRole, unicode(message, 'utf-8)')) # No longer hold the message in the table; we'll use a SQL query to display it as needed.
newItem.setFlags(
QtCore.Qt.ItemIsSelectable | QtCore.Qt.ItemIsEnabled)
self.ui.tableWidgetSent.setItem(0, 2, newItem)
@ -680,7 +680,7 @@ class MyForm(QtGui.QMainWindow):
where = "toaddress || fromaddress || subject || message"
sqlStatement = '''
SELECT msgid, toaddress, fromaddress, subject, received, message, read
SELECT msgid, toaddress, fromaddress, subject, received, read
FROM inbox WHERE folder="inbox" AND %s LIKE ?
ORDER BY received
''' % (where,)
@ -692,9 +692,9 @@ class MyForm(QtGui.QMainWindow):
font.setBold(True)
queryreturn = sqlQuery(sqlStatement, what)
for row in queryreturn:
msgid, toAddress, fromAddress, subject, received, message, read = row
msgid, toAddress, fromAddress, subject, received, read = row
subject = shared.fixPotentiallyInvalidUTF8Data(subject)
message = shared.fixPotentiallyInvalidUTF8Data(message)
#message = shared.fixPotentiallyInvalidUTF8Data(message)
try:
if toAddress == self.str_broadcast_subscribers:
toLabel = self.str_broadcast_subscribers
@ -750,7 +750,7 @@ class MyForm(QtGui.QMainWindow):
self.ui.tableWidgetInbox.setItem(0, 1, newItem)
newItem = QtGui.QTableWidgetItem(unicode(subject, 'utf-8'))
newItem.setToolTip(unicode(subject, 'utf-8'))
newItem.setData(Qt.UserRole, unicode(message, 'utf-8)'))
#newItem.setData(Qt.UserRole, unicode(message, 'utf-8)')) # No longer hold the message in the table (and thus in memory); we'll use a SQL query when we need to display it.
newItem.setFlags(
QtCore.Qt.ItemIsSelectable | QtCore.Qt.ItemIsEnabled)
if not read:
@ -1632,44 +1632,12 @@ class MyForm(QtGui.QMainWindow):
sqlExecute(
'''INSERT INTO sent VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)''', *t)
shared.workerQueue.put(('sendbroadcast', ''))
try:
fromLabel = shared.config.get(fromAddress, 'label')
except:
fromLabel = ''
if fromLabel == '':
fromLabel = fromAddress
toLabel = self.str_broadcast_subscribers
self.displayNewSentMessage(
toAddress, toLabel, fromAddress, subject, message, ackdata)
self.ui.tableWidgetSent.insertRow(0)
newItem = QtGui.QTableWidgetItem(unicode(toLabel, 'utf-8'))
newItem.setData(Qt.UserRole, str(toAddress))
self.ui.tableWidgetSent.setItem(0, 0, newItem)
if fromLabel == '':
newItem = QtGui.QTableWidgetItem(
unicode(fromAddress, 'utf-8'))
else:
newItem = QtGui.QTableWidgetItem(
unicode(fromLabel, 'utf-8'))
newItem.setData(Qt.UserRole, str(fromAddress))
self.ui.tableWidgetSent.setItem(0, 1, newItem)
newItem = QtGui.QTableWidgetItem(unicode(subject, 'utf-8)'))
newItem.setData(Qt.UserRole, unicode(message, 'utf-8)'))
self.ui.tableWidgetSent.setItem(0, 2, newItem)
# newItem = QtGui.QTableWidgetItem('Doing work necessary to
# send broadcast...'+
# unicode(strftime(config.get('bitmessagesettings',
# 'timeformat'),localtime(int(time.time()))),'utf-8'))
newItem = myTableWidgetItem(_translate("MainWindow", "Work is queued."))
newItem.setData(Qt.UserRole, QByteArray(ackdata))
newItem.setData(33, int(time.time()))
self.ui.tableWidgetSent.setItem(0, 3, newItem)
self.ui.textEditSentMessage.setPlainText(
self.ui.tableWidgetSent.item(0, 2).data(Qt.UserRole).toPyObject())
shared.workerQueue.put(('sendbroadcast', ''))
self.ui.comboBoxSendFrom.setCurrentIndex(0)
self.ui.labelFrom.setText('')
@ -1765,7 +1733,7 @@ class MyForm(QtGui.QMainWindow):
self.ui.tableWidgetSent.setItem(0, 1, newItem)
newItem = QtGui.QTableWidgetItem(unicode(subject, 'utf-8)'))
newItem.setToolTip(unicode(subject, 'utf-8)'))
newItem.setData(Qt.UserRole, unicode(message, 'utf-8)'))
#newItem.setData(Qt.UserRole, unicode(message, 'utf-8)')) # No longer hold the message in the table; we'll use a SQL query to display it as needed.
self.ui.tableWidgetSent.setItem(0, 2, newItem)
# newItem = QtGui.QTableWidgetItem('Doing work necessary to send
# broadcast...'+
@ -1778,13 +1746,12 @@ class MyForm(QtGui.QMainWindow):
newItem.setData(Qt.UserRole, QByteArray(ackdata))
newItem.setData(33, int(time.time()))
self.ui.tableWidgetSent.setItem(0, 3, newItem)
self.ui.textEditSentMessage.setPlainText(
self.ui.tableWidgetSent.item(0, 2).data(Qt.UserRole).toPyObject())
self.ui.textEditSentMessage.setPlainText(unicode(message, 'utf-8)'))
self.ui.tableWidgetSent.setSortingEnabled(True)
def displayNewInboxMessage(self, inventoryHash, toAddress, fromAddress, subject, message):
subject = shared.fixPotentiallyInvalidUTF8Data(subject)
message = shared.fixPotentiallyInvalidUTF8Data(message)
#message = shared.fixPotentiallyInvalidUTF8Data(message)
fromLabel = ''
queryreturn = sqlQuery(
'''select label from addressbook where address=?''', fromAddress)
@ -1838,7 +1805,7 @@ class MyForm(QtGui.QMainWindow):
self.ui.tableWidgetInbox.setItem(0, 1, newItem)
newItem = QtGui.QTableWidgetItem(unicode(subject, 'utf-8)'))
newItem.setToolTip(unicode(subject, 'utf-8)'))
newItem.setData(Qt.UserRole, unicode(message, 'utf-8)'))
#newItem.setData(Qt.UserRole, unicode(message, 'utf-8)')) # No longer hold the message in the table; we'll use a SQL query to display it as needed.
newItem.setFont(font)
self.ui.tableWidgetInbox.setItem(0, 2, newItem)
newItem = myTableWidgetItem(unicode(strftime(shared.config.get(
@ -2255,10 +2222,18 @@ class MyForm(QtGui.QMainWindow):
def on_action_InboxMessageForceHtml(self):
currentInboxRow = self.ui.tableWidgetInbox.currentRow()
lines = self.ui.tableWidgetInbox.item(
currentInboxRow, 2).data(Qt.UserRole).toPyObject().split('\n')
msgid = str(self.ui.tableWidgetInbox.item(
currentInboxRow, 3).data(Qt.UserRole).toPyObject())
queryreturn = sqlQuery(
'''select message from inbox where msgid=?''', msgid)
if queryreturn != []:
for row in queryreturn:
messageAtCurrentInboxRow, = row
lines = messageAtCurrentInboxRow.split('\n')
for i in xrange(len(lines)):
if lines[i].contains('Message ostensibly from '):
if 'Message ostensibly from ' in lines[i]:
lines[i] = '<p style="font-size: 12px; color: grey;">%s</span></p>' % (
lines[i])
elif lines[i] == '------------------------------------------------------':
@ -2292,6 +2267,13 @@ class MyForm(QtGui.QMainWindow):
currentInboxRow, 0).data(Qt.UserRole).toPyObject())
fromAddressAtCurrentInboxRow = str(self.ui.tableWidgetInbox.item(
currentInboxRow, 1).data(Qt.UserRole).toPyObject())
msgid = str(self.ui.tableWidgetInbox.item(
currentInboxRow, 3).data(Qt.UserRole).toPyObject())
queryreturn = sqlQuery(
'''select message from inbox where msgid=?''', msgid)
if queryreturn != []:
for row in queryreturn:
messageAtCurrentInboxRow, = row
if toAddressAtCurrentInboxRow == self.str_broadcast_subscribers:
self.ui.labelFrom.setText('')
elif not shared.config.has_section(toAddressAtCurrentInboxRow):
@ -2315,9 +2297,7 @@ class MyForm(QtGui.QMainWindow):
self.ui.lineEditTo.setText(str(toAddressAtCurrentInboxRow))
self.ui.comboBoxSendFrom.setCurrentIndex(0)
# self.ui.comboBoxSendFrom.setEditText(str(self.ui.tableWidgetInbox.item(currentInboxRow,0).text))
self.ui.textEditMessage.setText('\n\n------------------------------------------------------\n' + self.ui.tableWidgetInbox.item(
currentInboxRow, 2).data(Qt.UserRole).toPyObject())
self.ui.textEditMessage.setText('\n\n------------------------------------------------------\n' + unicode(messageAtCurrentInboxRow, 'utf-8)'))
if self.ui.tableWidgetInbox.item(currentInboxRow, 2).text()[0:3] in ['Re:', 'RE:']:
self.ui.lineEditSubject.setText(
self.ui.tableWidgetInbox.item(currentInboxRow, 2).text())
@ -2377,14 +2357,23 @@ class MyForm(QtGui.QMainWindow):
subjectAtCurrentInboxRow = str(self.ui.tableWidgetInbox.item(currentInboxRow,2).text())
except:
subjectAtCurrentInboxRow = ''
# Retrieve the message data out of the SQL database
msgid = str(self.ui.tableWidgetInbox.item(
currentInboxRow, 3).data(Qt.UserRole).toPyObject())
queryreturn = sqlQuery(
'''select message from inbox where msgid=?''', msgid)
if queryreturn != []:
for row in queryreturn:
message, = row
defaultFilename = "".join(x for x in subjectAtCurrentInboxRow if x.isalnum()) + '.txt'
data = self.ui.tableWidgetInbox.item(currentInboxRow,2).data(Qt.UserRole).toPyObject()
filename = QFileDialog.getSaveFileName(self, _translate("MainWindow","Save As..."), defaultFilename, "Text files (*.txt);;All files (*.*)")
if filename == '':
return
try:
f = open(filename, 'w')
f.write( self.ui.tableWidgetInbox.item(currentInboxRow,2).data(Qt.UserRole).toPyObject() )
f.write(message)
f.close()
except Exception, e:
sys.stderr.write('Write error: '+ e)
@ -2716,21 +2705,27 @@ class MyForm(QtGui.QMainWindow):
fromAddress = str(self.ui.tableWidgetInbox.item(
currentRow, 1).data(Qt.UserRole).toPyObject())
msgid = str(self.ui.tableWidgetInbox.item(
currentRow, 3).data(Qt.UserRole).toPyObject())
queryreturn = sqlQuery(
'''select message from inbox where msgid=?''', msgid)
if queryreturn != []:
for row in queryreturn:
message, = row
message = unicode(message, 'utf-8)')
# If we have received this message from either a broadcast address
# or from someone in our address book, display as HTML
if decodeAddress(fromAddress)[3] in shared.broadcastSendersForWhichImWatching or shared.isAddressInMyAddressBook(fromAddress):
if len(self.ui.tableWidgetInbox.item(currentRow, 2).data(Qt.UserRole).toPyObject()) < 30000:
self.ui.textEditInboxMessage.setText(self.ui.tableWidgetInbox.item(
currentRow, 2).data(Qt.UserRole).toPyObject()) # Only show the first 30K characters
if len(message) < 30000:
self.ui.textEditInboxMessage.setText(message) # Only show the first 30K characters
else:
self.ui.textEditInboxMessage.setText(self.ui.tableWidgetInbox.item(currentRow, 2).data(Qt.UserRole).toPyObject()[
self.ui.textEditInboxMessage.setText(message[
:30000] + '\n\nDisplay of the remainder of the message truncated because it is too long.') # Only show the first 30K characters
else:
if len(self.ui.tableWidgetInbox.item(currentRow, 2).data(Qt.UserRole).toPyObject()) < 30000:
self.ui.textEditInboxMessage.setPlainText(self.ui.tableWidgetInbox.item(
currentRow, 2).data(Qt.UserRole).toPyObject()) # Only show the first 30K characters
if len(message) < 30000:
self.ui.textEditInboxMessage.setPlainText(message) # Only show the first 30K characters
else:
self.ui.textEditInboxMessage.setPlainText(self.ui.tableWidgetInbox.item(currentRow, 2).data(Qt.UserRole).toPyObject()[
self.ui.textEditInboxMessage.setPlainText(message[
:30000] + '\n\nDisplay of the remainder of the message truncated because it is too long.') # Only show the first 30K characters
self.ui.tableWidgetInbox.item(currentRow, 0).setFont(font)
@ -2746,8 +2741,17 @@ class MyForm(QtGui.QMainWindow):
def tableWidgetSentItemClicked(self):
currentRow = self.ui.tableWidgetSent.currentRow()
if currentRow >= 0:
self.ui.textEditSentMessage.setPlainText(self.ui.tableWidgetSent.item(
currentRow, 2).data(Qt.UserRole).toPyObject())
ackdata = str(self.ui.tableWidgetSent.item(
currentRow, 3).data(Qt.UserRole).toPyObject())
queryreturn = sqlQuery(
'''select message from sent where ackdata=?''', ackdata)
if queryreturn != []:
for row in queryreturn:
message, = row
else:
message = "Error occurred: could not load message from disk."
message = unicode(message, 'utf-8)')
self.ui.textEditSentMessage.setPlainText(message)
def tableWidgetYourIdentitiesItemChanged(self):
currentRow = self.ui.tableWidgetYourIdentities.currentRow()

12
src/pyelliptic/openssl.py
View File

@ -268,7 +268,7 @@ class _OpenSSL:
self.EVP_MD_CTX_destroy.argtypes = [ctypes.c_void_p]
self.RAND_bytes = self._lib.RAND_bytes
self.RAND_bytes.restype = None
self.RAND_bytes.restype = ctypes.c_int
self.RAND_bytes.argtypes = [ctypes.c_void_p, ctypes.c_int]
@ -394,7 +394,15 @@ class _OpenSSL:
OpenSSL random function
"""
buffer = self.malloc(0, size)
self.RAND_bytes(buffer, size)
# This pyelliptic library, by default, didn't check the return value of RAND_bytes. It is
# evidently possible that it returned an error and not-actually-random data. However, in
# tests on various operating systems, while generating hundreds of gigabytes of random
# strings of various sizes I could not get an error to occur. Also Bitcoin doesn't check
# the return value of RAND_bytes either.
# Fixed in Bitmessage version 0.4.2 (in source code on 2013-10-13)
while self.RAND_bytes(buffer, size) != 1:
import time
time.sleep(1)
return buffer.raw
def malloc(self, data, size):

4
src/shared.py
View File

@ -279,6 +279,8 @@ def reloadBroadcastSendersForWhichImWatching():
def doCleanShutdown():
global shutdown
shutdown = 1 #Used to tell proof of work worker threads to exit.
broadcastToSendDataQueues((0, 'shutdown', 'all'))
knownNodesLock.acquire()
UISignalQueue.put(('updateStatusBar','Saving the knownNodes list of peers to disk...'))
output = open(appdata + 'knownnodes.dat', 'wb')
@ -290,8 +292,6 @@ def doCleanShutdown():
logger.info('Finished closing knownnodes.dat output file.')
UISignalQueue.put(('updateStatusBar','Done saving the knownNodes list of peers to disk.'))
broadcastToSendDataQueues((0, 'shutdown', 'all'))
logger.info('Flushing inventory in memory out to disk...')
UISignalQueue.put((
'updateStatusBar',