Merge pull request #532 from Atheros1/master

Don't store inbox & sent message data in memory

src/bitmessageqt/__init__.py

@@ -560,7 +560,7 @@ class MyForm(QtGui.QMainWindow):
             where = "toaddress || fromaddress || subject || message"
 
         sqlStatement = '''
-            SELECT toaddress, fromaddress, subject, message, status, ackdata, lastactiontime 
+            SELECT toaddress, fromaddress, subject, status, ackdata, lastactiontime 
             FROM sent WHERE folder="sent" AND %s LIKE ? 
             ORDER BY lastactiontime
             ''' % (where,)
@@ -570,9 +570,9 @@ class MyForm(QtGui.QMainWindow):
 
         queryreturn = sqlQuery(sqlStatement, what)
         for row in queryreturn:
-            toAddress, fromAddress, subject, message, status, ackdata, lastactiontime = row
+            toAddress, fromAddress, subject, status, ackdata, lastactiontime = row
             subject = shared.fixPotentiallyInvalidUTF8Data(subject)
-            message = shared.fixPotentiallyInvalidUTF8Data(message)
+            #message = shared.fixPotentiallyInvalidUTF8Data(message)
             try:
                 fromLabel = shared.config.get(fromAddress, 'label')
             except:
@@ -612,7 +612,7 @@ class MyForm(QtGui.QMainWindow):
             self.ui.tableWidgetSent.setItem(0, 1, newItem)
             newItem = QtGui.QTableWidgetItem(unicode(subject, 'utf-8'))
             newItem.setToolTip(unicode(subject, 'utf-8'))
-            newItem.setData(Qt.UserRole, unicode(message, 'utf-8)'))
+            #newItem.setData(Qt.UserRole, unicode(message, 'utf-8)')) # No longer hold the message in the table; we'll use a SQL query to display it as needed.
             newItem.setFlags(
                 QtCore.Qt.ItemIsSelectable | QtCore.Qt.ItemIsEnabled)
             self.ui.tableWidgetSent.setItem(0, 2, newItem)
@@ -680,7 +680,7 @@ class MyForm(QtGui.QMainWindow):
             where = "toaddress || fromaddress || subject || message"
 
         sqlStatement = '''
-            SELECT msgid, toaddress, fromaddress, subject, received, message, read 
+            SELECT msgid, toaddress, fromaddress, subject, received, read 
             FROM inbox WHERE folder="inbox" AND %s LIKE ? 
             ORDER BY received
             ''' % (where,)
@@ -692,9 +692,9 @@ class MyForm(QtGui.QMainWindow):
         font.setBold(True)
         queryreturn = sqlQuery(sqlStatement, what)
         for row in queryreturn:
-            msgid, toAddress, fromAddress, subject, received, message, read = row
+            msgid, toAddress, fromAddress, subject, received, read = row
             subject = shared.fixPotentiallyInvalidUTF8Data(subject)
-            message = shared.fixPotentiallyInvalidUTF8Data(message)
+            #message = shared.fixPotentiallyInvalidUTF8Data(message)
             try:
                 if toAddress == self.str_broadcast_subscribers:
                     toLabel = self.str_broadcast_subscribers
@@ -750,7 +750,7 @@ class MyForm(QtGui.QMainWindow):
             self.ui.tableWidgetInbox.setItem(0, 1, newItem)
             newItem = QtGui.QTableWidgetItem(unicode(subject, 'utf-8'))
             newItem.setToolTip(unicode(subject, 'utf-8'))
-            newItem.setData(Qt.UserRole, unicode(message, 'utf-8)'))
+            #newItem.setData(Qt.UserRole, unicode(message, 'utf-8)')) # No longer hold the message in the table (and thus in memory); we'll use a SQL query when we need to display it.
             newItem.setFlags(
                 QtCore.Qt.ItemIsSelectable | QtCore.Qt.ItemIsEnabled)
             if not read:
@@ -1632,44 +1632,12 @@ class MyForm(QtGui.QMainWindow):
                 sqlExecute(
                     '''INSERT INTO sent VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)''', *t)
 
-                shared.workerQueue.put(('sendbroadcast', ''))
-
-                try:
-                    fromLabel = shared.config.get(fromAddress, 'label')
-                except:
-                    fromLabel = ''
-                if fromLabel == '':
-                    fromLabel = fromAddress
-
                 toLabel = self.str_broadcast_subscribers
                 
-                self.ui.tableWidgetSent.insertRow(0)
+                self.displayNewSentMessage(
-                newItem = QtGui.QTableWidgetItem(unicode(toLabel, 'utf-8'))
+                    toAddress, toLabel, fromAddress, subject, message, ackdata)
-                newItem.setData(Qt.UserRole, str(toAddress))
-                self.ui.tableWidgetSent.setItem(0, 0, newItem)
 
-                if fromLabel == '':
+                shared.workerQueue.put(('sendbroadcast', ''))
-                    newItem = QtGui.QTableWidgetItem(
-                        unicode(fromAddress, 'utf-8'))
-                else:
-                    newItem = QtGui.QTableWidgetItem(
-                        unicode(fromLabel, 'utf-8'))
-                newItem.setData(Qt.UserRole, str(fromAddress))
-                self.ui.tableWidgetSent.setItem(0, 1, newItem)
-                newItem = QtGui.QTableWidgetItem(unicode(subject, 'utf-8)'))
-                newItem.setData(Qt.UserRole, unicode(message, 'utf-8)'))
-                self.ui.tableWidgetSent.setItem(0, 2, newItem)
-                # newItem =  QtGui.QTableWidgetItem('Doing work necessary to
-                # send broadcast...'+
-                # unicode(strftime(config.get('bitmessagesettings',
-                # 'timeformat'),localtime(int(time.time()))),'utf-8'))
-                newItem = myTableWidgetItem(_translate("MainWindow", "Work is queued."))
-                newItem.setData(Qt.UserRole, QByteArray(ackdata))
-                newItem.setData(33, int(time.time()))
-                self.ui.tableWidgetSent.setItem(0, 3, newItem)
-
-                self.ui.textEditSentMessage.setPlainText(
-                    self.ui.tableWidgetSent.item(0, 2).data(Qt.UserRole).toPyObject())
 
                 self.ui.comboBoxSendFrom.setCurrentIndex(0)
                 self.ui.labelFrom.setText('')
@@ -1765,7 +1733,7 @@ class MyForm(QtGui.QMainWindow):
         self.ui.tableWidgetSent.setItem(0, 1, newItem)
         newItem = QtGui.QTableWidgetItem(unicode(subject, 'utf-8)'))
         newItem.setToolTip(unicode(subject, 'utf-8)'))
-        newItem.setData(Qt.UserRole, unicode(message, 'utf-8)'))
+        #newItem.setData(Qt.UserRole, unicode(message, 'utf-8)')) # No longer hold the message in the table; we'll use a SQL query to display it as needed.
         self.ui.tableWidgetSent.setItem(0, 2, newItem)
         # newItem =  QtGui.QTableWidgetItem('Doing work necessary to send
         # broadcast...'+
@@ -1778,13 +1746,12 @@ class MyForm(QtGui.QMainWindow):
         newItem.setData(Qt.UserRole, QByteArray(ackdata))
         newItem.setData(33, int(time.time()))
         self.ui.tableWidgetSent.setItem(0, 3, newItem)
-        self.ui.textEditSentMessage.setPlainText(
+        self.ui.textEditSentMessage.setPlainText(unicode(message, 'utf-8)'))
-            self.ui.tableWidgetSent.item(0, 2).data(Qt.UserRole).toPyObject())
         self.ui.tableWidgetSent.setSortingEnabled(True)
 
     def displayNewInboxMessage(self, inventoryHash, toAddress, fromAddress, subject, message):
         subject = shared.fixPotentiallyInvalidUTF8Data(subject)
-        message = shared.fixPotentiallyInvalidUTF8Data(message)
+        #message = shared.fixPotentiallyInvalidUTF8Data(message)
         fromLabel = ''
         queryreturn = sqlQuery(
             '''select label from addressbook where address=?''', fromAddress)
@@ -1838,7 +1805,7 @@ class MyForm(QtGui.QMainWindow):
         self.ui.tableWidgetInbox.setItem(0, 1, newItem)
         newItem = QtGui.QTableWidgetItem(unicode(subject, 'utf-8)'))
         newItem.setToolTip(unicode(subject, 'utf-8)'))
-        newItem.setData(Qt.UserRole, unicode(message, 'utf-8)'))
+        #newItem.setData(Qt.UserRole, unicode(message, 'utf-8)')) # No longer hold the message in the table; we'll use a SQL query to display it as needed.
         newItem.setFont(font)
         self.ui.tableWidgetInbox.setItem(0, 2, newItem)
         newItem = myTableWidgetItem(unicode(strftime(shared.config.get(
@@ -2255,10 +2222,18 @@ class MyForm(QtGui.QMainWindow):
 
     def on_action_InboxMessageForceHtml(self):
         currentInboxRow = self.ui.tableWidgetInbox.currentRow()
-        lines = self.ui.tableWidgetInbox.item(
+
-            currentInboxRow, 2).data(Qt.UserRole).toPyObject().split('\n')
+        msgid = str(self.ui.tableWidgetInbox.item(
+            currentInboxRow, 3).data(Qt.UserRole).toPyObject())
+        queryreturn = sqlQuery(
+            '''select message from inbox where msgid=?''', msgid)
+        if queryreturn != []:
+            for row in queryreturn:
+                messageAtCurrentInboxRow, = row 
+
+        lines = messageAtCurrentInboxRow.split('\n')
         for i in xrange(len(lines)):
-            if lines[i].contains('Message ostensibly from '):
+            if 'Message ostensibly from ' in lines[i]:
                 lines[i] = '<p style="font-size: 12px; color: grey;">%s</span></p>' % (
                     lines[i])
             elif lines[i] == '------------------------------------------------------':
@@ -2292,6 +2267,13 @@ class MyForm(QtGui.QMainWindow):
             currentInboxRow, 0).data(Qt.UserRole).toPyObject())
         fromAddressAtCurrentInboxRow = str(self.ui.tableWidgetInbox.item(
             currentInboxRow, 1).data(Qt.UserRole).toPyObject())
+        msgid = str(self.ui.tableWidgetInbox.item(
+            currentInboxRow, 3).data(Qt.UserRole).toPyObject())
+        queryreturn = sqlQuery(
+            '''select message from inbox where msgid=?''', msgid)
+        if queryreturn != []:
+            for row in queryreturn:
+                messageAtCurrentInboxRow, = row
         if toAddressAtCurrentInboxRow == self.str_broadcast_subscribers:
             self.ui.labelFrom.setText('')
         elif not shared.config.has_section(toAddressAtCurrentInboxRow):
@@ -2315,9 +2297,7 @@ class MyForm(QtGui.QMainWindow):
                 self.ui.lineEditTo.setText(str(toAddressAtCurrentInboxRow))
 
         self.ui.comboBoxSendFrom.setCurrentIndex(0)
-        # self.ui.comboBoxSendFrom.setEditText(str(self.ui.tableWidgetInbox.item(currentInboxRow,0).text))
+        self.ui.textEditMessage.setText('\n\n------------------------------------------------------\n' + unicode(messageAtCurrentInboxRow, 'utf-8)'))
-        self.ui.textEditMessage.setText('\n\n------------------------------------------------------\n' + self.ui.tableWidgetInbox.item(
-            currentInboxRow, 2).data(Qt.UserRole).toPyObject())
         if self.ui.tableWidgetInbox.item(currentInboxRow, 2).text()[0:3] in ['Re:', 'RE:']:
             self.ui.lineEditSubject.setText(
                 self.ui.tableWidgetInbox.item(currentInboxRow, 2).text())
@@ -2377,14 +2357,23 @@ class MyForm(QtGui.QMainWindow):
             subjectAtCurrentInboxRow = str(self.ui.tableWidgetInbox.item(currentInboxRow,2).text())
         except:
             subjectAtCurrentInboxRow = ''
+
+        # Retrieve the message data out of the SQL database
+        msgid = str(self.ui.tableWidgetInbox.item(
+            currentInboxRow, 3).data(Qt.UserRole).toPyObject())
+        queryreturn = sqlQuery(
+            '''select message from inbox where msgid=?''', msgid)
+        if queryreturn != []:
+            for row in queryreturn:
+                message, = row
+
         defaultFilename = "".join(x for x in subjectAtCurrentInboxRow if x.isalnum()) + '.txt'
-        data = self.ui.tableWidgetInbox.item(currentInboxRow,2).data(Qt.UserRole).toPyObject()
         filename = QFileDialog.getSaveFileName(self, _translate("MainWindow","Save As..."), defaultFilename, "Text files (*.txt);;All files (*.*)")
         if filename == '':
             return
         try:
             f = open(filename, 'w')
-            f.write( self.ui.tableWidgetInbox.item(currentInboxRow,2).data(Qt.UserRole).toPyObject() )
+            f.write(message)
             f.close()
         except Exception, e:
             sys.stderr.write('Write error: '+ e)
@@ -2716,21 +2705,27 @@ class MyForm(QtGui.QMainWindow):
             
             fromAddress = str(self.ui.tableWidgetInbox.item(
                 currentRow, 1).data(Qt.UserRole).toPyObject())
+            msgid = str(self.ui.tableWidgetInbox.item(
+                currentRow, 3).data(Qt.UserRole).toPyObject())
+            queryreturn = sqlQuery(
+                '''select message from inbox where msgid=?''', msgid)
+            if queryreturn != []:
+                for row in queryreturn:
+                    message, = row
+            message = unicode(message, 'utf-8)')
             # If we have received this message from either a broadcast address
             # or from someone in our address book, display as HTML
             if decodeAddress(fromAddress)[3] in shared.broadcastSendersForWhichImWatching or shared.isAddressInMyAddressBook(fromAddress):
-                if len(self.ui.tableWidgetInbox.item(currentRow, 2).data(Qt.UserRole).toPyObject()) < 30000:
+                if len(message) < 30000:
-                    self.ui.textEditInboxMessage.setText(self.ui.tableWidgetInbox.item(
+                    self.ui.textEditInboxMessage.setText(message)  # Only show the first 30K characters
-                        currentRow, 2).data(Qt.UserRole).toPyObject())  # Only show the first 30K characters
                 else:
-                    self.ui.textEditInboxMessage.setText(self.ui.tableWidgetInbox.item(currentRow, 2).data(Qt.UserRole).toPyObject()[
+                    self.ui.textEditInboxMessage.setText(message[
                                                          :30000] + '\n\nDisplay of the remainder of the message truncated because it is too long.')  # Only show the first 30K characters
             else:
-                if len(self.ui.tableWidgetInbox.item(currentRow, 2).data(Qt.UserRole).toPyObject()) < 30000:
+                if len(message) < 30000:
-                    self.ui.textEditInboxMessage.setPlainText(self.ui.tableWidgetInbox.item(
+                    self.ui.textEditInboxMessage.setPlainText(message)  # Only show the first 30K characters
-                        currentRow, 2).data(Qt.UserRole).toPyObject())  # Only show the first 30K characters
                 else:
-                    self.ui.textEditInboxMessage.setPlainText(self.ui.tableWidgetInbox.item(currentRow, 2).data(Qt.UserRole).toPyObject()[
+                    self.ui.textEditInboxMessage.setPlainText(message[
                                                               :30000] + '\n\nDisplay of the remainder of the message truncated because it is too long.')  # Only show the first 30K characters
             
             self.ui.tableWidgetInbox.item(currentRow, 0).setFont(font)
@@ -2746,8 +2741,17 @@ class MyForm(QtGui.QMainWindow):
     def tableWidgetSentItemClicked(self):
         currentRow = self.ui.tableWidgetSent.currentRow()
         if currentRow >= 0:
-            self.ui.textEditSentMessage.setPlainText(self.ui.tableWidgetSent.item(
+            ackdata = str(self.ui.tableWidgetSent.item(
-                currentRow, 2).data(Qt.UserRole).toPyObject())
+                currentRow, 3).data(Qt.UserRole).toPyObject())
+            queryreturn = sqlQuery(
+                '''select message from sent where ackdata=?''', ackdata)
+            if queryreturn != []:
+                for row in queryreturn:
+                    message, = row
+            else:
+                message = "Error occurred: could not load message from disk."
+            message = unicode(message, 'utf-8)')
+            self.ui.textEditSentMessage.setPlainText(message)
 
     def tableWidgetYourIdentitiesItemChanged(self):
         currentRow = self.ui.tableWidgetYourIdentities.currentRow()

src/pyelliptic/openssl.py

@@ -268,7 +268,7 @@ class _OpenSSL:
         self.EVP_MD_CTX_destroy.argtypes = [ctypes.c_void_p]
 
         self.RAND_bytes = self._lib.RAND_bytes
-        self.RAND_bytes.restype = None
+        self.RAND_bytes.restype = ctypes.c_int
         self.RAND_bytes.argtypes = [ctypes.c_void_p, ctypes.c_int]
 
 
@@ -394,7 +394,15 @@ class _OpenSSL:
         OpenSSL random function
         """
         buffer = self.malloc(0, size)
-        self.RAND_bytes(buffer, size)
+        # This pyelliptic library, by default, didn't check the return value of RAND_bytes. It is 
+        # evidently possible that it returned an error and not-actually-random data. However, in
+        # tests on various operating systems, while generating hundreds of gigabytes of random 
+        # strings of various sizes I could not get an error to occur. Also Bitcoin doesn't check
+        # the return value of RAND_bytes either. 
+        # Fixed in Bitmessage version 0.4.2 (in source code on 2013-10-13)
+        while self.RAND_bytes(buffer, size) != 1:
+            import time
+            time.sleep(1)
         return buffer.raw
 
     def malloc(self, data, size):

src/shared.py

@@ -279,6 +279,8 @@ def reloadBroadcastSendersForWhichImWatching():
 def doCleanShutdown():
     global shutdown
     shutdown = 1 #Used to tell proof of work worker threads to exit.    
+    broadcastToSendDataQueues((0, 'shutdown', 'all'))
+    
     knownNodesLock.acquire()
     UISignalQueue.put(('updateStatusBar','Saving the knownNodes list of peers to disk...'))
     output = open(appdata + 'knownnodes.dat', 'wb')
@@ -290,8 +292,6 @@ def doCleanShutdown():
     logger.info('Finished closing knownnodes.dat output file.')
     UISignalQueue.put(('updateStatusBar','Done saving the knownNodes list of peers to disk.'))
 
-    broadcastToSendDataQueues((0, 'shutdown', 'all'))
-
     logger.info('Flushing inventory in memory out to disk...')
     UISignalQueue.put((
         'updateStatusBar',