From 59f3a2fbe7c1278e4ee073c5122c3908a096e8ab Mon Sep 17 00:00:00 2001 From: Peter Surda Date: Wed, 8 Feb 2017 13:41:56 +0100 Subject: [PATCH] Reduce cyclic dependencies - rearranged code to reduce cyclic dependencies - doCleanShutdown is separated in shutdown.py - shared queues are separated in queues.py - some default values were moved to defaults.py - knownnodes partially moved to knownnodes.py --- src/api.py | 67 ++++++++++---------- src/bitmessagecurses/__init__.py | 20 +++--- src/bitmessagemain.py | 3 +- src/bitmessageqt/__init__.py | 40 ++++++------ src/bitmessageqt/account.py | 4 +- src/bitmessageqt/addressvalidator.py | 2 +- src/bitmessageqt/newchandialog.py | 2 +- src/bitmessageqt/safehtmlparser.py | 2 +- src/bitmessageqt/support.py | 4 +- src/bitmessageqt/uisignaler.py | 5 +- src/class_addressGenerator.py | 35 ++++++----- src/class_objectProcessor.py | 33 +++++----- src/class_objectProcessorQueue.py | 2 - src/class_outgoingSynSender.py | 62 +++++++++--------- src/class_receiveDataThread.py | 66 +++++++++---------- src/class_singleCleaner.py | 40 ++++++------ src/class_singleWorker.py | 59 ++++++++--------- src/class_smtpDeliver.py | 6 +- src/class_smtpServer.py | 4 +- src/class_sqlThread.py | 30 ++++----- src/defaults.py | 8 +++ src/helper_bootstrap.py | 19 +++--- src/helper_generic.py | 7 ++- src/helper_inbox.py | 8 +-- src/knownnodes.py | 5 ++ src/message_data_reader.py | 4 +- src/namecoin.py | 6 +- src/proofofwork.py | 10 +-- src/queues.py | 16 +++++ src/shared.py | 94 +--------------------------- src/shutdown.py | 87 +++++++++++++++++++++++++ src/state.py | 1 + src/upnp.py | 5 +- 33 files changed, 401 insertions(+), 355 deletions(-) create mode 100644 src/defaults.py create mode 100644 src/knownnodes.py create mode 100644 src/queues.py create mode 100644 src/shutdown.py diff --git a/src/api.py b/src/api.py index f25972aa..101241ec 100644 --- a/src/api.py +++ b/src/api.py @@ -27,6 +27,7 @@ import hashlib import protocol import state from pyelliptic.openssl import OpenSSL +import queues from struct import pack # Classes @@ -217,9 +218,9 @@ class MySimpleXMLRPCRequestHandler(SimpleXMLRPCRequestHandler): raise APIError(16, 'You already have this address in your address book.') sqlExecute("INSERT INTO addressbook VALUES(?,?)", label, address) - shared.UISignalQueue.put(('rerenderMessagelistFromLabels','')) - shared.UISignalQueue.put(('rerenderMessagelistToLabels','')) - shared.UISignalQueue.put(('rerenderAddressBook','')) + queues.UISignalQueue.put(('rerenderMessagelistFromLabels','')) + queues.UISignalQueue.put(('rerenderMessagelistToLabels','')) + queues.UISignalQueue.put(('rerenderAddressBook','')) return "Added address %s to address book" % address def HandleDeleteAddressBookEntry(self, params): @@ -229,9 +230,9 @@ class MySimpleXMLRPCRequestHandler(SimpleXMLRPCRequestHandler): address = addBMIfNotPresent(address) self._verifyAddress(address) sqlExecute('DELETE FROM addressbook WHERE address=?', address) - shared.UISignalQueue.put(('rerenderMessagelistFromLabels','')) - shared.UISignalQueue.put(('rerenderMessagelistToLabels','')) - shared.UISignalQueue.put(('rerenderAddressBook','')) + queues.UISignalQueue.put(('rerenderMessagelistFromLabels','')) + queues.UISignalQueue.put(('rerenderMessagelistToLabels','')) + queues.UISignalQueue.put(('rerenderAddressBook','')) return "Deleted address book entry for %s if it existed" % address def HandleCreateRandomAddress(self, params): @@ -269,11 +270,11 @@ class MySimpleXMLRPCRequestHandler(SimpleXMLRPCRequestHandler): unicode(label, 'utf-8') except: raise APIError(17, 'Label is not valid UTF-8 data.') - shared.apiAddressGeneratorReturnQueue.queue.clear() + queues.apiAddressGeneratorReturnQueue.queue.clear() streamNumberForAddress = 1 - shared.addressGeneratorQueue.put(( + queues.addressGeneratorQueue.put(( 'createRandomAddress', 4, streamNumberForAddress, label, 1, "", eighteenByteRipe, nonceTrialsPerByte, payloadLengthExtraBytes)) - return shared.apiAddressGeneratorReturnQueue.get() + return queues.apiAddressGeneratorReturnQueue.get() def HandleCreateDeterministicAddresses(self, params): if len(params) == 0: @@ -349,13 +350,13 @@ class MySimpleXMLRPCRequestHandler(SimpleXMLRPCRequestHandler): raise APIError(4, 'Why would you ask me to generate 0 addresses for you?') if numberOfAddresses > 999: raise APIError(5, 'You have (accidentally?) specified too many addresses to make. Maximum 999. This check only exists to prevent mischief; if you really want to create more addresses than this, contact the Bitmessage developers and we can modify the check or you can do it yourself by searching the source code for this message.') - shared.apiAddressGeneratorReturnQueue.queue.clear() + queues.apiAddressGeneratorReturnQueue.queue.clear() logger.debug('Requesting that the addressGenerator create %s addresses.', numberOfAddresses) - shared.addressGeneratorQueue.put( + queues.addressGeneratorQueue.put( ('createDeterministicAddresses', addressVersionNumber, streamNumber, 'unused API address', numberOfAddresses, passphrase, eighteenByteRipe, nonceTrialsPerByte, payloadLengthExtraBytes)) data = '{"addresses":[' - queueReturn = shared.apiAddressGeneratorReturnQueue.get() + queueReturn = queues.apiAddressGeneratorReturnQueue.get() for item in queueReturn: if len(data) > 20: data += ',' @@ -376,12 +377,12 @@ class MySimpleXMLRPCRequestHandler(SimpleXMLRPCRequestHandler): raise APIError(2, 'The address version number currently must be 3 or 4. ' + addressVersionNumber + ' isn\'t supported.') if streamNumber != 1: raise APIError(3, ' The stream number must be 1. Others aren\'t supported.') - shared.apiAddressGeneratorReturnQueue.queue.clear() + queues.apiAddressGeneratorReturnQueue.queue.clear() logger.debug('Requesting that the addressGenerator create %s addresses.', numberOfAddresses) - shared.addressGeneratorQueue.put( + queues.addressGeneratorQueue.put( ('getDeterministicAddress', addressVersionNumber, streamNumber, 'unused API address', numberOfAddresses, passphrase, eighteenByteRipe)) - return shared.apiAddressGeneratorReturnQueue.get() + return queues.apiAddressGeneratorReturnQueue.get() def HandleCreateChan(self, params): if len(params) == 0: @@ -401,10 +402,10 @@ class MySimpleXMLRPCRequestHandler(SimpleXMLRPCRequestHandler): addressVersionNumber = 4 streamNumber = 1 - shared.apiAddressGeneratorReturnQueue.queue.clear() + queues.apiAddressGeneratorReturnQueue.queue.clear() logger.debug('Requesting that the addressGenerator create chan %s.', passphrase) - shared.addressGeneratorQueue.put(('createChan', addressVersionNumber, streamNumber, label, passphrase, True)) - queueReturn = shared.apiAddressGeneratorReturnQueue.get() + queues.addressGeneratorQueue.put(('createChan', addressVersionNumber, streamNumber, label, passphrase, True)) + queueReturn = queues.apiAddressGeneratorReturnQueue.get() if len(queueReturn) == 0: raise APIError(24, 'Chan address is already present.') address = queueReturn[0] @@ -428,9 +429,9 @@ class MySimpleXMLRPCRequestHandler(SimpleXMLRPCRequestHandler): status, addressVersionNumber, streamNumber, toRipe = self._verifyAddress(suppliedAddress) suppliedAddress = addBMIfNotPresent(suppliedAddress) - shared.apiAddressGeneratorReturnQueue.queue.clear() - shared.addressGeneratorQueue.put(('joinChan', suppliedAddress, label, passphrase, True)) - addressGeneratorReturnValue = shared.apiAddressGeneratorReturnQueue.get() + queues.apiAddressGeneratorReturnQueue.queue.clear() + queues.addressGeneratorQueue.put(('joinChan', suppliedAddress, label, passphrase, True)) + addressGeneratorReturnValue = queues.apiAddressGeneratorReturnQueue.get() if addressGeneratorReturnValue[0] == 'chan name does not match address': raise APIError(18, 'Chan name does not match address.') @@ -468,8 +469,8 @@ class MySimpleXMLRPCRequestHandler(SimpleXMLRPCRequestHandler): BMConfigParser().remove_section(address) with open(state.appdata + 'keys.dat', 'wb') as configfile: BMConfigParser().write(configfile) - shared.UISignalQueue.put(('rerenderMessagelistFromLabels','')) - shared.UISignalQueue.put(('rerenderMessagelistToLabels','')) + queues.UISignalQueue.put(('rerenderMessagelistFromLabels','')) + queues.UISignalQueue.put(('rerenderMessagelistToLabels','')) shared.reloadMyAddressHashes() return 'success' @@ -514,7 +515,7 @@ class MySimpleXMLRPCRequestHandler(SimpleXMLRPCRequestHandler): # UPDATE is slow, only update if status is different if queryreturn != [] and (queryreturn[0][0] == 1) != readStatus: sqlExecute('''UPDATE inbox set read = ? WHERE msgid=?''', readStatus, msgid) - shared.UISignalQueue.put(('changedInboxUnread', None)) + queues.UISignalQueue.put(('changedInboxUnread', None)) queryreturn = sqlQuery('''SELECT msgid, toaddress, fromaddress, subject, received, message, encodingtype, read FROM inbox WHERE msgid=?''', msgid) data = '{"inboxMessage":[' for row in queryreturn: @@ -695,10 +696,10 @@ class MySimpleXMLRPCRequestHandler(SimpleXMLRPCRequestHandler): for row in queryreturn: toLabel, = row # apiSignalQueue.put(('displayNewSentMessage',(toAddress,toLabel,fromAddress,subject,message,ackdata))) - shared.UISignalQueue.put(('displayNewSentMessage', ( + queues.UISignalQueue.put(('displayNewSentMessage', ( toAddress, toLabel, fromAddress, subject, message, ackdata))) - shared.workerQueue.put(('sendmessage', toAddress)) + queues.workerQueue.put(('sendmessage', toAddress)) return hexlify(ackdata) @@ -753,9 +754,9 @@ class MySimpleXMLRPCRequestHandler(SimpleXMLRPCRequestHandler): helper_sent.insert(t) toLabel = '[Broadcast subscribers]' - shared.UISignalQueue.put(('displayNewSentMessage', ( + queues.UISignalQueue.put(('displayNewSentMessage', ( toAddress, toLabel, fromAddress, subject, message, ackdata))) - shared.workerQueue.put(('sendbroadcast', '')) + queues.workerQueue.put(('sendbroadcast', '')) return hexlify(ackdata) @@ -799,8 +800,8 @@ class MySimpleXMLRPCRequestHandler(SimpleXMLRPCRequestHandler): raise APIError(16, 'You are already subscribed to that address.') sqlExecute('''INSERT INTO subscriptions VALUES (?,?,?)''',label, address, True) shared.reloadBroadcastSendersForWhichImWatching() - shared.UISignalQueue.put(('rerenderMessagelistFromLabels', '')) - shared.UISignalQueue.put(('rerenderSubscriptions', '')) + queues.UISignalQueue.put(('rerenderMessagelistFromLabels', '')) + queues.UISignalQueue.put(('rerenderSubscriptions', '')) return 'Added subscription.' def HandleDeleteSubscription(self, params): @@ -810,8 +811,8 @@ class MySimpleXMLRPCRequestHandler(SimpleXMLRPCRequestHandler): address = addBMIfNotPresent(address) sqlExecute('''DELETE FROM subscriptions WHERE address=?''', address) shared.reloadBroadcastSendersForWhichImWatching() - shared.UISignalQueue.put(('rerenderMessagelistFromLabels', '')) - shared.UISignalQueue.put(('rerenderSubscriptions', '')) + queues.UISignalQueue.put(('rerenderMessagelistFromLabels', '')) + queues.UISignalQueue.put(('rerenderSubscriptions', '')) return 'Deleted subscription if it existed.' def ListSubscriptions(self, params): @@ -972,7 +973,7 @@ class MySimpleXMLRPCRequestHandler(SimpleXMLRPCRequestHandler): def HandleStatusBar(self, params): message, = params - shared.UISignalQueue.put(('updateStatusBar', message)) + queues.UISignalQueue.put(('updateStatusBar', message)) def HandleDeleteAndVacuum(self, params): sqlStoredProcedure('deleteandvacuume') diff --git a/src/bitmessagecurses/__init__.py b/src/bitmessagecurses/__init__.py index 029f1725..61e2466a 100644 --- a/src/bitmessagecurses/__init__.py +++ b/src/bitmessagecurses/__init__.py @@ -21,13 +21,15 @@ import dialog from dialog import Dialog from helper_sql import * -import shared +from addresses import * import ConfigParser from configparser import BMConfigParser -from addresses import * -from pyelliptic.openssl import OpenSSL -import l10n from inventory import Inventory +import l10n +from pyelliptic.openssl import OpenSSL +import queues +import shared +import shutdown quit = False menutab = 1 @@ -446,7 +448,7 @@ def handlech(c, stdscr): choices=[("1", "Spend time shortening the address", 1 if shorten else 0)]) if r == d.DIALOG_OK and "1" in t: shorten = True - shared.addressGeneratorQueue.put(("createRandomAddress", 4, stream, label, 1, "", shorten)) + queues.addressGeneratorQueue.put(("createRandomAddress", 4, stream, label, 1, "", shorten)) elif t == "2": set_background_title(d, "Make deterministic addresses") r, t = d.passwordform("Enter passphrase", @@ -469,7 +471,7 @@ def handlech(c, stdscr): scrollbox(d, unicode("In addition to your passphrase, be sure to remember the following numbers:\n" "\n * Address version number: "+str(4)+"\n" " * Stream number: "+str(stream))) - shared.addressGeneratorQueue.put(('createDeterministicAddresses', 4, stream, "unused deterministic address", number, str(passphrase), shorten)) + queues.addressGeneratorQueue.put(('createDeterministicAddresses', 4, stream, "unused deterministic address", number, str(passphrase), shorten)) else: scrollbox(d, unicode("Passphrases do not match")) elif t == "2": # Send a message @@ -795,7 +797,7 @@ def sendMessage(sender="", recv="", broadcast=None, subject="", body="", reply=F "sent", 2, # encodingType BMConfigParser().getint('bitmessagesettings', 'ttl')) - shared.workerQueue.put(("sendmessage", addr)) + queues.workerQueue.put(("sendmessage", addr)) else: # Broadcast if recv == "": set_background_title(d, "Empty sender error") @@ -821,7 +823,7 @@ def sendMessage(sender="", recv="", broadcast=None, subject="", body="", reply=F "sent", # folder 2, # encodingType BMConfigParser().getint('bitmessagesettings', 'ttl')) - shared.workerQueue.put(('sendbroadcast', '')) + queues.workerQueue.put(('sendbroadcast', '')) def loadInbox(): sys.stdout = sys.__stdout__ @@ -1052,7 +1054,7 @@ def shutdown(): sys.stdout = sys.__stdout__ print("Shutting down...") sys.stdout = printlog - shared.doCleanShutdown() + shutdown.doCleanShutdown() sys.stdout = sys.__stdout__ sys.stderr = sys.__stderr__ diff --git a/src/bitmessagemain.py b/src/bitmessagemain.py index fd91d699..59d70cb5 100755 --- a/src/bitmessagemain.py +++ b/src/bitmessagemain.py @@ -30,6 +30,7 @@ import shared from helper_sql import sqlQuery import state import protocol +import shutdown import threading # Classes @@ -287,7 +288,7 @@ class Main: def stop(self): with shared.printLock: print('Stopping Bitmessage Deamon.') - shared.doCleanShutdown() + shutdown.doCleanShutdown() #TODO: nice function but no one is using this diff --git a/src/bitmessageqt/__init__.py b/src/bitmessageqt/__init__.py index 32f29eaf..7360fc05 100644 --- a/src/bitmessageqt/__init__.py +++ b/src/bitmessageqt/__init__.py @@ -73,14 +73,18 @@ import types from utils import * from collections import OrderedDict from account import * -from dialogs import AddAddressDialog from class_objectHashHolder import objectHashHolder from class_singleWorker import singleWorker +import defaults +from dialogs import AddAddressDialog from helper_generic import powQueueSize from inventory import PendingDownload, PendingUpload, PendingUploadDeadlineException +import knownnodes import paths from proofofwork import getPowType import protocol +import queues +import shutdown import state from statusbar import BMStatusBar import throttle @@ -1587,7 +1591,7 @@ class MyForm(settingsmixin.SMainWindow): QMessageBox.about(self, _translate("MainWindow", "Bad address version number"), _translate( "MainWindow", "Your address version number must be either 3 or 4.")) return - shared.addressGeneratorQueue.put(('createDeterministicAddresses', addressVersionNumber, streamNumberForAddress, "regenerated deterministic address", self.regenerateAddressesDialogInstance.ui.spinBoxNumberOfAddressesToMake.value( + queues.addressGeneratorQueue.put(('createDeterministicAddresses', addressVersionNumber, streamNumberForAddress, "regenerated deterministic address", self.regenerateAddressesDialogInstance.ui.spinBoxNumberOfAddressesToMake.value( ), self.regenerateAddressesDialogInstance.ui.lineEditPassphrase.text().toUtf8(), self.regenerateAddressesDialogInstance.ui.checkBoxEighteenByteRipe.isChecked())) self.ui.tabWidget.setCurrentIndex(3) @@ -2042,7 +2046,7 @@ class MyForm(settingsmixin.SMainWindow): self.displayNewSentMessage( toAddress, toLabel, fromAddress, subject, message, ackdata) - shared.workerQueue.put(('sendmessage', toAddress)) + queues.workerQueue.put(('sendmessage', toAddress)) self.ui.comboBoxSendFrom.setCurrentIndex(0) self.ui.lineEditTo.setText('') @@ -2093,7 +2097,7 @@ class MyForm(settingsmixin.SMainWindow): self.displayNewSentMessage( toAddress, toLabel, fromAddress, subject, message, ackdata) - shared.workerQueue.put(('sendbroadcast', '')) + queues.workerQueue.put(('sendbroadcast', '')) self.ui.comboBoxSendFromBroadcast.setCurrentIndex(0) self.ui.lineEditSubjectBroadcast.setText('') @@ -2304,7 +2308,7 @@ class MyForm(settingsmixin.SMainWindow): addressVersion) + encodeVarint(streamNumber) + ripe).digest()).digest() tag = doubleHashOfAddressData[32:] for value in shared.inventory.by_type_and_tag(3, tag): - shared.objectProcessorQueue.put((value.type, value.payload)) + queues.objectProcessorQueue.put((value.type, value.payload)) def click_pushButtonStatusIcon(self): logger.debug('click_pushButtonStatusIcon') @@ -2443,7 +2447,7 @@ class MyForm(settingsmixin.SMainWindow): # mark them as toodifficult if the receiver's required difficulty is still higher than # we are willing to do. sqlExecute('''UPDATE sent SET status='msgqueued' WHERE status='toodifficult' ''') - shared.workerQueue.put(('sendmessage', '')) + queues.workerQueue.put(('sendmessage', '')) #start:UI setting to stop trying to send messages after X days/months # I'm open to changing this UI to something else if someone has a better idea. @@ -2503,11 +2507,11 @@ class MyForm(settingsmixin.SMainWindow): with open(paths.lookupExeFolder() + 'keys.dat', 'wb') as configfile: BMConfigParser().write(configfile) # Write the knownnodes.dat file to disk in the new location - shared.knownNodesLock.acquire() + knownnodes.knownNodesLock.acquire() output = open(paths.lookupExeFolder() + 'knownnodes.dat', 'wb') - pickle.dump(shared.knownNodes, output) + pickle.dump(knownnodes.knownNodes, output) output.close() - shared.knownNodesLock.release() + knownnodes.knownNodesLock.release() os.remove(state.appdata + 'keys.dat') os.remove(state.appdata + 'knownnodes.dat') previousAppdataLocation = state.appdata @@ -2527,11 +2531,11 @@ class MyForm(settingsmixin.SMainWindow): # Write the keys.dat file to disk in the new location BMConfigParser().save() # Write the knownnodes.dat file to disk in the new location - shared.knownNodesLock.acquire() + knownnodes.knownNodesLock.acquire() output = open(state.appdata + 'knownnodes.dat', 'wb') - pickle.dump(shared.knownNodes, output) + pickle.dump(knownnodes.knownNodes, output) output.close() - shared.knownNodesLock.release() + knownnodes.knownNodesLock.release() os.remove(paths.lookupExeFolder() + 'keys.dat') os.remove(paths.lookupExeFolder() + 'knownnodes.dat') debug.restartLoggingInUpdatedAppdataLocation() @@ -2681,7 +2685,7 @@ class MyForm(settingsmixin.SMainWindow): # address.' streamNumberForAddress = decodeAddress( self.dialog.ui.comboBoxExisting.currentText())[2] - shared.addressGeneratorQueue.put(('createRandomAddress', 4, streamNumberForAddress, str( + queues.addressGeneratorQueue.put(('createRandomAddress', 4, streamNumberForAddress, str( self.dialog.ui.newaddresslabel.text().toUtf8()), 1, "", self.dialog.ui.checkBoxEighteenByteRipe.isChecked())) else: if self.dialog.ui.lineEditPassphrase.text() != self.dialog.ui.lineEditPassphraseAgain.text(): @@ -2692,7 +2696,7 @@ class MyForm(settingsmixin.SMainWindow): "MainWindow", "Choose a passphrase"), _translate("MainWindow", "You really do need a passphrase.")) else: streamNumberForAddress = 1 # this will eventually have to be replaced by logic to determine the most available stream number. - shared.addressGeneratorQueue.put(('createDeterministicAddresses', 4, streamNumberForAddress, "unused deterministic address", self.dialog.ui.spinBoxNumberOfAddressesToMake.value( + queues.addressGeneratorQueue.put(('createDeterministicAddresses', 4, streamNumberForAddress, "unused deterministic address", self.dialog.ui.spinBoxNumberOfAddressesToMake.value( ), self.dialog.ui.lineEditPassphrase.text().toUtf8(), self.dialog.ui.checkBoxEighteenByteRipe.isChecked())) else: logger.debug('new address dialog box rejected') @@ -2817,7 +2821,7 @@ class MyForm(settingsmixin.SMainWindow): self.statusBar().showMessage(_translate("MainWindow", "Shutting down core... %1%").arg(str(80))) QtCore.QCoreApplication.processEvents(QtCore.QEventLoop.AllEvents, 1000) - shared.doCleanShutdown() + shutdown.doCleanShutdown() self.statusBar().showMessage(_translate("MainWindow", "Stopping notifications... %1%").arg(str(90))) self.tray.hide() # unregister the messaging system @@ -3217,9 +3221,9 @@ class MyForm(settingsmixin.SMainWindow): queryreturn = sqlQuery('''select ackdata FROM sent WHERE status='forcepow' ''') for row in queryreturn: ackdata, = row - shared.UISignalQueue.put(('updateSentItemStatusByAckdata', ( + queues.UISignalQueue.put(('updateSentItemStatusByAckdata', ( ackdata, 'Overriding maximum-difficulty setting. Work queued.'))) - shared.workerQueue.put(('sendmessage', '')) + queues.workerQueue.put(('sendmessage', '')) def on_action_SentClipboard(self): currentRow = self.ui.tableWidgetInbox.currentRow() @@ -4228,7 +4232,7 @@ class settingsDialog(QtGui.QDialog): self.ui.labelNamecoinPassword.setEnabled(isNamecoind) if isNamecoind: - self.ui.lineEditNamecoinPort.setText(shared.namecoinDefaultRpcPort) + self.ui.lineEditNamecoinPort.setText(defaults.namecoinDefaultRpcPort) else: self.ui.lineEditNamecoinPort.setText("9000") diff --git a/src/bitmessageqt/account.py b/src/bitmessageqt/account.py index bdfeaed3..897cddef 100644 --- a/src/bitmessageqt/account.py +++ b/src/bitmessageqt/account.py @@ -1,6 +1,6 @@ from PyQt4 import QtCore, QtGui -import shared +import queues import re import sys import inspect @@ -172,7 +172,7 @@ class GatewayAccount(BMAccount): min(BMConfigParser().getint('bitmessagesettings', 'ttl'), 86400 * 2) # not necessary to have a TTL higher than 2 days ) - shared.workerQueue.put(('sendmessage', self.toAddress)) + queues.workerQueue.put(('sendmessage', self.toAddress)) def parseMessage(self, toAddress, fromAddress, subject, message): super(GatewayAccount, self).parseMessage(toAddress, fromAddress, subject, message) diff --git a/src/bitmessageqt/addressvalidator.py b/src/bitmessageqt/addressvalidator.py index c2c686f7..56352b72 100644 --- a/src/bitmessageqt/addressvalidator.py +++ b/src/bitmessageqt/addressvalidator.py @@ -3,7 +3,7 @@ from Queue import Empty from addresses import decodeAddress, addBMIfNotPresent from account import getSortedAccounts -from shared import apiAddressGeneratorReturnQueue, addressGeneratorQueue +from queues import apiAddressGeneratorReturnQueue, addressGeneratorQueue from tr import _translate from utils import str_chan diff --git a/src/bitmessageqt/newchandialog.py b/src/bitmessageqt/newchandialog.py index d9f70f87..64ed3a8c 100644 --- a/src/bitmessageqt/newchandialog.py +++ b/src/bitmessageqt/newchandialog.py @@ -2,7 +2,7 @@ from PyQt4 import QtCore, QtGui from addresses import addBMIfNotPresent from addressvalidator import AddressValidator, PassPhraseValidator -from shared import apiAddressGeneratorReturnQueue, addressGeneratorQueue, UISignalQueue +from queues import apiAddressGeneratorReturnQueue, addressGeneratorQueue, UISignalQueue from retranslateui import RetranslateMixin from tr import _translate from utils import str_chan diff --git a/src/bitmessageqt/safehtmlparser.py b/src/bitmessageqt/safehtmlparser.py index 8009f02c..1692844e 100644 --- a/src/bitmessageqt/safehtmlparser.py +++ b/src/bitmessageqt/safehtmlparser.py @@ -6,7 +6,7 @@ import Queue from urllib import quote, quote_plus from urlparse import urlparse from debug import logger -from shared import parserInputQueue, parserOutputQueue, parserProcess, parserLock +from queues import parserInputQueue, parserOutputQueue, parserProcess, parserLock def regexpSubprocess(parserInputQueue, parserOutputQueue): for data in iter(parserInputQueue.get, None): diff --git a/src/bitmessageqt/support.py b/src/bitmessageqt/support.py index ba750423..041d50cc 100644 --- a/src/bitmessageqt/support.py +++ b/src/bitmessageqt/support.py @@ -1,6 +1,5 @@ import ctypes from PyQt4 import QtCore, QtGui -from os import path import ssl import sys import time @@ -16,6 +15,7 @@ import paths from proofofwork import bmpow import protocol from pyelliptic.openssl import OpenSSL +import queues import shared import state from version import softwareVersion @@ -67,7 +67,7 @@ def checkHasNormalAddress(): def createAddressIfNeeded(myapp): if not checkHasNormalAddress(): - shared.addressGeneratorQueue.put(('createRandomAddress', 4, 1, str(QtGui.QApplication.translate("Support", SUPPORT_MY_LABEL)), 1, "", False, protocol.networkDefaultProofOfWorkNonceTrialsPerByte, protocol.networkDefaultPayloadLengthExtraBytes)) + queues.addressGeneratorQueue.put(('createRandomAddress', 4, 1, str(QtGui.QApplication.translate("Support", SUPPORT_MY_LABEL)), 1, "", False, protocol.networkDefaultProofOfWorkNonceTrialsPerByte, protocol.networkDefaultPayloadLengthExtraBytes)) while state.shutdown == 0 and not checkHasNormalAddress(): time.sleep(.2) myapp.rerenderComboBoxSendFrom() diff --git a/src/bitmessageqt/uisignaler.py b/src/bitmessageqt/uisignaler.py index ea18f0b0..aabfaf68 100644 --- a/src/bitmessageqt/uisignaler.py +++ b/src/bitmessageqt/uisignaler.py @@ -1,8 +1,9 @@ from PyQt4.QtCore import QThread, SIGNAL -import shared import sys +import queues + class UISignaler(QThread): _instance = None @@ -18,7 +19,7 @@ class UISignaler(QThread): def run(self): while True: - command, data = shared.UISignalQueue.get() + command, data = queues.UISignalQueue.get() if command == 'writeNewAddressToTable': label, address, streamNumber = data self.emit(SIGNAL( diff --git a/src/class_addressGenerator.py b/src/class_addressGenerator.py index c3dfac3e..a3498694 100644 --- a/src/class_addressGenerator.py +++ b/src/class_addressGenerator.py @@ -14,6 +14,7 @@ import protocol from pyelliptic import arithmetic import tr from binascii import hexlify +import queues import state class addressGenerator(threading.Thread, StoppableThread): @@ -25,14 +26,14 @@ class addressGenerator(threading.Thread, StoppableThread): def stopThread(self): try: - shared.addressGeneratorQueue.put(("stopThread", "data")) + queues.addressGeneratorQueue.put(("stopThread", "data")) except: pass super(addressGenerator, self).stopThread() def run(self): while state.shutdown == 0: - queueValue = shared.addressGeneratorQueue.get() + queueValue = queues.addressGeneratorQueue.get() nonceTrialsPerByte = 0 payloadLengthExtraBytes = 0 live = True @@ -87,7 +88,7 @@ class addressGenerator(threading.Thread, StoppableThread): if payloadLengthExtraBytes < protocol.networkDefaultPayloadLengthExtraBytes: payloadLengthExtraBytes = protocol.networkDefaultPayloadLengthExtraBytes if command == 'createRandomAddress': - shared.UISignalQueue.put(( + queues.UISignalQueue.put(( 'updateStatusBar', tr._translate("MainWindow", "Generating one new address"))) # This next section is a little bit strange. We're going to generate keys over and over until we # find one that starts with either \x00 or \x00\x00. Then when we pack them into a Bitmessage address, @@ -147,18 +148,18 @@ class addressGenerator(threading.Thread, StoppableThread): # The API and the join and create Chan functionality # both need information back from the address generator. - shared.apiAddressGeneratorReturnQueue.put(address) + queues.apiAddressGeneratorReturnQueue.put(address) - shared.UISignalQueue.put(( + queues.UISignalQueue.put(( 'updateStatusBar', tr._translate("MainWindow", "Done generating address. Doing work necessary to broadcast it..."))) - shared.UISignalQueue.put(('writeNewAddressToTable', ( + queues.UISignalQueue.put(('writeNewAddressToTable', ( label, address, streamNumber))) shared.reloadMyAddressHashes() if addressVersionNumber == 3: - shared.workerQueue.put(( + queues.workerQueue.put(( 'sendOutOrStoreMyV3Pubkey', ripe.digest())) elif addressVersionNumber == 4: - shared.workerQueue.put(( + queues.workerQueue.put(( 'sendOutOrStoreMyV4Pubkey', address)) elif command == 'createDeterministicAddresses' or command == 'getDeterministicAddress' or command == 'createChan' or command == 'joinChan': @@ -166,7 +167,7 @@ class addressGenerator(threading.Thread, StoppableThread): sys.stderr.write( 'WARNING: You are creating deterministic address(es) using a blank passphrase. Bitmessage will do it but it is rather stupid.') if command == 'createDeterministicAddresses': - shared.UISignalQueue.put(( + queues.UISignalQueue.put(( 'updateStatusBar', tr._translate("MainWindow","Generating %1 new addresses.").arg(str(numberOfAddressesToMake)))) signingKeyNonce = 0 encryptionKeyNonce = 1 @@ -243,7 +244,7 @@ class addressGenerator(threading.Thread, StoppableThread): if addressAlreadyExists: logger.info('%s already exists. Not adding it again.' % address) - shared.UISignalQueue.put(( + queues.UISignalQueue.put(( 'updateStatusBar', tr._translate("MainWindow","%1 is already in 'Your Identities'. Not adding it again.").arg(address))) else: logger.debug('label: %s' % label) @@ -262,7 +263,7 @@ class addressGenerator(threading.Thread, StoppableThread): address, 'privEncryptionKey', privEncryptionKeyWIF) BMConfigParser().save() - shared.UISignalQueue.put(('writeNewAddressToTable', ( + queues.UISignalQueue.put(('writeNewAddressToTable', ( label, address, str(streamNumber)))) listOfNewAddressesToSendOutThroughTheAPI.append( address) @@ -273,24 +274,24 @@ class addressGenerator(threading.Thread, StoppableThread): addressVersionNumber) + encodeVarint(streamNumber) + ripe.digest()).digest()).digest()[32:] shared.myAddressesByTag[tag] = address if addressVersionNumber == 3: - shared.workerQueue.put(( + queues.workerQueue.put(( 'sendOutOrStoreMyV3Pubkey', ripe.digest())) # If this is a chan address, # the worker thread won't send out the pubkey over the network. elif addressVersionNumber == 4: - shared.workerQueue.put(( + queues.workerQueue.put(( 'sendOutOrStoreMyV4Pubkey', address)) - shared.UISignalQueue.put(( + queues.UISignalQueue.put(( 'updateStatusBar', tr._translate("MainWindow", "Done generating address"))) elif saveAddressToDisk and not live and not BMConfigParser().has_section(address): listOfNewAddressesToSendOutThroughTheAPI.append(address) # Done generating addresses. if command == 'createDeterministicAddresses' or command == 'joinChan' or command == 'createChan': - shared.apiAddressGeneratorReturnQueue.put( + queues.apiAddressGeneratorReturnQueue.put( listOfNewAddressesToSendOutThroughTheAPI) elif command == 'getDeterministicAddress': - shared.apiAddressGeneratorReturnQueue.put(address) + queues.apiAddressGeneratorReturnQueue.put(address) else: raise Exception( "Error in the addressGenerator thread. Thread was given a command it could not understand: " + command) - shared.addressGeneratorQueue.task_done() + queues.addressGeneratorQueue.task_done() diff --git a/src/class_objectProcessor.py b/src/class_objectProcessor.py index 00e5fbd4..b9969d9e 100644 --- a/src/class_objectProcessor.py +++ b/src/class_objectProcessor.py @@ -22,6 +22,7 @@ import helper_msgcoding import helper_sent from helper_sql import * import protocol +import queues import state import tr from debug import logger @@ -46,14 +47,14 @@ class objectProcessor(threading.Thread): '''SELECT objecttype, data FROM objectprocessorqueue''') for row in queryreturn: objectType, data = row - shared.objectProcessorQueue.put((objectType,data)) + queues.objectProcessorQueue.put((objectType,data)) sqlExecute('''DELETE FROM objectprocessorqueue''') logger.debug('Loaded %s objects from disk into the objectProcessorQueue.' % str(len(queryreturn))) def run(self): while True: - objectType, data = shared.objectProcessorQueue.get() + objectType, data = queues.objectProcessorQueue.get() try: if objectType == 0: # getpubkey @@ -77,8 +78,8 @@ class objectProcessor(threading.Thread): time.sleep(.5) # Wait just a moment for most of the connections to close numberOfObjectsThatWereInTheObjectProcessorQueue = 0 with SqlBulkExecute() as sql: - while shared.objectProcessorQueue.curSize > 0: - objectType, data = shared.objectProcessorQueue.get() + while queues.objectProcessorQueue.curSize > 0: + objectType, data = queues.objectProcessorQueue.get() sql.execute('''INSERT INTO objectprocessorqueue VALUES (?,?)''', objectType,data) numberOfObjectsThatWereInTheObjectProcessorQueue += 1 @@ -146,19 +147,19 @@ class objectProcessor(threading.Thread): return logger.info('Found getpubkey-requested-hash in my list of EC hashes. Telling Worker thread to do the POW for a pubkey message and send it out.') if requestedAddressVersionNumber == 2: - shared.workerQueue.put(( + queues.workerQueue.put(( 'doPOWForMyV2Pubkey', requestedHash)) elif requestedAddressVersionNumber == 3: - shared.workerQueue.put(( + queues.workerQueue.put(( 'sendOutOrStoreMyV3Pubkey', requestedHash)) elif requestedAddressVersionNumber == 4: - shared.workerQueue.put(( + queues.workerQueue.put(( 'sendOutOrStoreMyV4Pubkey', myAddress)) def processpubkey(self, data): pubkeyProcessingStartTime = time.time() shared.numberOfPubkeysProcessed += 1 - shared.UISignalQueue.put(( + queues.UISignalQueue.put(( 'updateNumberOfPubkeysProcessed', 'no data')) embeddedTime, = unpack('>Q', data[8:16]) readPosition = 20 # bypass the nonce, time, and object type @@ -307,7 +308,7 @@ class objectProcessor(threading.Thread): def processmsg(self, data): messageProcessingStartTime = time.time() shared.numberOfMessagesProcessed += 1 - shared.UISignalQueue.put(( + queues.UISignalQueue.put(( 'updateNumberOfMessagesProcessed', 'no data')) readPosition = 20 # bypass the nonce, time, and object type msgVersion, msgVersionLength = decodeVarint(data[readPosition:readPosition + 9]) @@ -329,7 +330,7 @@ class objectProcessor(threading.Thread): 'ackreceived', int(time.time()), data[-32:]) - shared.UISignalQueue.put(('updateSentItemStatusByAckdata', (data[-32:], tr._translate("MainWindow",'Acknowledgement of the message received %1').arg(l10n.formatTimestamp())))) + queues.UISignalQueue.put(('updateSentItemStatusByAckdata', (data[-32:], tr._translate("MainWindow",'Acknowledgement of the message received %1').arg(l10n.formatTimestamp())))) return else: logger.info('This was NOT an acknowledgement bound for me.') @@ -505,7 +506,7 @@ class objectProcessor(threading.Thread): time.time()), body, 'inbox', messageEncodingType, 0, sigHash) helper_inbox.insert(t) - shared.UISignalQueue.put(('displayNewInboxMessage', ( + queues.UISignalQueue.put(('displayNewInboxMessage', ( inventoryHash, toAddress, fromAddress, subject, body))) # If we are behaving as an API then we might need to run an @@ -561,9 +562,9 @@ class objectProcessor(threading.Thread): TTL) helper_sent.insert(t) - shared.UISignalQueue.put(('displayNewSentMessage', ( + queues.UISignalQueue.put(('displayNewSentMessage', ( toAddress, '[Broadcast subscribers]', fromAddress, subject, message, ackdataForBroadcast))) - shared.workerQueue.put(('sendbroadcast', '')) + queues.workerQueue.put(('sendbroadcast', '')) # Don't send ACK if invalid, blacklisted senders, invisible messages, disabled or chan if self.ackDataHasAValidHeader(ackData) and \ @@ -590,7 +591,7 @@ class objectProcessor(threading.Thread): def processbroadcast(self, data): messageProcessingStartTime = time.time() shared.numberOfBroadcastsProcessed += 1 - shared.UISignalQueue.put(( + queues.UISignalQueue.put(( 'updateNumberOfBroadcastsProcessed', 'no data')) inventoryHash = calculateInventoryHash(data) readPosition = 20 # bypass the nonce, time, and object type @@ -753,7 +754,7 @@ class objectProcessor(threading.Thread): time.time()), body, 'inbox', messageEncodingType, 0, sigHash) helper_inbox.insert(t) - shared.UISignalQueue.put(('displayNewInboxMessage', ( + queues.UISignalQueue.put(('displayNewInboxMessage', ( inventoryHash, toAddress, fromAddress, subject, body))) # If we are behaving as an API then we might need to run an @@ -808,7 +809,7 @@ class objectProcessor(threading.Thread): sqlExecute( '''UPDATE sent SET status='doingmsgpow', retrynumber=0 WHERE toaddress=? AND (status='awaitingpubkey' or status='doingpubkeypow') AND folder='sent' ''', address) - shared.workerQueue.put(('sendmessage', '')) + queues.workerQueue.put(('sendmessage', '')) def ackDataHasAValidHeader(self, ackData): if len(ackData) < protocol.Header.size: diff --git a/src/class_objectProcessorQueue.py b/src/class_objectProcessorQueue.py index 9bf3f82a..6309e994 100644 --- a/src/class_objectProcessorQueue.py +++ b/src/class_objectProcessorQueue.py @@ -1,5 +1,3 @@ -import shared - import Queue import threading import time diff --git a/src/class_outgoingSynSender.py b/src/class_outgoingSynSender.py index f154f455..85e9fa50 100644 --- a/src/class_outgoingSynSender.py +++ b/src/class_outgoingSynSender.py @@ -13,6 +13,8 @@ from class_sendDataThread import * from class_receiveDataThread import * from configparser import BMConfigParser from helper_threading import * +import knownnodes +import queues import state # For each stream to which we connect, several outgoingSynSender threads @@ -33,21 +35,21 @@ class outgoingSynSender(threading.Thread, StoppableThread): # ever connect to that. Otherwise we'll pick a random one from # the known nodes if state.trustedPeer: - shared.knownNodesLock.acquire() + knownnodes.knownNodesLock.acquire() peer = state.trustedPeer - shared.knownNodes[self.streamNumber][peer] = time.time() - shared.knownNodesLock.release() + knownnodes.knownNodes[self.streamNumber][peer] = time.time() + knownnodes.knownNodesLock.release() else: while not state.shutdown: - shared.knownNodesLock.acquire() + knownnodes.knownNodesLock.acquire() try: - peer, = random.sample(shared.knownNodes[self.streamNumber], 1) + peer, = random.sample(knownnodes.knownNodes[self.streamNumber], 1) except ValueError: # no known nodes - shared.knownNodesLock.release() + knownnodes.knownNodesLock.release() self.stop.wait(1) continue - priority = (183600 - (time.time() - shared.knownNodes[self.streamNumber][peer])) / 183600 # 2 days and 3 hours - shared.knownNodesLock.release() + priority = (183600 - (time.time() - knownnodes.knownNodes[self.streamNumber][peer])) / 183600 # 2 days and 3 hours + knownnodes.knownNodesLock.release() if BMConfigParser().get('bitmessagesettings', 'socksproxytype') != 'none': if peer.host.find(".onion") == -1: priority /= 10 # hidden services have 10x priority over plain net @@ -131,13 +133,13 @@ class outgoingSynSender(threading.Thread, StoppableThread): So let us remove the offending address from our knownNodes file. """ - shared.knownNodesLock.acquire() + knownnodes.knownNodesLock.acquire() try: - del shared.knownNodes[self.streamNumber][peer] + del knownnodes.knownNodes[self.streamNumber][peer] except: pass - shared.knownNodesLock.release() - logger.debug('deleting ' + str(peer) + ' from shared.knownNodes because it caused a socks.socksocket exception. We must not be 64-bit compatible.') + knownnodes.knownNodesLock.release() + logger.debug('deleting ' + str(peer) + ' from knownnodes.knownNodes because it caused a socks.socksocket exception. We must not be 64-bit compatible.') continue # This option apparently avoids the TIME_WAIT state so that we # can rebind faster @@ -220,7 +222,7 @@ class outgoingSynSender(threading.Thread, StoppableThread): except socks.GeneralProxyError as err: if err[0][0] in [7, 8, 9]: logger.error('Error communicating with proxy: %s', str(err)) - shared.UISignalQueue.put(( + queues.UISignalQueue.put(( 'updateStatusBar', tr._translate( "MainWindow", "Problem communicating with proxy: %1. Please check your network settings.").arg(str(err[0][1])) @@ -231,25 +233,25 @@ class outgoingSynSender(threading.Thread, StoppableThread): logger.debug('Could NOT connect to ' + str(peer) + ' during outgoing attempt. ' + str(err)) deletedPeer = None - with shared.knownNodesLock: + with knownnodes.knownNodesLock: """ - It is remotely possible that peer is no longer in shared.knownNodes. + It is remotely possible that peer is no longer in knownnodes.knownNodes. This could happen if two outgoingSynSender threads both try to connect to the same peer, both fail, and then both try to remove - it from shared.knownNodes. This is unlikely because of the + it from knownnodes.knownNodes. This is unlikely because of the alreadyAttemptedConnectionsList but because we clear that list once every half hour, it can happen. """ - if peer in shared.knownNodes[self.streamNumber]: - timeLastSeen = shared.knownNodes[self.streamNumber][peer] - if (int(time.time()) - timeLastSeen) > 172800 and len(shared.knownNodes[self.streamNumber]) > 1000: # for nodes older than 48 hours old if we have more than 1000 hosts in our list, delete from the shared.knownNodes data-structure. - del shared.knownNodes[self.streamNumber][peer] + if peer in knownnodes.knownNodes[self.streamNumber]: + timeLastSeen = knownnodes.knownNodes[self.streamNumber][peer] + if (int(time.time()) - timeLastSeen) > 172800 and len(knownnodes.knownNodes[self.streamNumber]) > 1000: # for nodes older than 48 hours old if we have more than 1000 hosts in our list, delete from the knownnodes.knownNodes data-structure. + del knownnodes.knownNodes[self.streamNumber][peer] deletedPeer = peer if deletedPeer: - str ('deleting ' + str(peer) + ' from shared.knownNodes because it is more than 48 hours old and we could not connect to it.') + str ('deleting ' + str(peer) + ' from knownnodes.knownNodes because it is more than 48 hours old and we could not connect to it.') except socks.Socks5AuthError as err: - shared.UISignalQueue.put(( + queues.UISignalQueue.put(( 'updateStatusBar', tr._translate( "MainWindow", "SOCKS5 Authentication problem: %1. Please check your SOCKS5 settings.").arg(str(err)))) except socks.Socks5Error as err: @@ -272,22 +274,22 @@ class outgoingSynSender(threading.Thread, StoppableThread): logger.debug('Could NOT connect to ' + str(peer) + 'during outgoing attempt. ' + str(err)) deletedPeer = None - with shared.knownNodesLock: + with knownnodes.knownNodesLock: """ - It is remotely possible that peer is no longer in shared.knownNodes. + It is remotely possible that peer is no longer in knownnodes.knownNodes. This could happen if two outgoingSynSender threads both try to connect to the same peer, both fail, and then both try to remove - it from shared.knownNodes. This is unlikely because of the + it from knownnodes.knownNodes. This is unlikely because of the alreadyAttemptedConnectionsList but because we clear that list once every half hour, it can happen. """ - if peer in shared.knownNodes[self.streamNumber]: - timeLastSeen = shared.knownNodes[self.streamNumber][peer] - if (int(time.time()) - timeLastSeen) > 172800 and len(shared.knownNodes[self.streamNumber]) > 1000: # for nodes older than 48 hours old if we have more than 1000 hosts in our list, delete from the shared.knownNodes data-structure. - del shared.knownNodes[self.streamNumber][peer] + if peer in knownnodes.knownNodes[self.streamNumber]: + timeLastSeen = knownnodes.knownNodes[self.streamNumber][peer] + if (int(time.time()) - timeLastSeen) > 172800 and len(knownnodes.knownNodes[self.streamNumber]) > 1000: # for nodes older than 48 hours old if we have more than 1000 hosts in our list, delete from the knownnodes.knownNodes data-structure. + del knownnodes.knownNodes[self.streamNumber][peer] deletedPeer = peer if deletedPeer: - logger.debug('deleting ' + str(peer) + ' from shared.knownNodes because it is more than 48 hours old and we could not connect to it.') + logger.debug('deleting ' + str(peer) + ' from knownnodes.knownNodes because it is more than 48 hours old and we could not connect to it.') except Exception as err: import traceback diff --git a/src/class_receiveDataThread.py b/src/class_receiveDataThread.py index 2793f612..ef7e32e3 100644 --- a/src/class_receiveDataThread.py +++ b/src/class_receiveDataThread.py @@ -28,10 +28,12 @@ from configparser import BMConfigParser from class_objectHashHolder import objectHashHolder from helper_generic import addDataPadding, isHostInPrivateIPRange from helper_sql import sqlQuery +import knownnodes from debug import logger import paths import protocol from inventory import Inventory, PendingDownload, PendingUpload +import queues import state import throttle import tr @@ -147,13 +149,13 @@ class receiveDataThread(threading.Thread): logger.error('Could not delete ' + str(self.hostIdent) + ' from shared.connectedHostsList.' + str(err)) PendingDownload().threadEnd() - shared.UISignalQueue.put(('updateNetworkStatusTab', 'no data')) + queues.UISignalQueue.put(('updateNetworkStatusTab', 'no data')) self.checkTimeOffsetNotification() logger.debug('receiveDataThread ending. ID ' + str(id(self)) + '. The size of the shared.connectedHostsList is now ' + str(len(shared.connectedHostsList))) def antiIntersectionDelay(self, initial = False): # estimated time for a small object to propagate across the whole network - delay = math.ceil(math.log(max(len(shared.knownNodes[x]) for x in shared.knownNodes) + 2, 20)) * (0.2 + objectHashHolder.size/2) + delay = math.ceil(math.log(max(len(knownnodes.knownNodes[x]) for x in knownnodes.knownNodes) + 2, 20)) * (0.2 + objectHashHolder.size/2) # take the stream with maximum amount of nodes # +2 is to avoid problems with log(0) and log(1) # 20 is avg connected nodes count @@ -168,7 +170,7 @@ class receiveDataThread(threading.Thread): def checkTimeOffsetNotification(self): if shared.timeOffsetWrongCount >= 4 and not self.connectionIsOrWasFullyEstablished: - shared.UISignalQueue.put(('updateStatusBar', tr._translate("MainWindow", "The time on your computer, %1, may be wrong. Please verify your settings.").arg(datetime.datetime.now().strftime("%H:%M:%S")))) + queues.UISignalQueue.put(('updateStatusBar', tr._translate("MainWindow", "The time on your computer, %1, may be wrong. Please verify your settings.").arg(datetime.datetime.now().strftime("%H:%M:%S")))) def processData(self): if len(self.data) < protocol.Header.size: # if so little of the data has arrived that we can't even read the checksum then wait for more data. @@ -198,9 +200,9 @@ class receiveDataThread(threading.Thread): # just received valid data from it. So update the knownNodes list so # that other peers can be made aware of its existance. if self.initiatedConnection and self.connectionIsOrWasFullyEstablished: # The remote port is only something we should share with others if it is the remote node's incoming port (rather than some random operating-system-assigned outgoing port). - with shared.knownNodesLock: + with knownnodes.knownNodesLock: for stream in self.streamNumber: - shared.knownNodes[stream][self.peer] = int(time.time()) + knownnodes.knownNodes[stream][self.peer] = int(time.time()) #Strip the nulls command = command.rstrip('\x00') @@ -359,10 +361,10 @@ class receiveDataThread(threading.Thread): if not self.initiatedConnection: shared.clientHasReceivedIncomingConnections = True - shared.UISignalQueue.put(('setStatusIcon', 'green')) + queues.UISignalQueue.put(('setStatusIcon', 'green')) self.sock.settimeout( 600) # We'll send out a ping every 5 minutes to make sure the connection stays alive if there has been no other traffic to send lately. - shared.UISignalQueue.put(('updateNetworkStatusTab', 'no data')) + queues.UISignalQueue.put(('updateNetworkStatusTab', 'no data')) logger.debug('Connection fully established with ' + str(self.peer) + "\n" + \ 'The size of the connectedHostsList is now ' + str(len(shared.connectedHostsList)) + "\n" + \ 'The length of sendDataQueues is now: ' + str(len(state.sendDataQueues)) + "\n" + \ @@ -611,14 +613,14 @@ class receiveDataThread(threading.Thread): continue timeSomeoneElseReceivedMessageFromThisNode, = unpack('>Q', data[lengthOfNumberOfAddresses + ( 38 * i):8 + lengthOfNumberOfAddresses + (38 * i)]) # This is the 'time' value in the received addr message. 64-bit. - if recaddrStream not in shared.knownNodes: # knownNodes is a dictionary of dictionaries with one outer dictionary for each stream. If the outer stream dictionary doesn't exist yet then we must make it. - with shared.knownNodesLock: - shared.knownNodes[recaddrStream] = {} + if recaddrStream not in knownnodes.knownNodes: # knownNodes is a dictionary of dictionaries with one outer dictionary for each stream. If the outer stream dictionary doesn't exist yet then we must make it. + with knownnodes.knownNodesLock: + knownnodes.knownNodes[recaddrStream] = {} peerFromAddrMessage = state.Peer(hostStandardFormat, recaddrPort) - if peerFromAddrMessage not in shared.knownNodes[recaddrStream]: - if len(shared.knownNodes[recaddrStream]) < 20000 and timeSomeoneElseReceivedMessageFromThisNode > (int(time.time()) - 10800) and timeSomeoneElseReceivedMessageFromThisNode < (int(time.time()) + 10800): # If we have more than 20000 nodes in our list already then just forget about adding more. Also, make sure that the time that someone else received a message from this node is within three hours from now. - with shared.knownNodesLock: - shared.knownNodes[recaddrStream][peerFromAddrMessage] = timeSomeoneElseReceivedMessageFromThisNode + if peerFromAddrMessage not in knownnodes.knownNodes[recaddrStream]: + if len(knownnodes.knownNodes[recaddrStream]) < 20000 and timeSomeoneElseReceivedMessageFromThisNode > (int(time.time()) - 10800) and timeSomeoneElseReceivedMessageFromThisNode < (int(time.time()) + 10800): # If we have more than 20000 nodes in our list already then just forget about adding more. Also, make sure that the time that someone else received a message from this node is within three hours from now. + with knownnodes.knownNodesLock: + knownnodes.knownNodes[recaddrStream][peerFromAddrMessage] = timeSomeoneElseReceivedMessageFromThisNode logger.debug('added new node ' + str(peerFromAddrMessage) + ' to knownNodes in stream ' + str(recaddrStream)) shared.needToWriteKnownNodesToDisk = True @@ -628,14 +630,14 @@ class receiveDataThread(threading.Thread): protocol.broadcastToSendDataQueues(( recaddrStream, 'advertisepeer', hostDetails)) else: - timeLastReceivedMessageFromThisNode = shared.knownNodes[recaddrStream][ + timeLastReceivedMessageFromThisNode = knownnodes.knownNodes[recaddrStream][ peerFromAddrMessage] if (timeLastReceivedMessageFromThisNode < timeSomeoneElseReceivedMessageFromThisNode) and (timeSomeoneElseReceivedMessageFromThisNode < int(time.time())+900): # 900 seconds for wiggle-room in case other nodes' clocks aren't quite right. - with shared.knownNodesLock: - shared.knownNodes[recaddrStream][peerFromAddrMessage] = timeSomeoneElseReceivedMessageFromThisNode + with knownnodes.knownNodesLock: + knownnodes.knownNodes[recaddrStream][peerFromAddrMessage] = timeSomeoneElseReceivedMessageFromThisNode for stream in self.streamNumber: - logger.debug('knownNodes currently has %i nodes for stream %i', len(shared.knownNodes[stream]), stream) + logger.debug('knownNodes currently has %i nodes for stream %i', len(knownnodes.knownNodes[stream]), stream) # Send a huge addr message to our peer. This is only used @@ -651,8 +653,8 @@ class receiveDataThread(threading.Thread): addrsInChildStreamLeft = {} addrsInChildStreamRight = {} - with shared.knownNodesLock: - if len(shared.knownNodes[stream]) > 0: + with knownnodes.knownNodesLock: + if len(knownnodes.knownNodes[stream]) > 0: ownPosition = random.randint(0, 499) sentOwn = False for i in range(500): @@ -662,29 +664,29 @@ class receiveDataThread(threading.Thread): peer = state.Peer(BMConfigParser().get("bitmessagesettings", "onionhostname"), BMConfigParser().getint("bitmessagesettings", "onionport")) else: # still may contain own onion address, but we don't change it - peer, = random.sample(shared.knownNodes[stream], 1) + peer, = random.sample(knownnodes.knownNodes[stream], 1) if isHostInPrivateIPRange(peer.host): continue if peer.host == BMConfigParser().get("bitmessagesettings", "onionhostname") and peer.port == BMConfigParser().getint("bitmessagesettings", "onionport") : sentOwn = True - addrsInMyStream[peer] = shared.knownNodes[ + addrsInMyStream[peer] = knownnodes.knownNodes[ stream][peer] # sent 250 only if the remote isn't interested in it - if len(shared.knownNodes[stream * 2]) > 0 and stream not in self.streamNumber: + if len(knownnodes.knownNodes[stream * 2]) > 0 and stream not in self.streamNumber: for i in range(250): - peer, = random.sample(shared.knownNodes[ + peer, = random.sample(knownnodes.knownNodes[ stream * 2], 1) if isHostInPrivateIPRange(peer.host): continue - addrsInChildStreamLeft[peer] = shared.knownNodes[ + addrsInChildStreamLeft[peer] = knownnodes.knownNodes[ stream * 2][peer] - if len(shared.knownNodes[(stream * 2) + 1]) > 0 and stream not in self.streamNumber: + if len(knownnodes.knownNodes[(stream * 2) + 1]) > 0 and stream not in self.streamNumber: for i in range(250): - peer, = random.sample(shared.knownNodes[ + peer, = random.sample(knownnodes.knownNodes[ (stream * 2) + 1], 1) if isHostInPrivateIPRange(peer.host): continue - addrsInChildStreamRight[peer] = shared.knownNodes[ + addrsInChildStreamRight[peer] = knownnodes.knownNodes[ (stream * 2) + 1][peer] numberOfAddressesInAddrMessage = 0 payload = '' @@ -772,7 +774,7 @@ class receiveDataThread(threading.Thread): try: if cmp(remoteVersion, myVersion) > 0 and \ (myVersion[1] % 2 == remoteVersion[1] % 2): - shared.UISignalQueue.put(('newVersionAvailable', remoteVersion)) + queues.UISignalQueue.put(('newVersionAvailable', remoteVersion)) except: pass @@ -804,11 +806,11 @@ class receiveDataThread(threading.Thread): self.sendDataThreadQueue.put((0, 'setRemoteProtocolVersion', self.remoteProtocolVersion)) if not isHostInPrivateIPRange(self.peer.host): - with shared.knownNodesLock: + with knownnodes.knownNodesLock: for stream in self.remoteStreams: - shared.knownNodes[stream][state.Peer(self.peer.host, self.remoteNodeIncomingPort)] = int(time.time()) + knownnodes.knownNodes[stream][state.Peer(self.peer.host, self.remoteNodeIncomingPort)] = int(time.time()) if not self.initiatedConnection: - shared.knownNodes[stream][state.Peer(self.peer.host, self.remoteNodeIncomingPort)] -= 162000 # penalise inbound, 2 days minus 3 hours + knownnodes.knownNodes[stream][state.Peer(self.peer.host, self.remoteNodeIncomingPort)] -= 162000 # penalise inbound, 2 days minus 3 hours shared.needToWriteKnownNodesToDisk = True self.sendverack() diff --git a/src/class_singleCleaner.py b/src/class_singleCleaner.py index 733e4607..648646a7 100644 --- a/src/class_singleCleaner.py +++ b/src/class_singleCleaner.py @@ -11,6 +11,8 @@ from helper_sql import * from helper_threading import * from inventory import Inventory from debug import logger +import knownnodes +import queues import protocol import state @@ -49,10 +51,10 @@ class singleCleaner(threading.Thread, StoppableThread): shared.maximumLengthOfTimeToBotherResendingMessages = float('inf') while state.shutdown == 0: - shared.UISignalQueue.put(( + queues.UISignalQueue.put(( 'updateStatusBar', 'Doing housekeeping (Flushing inventory in memory to disk...)')) Inventory().flush() - shared.UISignalQueue.put(('updateStatusBar', '')) + queues.UISignalQueue.put(('updateStatusBar', '')) protocol.broadcastToSendDataQueues(( 0, 'pong', 'no data')) # commands the sendData threads to send out a pong message if they haven't sent anything else in the last five minutes. The socket timeout-time is 10 minutes. @@ -60,7 +62,7 @@ class singleCleaner(threading.Thread, StoppableThread): # queue which will never be handled by a UI. We should clear it to # save memory. if BMConfigParser().safeGetBoolean('bitmessagesettings', 'daemon'): - shared.UISignalQueue.queue.clear() + queues.UISignalQueue.queue.clear() if timeWeLastClearedInventoryAndPubkeysTables < int(time.time()) - 7380: timeWeLastClearedInventoryAndPubkeysTables = int(time.time()) Inventory().clean() @@ -88,28 +90,28 @@ class singleCleaner(threading.Thread, StoppableThread): # cleanup old nodes now = int(time.time()) toDelete = [] - shared.knownNodesLock.acquire() - for stream in shared.knownNodes: - for node in shared.knownNodes[stream].keys(): - if now - shared.knownNodes[stream][node] > 2419200: # 28 days + knownnodes.knownNodesLock.acquire() + for stream in knownnodes.knownNodes: + for node in knownnodes.knownNodes[stream].keys(): + if now - knownnodes.knownNodes[stream][node] > 2419200: # 28 days shared.needToWriteKownNodesToDisk = True - del shared.knownNodes[stream][node] - shared.knownNodesLock.release() + del knownnodes.knownNodes[stream][node] + knownnodes.knownNodesLock.release() # Let us write out the knowNodes to disk if there is anything new to write out. if shared.needToWriteKnownNodesToDisk: - shared.knownNodesLock.acquire() + knownnodes.knownNodesLock.acquire() output = open(state.appdata + 'knownnodes.dat', 'wb') try: - pickle.dump(shared.knownNodes, output) + pickle.dump(knownnodes.knownNodes, output) output.close() except Exception as err: if "Errno 28" in str(err): - logger.fatal('(while receiveDataThread shared.needToWriteKnownNodesToDisk) Alert: Your disk or data storage volume is full. ') - shared.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True))) + logger.fatal('(while receiveDataThread knownnodes.needToWriteKnownNodesToDisk) Alert: Your disk or data storage volume is full. ') + queues.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True))) if shared.daemon: os._exit(0) - shared.knownNodesLock.release() + knownnodes.knownNodesLock.release() shared.needToWriteKnownNodesToDisk = False # TODO: cleanup pending upload / download @@ -122,22 +124,22 @@ def resendPubkeyRequest(address): logger.debug('It has been a long time and we haven\'t heard a response to our getpubkey request. Sending again.') try: del state.neededPubkeys[ - address] # We need to take this entry out of the neededPubkeys structure because the shared.workerQueue checks to see whether the entry is already present and will not do the POW and send the message because it assumes that it has already done it recently. + address] # We need to take this entry out of the neededPubkeys structure because the queues.workerQueue checks to see whether the entry is already present and will not do the POW and send the message because it assumes that it has already done it recently. except: pass - shared.UISignalQueue.put(( + queues.UISignalQueue.put(( 'updateStatusBar', 'Doing work necessary to again attempt to request a public key...')) sqlExecute( '''UPDATE sent SET status='msgqueued' WHERE toaddress=?''', address) - shared.workerQueue.put(('sendmessage', '')) + queues.workerQueue.put(('sendmessage', '')) def resendMsg(ackdata): logger.debug('It has been a long time and we haven\'t heard an acknowledgement to our msg. Sending again.') sqlExecute( '''UPDATE sent SET status='msgqueued' WHERE ackdata=?''', ackdata) - shared.workerQueue.put(('sendmessage', '')) - shared.UISignalQueue.put(( + queues.workerQueue.put(('sendmessage', '')) + queues.UISignalQueue.put(( 'updateStatusBar', 'Doing work necessary to again attempt to deliver a message...')) diff --git a/src/class_singleWorker.py b/src/class_singleWorker.py index ac36e0b7..cd5cf283 100644 --- a/src/class_singleWorker.py +++ b/src/class_singleWorker.py @@ -21,6 +21,7 @@ from helper_threading import * from inventory import Inventory, PendingUpload import l10n import protocol +import queues import state from binascii import hexlify, unhexlify @@ -43,7 +44,7 @@ class singleWorker(threading.Thread, StoppableThread): def stopThread(self): try: - shared.workerQueue.put(("stopThread", "data")) + queues.workerQueue.put(("stopThread", "data")) except: pass super(singleWorker, self).stopThread() @@ -79,14 +80,14 @@ class singleWorker(threading.Thread, StoppableThread): if state.shutdown == 0: # just in case there are any pending tasks for msg # messages that have yet to be sent. - shared.workerQueue.put(('sendmessage', '')) + queues.workerQueue.put(('sendmessage', '')) # just in case there are any tasks for Broadcasts # that have yet to be sent. - shared.workerQueue.put(('sendbroadcast', '')) + queues.workerQueue.put(('sendbroadcast', '')) while state.shutdown == 0: self.busy = 0 - command, data = shared.workerQueue.get() + command, data = queues.workerQueue.get() self.busy = 1 if command == 'sendmessage': try: @@ -119,7 +120,7 @@ class singleWorker(threading.Thread, StoppableThread): else: logger.error('Probable programming error: The command sent to the workerThread is weird. It is: %s\n' % command) - shared.workerQueue.task_done() + queues.workerQueue.task_done() logger.info("Quitting...") def doPOWForMyV2Pubkey(self, hash): # This function also broadcasts out the pubkey message once it is done with the POW @@ -182,7 +183,7 @@ class singleWorker(threading.Thread, StoppableThread): protocol.broadcastToSendDataQueues(( streamNumber, 'advertiseobject', inventoryHash)) - shared.UISignalQueue.put(('updateStatusBar', '')) + queues.UISignalQueue.put(('updateStatusBar', '')) try: BMConfigParser().set( myAddress, 'lastpubkeysendtime', str(int(time.time()))) @@ -273,7 +274,7 @@ class singleWorker(threading.Thread, StoppableThread): protocol.broadcastToSendDataQueues(( streamNumber, 'advertiseobject', inventoryHash)) - shared.UISignalQueue.put(('updateStatusBar', '')) + queues.UISignalQueue.put(('updateStatusBar', '')) try: BMConfigParser().set( myAddress, 'lastpubkeysendtime', str(int(time.time()))) @@ -364,7 +365,7 @@ class singleWorker(threading.Thread, StoppableThread): protocol.broadcastToSendDataQueues(( streamNumber, 'advertiseobject', inventoryHash)) - shared.UISignalQueue.put(('updateStatusBar', '')) + queues.UISignalQueue.put(('updateStatusBar', '')) try: BMConfigParser().set( myAddress, 'lastpubkeysendtime', str(int(time.time()))) @@ -394,7 +395,7 @@ class singleWorker(threading.Thread, StoppableThread): privEncryptionKeyBase58 = BMConfigParser().get( fromaddress, 'privencryptionkey') except: - shared.UISignalQueue.put(('updateSentItemStatusByAckdata', ( + queues.UISignalQueue.put(('updateSentItemStatusByAckdata', ( ackdata, tr._translate("MainWindow", "Error! Could not find sender address (your address) in the keys.dat file.")))) continue @@ -471,7 +472,7 @@ class singleWorker(threading.Thread, StoppableThread): target = 2 ** 64 / (protocol.networkDefaultProofOfWorkNonceTrialsPerByte*(len(payload) + 8 + protocol.networkDefaultPayloadLengthExtraBytes + ((TTL*(len(payload)+8+protocol.networkDefaultPayloadLengthExtraBytes))/(2 ** 16)))) logger.info('(For broadcast message) Doing proof of work...') - shared.UISignalQueue.put(('updateSentItemStatusByAckdata', ( + queues.UISignalQueue.put(('updateSentItemStatusByAckdata', ( ackdata, tr._translate("MainWindow", "Doing work necessary to send broadcast...")))) initialHash = hashlib.sha512(payload).digest() trialValue, nonce = proofofwork.run(target, initialHash) @@ -495,7 +496,7 @@ class singleWorker(threading.Thread, StoppableThread): protocol.broadcastToSendDataQueues(( streamNumber, 'advertiseobject', inventoryHash)) - shared.UISignalQueue.put(('updateSentItemStatusByAckdata', (ackdata, tr._translate("MainWindow", "Broadcast sent on %1").arg(l10n.formatTimestamp())))) + queues.UISignalQueue.put(('updateSentItemStatusByAckdata', (ackdata, tr._translate("MainWindow", "Broadcast sent on %1").arg(l10n.formatTimestamp())))) # Update the status of the message in the 'sent' table to have # a 'broadcastsent' status @@ -563,7 +564,7 @@ class singleWorker(threading.Thread, StoppableThread): '''UPDATE sent SET status='awaitingpubkey', sleeptill=? WHERE toaddress=? AND status='msgqueued' ''', int(time.time()) + 2.5*24*60*60, toaddress) - shared.UISignalQueue.put(('updateSentItemStatusByToAddress', ( + queues.UISignalQueue.put(('updateSentItemStatusByToAddress', ( toaddress, tr._translate("MainWindow",'Encryption key was requested earlier.')))) continue #on with the next msg on which we can do some work else: @@ -600,7 +601,7 @@ class singleWorker(threading.Thread, StoppableThread): sqlExecute( '''UPDATE sent SET status='doingpubkeypow' WHERE toaddress=? AND status='msgqueued' ''', toaddress) - shared.UISignalQueue.put(('updateSentItemStatusByToAddress', ( + queues.UISignalQueue.put(('updateSentItemStatusByToAddress', ( toaddress, tr._translate("MainWindow",'Sending a request for the recipient\'s encryption key.')))) self.requestPubKey(toaddress) continue #on with the next msg on which we can do some work @@ -617,7 +618,7 @@ class singleWorker(threading.Thread, StoppableThread): if not BMConfigParser().has_section(toaddress): # if we aren't sending this to ourselves or a chan shared.ackdataForWhichImWatching[ackdata] = 0 - shared.UISignalQueue.put(('updateSentItemStatusByAckdata', ( + queues.UISignalQueue.put(('updateSentItemStatusByAckdata', ( ackdata, tr._translate("MainWindow", "Looking up the receiver\'s public key")))) logger.info('Sending a message.') logger.debug('First 150 characters of message: ' + repr(message[:150])) @@ -651,7 +652,7 @@ class singleWorker(threading.Thread, StoppableThread): if shared.isBitSetWithinBitfield(behaviorBitfield,30): # if receiver is a mobile device who expects that their address RIPE is included unencrypted on the front of the message.. if not shared.BMConfigParser().safeGetBoolean('bitmessagesettings','willinglysendtomobile'): # if we are Not willing to include the receiver's RIPE hash on the message.. logger.info('The receiver is a mobile user but the sender (you) has not selected that you are willing to send to mobiles. Aborting send.') - shared.UISignalQueue.put(('updateSentItemStatusByAckdata',(ackdata,tr._translate("MainWindow",'Problem: Destination is a mobile device who requests that the destination be included in the message but this is disallowed in your settings. %1').arg(l10n.formatTimestamp())))) + queues.UISignalQueue.put(('updateSentItemStatusByAckdata',(ackdata,tr._translate("MainWindow",'Problem: Destination is a mobile device who requests that the destination be included in the message but this is disallowed in your settings. %1').arg(l10n.formatTimestamp())))) # if the human changes their setting and then sends another message or restarts their client, this one will send at that time. continue readPosition += 4 # to bypass the bitfield of behaviors @@ -665,7 +666,7 @@ class singleWorker(threading.Thread, StoppableThread): if toAddressVersionNumber == 2: requiredAverageProofOfWorkNonceTrialsPerByte = protocol.networkDefaultProofOfWorkNonceTrialsPerByte requiredPayloadLengthExtraBytes = protocol.networkDefaultPayloadLengthExtraBytes - shared.UISignalQueue.put(('updateSentItemStatusByAckdata', ( + queues.UISignalQueue.put(('updateSentItemStatusByAckdata', ( ackdata, tr._translate("MainWindow", "Doing work necessary to send message.\nThere is no required difficulty for version 2 addresses like this.")))) elif toAddressVersionNumber >= 3: requiredAverageProofOfWorkNonceTrialsPerByte, varintLength = decodeVarint( @@ -679,7 +680,7 @@ class singleWorker(threading.Thread, StoppableThread): if requiredPayloadLengthExtraBytes < protocol.networkDefaultPayloadLengthExtraBytes: requiredPayloadLengthExtraBytes = protocol.networkDefaultPayloadLengthExtraBytes logger.debug('Using averageProofOfWorkNonceTrialsPerByte: %s and payloadLengthExtraBytes: %s.' % (requiredAverageProofOfWorkNonceTrialsPerByte, requiredPayloadLengthExtraBytes)) - shared.UISignalQueue.put(('updateSentItemStatusByAckdata', (ackdata, tr._translate("MainWindow", "Doing work necessary to send message.\nReceiver\'s required difficulty: %1 and %2").arg(str(float( + queues.UISignalQueue.put(('updateSentItemStatusByAckdata', (ackdata, tr._translate("MainWindow", "Doing work necessary to send message.\nReceiver\'s required difficulty: %1 and %2").arg(str(float( requiredAverageProofOfWorkNonceTrialsPerByte) / protocol.networkDefaultProofOfWorkNonceTrialsPerByte)).arg(str(float(requiredPayloadLengthExtraBytes) / protocol.networkDefaultPayloadLengthExtraBytes))))) if status != 'forcepow': if (requiredAverageProofOfWorkNonceTrialsPerByte > BMConfigParser().getint('bitmessagesettings', 'maxacceptablenoncetrialsperbyte') and BMConfigParser().getint('bitmessagesettings', 'maxacceptablenoncetrialsperbyte') != 0) or (requiredPayloadLengthExtraBytes > BMConfigParser().getint('bitmessagesettings', 'maxacceptablepayloadlengthextrabytes') and BMConfigParser().getint('bitmessagesettings', 'maxacceptablepayloadlengthextrabytes') != 0): @@ -688,7 +689,7 @@ class singleWorker(threading.Thread, StoppableThread): sqlExecute( '''UPDATE sent SET status='toodifficult' WHERE ackdata=? ''', ackdata) - shared.UISignalQueue.put(('updateSentItemStatusByAckdata', (ackdata, tr._translate("MainWindow", "Problem: The work demanded by the recipient (%1 and %2) is more difficult than you are willing to do. %3").arg(str(float(requiredAverageProofOfWorkNonceTrialsPerByte) / protocol.networkDefaultProofOfWorkNonceTrialsPerByte)).arg(str(float( + queues.UISignalQueue.put(('updateSentItemStatusByAckdata', (ackdata, tr._translate("MainWindow", "Problem: The work demanded by the recipient (%1 and %2) is more difficult than you are willing to do. %3").arg(str(float(requiredAverageProofOfWorkNonceTrialsPerByte) / protocol.networkDefaultProofOfWorkNonceTrialsPerByte)).arg(str(float( requiredPayloadLengthExtraBytes) / protocol.networkDefaultPayloadLengthExtraBytes)).arg(l10n.formatTimestamp())))) continue else: # if we are sending a message to ourselves or a chan.. @@ -700,7 +701,7 @@ class singleWorker(threading.Thread, StoppableThread): privEncryptionKeyBase58 = BMConfigParser().get( toaddress, 'privencryptionkey') except Exception as err: - shared.UISignalQueue.put(('updateSentItemStatusByAckdata',(ackdata,tr._translate("MainWindow",'Problem: You are trying to send a message to yourself or a chan but your encryption key could not be found in the keys.dat file. Could not encrypt message. %1').arg(l10n.formatTimestamp())))) + queues.UISignalQueue.put(('updateSentItemStatusByAckdata',(ackdata,tr._translate("MainWindow",'Problem: You are trying to send a message to yourself or a chan but your encryption key could not be found in the keys.dat file. Could not encrypt message. %1').arg(l10n.formatTimestamp())))) logger.error('Error within sendMsg. Could not read the keys from the keys.dat file for our own address. %s\n' % err) continue privEncryptionKeyHex = hexlify(shared.decodeWalletImportFormat( @@ -709,7 +710,7 @@ class singleWorker(threading.Thread, StoppableThread): privEncryptionKeyHex))[1:] requiredAverageProofOfWorkNonceTrialsPerByte = protocol.networkDefaultProofOfWorkNonceTrialsPerByte requiredPayloadLengthExtraBytes = protocol.networkDefaultPayloadLengthExtraBytes - shared.UISignalQueue.put(('updateSentItemStatusByAckdata', ( + queues.UISignalQueue.put(('updateSentItemStatusByAckdata', ( ackdata, tr._translate("MainWindow", "Doing work necessary to send message.")))) # Now we can start to assemble our message. @@ -725,7 +726,7 @@ class singleWorker(threading.Thread, StoppableThread): privEncryptionKeyBase58 = BMConfigParser().get( fromaddress, 'privencryptionkey') except: - shared.UISignalQueue.put(('updateSentItemStatusByAckdata', ( + queues.UISignalQueue.put(('updateSentItemStatusByAckdata', ( ackdata, tr._translate("MainWindow", "Error! Could not find sender address (your address) in the keys.dat file.")))) continue @@ -785,7 +786,7 @@ class singleWorker(threading.Thread, StoppableThread): encrypted = highlevelcrypto.encrypt(payload,"04"+hexlify(pubEncryptionKeyBase256)) except: sqlExecute('''UPDATE sent SET status='badkey' WHERE ackdata=?''', ackdata) - shared.UISignalQueue.put(('updateSentItemStatusByAckdata',(ackdata,tr._translate("MainWindow",'Problem: The recipient\'s encryption key is no good. Could not encrypt message. %1').arg(l10n.formatTimestamp())))) + queues.UISignalQueue.put(('updateSentItemStatusByAckdata',(ackdata,tr._translate("MainWindow",'Problem: The recipient\'s encryption key is no good. Could not encrypt message. %1').arg(l10n.formatTimestamp())))) continue encryptedPayload = pack('>Q', embeddedTime) @@ -819,10 +820,10 @@ class singleWorker(threading.Thread, StoppableThread): objectType, toStreamNumber, encryptedPayload, embeddedTime, '') PendingUpload().add(inventoryHash) if BMConfigParser().has_section(toaddress) or not protocol.checkBitfield(behaviorBitfield, protocol.BITFIELD_DOESACK): - shared.UISignalQueue.put(('updateSentItemStatusByAckdata', (ackdata, tr._translate("MainWindow", "Message sent. Sent at %1").arg(l10n.formatTimestamp())))) + queues.UISignalQueue.put(('updateSentItemStatusByAckdata', (ackdata, tr._translate("MainWindow", "Message sent. Sent at %1").arg(l10n.formatTimestamp())))) else: # not sending to a chan or one of my addresses - shared.UISignalQueue.put(('updateSentItemStatusByAckdata', (ackdata, tr._translate("MainWindow", "Message sent. Waiting for acknowledgement. Sent on %1").arg(l10n.formatTimestamp())))) + queues.UISignalQueue.put(('updateSentItemStatusByAckdata', (ackdata, tr._translate("MainWindow", "Message sent. Waiting for acknowledgement. Sent on %1").arg(l10n.formatTimestamp())))) logger.info('Broadcasting inv for my msg(within sendmsg function):' + hexlify(inventoryHash)) protocol.broadcastToSendDataQueues(( toStreamNumber, 'advertiseobject', inventoryHash)) @@ -852,7 +853,7 @@ class singleWorker(threading.Thread, StoppableThread): time.time()), message, 'inbox', encoding, 0, sigHash) helper_inbox.insert(t) - shared.UISignalQueue.put(('displayNewInboxMessage', ( + queues.UISignalQueue.put(('displayNewInboxMessage', ( inventoryHash, toaddress, fromaddress, subject, message))) # If we are behaving as an API then we might need to run an @@ -913,8 +914,8 @@ class singleWorker(threading.Thread, StoppableThread): # print 'trial value', trialValue statusbar = 'Doing the computations necessary to request the recipient\'s public key.' - shared.UISignalQueue.put(('updateStatusBar', statusbar)) - shared.UISignalQueue.put(('updateSentItemStatusByToAddress', ( + queues.UISignalQueue.put(('updateStatusBar', statusbar)) + queues.UISignalQueue.put(('updateSentItemStatusByToAddress', ( toAddress, tr._translate("MainWindow",'Doing work necessary to request encryption key.')))) target = 2 ** 64 / (protocol.networkDefaultProofOfWorkNonceTrialsPerByte*(len(payload) + 8 + protocol.networkDefaultPayloadLengthExtraBytes + ((TTL*(len(payload)+8+protocol.networkDefaultPayloadLengthExtraBytes))/(2 ** 16)))) @@ -943,9 +944,9 @@ class singleWorker(threading.Thread, StoppableThread): sleeptill, toAddress) - shared.UISignalQueue.put(( + queues.UISignalQueue.put(( 'updateStatusBar', tr._translate("MainWindow",'Broadcasting the public key request. This program will auto-retry if they are offline.'))) - shared.UISignalQueue.put(('updateSentItemStatusByToAddress', (toAddress, tr._translate("MainWindow",'Sending public key request. Waiting for reply. Requested at %1').arg(l10n.formatTimestamp())))) + queues.UISignalQueue.put(('updateSentItemStatusByToAddress', (toAddress, tr._translate("MainWindow",'Sending public key request. Waiting for reply. Requested at %1').arg(l10n.formatTimestamp())))) def generateFullAckMessage(self, ackdata, toStreamNumber, TTL): diff --git a/src/class_smtpDeliver.py b/src/class_smtpDeliver.py index 2314238f..5c320f51 100644 --- a/src/class_smtpDeliver.py +++ b/src/class_smtpDeliver.py @@ -9,7 +9,7 @@ from configparser import BMConfigParser from debug import logger from helper_threading import * from bitmessageqt.uisignaler import UISignaler -import shared +import queues import state SMTPDOMAIN = "bmaddr.lan" @@ -23,7 +23,7 @@ class smtpDeliver(threading.Thread, StoppableThread): def stopThread(self): try: - shared.UISignallerQueue.put(("stopThread", "data")) + queues.UISignallerQueue.put(("stopThread", "data")) except: pass super(smtpDeliver, self).stopThread() @@ -36,7 +36,7 @@ class smtpDeliver(threading.Thread, StoppableThread): def run(self): while state.shutdown == 0: - command, data = shared.UISignalQueue.get() + command, data = queues.UISignalQueue.get() if command == 'writeNewAddressToTable': label, address, streamNumber = data pass diff --git a/src/class_smtpServer.py b/src/class_smtpServer.py index 65a7e1fc..359c8625 100644 --- a/src/class_smtpServer.py +++ b/src/class_smtpServer.py @@ -16,7 +16,7 @@ from debug import logger from helper_sql import sqlExecute from helper_threading import StoppableThread from pyelliptic.openssl import OpenSSL -import shared +import queues from version import softwareVersion SMTPDOMAIN = "bmaddr.lan" @@ -86,7 +86,7 @@ class smtpServerPyBitmessage(smtpd.SMTPServer): min(BMConfigParser().getint('bitmessagesettings', 'ttl'), 86400 * 2) # not necessary to have a TTL higher than 2 days ) - shared.workerQueue.put(('sendmessage', toAddress)) + queues.workerQueue.put(('sendmessage', toAddress)) def decode_header(self, hdr): ret = [] diff --git a/src/class_sqlThread.py b/src/class_sqlThread.py index 9b0ba696..b438bc15 100644 --- a/src/class_sqlThread.py +++ b/src/class_sqlThread.py @@ -1,15 +1,17 @@ import threading from configparser import BMConfigParser -import shared import sqlite3 import time import shutil # used for moving the messages.dat file import sys import os from debug import logger +import defaults +import helper_sql from namecoin import ensureNamecoinOptions import paths import protocol +import queues import random import state import string @@ -334,9 +336,9 @@ class sqlThread(threading.Thread): # sanity check if BMConfigParser().getint('bitmessagesettings', 'maxacceptablenoncetrialsperbyte') == 0: - BMConfigParser().set('bitmessagesettings','maxacceptablenoncetrialsperbyte', str(shared.ridiculousDifficulty * protocol.networkDefaultProofOfWorkNonceTrialsPerByte)) + BMConfigParser().set('bitmessagesettings','maxacceptablenoncetrialsperbyte', str(defaults.ridiculousDifficulty * protocol.networkDefaultProofOfWorkNonceTrialsPerByte)) if BMConfigParser().getint('bitmessagesettings', 'maxacceptablepayloadlengthextrabytes') == 0: - BMConfigParser().set('bitmessagesettings','maxacceptablepayloadlengthextrabytes', str(shared.ridiculousDifficulty * protocol.networkDefaultPayloadLengthExtraBytes)) + BMConfigParser().set('bitmessagesettings','maxacceptablepayloadlengthextrabytes', str(defaults.ridiculousDifficulty * protocol.networkDefaultPayloadLengthExtraBytes)) # The format of data stored in the pubkeys table has changed. Let's # clear it, and the pubkeys from inventory, so that they'll be re-downloaded. @@ -457,7 +459,7 @@ class sqlThread(threading.Thread): except Exception as err: if str(err) == 'database or disk is full': logger.fatal('(While null value test) Alert: Your disk or data storage volume is full. sqlThread will now exit.') - shared.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True))) + queues.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True))) os._exit(0) else: logger.error(err) @@ -477,21 +479,21 @@ class sqlThread(threading.Thread): except Exception as err: if str(err) == 'database or disk is full': logger.fatal('(While VACUUM) Alert: Your disk or data storage volume is full. sqlThread will now exit.') - shared.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True))) + queues.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True))) os._exit(0) item = '''update settings set value=? WHERE key='lastvacuumtime';''' parameters = (int(time.time()),) self.cur.execute(item, parameters) while True: - item = shared.sqlSubmitQueue.get() + item = helper_sql.sqlSubmitQueue.get() if item == 'commit': try: self.conn.commit() except Exception as err: if str(err) == 'database or disk is full': logger.fatal('(While committing) Alert: Your disk or data storage volume is full. sqlThread will now exit.') - shared.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True))) + queues.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True))) os._exit(0) elif item == 'exit': self.conn.close() @@ -506,7 +508,7 @@ class sqlThread(threading.Thread): except Exception as err: if str(err) == 'database or disk is full': logger.fatal('(while movemessagstoprog) Alert: Your disk or data storage volume is full. sqlThread will now exit.') - shared.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True))) + queues.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True))) os._exit(0) self.conn.close() shutil.move( @@ -522,7 +524,7 @@ class sqlThread(threading.Thread): except Exception as err: if str(err) == 'database or disk is full': logger.fatal('(while movemessagstoappdata) Alert: Your disk or data storage volume is full. sqlThread will now exit.') - shared.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True))) + queues.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True))) os._exit(0) self.conn.close() shutil.move( @@ -539,10 +541,10 @@ class sqlThread(threading.Thread): except Exception as err: if str(err) == 'database or disk is full': logger.fatal('(while deleteandvacuume) Alert: Your disk or data storage volume is full. sqlThread will now exit.') - shared.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True))) + queues.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True))) os._exit(0) else: - parameters = shared.sqlSubmitQueue.get() + parameters = helper_sql.sqlSubmitQueue.get() rowcount = 0 # print 'item', item # print 'parameters', parameters @@ -552,7 +554,7 @@ class sqlThread(threading.Thread): except Exception as err: if str(err) == 'database or disk is full': logger.fatal('(while cur.execute) Alert: Your disk or data storage volume is full. sqlThread will now exit.') - shared.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True))) + queues.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True))) os._exit(0) else: logger.fatal('Major error occurred when trying to execute a SQL statement within the sqlThread. Please tell Atheros about this error message or post it in the forum! Error occurred while trying to execute statement: "%s" Here are the parameters; you might want to censor this data with asterisks (***) as it can contain private information: %s. Here is the actual error message thrown by the sqlThread: %s', str(item), str(repr(parameters)), str(err)) @@ -560,5 +562,5 @@ class sqlThread(threading.Thread): os._exit(0) - shared.sqlReturnQueue.put((self.cur.fetchall(), rowcount)) - # shared.sqlSubmitQueue.task_done() + helper_sql.sqlReturnQueue.put((self.cur.fetchall(), rowcount)) + # helper_sql.sqlSubmitQueue.task_done() diff --git a/src/defaults.py b/src/defaults.py new file mode 100644 index 00000000..99982acd --- /dev/null +++ b/src/defaults.py @@ -0,0 +1,8 @@ +# sanity check, prevent doing ridiculous PoW +# 20 million PoWs equals approximately 2 days on dev's dual R9 290 +ridiculousDifficulty = 20000000 + +# Remember here the RPC port read from namecoin.conf so we can restore to +# it as default whenever the user changes the "method" selection for +# namecoin integration to "namecoind". +namecoinDefaultRpcPort = "8336" diff --git a/src/helper_bootstrap.py b/src/helper_bootstrap.py index 5361b9be..a3f74288 100644 --- a/src/helper_bootstrap.py +++ b/src/helper_bootstrap.py @@ -6,12 +6,13 @@ import time from configparser import BMConfigParser from debug import logger +import knownnodes import socks import state def knownNodes(): try: - # We shouldn't have to use the shared.knownNodesLock because this had + # We shouldn't have to use the knownnodes.knownNodesLock because this had # better be the only thread accessing knownNodes right now. pickleFile = open(state.appdata + 'knownnodes.dat', 'rb') loadedKnownNodes = pickle.load(pickleFile) @@ -20,19 +21,19 @@ def knownNodes(): # mapping. The new format is as 'Peer: time' pairs. If we loaded # data in the old format, transform it to the new style. for stream, nodes in loadedKnownNodes.items(): - shared.knownNodes[stream] = {} + knownnodes.knownNodes[stream] = {} for node_tuple in nodes.items(): try: host, (port, lastseen) = node_tuple peer = state.Peer(host, port) except: peer, lastseen = node_tuple - shared.knownNodes[stream][peer] = lastseen + knownnodes.knownNodes[stream][peer] = lastseen except: - shared.knownNodes = defaultKnownNodes.createDefaultKnownNodes(state.appdata) + knownnodes.knownNodes = defaultKnownNodes.createDefaultKnownNodes(state.appdata) # your own onion address, if setup if BMConfigParser().has_option('bitmessagesettings', 'onionhostname') and ".onion" in BMConfigParser().get('bitmessagesettings', 'onionhostname'): - shared.knownNodes[1][state.Peer(BMConfigParser().get('bitmessagesettings', 'onionhostname'), BMConfigParser().getint('bitmessagesettings', 'onionport'))] = int(time.time()) + knownnodes.knownNodes[1][state.Peer(BMConfigParser().get('bitmessagesettings', 'onionhostname'), BMConfigParser().getint('bitmessagesettings', 'onionport'))] = int(time.time()) if BMConfigParser().getint('bitmessagesettings', 'settingsversion') > 10: logger.error('Bitmessage cannot read future versions of the keys file (keys.dat). Run the newer version of Bitmessage.') raise SystemExit @@ -47,17 +48,17 @@ def dns(): try: for item in socket.getaddrinfo('bootstrap8080.bitmessage.org', 80): logger.info('Adding ' + item[4][0] + ' to knownNodes based on DNS bootstrap method') - shared.knownNodes[1][state.Peer(item[4][0], 8080)] = int(time.time()) + knownnodes.knownNodes[1][state.Peer(item[4][0], 8080)] = int(time.time()) except: logger.error('bootstrap8080.bitmessage.org DNS bootstrapping failed.') try: for item in socket.getaddrinfo('bootstrap8444.bitmessage.org', 80): logger.info ('Adding ' + item[4][0] + ' to knownNodes based on DNS bootstrap method') - shared.knownNodes[1][state.Peer(item[4][0], 8444)] = int(time.time()) + knownnodes.knownNodes[1][state.Peer(item[4][0], 8444)] = int(time.time()) except: logger.error('bootstrap8444.bitmessage.org DNS bootstrapping failed.') elif BMConfigParser().get('bitmessagesettings', 'socksproxytype') == 'SOCKS5': - shared.knownNodes[1][state.Peer('quzwelsuziwqgpt2.onion', 8444)] = int(time.time()) + knownnodes.knownNodes[1][state.Peer('quzwelsuziwqgpt2.onion', 8444)] = int(time.time()) logger.debug("Adding quzwelsuziwqgpt2.onion:8444 to knownNodes.") for port in [8080, 8444]: logger.debug("Resolving %i through SOCKS...", port) @@ -90,7 +91,7 @@ def dns(): else: if ip is not None: logger.info ('Adding ' + ip + ' to knownNodes based on SOCKS DNS bootstrap method') - shared.knownNodes[1][state.Peer(ip, port)] = time.time() + knownnodes.knownNodes[1][state.Peer(ip, port)] = time.time() else: logger.info('DNS bootstrap skipped because the proxy type does not support DNS resolution.') diff --git a/src/helper_generic.py b/src/helper_generic.py index d710f429..1e70b3e0 100644 --- a/src/helper_generic.py +++ b/src/helper_generic.py @@ -7,10 +7,11 @@ from threading import current_thread, enumerate from configparser import BMConfigParser from debug import logger -import shared +import queues +import shutdown def powQueueSize(): - curWorkerQueue = shared.workerQueue.qsize() + curWorkerQueue = queues.workerQueue.qsize() for thread in enumerate(): try: if thread.name == "singleWorker": @@ -43,7 +44,7 @@ def signal_handler(signal, frame): return logger.error("Got signal %i", signal) if BMConfigParser().safeGetBoolean('bitmessagesettings', 'daemon'): - shared.doCleanShutdown() + shutdown.doCleanShutdown() else: print 'Unfortunately you cannot use Ctrl+C when running the UI because the UI captures the signal.' diff --git a/src/helper_inbox.py b/src/helper_inbox.py index a3ad9755..3506150a 100644 --- a/src/helper_inbox.py +++ b/src/helper_inbox.py @@ -1,16 +1,16 @@ from helper_sql import * -import shared +import queues def insert(t): sqlExecute('''INSERT INTO inbox VALUES (?,?,?,?,?,?,?,?,?,?)''', *t) #shouldn't emit changedInboxUnread and displayNewInboxMessage at the same time - #shared.UISignalQueue.put(('changedInboxUnread', None)) + #queues.UISignalQueue.put(('changedInboxUnread', None)) def trash(msgid): sqlExecute('''UPDATE inbox SET folder='trash' WHERE msgid=?''', msgid) - shared.UISignalQueue.put(('removeInboxRowByMsgid',msgid)) + queues.UISignalQueue.put(('removeInboxRowByMsgid',msgid)) def isMessageAlreadyInInbox(sigHash): queryReturn = sqlQuery( '''SELECT COUNT(*) FROM inbox WHERE sighash=?''', sigHash) - return queryReturn[0][0] != 0 \ No newline at end of file + return queryReturn[0][0] != 0 diff --git a/src/knownnodes.py b/src/knownnodes.py new file mode 100644 index 00000000..051b2593 --- /dev/null +++ b/src/knownnodes.py @@ -0,0 +1,5 @@ +import threading + +knownNodesLock = threading.Lock() +knownNodes = {} + diff --git a/src/message_data_reader.py b/src/message_data_reader.py index 79a3a607..a0659807 100644 --- a/src/message_data_reader.py +++ b/src/message_data_reader.py @@ -5,8 +5,8 @@ import sqlite3 from time import strftime, localtime import sys -import shared import paths +import queues import state from binascii import hexlify @@ -87,7 +87,7 @@ def markAllInboxMessagesAsUnread(): cur.execute(item, parameters) output = cur.fetchall() conn.commit() - shared.UISignalQueue.put(('changedInboxUnread', None)) + queues.UISignalQueue.put(('changedInboxUnread', None)) print 'done' def vacuum(): diff --git a/src/namecoin.py b/src/namecoin.py index 7fca0162..1834458c 100644 --- a/src/namecoin.py +++ b/src/namecoin.py @@ -27,7 +27,7 @@ import sys import os from configparser import BMConfigParser -import shared +import defaults import tr # translate # FIXME: from debug import logger crashes PyBitmessage due to a circular @@ -293,7 +293,7 @@ def ensureNamecoinOptions (): if key == "rpcpassword" and not hasPass: defaultPass = val if key == "rpcport": - shared.namecoinDefaultRpcPort = val + defaults.namecoinDefaultRpcPort = val nmc.close () except IOError: @@ -310,4 +310,4 @@ def ensureNamecoinOptions (): # Set default port now, possibly to found value. if (not hasPort): BMConfigParser().set (configSection, "namecoinrpcport", - shared.namecoinDefaultRpcPort) + defaults.namecoinDefaultRpcPort) diff --git a/src/proofofwork.py b/src/proofofwork.py index da9282f2..8cb6e7e8 100644 --- a/src/proofofwork.py +++ b/src/proofofwork.py @@ -9,8 +9,8 @@ import time from configparser import BMConfigParser from debug import logger import paths -import shared import openclpow +import queues import tr import os import ctypes @@ -115,7 +115,7 @@ def _doGPUPoW(target, initialHash): #print "{} - value {} < {}".format(nonce, trialValue, target) if trialValue > target: deviceNames = ", ".join(gpu.name for gpu in openclpow.enabledGpus) - shared.UISignalQueue.put(('updateStatusBar', (tr._translate("MainWindow",'Your GPU(s) did not calculate correctly, disabling OpenCL. Please report to the developers.'), 1))) + queues.UISignalQueue.put(('updateStatusBar', (tr._translate("MainWindow",'Your GPU(s) did not calculate correctly, disabling OpenCL. Please report to the developers.'), 1))) logger.error("Your GPUs (%s) did not calculate correctly, disabling OpenCL. Please report to the developers.", deviceNames) openclpow.enabledGpus = [] raise Exception("GPU did not calculate correctly.") @@ -160,11 +160,11 @@ def notifyBuild(tried=False): global bmpow if bmpow: - shared.UISignalQueue.put(('updateStatusBar', (tr._translate("proofofwork", "C PoW module built successfully."), 1))) + queues.UISignalQueue.put(('updateStatusBar', (tr._translate("proofofwork", "C PoW module built successfully."), 1))) elif tried: - shared.UISignalQueue.put(('updateStatusBar', (tr._translate("proofofwork", "Failed to build C PoW module. Please build it manually."), 1))) + queues.UISignalQueue.put(('updateStatusBar', (tr._translate("proofofwork", "Failed to build C PoW module. Please build it manually."), 1))) else: - shared.UISignalQueue.put(('updateStatusBar', (tr._translate("proofofwork", "C PoW module unavailable. Please build it."), 1))) + queues.UISignalQueue.put(('updateStatusBar', (tr._translate("proofofwork", "C PoW module unavailable. Please build it."), 1))) def buildCPoW(): global bmpow diff --git a/src/queues.py b/src/queues.py new file mode 100644 index 00000000..f50a0f1c --- /dev/null +++ b/src/queues.py @@ -0,0 +1,16 @@ +import Queue +from multiprocessing import Queue as mpQueue, Lock as mpLock +from class_objectProcessorQueue import ObjectProcessorQueue + +workerQueue = Queue.Queue() +UISignalQueue = Queue.Queue() +addressGeneratorQueue = Queue.Queue() +# receiveDataThreads dump objects they hear on the network into this queue to be processed. +objectProcessorQueue = ObjectProcessorQueue() +apiAddressGeneratorReturnQueue = Queue.Queue( + ) # The address generator thread uses this queue to get information back to the API thread. + +parserProcess = None +parserLock = mpLock() +parserInputQueue = mpQueue() +parserOutputQueue = mpQueue() diff --git a/src/shared.py b/src/shared.py index d84ec191..a41271bc 100644 --- a/src/shared.py +++ b/src/shared.py @@ -21,13 +21,13 @@ from binascii import hexlify # Project imports. from addresses import * -from class_objectProcessorQueue import ObjectProcessorQueue from configparser import BMConfigParser import highlevelcrypto #import helper_startup from helper_sql import * from helper_threading import * from inventory import Inventory, PendingDownload +from queues import objectProcessorQueue import protocol import state @@ -37,15 +37,6 @@ MyECSubscriptionCryptorObjects = {} myAddressesByHash = {} #The key in this dictionary is the RIPE hash which is encoded in an address and value is the address itself. myAddressesByTag = {} # The key in this dictionary is the tag generated from the address. broadcastSendersForWhichImWatching = {} -workerQueue = Queue.Queue() -UISignalQueue = Queue.Queue() -parserInputQueue = mpQueue() -parserOutputQueue = mpQueue() -parserProcess = None -parserLock = mpLock() -addressGeneratorQueue = Queue.Queue() -knownNodesLock = threading.Lock() -knownNodes = {} printLock = threading.Lock() statusIconColor = 'red' connectedHostsList = {} #List of hosts to which we are connected. Used to guarantee that the outgoingSynSender threads won't connect to the same remote node twice. @@ -57,8 +48,6 @@ alreadyAttemptedConnectionsListResetTime = int( time.time()) # used to clear out the alreadyAttemptedConnectionsList periodically so that we will retry connecting to hosts to which we have already tried to connect. successfullyDecryptMessageTimings = [ ] # A list of the amounts of time it took to successfully decrypt msg messages -apiAddressGeneratorReturnQueue = Queue.Queue( - ) # The address generator thread uses this queue to get information back to the API thread. ackdataForWhichImWatching = {} clientHasReceivedIncomingConnections = False #used by API command clientStatus numberOfMessagesProcessed = 0 @@ -67,18 +56,8 @@ numberOfPubkeysProcessed = 0 daemon = False needToWriteKnownNodesToDisk = False # If True, the singleCleaner will write it to disk eventually. maximumLengthOfTimeToBotherResendingMessages = 0 -objectProcessorQueue = ObjectProcessorQueue() # receiveDataThreads dump objects they hear on the network into this queue to be processed. timeOffsetWrongCount = 0 -# sanity check, prevent doing ridiculous PoW -# 20 million PoWs equals approximately 2 days on dev's dual R9 290 -ridiculousDifficulty = 20000000 - -# Remember here the RPC port read from namecoin.conf so we can restore to -# it as default whenever the user changes the "method" selection for -# namecoin integration to "namecoind". -namecoinDefaultRpcPort = "8336" - def isAddressInMyAddressBook(address): queryreturn = sqlQuery( '''select address from addressbook where address=?''', @@ -184,77 +163,6 @@ def reloadBroadcastSendersForWhichImWatching(): privEncryptionKey = doubleHashOfAddressData[:32] MyECSubscriptionCryptorObjects[tag] = highlevelcrypto.makeCryptor(hexlify(privEncryptionKey)) -def doCleanShutdown(): - state.shutdown = 1 #Used to tell proof of work worker threads and the objectProcessorThread to exit. - try: - parserInputQueue.put(None, False) - except Queue.Full: - pass - protocol.broadcastToSendDataQueues((0, 'shutdown', 'no data')) - objectProcessorQueue.put(('checkShutdownVariable', 'no data')) - for thread in threading.enumerate(): - if thread.isAlive() and isinstance(thread, StoppableThread): - thread.stopThread() - - knownNodesLock.acquire() - UISignalQueue.put(('updateStatusBar','Saving the knownNodes list of peers to disk...')) - output = open(state.appdata + 'knownnodes.dat', 'wb') - logger.info('finished opening knownnodes.dat. Now pickle.dump') - pickle.dump(knownNodes, output) - logger.info('Completed pickle.dump. Closing output...') - output.close() - knownNodesLock.release() - logger.info('Finished closing knownnodes.dat output file.') - UISignalQueue.put(('updateStatusBar','Done saving the knownNodes list of peers to disk.')) - - logger.info('Flushing inventory in memory out to disk...') - UISignalQueue.put(( - 'updateStatusBar', - 'Flushing inventory in memory out to disk. This should normally only take a second...')) - Inventory().flush() - - # Verify that the objectProcessor has finished exiting. It should have incremented the - # shutdown variable from 1 to 2. This must finish before we command the sqlThread to exit. - while state.shutdown == 1: - time.sleep(.1) - - # This one last useless query will guarantee that the previous flush committed and that the - # objectProcessorThread committed before we close the program. - sqlQuery('SELECT address FROM subscriptions') - logger.info('Finished flushing inventory.') - sqlStoredProcedure('exit') - - # Wait long enough to guarantee that any running proof of work worker threads will check the - # shutdown variable and exit. If the main thread closes before they do then they won't stop. - time.sleep(.25) - - from class_outgoingSynSender import outgoingSynSender - from class_sendDataThread import sendDataThread - for thread in threading.enumerate(): - if isinstance(thread, sendDataThread): - thread.sendDataThreadQueue.put((0, 'shutdown','no data')) - if thread is not threading.currentThread() and isinstance(thread, StoppableThread) and not isinstance(thread, outgoingSynSender): - logger.debug("Waiting for thread %s", thread.name) - thread.join() - - # flush queued - for queue in (workerQueue, UISignalQueue, addressGeneratorQueue, objectProcessorQueue): - while True: - try: - queue.get(False) - queue.task_done() - except Queue.Empty: - break - - if BMConfigParser().safeGetBoolean('bitmessagesettings','daemon'): - logger.info('Clean shutdown complete.') - thisapp.cleanup() - os._exit(0) - else: - logger.info('Core shutdown complete.') - for thread in threading.enumerate(): - logger.debug("Thread %s still running", thread.name) - def fixPotentiallyInvalidUTF8Data(text): try: unicode(text,'utf-8') diff --git a/src/shutdown.py b/src/shutdown.py new file mode 100644 index 00000000..3a6c1386 --- /dev/null +++ b/src/shutdown.py @@ -0,0 +1,87 @@ +import os +import pickle +import Queue +import threading +import time + +from class_outgoingSynSender import outgoingSynSender +from class_sendDataThread import sendDataThread +from configparser import BMConfigParser +from debug import logger +from helper_sql import sqlQuery, sqlStoredProcedure +from helper_threading import StoppableThread +from knownnodes import knownNodes, knownNodesLock +from inventory import Inventory +import protocol +from queues import addressGeneratorQueue, objectProcessorQueue, parserInputQueue, UISignalQueue, workerQueue +import shared +import state + +def doCleanShutdown(): + state.shutdown = 1 #Used to tell proof of work worker threads and the objectProcessorThread to exit. + try: + parserInputQueue.put(None, False) + except Queue.Full: + pass + protocol.broadcastToSendDataQueues((0, 'shutdown', 'no data')) + objectProcessorQueue.put(('checkShutdownVariable', 'no data')) + for thread in threading.enumerate(): + if thread.isAlive() and isinstance(thread, StoppableThread): + thread.stopThread() + + knownNodesLock.acquire() + UISignalQueue.put(('updateStatusBar','Saving the knownNodes list of peers to disk...')) + output = open(state.appdata + 'knownnodes.dat', 'wb') + logger.info('finished opening knownnodes.dat. Now pickle.dump') + pickle.dump(knownNodes, output) + logger.info('Completed pickle.dump. Closing output...') + output.close() + knownNodesLock.release() + logger.info('Finished closing knownnodes.dat output file.') + UISignalQueue.put(('updateStatusBar','Done saving the knownNodes list of peers to disk.')) + + logger.info('Flushing inventory in memory out to disk...') + UISignalQueue.put(( + 'updateStatusBar', + 'Flushing inventory in memory out to disk. This should normally only take a second...')) + Inventory().flush() + + # Verify that the objectProcessor has finished exiting. It should have incremented the + # shutdown variable from 1 to 2. This must finish before we command the sqlThread to exit. + while state.shutdown == 1: + time.sleep(.1) + + # This one last useless query will guarantee that the previous flush committed and that the + # objectProcessorThread committed before we close the program. + sqlQuery('SELECT address FROM subscriptions') + logger.info('Finished flushing inventory.') + sqlStoredProcedure('exit') + + # Wait long enough to guarantee that any running proof of work worker threads will check the + # shutdown variable and exit. If the main thread closes before they do then they won't stop. + time.sleep(.25) + + for thread in threading.enumerate(): + if isinstance(thread, sendDataThread): + thread.sendDataThreadQueue.put((0, 'shutdown','no data')) + if thread is not threading.currentThread() and isinstance(thread, StoppableThread) and not isinstance(thread, outgoingSynSender): + logger.debug("Waiting for thread %s", thread.name) + thread.join() + + # flush queued + for queue in (workerQueue, UISignalQueue, addressGeneratorQueue, objectProcessorQueue): + while True: + try: + queue.get(False) + queue.task_done() + except Queue.Empty: + break + + if BMConfigParser().safeGetBoolean('bitmessagesettings','daemon'): + logger.info('Clean shutdown complete.') + shared.thisapp.cleanup() + os._exit(0) + else: + logger.info('Core shutdown complete.') + for thread in threading.enumerate(): + logger.debug("Thread %s still running", thread.name) diff --git a/src/state.py b/src/state.py index 3d5209c4..e1a5adad 100644 --- a/src/state.py +++ b/src/state.py @@ -1,4 +1,5 @@ import collections +import threading neededPubkeys = {} streamsInWhichIAmParticipating = [] diff --git a/src/upnp.py b/src/upnp.py index b8d41f4b..fa969f36 100644 --- a/src/upnp.py +++ b/src/upnp.py @@ -8,6 +8,7 @@ import threading import time from configparser import BMConfigParser from helper_threading import * +import queues import shared import state import tr @@ -218,7 +219,7 @@ class uPnPThread(threading.Thread, StoppableThread): logger.debug("Found UPnP router at %s", ip) self.routers.append(newRouter) self.createPortMapping(newRouter) - shared.UISignalQueue.put(('updateStatusBar', tr._translate("MainWindow",'UPnP port mapping established on port %1').arg(str(self.extPort)))) + queues.UISignalQueue.put(('updateStatusBar', tr._translate("MainWindow",'UPnP port mapping established on port %1').arg(str(self.extPort)))) break except socket.timeout as e: pass @@ -242,7 +243,7 @@ class uPnPThread(threading.Thread, StoppableThread): self.deletePortMapping(router) shared.extPort = None if deleted: - shared.UISignalQueue.put(('updateStatusBar', tr._translate("MainWindow",'UPnP port mapping removed'))) + queues.UISignalQueue.put(('updateStatusBar', tr._translate("MainWindow",'UPnP port mapping removed'))) logger.debug("UPnP thread done") def getLocalIP(self):