You may generate addresses by using either random numbers or by using a passphrase. " - "If you use a passphrase, the address is called a "deterministic" address. " - "The \'Random Number\' option is selected by default but deterministic addresses have several pros and cons:
" - "| Pros: | Cons: |
| You can recreate your addresses on any computer from memory. " - "You need-not worry about backing up your keys.dat file as long as you can remember your passphrase. | " - "You must remember (or write down) your passphrase if you expect to be able " - "to recreate your keys if they are lost. " -# "You must remember the address version number and the stream number along with your passphrase. " - "If you choose a weak passphrase and someone on the Internet can brute-force it, they can read your messages and send messages as you." - "") - label.setWordWrap(True) - - self.randomAddress = QtGui.QRadioButton("Use a random number generator to make an address") - self.deterministicAddress = QtGui.QRadioButton("Use a passphrase to make an address") - self.randomAddress.setChecked(True) - - layout = QtGui.QVBoxLayout() - layout.addWidget(label) - layout.addWidget(self.randomAddress) - layout.addWidget(self.deterministicAddress) - self.setLayout(layout) - - def nextId(self): - if self.randomAddress.isChecked(): - return 2 - else: - return 3 - -class NewAddressWizardRandomPage(QtGui.QWizardPage): - def __init__(self, addresses): - super(QtGui.QWizardPage, self).__init__() - self.setTitle("Random") - - label = QtGui.QLabel("Random address.") - label.setWordWrap(True) - - labelLabel = QtGui.QLabel("Label (not shown to anyone except you):") - self.labelLineEdit = QtGui.QLineEdit() - - self.radioButtonMostAvailable = QtGui.QRadioButton("Use the most available stream\n" - "(best if this is the first of many addresses you will create)") - self.radioButtonExisting = QtGui.QRadioButton("Use the same stream as an existing address\n" - "(saves you some bandwidth and processing power)") - self.radioButtonMostAvailable.setChecked(True) - self.comboBoxExisting = QtGui.QComboBox() - self.comboBoxExisting.setEnabled(False) - self.comboBoxExisting.setEditable(True) - - for address in addresses: - self.comboBoxExisting.addItem(address) - -# self.comboBoxExisting.setObjectName(_fromUtf8("comboBoxExisting")) - self.checkBoxEighteenByteRipe = QtGui.QCheckBox("Spend several minutes of extra computing time to make the address(es) 1 or 2 characters shorter") - - layout = QtGui.QGridLayout() - layout.addWidget(label, 0, 0) - layout.addWidget(labelLabel, 1, 0) - layout.addWidget(self.labelLineEdit, 2, 0) - layout.addWidget(self.radioButtonMostAvailable, 3, 0) - layout.addWidget(self.radioButtonExisting, 4, 0) - layout.addWidget(self.comboBoxExisting, 5, 0) - layout.addWidget(self.checkBoxEighteenByteRipe, 6, 0) - self.setLayout(layout) - - QtCore.QObject.connect(self.radioButtonExisting, QtCore.SIGNAL("toggled(bool)"), self.comboBoxExisting.setEnabled) - - self.registerField("label", self.labelLineEdit) - self.registerField("radioButtonMostAvailable", self.radioButtonMostAvailable) - self.registerField("radioButtonExisting", self.radioButtonExisting) - self.registerField("comboBoxExisting", self.comboBoxExisting) - -# self.emailAsWell = QtGui.QRadioButton("Combined email and bitmessage account") -# self.onlyBM = QtGui.QRadioButton("Bitmessage-only account (no email)") -# self.emailAsWell.setChecked(True) - - def nextId(self): - return 6 - - -class NewAddressWizardPassphrasePage(QtGui.QWizardPage): - def __init__(self): - super(QtGui.QWizardPage, self).__init__() - self.setTitle("Passphrase") - - label = QtGui.QLabel("Deterministric address.") - label.setWordWrap(True) - - passphraseLabel = QtGui.QLabel("Passphrase") - self.lineEditPassphrase = QtGui.QLineEdit() - self.lineEditPassphrase.setEchoMode(QtGui.QLineEdit.Password) - self.lineEditPassphrase.setInputMethodHints(QtCore.Qt.ImhHiddenText|QtCore.Qt.ImhNoAutoUppercase|QtCore.Qt.ImhNoPredictiveText) - retypePassphraseLabel = QtGui.QLabel("Retype passphrase") - self.lineEditPassphraseAgain = QtGui.QLineEdit() - self.lineEditPassphraseAgain.setEchoMode(QtGui.QLineEdit.Password) - - numberLabel = QtGui.QLabel("Number of addresses to make based on your passphrase:") - self.spinBoxNumberOfAddressesToMake = QtGui.QSpinBox() - self.spinBoxNumberOfAddressesToMake.setMinimum(1) - self.spinBoxNumberOfAddressesToMake.setProperty("value", 8) -# self.spinBoxNumberOfAddressesToMake.setObjectName(_fromUtf8("spinBoxNumberOfAddressesToMake")) - label2 = QtGui.QLabel("In addition to your passphrase, you must remember these numbers:") - label3 = QtGui.QLabel("Address version number: 4") - label4 = QtGui.QLabel("Stream number: 1") - - layout = QtGui.QGridLayout() - layout.addWidget(label, 0, 0, 1, 4) - layout.addWidget(passphraseLabel, 1, 0, 1, 4) - layout.addWidget(self.lineEditPassphrase, 2, 0, 1, 4) - layout.addWidget(retypePassphraseLabel, 3, 0, 1, 4) - layout.addWidget(self.lineEditPassphraseAgain, 4, 0, 1, 4) - layout.addWidget(numberLabel, 5, 0, 1, 3) - layout.addWidget(self.spinBoxNumberOfAddressesToMake, 5, 3) - layout.setColumnMinimumWidth(3, 1) - layout.addWidget(label2, 6, 0, 1, 4) - layout.addWidget(label3, 7, 0, 1, 2) - layout.addWidget(label4, 7, 2, 1, 2) - self.setLayout(layout) - - def nextId(self): - return 6 - - -class NewAddressWizardEmailProviderPage(QtGui.QWizardPage): - def __init__(self): - super(QtGui.QWizardPage, self).__init__() - self.setTitle("Choose email provider") - - label = QtGui.QLabel("Currently only Mailchuck email gateway is available " - "(@mailchuck.com email address). In the future, maybe other gateways will be available. " - "Press Next.") - label.setWordWrap(True) - -# self.mailchuck = QtGui.QRadioButton("Mailchuck email gateway (@mailchuck.com)") -# self.mailchuck.setChecked(True) - - layout = QtGui.QVBoxLayout() - layout.addWidget(label) -# layout.addWidget(self.mailchuck) - self.setLayout(layout) - - def nextId(self): - return 5 - - -class NewAddressWizardEmailAddressPage(QtGui.QWizardPage): - def __init__(self): - super(QtGui.QWizardPage, self).__init__() - self.setTitle("Email address") - - label = QtGui.QLabel("Choosing an email address. Address must end with @mailchuck.com") - label.setWordWrap(True) - - self.specificEmail = QtGui.QRadioButton("Pick your own email address:") - self.specificEmail.setChecked(True) - self.emailLineEdit = QtGui.QLineEdit() - self.randomEmail = QtGui.QRadioButton("Generate a random email address") - - QtCore.QObject.connect(self.specificEmail, QtCore.SIGNAL("toggled(bool)"), self.emailLineEdit.setEnabled) - - layout = QtGui.QVBoxLayout() - layout.addWidget(label) - layout.addWidget(self.specificEmail) - layout.addWidget(self.emailLineEdit) - layout.addWidget(self.randomEmail) - self.setLayout(layout) - - def nextId(self): - return 6 - - -class NewAddressWizardWaitPage(QtGui.QWizardPage): - def __init__(self): - super(QtGui.QWizardPage, self).__init__() - self.setTitle("Wait") - - self.label = QtGui.QLabel("Wait!") - self.label.setWordWrap(True) - self.progressBar = QtGui.QProgressBar() - self.progressBar.setMinimum(0) - self.progressBar.setMaximum(100) - self.progressBar.setValue(0) - -# self.emailAsWell = QtGui.QRadioButton("Combined email and bitmessage account") -# self.onlyBM = QtGui.QRadioButton("Bitmessage-only account (no email)") -# self.emailAsWell.setChecked(True) - - layout = QtGui.QVBoxLayout() - layout.addWidget(self.label) - layout.addWidget(self.progressBar) -# layout.addWidget(self.emailAsWell) -# layout.addWidget(self.onlyBM) - self.setLayout(layout) - - def update(self, i): - if i == 101 and self.wizard().currentId() == 6: - self.wizard().button(QtGui.QWizard.NextButton).click() - return - elif i == 101: - print "haha" - return - self.progressBar.setValue(i) - if i == 50: - self.emit(QtCore.SIGNAL('completeChanged()')) - - def isComplete(self): -# print "val = " + str(self.progressBar.value()) - if self.progressBar.value() >= 50: - return True - else: - return False - - def initializePage(self): - if self.field("emailAsWell").toBool(): - val = "yes/" - else: - val = "no/" - if self.field("onlyBM").toBool(): - val += "yes" - else: - val += "no" - - self.label.setText("Wait! " + val) -# self.wizard().button(QtGui.QWizard.NextButton).setEnabled(False) - self.progressBar.setValue(0) - self.thread = NewAddressThread() - self.connect(self.thread, self.thread.signal, self.update) - self.thread.start() - - def nextId(self): - return 10 - - -class NewAddressWizardConclusionPage(QtGui.QWizardPage): - def __init__(self): - super(QtGui.QWizardPage, self).__init__() - self.setTitle("All done!") - - label = QtGui.QLabel("You successfully created a new address.") - label.setWordWrap(True) - - layout = QtGui.QVBoxLayout() - layout.addWidget(label) - self.setLayout(layout) - -class Ui_NewAddressWizard(QtGui.QWizard): - def __init__(self, addresses): - super(QtGui.QWizard, self).__init__() - - self.pages = {} - - page = NewAddressWizardIntroPage() - self.setPage(0, page) - self.setStartId(0) - page = NewAddressWizardRngPassphrasePage() - self.setPage(1, page) - page = NewAddressWizardRandomPage(addresses) - self.setPage(2, page) - page = NewAddressWizardPassphrasePage() - self.setPage(3, page) - page = NewAddressWizardEmailProviderPage() - self.setPage(4, page) - page = NewAddressWizardEmailAddressPage() - self.setPage(5, page) - page = NewAddressWizardWaitPage() - self.setPage(6, page) - page = NewAddressWizardConclusionPage() - self.setPage(10, page) - - self.setWindowTitle("New address wizard") - self.adjustSize() - self.show() - -class NewAddressThread(QtCore.QThread): - def __init__(self): - QtCore.QThread.__init__(self) - self.signal = QtCore.SIGNAL("signal") - - def __del__(self): - self.wait() - - def createDeterministic(self): - pass - - def createPassphrase(self): - pass - - def broadcastAddress(self): - pass - - def registerMailchuck(self): - pass - - def waitRegistration(self): - pass - - def run(self): - import time - for i in range(1, 101): - time.sleep(0.1) # artificial time delay - self.emit(self.signal, i) - self.emit(self.signal, 101) -# self.terminate() - -if __name__ == '__main__': - - import sys - - app = QtGui.QApplication(sys.argv) - - wizard = Ui_NewAddressWizard(["a", "b", "c", "d"]) - if (wizard.exec_()): - print "Email: " + ("yes" if wizard.field("emailAsWell").toBool() else "no") - print "BM: " + ("yes" if wizard.field("onlyBM").toBool() else "no") - else: - print "Wizard cancelled" - sys.exit() diff --git a/src/class_sqlThread.py b/src/class_sqlThread.py index a45571e0..91dfa206 100644 --- a/src/class_sqlThread.py +++ b/src/class_sqlThread.py @@ -1,29 +1,45 @@ -import threading -from bmconfigparser import BMConfigParser -import sqlite3 -import time -import shutil # used for moving the messages.dat file -import sys +""" +src/class_sqlThread.py +====================== +""" +# pylint: disable=attribute-defined-outside-init,protected-access + import os -from debug import logger +import shutil # used for moving the messages.dat file +import sqlite3 +import sys +import threading +import time + import helper_sql import helper_startup import paths import queues import state import tr - -# This thread exists because SQLITE3 is so un-threadsafe that we must -# submit queries to it and it puts results back in a different queue. They -# won't let us just use locks. +from bmconfigparser import BMConfigParser +from debug import logger class sqlThread(threading.Thread): + """ + Implement SQL operations in a dedicated thread. + + This thread exists because SQLITE3 is so un-threadsafe that we must submit queries to it and it puts results back + in a different queue. They won't let us just use locks. + + This actually only applies for certain deployments, and/or really old version of sqlite. I haven't actually seen it + anywhere. Current versions do have support for threading and multiprocessing. I don't see an urgent reason to + refactor this, but it should be noted in the comment that the problem is mostly not valid. Sadly, last time I + checked, there is no reliable way to check whether the library is or isn't thread-safe. + """ def __init__(self): threading.Thread.__init__(self, name="SQL") def run(self): + # pylint: disable=too-many-locals,too-many-branches,too-many-locals,too-many-statements + self.conn = sqlite3.connect(state.appdata + 'messages.dat') self.conn.text_factory = str self.cur = self.conn.cursor() @@ -32,30 +48,38 @@ class sqlThread(threading.Thread): try: self.cur.execute( - '''CREATE TABLE inbox (msgid blob, toaddress text, fromaddress text, subject text, received text, message text, folder text, encodingtype int, read bool, sighash blob, UNIQUE(msgid) ON CONFLICT REPLACE)''' ) + '''CREATE TABLE inbox (msgid blob, toaddress text, fromaddress text, subject text, received text,''' + ''' message text, folder text, encodingtype int, read bool, sighash blob, UNIQUE(msgid)''' + ''' ON CONFLICT REPLACE)''') self.cur.execute( - '''CREATE TABLE sent (msgid blob, toaddress text, toripe blob, fromaddress text, subject text, message text, ackdata blob, senttime integer, lastactiontime integer, sleeptill integer, status text, retrynumber integer, folder text, encodingtype int, ttl int)''' ) + '''CREATE TABLE sent (msgid blob, toaddress text, toripe blob, fromaddress text, subject text,''' + ''' message text, ackdata blob, senttime integer, lastactiontime integer, sleeptill integer,''' + ''' status text, retrynumber integer, folder text, encodingtype int, ttl int)''') self.cur.execute( - '''CREATE TABLE subscriptions (label text, address text, enabled bool)''' ) + '''CREATE TABLE subscriptions (label text, address text, enabled bool)''') self.cur.execute( - '''CREATE TABLE addressbook (label text, address text)''' ) + '''CREATE TABLE addressbook (label text, address text)''') self.cur.execute( - '''CREATE TABLE blacklist (label text, address text, enabled bool)''' ) + '''CREATE TABLE blacklist (label text, address text, enabled bool)''') self.cur.execute( - '''CREATE TABLE whitelist (label text, address text, enabled bool)''' ) + '''CREATE TABLE whitelist (label text, address text, enabled bool)''') self.cur.execute( - '''CREATE TABLE pubkeys (address text, addressversion int, transmitdata blob, time int, usedpersonally text, UNIQUE(address) ON CONFLICT REPLACE)''' ) + '''CREATE TABLE pubkeys (address text, addressversion int, transmitdata blob, time int,''' + ''' usedpersonally text, UNIQUE(address) ON CONFLICT REPLACE)''') self.cur.execute( - '''CREATE TABLE inventory (hash blob, objecttype int, streamnumber int, payload blob, expirestime integer, tag blob, UNIQUE(hash) ON CONFLICT REPLACE)''' ) + '''CREATE TABLE inventory (hash blob, objecttype int, streamnumber int, payload blob,''' + ''' expirestime integer, tag blob, UNIQUE(hash) ON CONFLICT REPLACE)''') self.cur.execute( - '''INSERT INTO subscriptions VALUES('Bitmessage new releases/announcements','BM-GtovgYdgs7qXPkoYaRgrLFuFKz1SFpsw',1)''') + '''INSERT INTO subscriptions VALUES''' + '''('Bitmessage new releases/announcements','BM-GtovgYdgs7qXPkoYaRgrLFuFKz1SFpsw',1)''') self.cur.execute( - '''CREATE TABLE settings (key blob, value blob, UNIQUE(key) ON CONFLICT REPLACE)''' ) - self.cur.execute( '''INSERT INTO settings VALUES('version','10')''') - self.cur.execute( '''INSERT INTO settings VALUES('lastvacuumtime',?)''', ( + '''CREATE TABLE settings (key blob, value blob, UNIQUE(key) ON CONFLICT REPLACE)''') + self.cur.execute('''INSERT INTO settings VALUES('version','10')''') + self.cur.execute('''INSERT INTO settings VALUES('lastvacuumtime',?)''', ( int(time.time()),)) self.cur.execute( - '''CREATE TABLE objectprocessorqueue (objecttype int, data blob, UNIQUE(objecttype, data) ON CONFLICT REPLACE)''' ) + '''CREATE TABLE objectprocessorqueue''' + ''' (objecttype int, data blob, UNIQUE(objecttype, data) ON CONFLICT REPLACE)''') self.conn.commit() logger.info('Created messages database file') except Exception as err: @@ -120,33 +144,37 @@ class sqlThread(threading.Thread): logger.debug( "In messages.dat database, creating new 'settings' table.") self.cur.execute( - '''CREATE TABLE settings (key text, value blob, UNIQUE(key) ON CONFLICT REPLACE)''' ) - self.cur.execute( '''INSERT INTO settings VALUES('version','1')''') - self.cur.execute( '''INSERT INTO settings VALUES('lastvacuumtime',?)''', ( + '''CREATE TABLE settings (key text, value blob, UNIQUE(key) ON CONFLICT REPLACE)''') + self.cur.execute('''INSERT INTO settings VALUES('version','1')''') + self.cur.execute('''INSERT INTO settings VALUES('lastvacuumtime',?)''', ( int(time.time()),)) logger.debug('In messages.dat database, removing an obsolete field from the pubkeys table.') self.cur.execute( - '''CREATE TEMPORARY TABLE pubkeys_backup(hash blob, transmitdata blob, time int, usedpersonally text, UNIQUE(hash) ON CONFLICT REPLACE);''') + '''CREATE TEMPORARY TABLE pubkeys_backup(hash blob, transmitdata blob, time int,''' + ''' usedpersonally text, UNIQUE(hash) ON CONFLICT REPLACE);''') self.cur.execute( '''INSERT INTO pubkeys_backup SELECT hash, transmitdata, time, usedpersonally FROM pubkeys;''') - self.cur.execute( '''DROP TABLE pubkeys''') + self.cur.execute('''DROP TABLE pubkeys''') self.cur.execute( - '''CREATE TABLE pubkeys (hash blob, transmitdata blob, time int, usedpersonally text, UNIQUE(hash) ON CONFLICT REPLACE)''' ) + '''CREATE TABLE pubkeys (hash blob, transmitdata blob, time int, usedpersonally text, UNIQUE(hash)''' + ''' ON CONFLICT REPLACE)''') self.cur.execute( '''INSERT INTO pubkeys SELECT hash, transmitdata, time, usedpersonally FROM pubkeys_backup;''') - self.cur.execute( '''DROP TABLE pubkeys_backup;''') - logger.debug('Deleting all pubkeys from inventory. They will be redownloaded and then saved with the correct times.') + self.cur.execute('''DROP TABLE pubkeys_backup;''') + logger.debug('Deleting all pubkeys from inventory. They will be redownloaded and then saved with the' + ' correct times.') self.cur.execute( '''delete from inventory where objecttype = 'pubkey';''') logger.debug('replacing Bitmessage announcements mailing list with a new one.') self.cur.execute( '''delete from subscriptions where address='BM-BbkPSZbzPwpVcYZpU4yHwf9ZPEapN5Zx' ''') self.cur.execute( - '''INSERT INTO subscriptions VALUES('Bitmessage new releases/announcements','BM-GtovgYdgs7qXPkoYaRgrLFuFKz1SFpsw',1)''') + '''INSERT INTO subscriptions''' + ''' VALUES('Bitmessage new releases/announcements','BM-GtovgYdgs7qXPkoYaRgrLFuFKz1SFpsw',1)''') logger.debug('Commiting.') self.conn.commit() logger.debug('Vacuuming message.dat. You might notice that the file size gets much smaller.') - self.cur.execute( ''' VACUUM ''') + self.cur.execute(''' VACUUM ''') # After code refactoring, the possible status values for sent messages # have changed. @@ -170,15 +198,19 @@ class sqlThread(threading.Thread): 'In messages.dat database, removing an obsolete field from' ' the inventory table.') self.cur.execute( - '''CREATE TEMPORARY TABLE inventory_backup(hash blob, objecttype text, streamnumber int, payload blob, receivedtime integer, UNIQUE(hash) ON CONFLICT REPLACE);''') + '''CREATE TEMPORARY TABLE inventory_backup(hash blob, objecttype text, streamnumber int,''' + ''' payload blob, receivedtime integer, UNIQUE(hash) ON CONFLICT REPLACE);''') self.cur.execute( - '''INSERT INTO inventory_backup SELECT hash, objecttype, streamnumber, payload, receivedtime FROM inventory;''') - self.cur.execute( '''DROP TABLE inventory''') + '''INSERT INTO inventory_backup SELECT hash, objecttype, streamnumber, payload,''' + ''' receivedtime FROM inventory;''') + self.cur.execute('''DROP TABLE inventory''') self.cur.execute( - '''CREATE TABLE inventory (hash blob, objecttype text, streamnumber int, payload blob, receivedtime integer, UNIQUE(hash) ON CONFLICT REPLACE)''' ) + '''CREATE TABLE inventory (hash blob, objecttype text, streamnumber int, payload blob,''' + ''' receivedtime integer, UNIQUE(hash) ON CONFLICT REPLACE)''') self.cur.execute( - '''INSERT INTO inventory SELECT hash, objecttype, streamnumber, payload, receivedtime FROM inventory_backup;''') - self.cur.execute( '''DROP TABLE inventory_backup;''') + '''INSERT INTO inventory SELECT hash, objecttype, streamnumber, payload,''' + ''' receivedtime FROM inventory_backup;''') + self.cur.execute('''DROP TABLE inventory_backup;''') item = '''update settings set value=? WHERE key='version';''' parameters = (3,) self.cur.execute(item, parameters) @@ -208,7 +240,8 @@ class sqlThread(threading.Thread): if currentVersion == 4: self.cur.execute('''DROP TABLE pubkeys''') self.cur.execute( - '''CREATE TABLE pubkeys (hash blob, addressversion int, transmitdata blob, time int, usedpersonally text, UNIQUE(hash, addressversion) ON CONFLICT REPLACE)''') + '''CREATE TABLE pubkeys (hash blob, addressversion int, transmitdata blob, time int,''' + ''' usedpersonally text, UNIQUE(hash, addressversion) ON CONFLICT REPLACE)''') self.cur.execute( '''delete from inventory where objecttype = 'pubkey';''') item = '''update settings set value=? WHERE key='version';''' @@ -224,7 +257,8 @@ class sqlThread(threading.Thread): if currentVersion == 5: self.cur.execute('''DROP TABLE knownnodes''') self.cur.execute( - '''CREATE TABLE objectprocessorqueue (objecttype text, data blob, UNIQUE(objecttype, data) ON CONFLICT REPLACE)''') + '''CREATE TABLE objectprocessorqueue (objecttype text, data blob, UNIQUE(objecttype, data)''' + ''' ON CONFLICT REPLACE)''') item = '''update settings set value=? WHERE key='version';''' parameters = (6,) self.cur.execute(item, parameters) @@ -240,10 +274,14 @@ class sqlThread(threading.Thread): logger.debug( 'In messages.dat database, dropping and recreating' ' the inventory table.') - self.cur.execute( '''DROP TABLE inventory''') - self.cur.execute( '''CREATE TABLE inventory (hash blob, objecttype int, streamnumber int, payload blob, expirestime integer, tag blob, UNIQUE(hash) ON CONFLICT REPLACE)''' ) - self.cur.execute( '''DROP TABLE objectprocessorqueue''') - self.cur.execute( '''CREATE TABLE objectprocessorqueue (objecttype int, data blob, UNIQUE(objecttype, data) ON CONFLICT REPLACE)''' ) + self.cur.execute('''DROP TABLE inventory''') + self.cur.execute( + '''CREATE TABLE inventory (hash blob, objecttype int, streamnumber int, payload blob,''' + ''' expirestime integer, tag blob, UNIQUE(hash) ON CONFLICT REPLACE)''') + self.cur.execute('''DROP TABLE objectprocessorqueue''') + self.cur.execute( + '''CREATE TABLE objectprocessorqueue (objecttype int, data blob, UNIQUE(objecttype, data)''' + ''' ON CONFLICT REPLACE)''') item = '''update settings set value=? WHERE key='version';''' parameters = (7,) self.cur.execute(item, parameters) @@ -305,15 +343,21 @@ class sqlThread(threading.Thread): ' fields into the retrynumber field and adding the' ' sleeptill and ttl fields...') self.cur.execute( - '''CREATE TEMPORARY TABLE sent_backup (msgid blob, toaddress text, toripe blob, fromaddress text, subject text, message text, ackdata blob, lastactiontime integer, status text, retrynumber integer, folder text, encodingtype int)''' ) + '''CREATE TEMPORARY TABLE sent_backup (msgid blob, toaddress text, toripe blob, fromaddress text,''' + ''' subject text, message text, ackdata blob, lastactiontime integer, status text,''' + ''' retrynumber integer, folder text, encodingtype int)''') self.cur.execute( - '''INSERT INTO sent_backup SELECT msgid, toaddress, toripe, fromaddress, subject, message, ackdata, lastactiontime, status, 0, folder, encodingtype FROM sent;''') - self.cur.execute( '''DROP TABLE sent''') + '''INSERT INTO sent_backup SELECT msgid, toaddress, toripe, fromaddress, subject, message, ackdata,''' + ''' lastactiontime, status, 0, folder, encodingtype FROM sent;''') + self.cur.execute('''DROP TABLE sent''') self.cur.execute( - '''CREATE TABLE sent (msgid blob, toaddress text, toripe blob, fromaddress text, subject text, message text, ackdata blob, senttime integer, lastactiontime integer, sleeptill int, status text, retrynumber integer, folder text, encodingtype int, ttl int)''' ) + '''CREATE TABLE sent (msgid blob, toaddress text, toripe blob, fromaddress text, subject text,''' + ''' message text, ackdata blob, senttime integer, lastactiontime integer, sleeptill int,''' + ''' status text, retrynumber integer, folder text, encodingtype int, ttl int)''') self.cur.execute( - '''INSERT INTO sent SELECT msgid, toaddress, toripe, fromaddress, subject, message, ackdata, lastactiontime, lastactiontime, 0, status, 0, folder, encodingtype, 216000 FROM sent_backup;''') - self.cur.execute( '''DROP TABLE sent_backup''') + '''INSERT INTO sent SELECT msgid, toaddress, toripe, fromaddress, subject, message, ackdata,''' + ''' lastactiontime, lastactiontime, 0, status, 0, folder, encodingtype, 216000 FROM sent_backup;''') + self.cur.execute('''DROP TABLE sent_backup''') logger.info('In messages.dat database, finished making TTL-related changes.') logger.debug('In messages.dat database, adding address field to the pubkeys table.') # We're going to have to calculate the address for each row in the pubkeys @@ -330,16 +374,22 @@ class sqlThread(threading.Thread): self.cur.execute(item, parameters) # Now we can remove the hash field from the pubkeys table. self.cur.execute( - '''CREATE TEMPORARY TABLE pubkeys_backup (address text, addressversion int, transmitdata blob, time int, usedpersonally text, UNIQUE(address) ON CONFLICT REPLACE)''' ) + '''CREATE TEMPORARY TABLE pubkeys_backup (address text, addressversion int, transmitdata blob,''' + ''' time int, usedpersonally text, UNIQUE(address) ON CONFLICT REPLACE)''') self.cur.execute( - '''INSERT INTO pubkeys_backup SELECT address, addressversion, transmitdata, time, usedpersonally FROM pubkeys;''') - self.cur.execute( '''DROP TABLE pubkeys''') + '''INSERT INTO pubkeys_backup SELECT address, addressversion, transmitdata, time,''' + ''' usedpersonally FROM pubkeys;''') + self.cur.execute('''DROP TABLE pubkeys''') self.cur.execute( - '''CREATE TABLE pubkeys (address text, addressversion int, transmitdata blob, time int, usedpersonally text, UNIQUE(address) ON CONFLICT REPLACE)''' ) + '''CREATE TABLE pubkeys (address text, addressversion int, transmitdata blob, time int,''' + ''' usedpersonally text, UNIQUE(address) ON CONFLICT REPLACE)''') self.cur.execute( - '''INSERT INTO pubkeys SELECT address, addressversion, transmitdata, time, usedpersonally FROM pubkeys_backup;''') - self.cur.execute( '''DROP TABLE pubkeys_backup''') - logger.debug('In messages.dat database, done adding address field to the pubkeys table and removing the hash field.') + '''INSERT INTO pubkeys SELECT address, addressversion, transmitdata, time, usedpersonally''' + ''' FROM pubkeys_backup;''') + self.cur.execute('''DROP TABLE pubkeys_backup''') + logger.debug( + 'In messages.dat database, done adding address field to the pubkeys table and removing the hash field.' + ) self.cur.execute('''update settings set value=10 WHERE key='version';''') # Are you hoping to add a new option to the keys.dat file of existing @@ -349,7 +399,7 @@ class sqlThread(threading.Thread): try: testpayload = '\x00\x00' t = ('1234', 1, testpayload, '12345678', 'no') - self.cur.execute( '''INSERT INTO pubkeys VALUES(?,?,?,?,?)''', t) + self.cur.execute('''INSERT INTO pubkeys VALUES(?,?,?,?,?)''', t) self.conn.commit() self.cur.execute( '''SELECT transmitdata FROM pubkeys WHERE address='1234' ''') @@ -359,13 +409,25 @@ class sqlThread(threading.Thread): self.cur.execute('''DELETE FROM pubkeys WHERE address='1234' ''') self.conn.commit() if transmitdata == '': - logger.fatal('Problem: The version of SQLite you have cannot store Null values. Please download and install the latest revision of your version of Python (for example, the latest Python 2.7 revision) and try again.\n') - logger.fatal('PyBitmessage will now exit very abruptly. You may now see threading errors related to this abrupt exit but the problem you need to solve is related to SQLite.\n\n') + logger.fatal( + 'Problem: The version of SQLite you have cannot store Null values. Please download and install' + ' the latest revision of your version of Python (for example, the latest Python 2.7 revision)' + ' and try again.\n') + logger.fatal( + 'PyBitmessage will now exit very abruptly. You may now see threading errors related to this' + ' abrupt exit but the problem you need to solve is related to SQLite.\n\n') os._exit(0) except Exception as err: if str(err) == 'database or disk is full': - logger.fatal('(While null value test) Alert: Your disk or data storage volume is full. sqlThread will now exit.') - queues.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True))) + logger.fatal( + '(While null value test) Alert: Your disk or data storage volume is full. sqlThread will now exit.' + ) + queues.UISignalQueue.put(( + 'alert', ( + tr._translate("MainWindow", "Disk full"), + tr._translate( + "MainWindow", + 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True))) os._exit(0) else: logger.error(err) @@ -379,13 +441,22 @@ class sqlThread(threading.Thread): for row in queryreturn: value, = row if int(value) < int(time.time()) - 86400: - logger.info('It has been a long time since the messages.dat file has been vacuumed. Vacuuming now...') + logger.info( + 'It has been a long time since the messages.dat file has been vacuumed. Vacuuming now...') try: - self.cur.execute( ''' VACUUM ''') + self.cur.execute(''' VACUUM ''') except Exception as err: if str(err) == 'database or disk is full': - logger.fatal('(While VACUUM) Alert: Your disk or data storage volume is full. sqlThread will now exit.') - queues.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True))) + logger.fatal( + '(While VACUUM) Alert: Your disk or data storage volume is full. sqlThread will now exit.' + ) + queues.UISignalQueue.put(( + 'alert', ( + tr._translate("MainWindow", "Disk full"), + tr._translate( + "MainWindow", + 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.' + ), True))) os._exit(0) item = '''update settings set value=? WHERE key='lastvacuumtime';''' parameters = (int(time.time()),) @@ -400,8 +471,15 @@ class sqlThread(threading.Thread): self.conn.commit() except Exception as err: if str(err) == 'database or disk is full': - logger.fatal('(While committing) Alert: Your disk or data storage volume is full. sqlThread will now exit.') - queues.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True))) + logger.fatal( + '(While committing) Alert: Your disk or data storage volume is full.' + ' sqlThread will now exit.') + queues.UISignalQueue.put(( + 'alert', ( + tr._translate("MainWindow", "Disk full"), + tr._translate( + "MainWindow", 'Alert: Your disk or data storage volume is full.' + ' Bitmessage will now exit.'), True))) os._exit(0) elif item == 'exit': self.conn.close() @@ -415,8 +493,16 @@ class sqlThread(threading.Thread): self.conn.commit() except Exception as err: if str(err) == 'database or disk is full': - logger.fatal('(while movemessagstoprog) Alert: Your disk or data storage volume is full. sqlThread will now exit.') - queues.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True))) + logger.fatal( + '(while movemessagstoprog) Alert: Your disk or data storage volume is full.' + ' sqlThread will now exit.') + queues.UISignalQueue.put(( + 'alert', ( + tr._translate("MainWindow", "Disk full"), + tr._translate( + "MainWindow", + 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.' + ), True))) os._exit(0) self.conn.close() shutil.move( @@ -431,8 +517,16 @@ class sqlThread(threading.Thread): self.conn.commit() except Exception as err: if str(err) == 'database or disk is full': - logger.fatal('(while movemessagstoappdata) Alert: Your disk or data storage volume is full. sqlThread will now exit.') - queues.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True))) + logger.fatal( + '(while movemessagstoappdata) Alert: Your disk or data storage volume is full.' + ' sqlThread will now exit.') + queues.UISignalQueue.put(( + 'alert', ( + tr._translate("MainWindow", "Disk full"), + tr._translate( + "MainWindow", + 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.' + ), True))) os._exit(0) self.conn.close() shutil.move( @@ -445,11 +539,19 @@ class sqlThread(threading.Thread): self.cur.execute('''delete from sent where folder='trash' ''') self.conn.commit() try: - self.cur.execute( ''' VACUUM ''') + self.cur.execute(''' VACUUM ''') except Exception as err: if str(err) == 'database or disk is full': - logger.fatal('(while deleteandvacuume) Alert: Your disk or data storage volume is full. sqlThread will now exit.') - queues.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True))) + logger.fatal( + '(while deleteandvacuume) Alert: Your disk or data storage volume is full.' + ' sqlThread will now exit.') + queues.UISignalQueue.put(( + 'alert', ( + tr._translate("MainWindow", "Disk full"), + tr._translate( + "MainWindow", + 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.' + ), True))) os._exit(0) else: parameters = helper_sql.sqlSubmitQueue.get() @@ -461,11 +563,27 @@ class sqlThread(threading.Thread): rowcount = self.cur.rowcount except Exception as err: if str(err) == 'database or disk is full': - logger.fatal('(while cur.execute) Alert: Your disk or data storage volume is full. sqlThread will now exit.') - queues.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True))) + logger.fatal( + '(while cur.execute) Alert: Your disk or data storage volume is full.' + ' sqlThread will now exit.') + queues.UISignalQueue.put(( + 'alert', ( + tr._translate("MainWindow", "Disk full"), + tr._translate( + "MainWindow", + 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.' + ), True))) os._exit(0) else: - logger.fatal('Major error occurred when trying to execute a SQL statement within the sqlThread. Please tell Atheros about this error message or post it in the forum! Error occurred while trying to execute statement: "%s" Here are the parameters; you might want to censor this data with asterisks (***) as it can contain private information: %s. Here is the actual error message thrown by the sqlThread: %s', str(item), str(repr(parameters)), str(err)) + logger.fatal( + 'Major error occurred when trying to execute a SQL statement within the sqlThread. Please' + ' tell Atheros about this error message or post it in the forum! Error occurred while' + ' trying to execute statement: "%s" Here are the parameters; you might want to censor' + ' this data with asterisks (***) as it can contain private information: %s. Here is the' + ' actual error message thrown by the sqlThread: %s', + str(item), + str(repr(parameters)), + str(err)) logger.fatal('This program shall now abruptly exit!') os._exit(0) diff --git a/src/network/bmproto.py b/src/network/bmproto.py index ccb22ef5..3a0cdf23 100644 --- a/src/network/bmproto.py +++ b/src/network/bmproto.py @@ -1,44 +1,59 @@ +# pylint: disable=too-many-return-statements,too-many-public-methods,attribute-defined-outside-init,too-many-branches +# pylint: disable=too-many-instance-attributes,too-many-statements +""" +src/network/bmproto.py +====================== +""" + import base64 import hashlib -import random import socket import struct import time from binascii import hexlify +import addresses +import helper_random +import knownnodes +import network.connectionpool +import protocol +import shared +import state from bmconfigparser import BMConfigParser from debug import logger from inventory import Inventory -import knownnodes from network.advanceddispatcher import AdvancedDispatcher +from network.bmobject import ( + BMObject, BMObjectAlreadyHaveError, BMObjectExpiredError, BMObjectInsufficientPOWError, BMObjectInvalidDataError, + BMObjectInvalidError, BMObjectUnwantedStreamError +) from network.dandelion import Dandelion -from network.bmobject import BMObject, BMObjectInsufficientPOWError, BMObjectInvalidDataError, \ - BMObjectExpiredError, BMObjectUnwantedStreamError, BMObjectInvalidError, BMObjectAlreadyHaveError -import network.connectionpool from network.node import Node from network.objectracker import ObjectTracker -from network.proxy import Proxy, ProxyError, GeneralProxyError +from network.proxy import ProxyError +from queues import addrQueue, invQueue, objectProcessorQueue, portCheckerQueue -import addresses -from queues import objectProcessorQueue, portCheckerQueue, invQueue, addrQueue -import shared -import state -import protocol -import helper_random class BMProtoError(ProxyError): + """A Bitmessage Protocol Base Error""" errorCodes = ("Protocol error") class BMProtoInsufficientDataError(BMProtoError): + """A Bitmessage Protocol Insufficient Data Error""" + errorCodes = ("Insufficient data") class BMProtoExcessiveDataError(BMProtoError): + """A Bitmessage Protocol Excessive Data Error""" + errorCodes = ("Too much data") class BMProto(AdvancedDispatcher, ObjectTracker): + """A parser for the Bitmessage Protocol""" + # ~1.6 MB which is the maximum possible size of an inv message. maxMessageSize = 1600100 # 2**18 = 256kB is the maximum size of an object payload @@ -53,12 +68,15 @@ class BMProto(AdvancedDispatcher, ObjectTracker): maxTimeOffset = 3600 def __init__(self, address=None, sock=None): + # pylint: disable=super-init-not-called,unused-argument + AdvancedDispatcher.__init__(self, sock) self.isOutbound = False # packet/connection from a local IP self.local = False def bm_proto_reset(self): + """Reset the bitmessage object parser""" self.magic = None self.command = None self.payloadLength = 0 @@ -70,7 +88,10 @@ class BMProto(AdvancedDispatcher, ObjectTracker): self.object = None def state_bm_header(self): - self.magic, self.command, self.payloadLength, self.checksum = protocol.Header.unpack(self.read_buf[:protocol.Header.size]) + """Predicate (with logging) to indicate the prescence of a header""" + + self.magic, self.command, self.payloadLength, self.checksum = protocol.Header.unpack( + self.read_buf[:protocol.Header.size]) self.command = self.command.rstrip('\x00') if self.magic != 0xE9BEB4D9: # skip 1 byte in order to sync @@ -85,8 +106,9 @@ class BMProto(AdvancedDispatcher, ObjectTracker): self.invalid = True self.set_state("bm_command", length=protocol.Header.size, expectBytes=self.payloadLength) return True - + def state_bm_command(self): + """Predicate (with logging) to indicate the presence of a command""" self.payload = self.read_buf[:self.payloadLength] if self.checksum != hashlib.sha512(self.payload).digest()[0:4]: logger.debug("Bad checksum, ignoring") @@ -123,7 +145,7 @@ class BMProto(AdvancedDispatcher, ObjectTracker): # broken read, ignore pass else: - #print "Skipping command %s due to invalid data" % (self.command) + # print "Skipping command %s due to invalid data" % (self.command) logger.debug("Closing due to invalid command %s", self.command) self.close_reason = "Invalid command %s" % (self.command) self.set_state("close") @@ -135,16 +157,21 @@ class BMProto(AdvancedDispatcher, ObjectTracker): return True def decode_payload_string(self, length): - value = self.payload[self.payloadOffset:self.payloadOffset+length] + """Read and return `length` bytes from payload""" + + value = self.payload[self.payloadOffset:self.payloadOffset + length] self.payloadOffset += length return value def decode_payload_varint(self): + """Decode a varint from the payload""" + value, offset = addresses.decodeVarint(self.payload[self.payloadOffset:]) self.payloadOffset += offset return value def decode_payload_node(self): + """Decode node details from the payload""" services, host, port = self.decode_payload_content("Q16sH") if host[0:12] == '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xFF\xFF': host = socket.inet_ntop(socket.AF_INET, str(host[12:16])) @@ -154,38 +181,45 @@ class BMProto(AdvancedDispatcher, ObjectTracker): else: host = socket.inet_ntop(socket.AF_INET6, str(host)) if host == "": - # This can happen on Windows systems which are not 64-bit compatible - # so let us drop the IPv6 address. + # This can happen on Windows systems which are not 64-bit compatible + # so let us drop the IPv6 address. host = socket.inet_ntop(socket.AF_INET, str(host[12:16])) return Node(services, host, port) - def decode_payload_content(self, pattern = "v"): - # L = varint indicating the length of the next array - # l = varint indicating the length of the next item - # v = varint (or array) - # H = uint16 - # I = uint32 - # Q = uint64 - # i = net_addr (without time and stream number) - # s = string - # 0-9 = length of the next item - # , = end of array + def decode_payload_content(self, pattern="v"): + """ + Decode the payload + + L = varint indicating the length of the next array + l = varint indicating the length of the next item + v = varint (or array) + H = uint16 + I = uint32 + Q = uint64 + i = net_addr (without time and stream number) + s = string + 0-9 = length of the next item + , = end of array + + """ def decode_simple(self, char="v"): + """Some expected objects can be decoded very straightforwardly""" if char == "v": return self.decode_payload_varint() if char == "i": return self.decode_payload_node() if char == "H": self.payloadOffset += 2 - return struct.unpack(">H", self.payload[self.payloadOffset-2:self.payloadOffset])[0] + return struct.unpack(">H", self.payload[self.payloadOffset - 2:self.payloadOffset])[0] if char == "I": self.payloadOffset += 4 - return struct.unpack(">I", self.payload[self.payloadOffset-4:self.payloadOffset])[0] + return struct.unpack(">I", self.payload[self.payloadOffset - 4:self.payloadOffset])[0] if char == "Q": self.payloadOffset += 8 - return struct.unpack(">Q", self.payload[self.payloadOffset-8:self.payloadOffset])[0] + return struct.unpack(">Q", self.payload[self.payloadOffset - 8:self.payloadOffset])[0] + return None size = None isArray = False @@ -198,27 +232,19 @@ class BMProto(AdvancedDispatcher, ObjectTracker): # retval (array) parserStack = [[1, 1, False, pattern, 0, []]] - #try: - # sys._getframe(200) - # logger.error("Stack depth warning, pattern: %s", pattern) - # return - #except ValueError: - # pass - while True: i = parserStack[-1][3][parserStack[-1][4]] - if i in "0123456789" and (size is None or parserStack[-1][3][parserStack[-1][4]-1] not in "lL"): + if i in "0123456789" and (size is None or parserStack[-1][3][parserStack[-1][4] - 1] not in "lL"): try: size = size * 10 + int(i) except TypeError: size = int(i) isArray = False + elif i in "Ll" and size is None: size = self.decode_payload_varint() - if i == "L": - isArray = True - else: - isArray = False + isArray = bool(i == "L") + elif size is not None: if isArray: parserStack.append([size, size, isArray, parserStack[-1][3][parserStack[-1][4]:], 0, []]) @@ -227,25 +253,26 @@ class BMProto(AdvancedDispatcher, ObjectTracker): for j in range(parserStack[-1][4], len(parserStack[-1][3])): if parserStack[-1][3][j] not in "lL0123456789": break - parserStack.append([size, size, isArray, parserStack[-1][3][parserStack[-1][4]:j+1], 0, []]) + # pylint: disable=undefined-loop-variable + parserStack.append([size, size, isArray, parserStack[-1][3][parserStack[-1][4]:j + 1], 0, []]) parserStack[-2][4] += len(parserStack[-1][3]) - 1 size = None continue + elif i == "s": - #if parserStack[-2][2]: - # parserStack[-1][5].append(self.payload[self.payloadOffset:self.payloadOffset + parserStack[-1][0]]) - #else: parserStack[-1][5] = self.payload[self.payloadOffset:self.payloadOffset + parserStack[-1][0]] self.payloadOffset += parserStack[-1][0] parserStack[-1][1] = 0 parserStack[-1][2] = True - #del parserStack[-1] size = None + elif i in "viHIQ": parserStack[-1][5].append(decode_simple(self, parserStack[-1][3][parserStack[-1][4]])) size = None + else: size = None + for depth in range(len(parserStack) - 1, -1, -1): parserStack[depth][4] += 1 if parserStack[depth][4] >= len(parserStack[depth][3]): @@ -270,16 +297,22 @@ class BMProto(AdvancedDispatcher, ObjectTracker): raise BMProtoInsufficientDataError() def bm_command_error(self): + """Decode an error message and log it""" + # pylint: disable=unused-variable fatalStatus, banTime, inventoryVector, errorText = self.decode_payload_content("vvlsls") logger.error("%s:%i error: %i, %s", self.destination.host, self.destination.port, fatalStatus, errorText) return True def bm_command_getdata(self): + """ + Incoming request for object(s), if we have them and some other conditions are fulfilled, append them to the + write queue. + """ items = self.decode_payload_content("l32s") # skip? if time.time() < self.skipUntil: return True - #TODO make this more asynchronous + # .. todo:: make this more asynchronous helper_random.randomshuffle(items) for i in map(str, items): if Dandelion().hasHash(i) and \ @@ -321,21 +354,24 @@ class BMProto(AdvancedDispatcher, ObjectTracker): return True def bm_command_inv(self): + """Non-dandelion announce""" return self._command_inv(False) def bm_command_dinv(self): - """ - Dandelion stem announce - """ + """Dandelion stem announce""" return self._command_inv(True) def bm_command_object(self): + """Incoming object, process it""" objectOffset = self.payloadOffset nonce, expiresTime, objectType, version, streamNumber = self.decode_payload_content("QQIvv") self.object = BMObject(nonce, expiresTime, objectType, version, streamNumber, self.payload, self.payloadOffset) if len(self.payload) - self.payloadOffset > BMProto.maxObjectPayloadSize: - logger.info('The payload length of this object is too large (%d bytes). Ignoring it.' % (len(self.payload) - self.payloadOffset)) + logger.info( + 'The payload length of this object is too large (%d bytes). Ignoring it.', + len(self.payload) - self.payloadOffset + ) raise BMProtoExcessiveDataError() try: @@ -348,7 +384,9 @@ class BMProto(AdvancedDispatcher, ObjectTracker): try: self.object.checkStream() except (BMObjectUnwantedStreamError,) as e: - BMProto.stopDownloadingObject(self.object.inventoryHash, BMConfigParser().get("inventory", "acceptmismatch")) + BMProto.stopDownloadingObject( + self.object.inventoryHash, BMConfigParser().get( + "inventory", "acceptmismatch")) if not BMConfigParser().get("inventory", "acceptmismatch"): raise e @@ -367,7 +405,10 @@ class BMProto(AdvancedDispatcher, ObjectTracker): Dandelion().removeHash(self.object.inventoryHash, "cycle detection") Inventory()[self.object.inventoryHash] = ( - self.object.objectType, self.object.streamNumber, buffer(self.payload[objectOffset:]), self.object.expiresTime, buffer(self.object.tag)) + self.object.objectType, self.object.streamNumber, + buffer(self.payload[objectOffset:]), self.object.expiresTime, + buffer(self.object.tag) + ) self.handleReceivedObject(self.object.streamNumber, self.object.inventoryHash) invQueue.put((self.object.streamNumber, self.object.inventoryHash, self.destination)) return True @@ -376,16 +417,17 @@ class BMProto(AdvancedDispatcher, ObjectTracker): return self.decode_payload_content("LQIQ16sH") def bm_command_addr(self): - addresses = self._decode_addr() - for i in addresses: - seenTime, stream, services, ip, port = i + """Incoming address, process it""" + these_addresses = self._decode_addr() + for i in these_addresses: + seenTime, stream, _, ip, port = i decodedIP = protocol.checkIPAddress(str(ip)) if stream not in state.streamsInWhichIAmParticipating: continue if ( - decodedIP and time.time() - seenTime > 0 and - seenTime > time.time() - BMProto.addressAlive and - port > 0 + decodedIP and time.time() - seenTime > 0 and + seenTime > time.time() - BMProto.addressAlive and + port > 0 ): peer = state.Peer(decodedIP, port) try: @@ -407,18 +449,27 @@ class BMProto(AdvancedDispatcher, ObjectTracker): return True def bm_command_portcheck(self): + """Incoming port check request, queue it.""" portCheckerQueue.put(state.Peer(self.destination, self.peerNode.port)) return True def bm_command_ping(self): + """Incoming ping, respond to it.""" self.append_write_buf(protocol.CreatePacket('pong')) return True def bm_command_pong(self): - # nothing really + """ + Incoming pong, ignore it. + + PyBitmessage pings connections after about 5 minutes of inactivity, and leaves it to the TCP stack to handle + actual timeouts. So there is no need to do anything when a pong arrives. + """ + # pylint: disable=no-self-use return True def bm_command_verack(self): + """Incoming version. Parse and log, remember important things, like streams, bitfields, etc.""" self.verackReceived = True if self.verackSent: if self.isSSL: @@ -429,6 +480,8 @@ class BMProto(AdvancedDispatcher, ObjectTracker): return True def bm_command_version(self): + """Determine and log protocol version and other details""" + self.remoteProtocolVersion, self.services, self.timestamp, self.sockNode, self.peerNode, self.nonce, \ self.userAgent, self.streams = self.decode_payload_content("IQQiiQlsLv") self.nonce = struct.pack('>Q', self.nonce) @@ -439,17 +492,19 @@ class BMProto(AdvancedDispatcher, ObjectTracker): logger.debug("my external IP: %s", self.sockNode.host) logger.debug("remote node incoming address: %s:%i", self.destination.host, self.peerNode.port) logger.debug("user agent: %s", self.userAgent) - logger.debug("streams: [%s]", ",".join(map(str,self.streams))) + logger.debug("streams: [%s]", ",".join(map(str, self.streams))) if not self.peerValidityChecks(): - # TODO ABORT return True - #shared.connectedHostsList[self.destination] = self.streams[0] self.append_write_buf(protocol.CreatePacket('verack')) self.verackSent = True if not self.isOutbound: - self.append_write_buf(protocol.assembleVersionMessage(self.destination.host, self.destination.port, \ - network.connectionpool.BMConnectionPool().streams, True, nodeid=self.nodeid)) - #print "%s:%i: Sending version" % (self.destination.host, self.destination.port) + self.append_write_buf( + protocol.assembleVersionMessage( + self.destination.host, + self.destination.port, + network.connectionpool.BMConnectionPool().streams, + True, + nodeid=self.nodeid)) if ((self.services & protocol.NODE_SSL == protocol.NODE_SSL) and protocol.haveSSL(not self.isOutbound)): self.isSSL = True @@ -462,69 +517,76 @@ class BMProto(AdvancedDispatcher, ObjectTracker): return True def peerValidityChecks(self): + """Check the validity of the peer""" if self.remoteProtocolVersion < 3: - self.append_write_buf(protocol.assembleErrorMessage(fatal=2, - errorText="Your is using an old protocol. Closing connection.")) - logger.debug ('Closing connection to old protocol version %s, node: %s', - str(self.remoteProtocolVersion), str(self.destination)) + self.append_write_buf(protocol.assembleErrorMessage( + fatal=2, errorText="Your is using an old protocol. Closing connection.")) + logger.debug('Closing connection to old protocol version %s, node: %s', + str(self.remoteProtocolVersion), str(self.destination)) return False if self.timeOffset > BMProto.maxTimeOffset: - self.append_write_buf(protocol.assembleErrorMessage(fatal=2, - errorText="Your time is too far in the future compared to mine. Closing connection.")) + self.append_write_buf( + protocol.assembleErrorMessage( + fatal=2, + errorText="Your time is too far in the future compared to mine. Closing connection.")) logger.info("%s's time is too far in the future (%s seconds). Closing connection to it.", - self.destination, self.timeOffset) + self.destination, self.timeOffset) shared.timeOffsetWrongCount += 1 return False elif self.timeOffset < -BMProto.maxTimeOffset: - self.append_write_buf(protocol.assembleErrorMessage(fatal=2, - errorText="Your time is too far in the past compared to mine. Closing connection.")) + self.append_write_buf(protocol.assembleErrorMessage( + fatal=2, errorText="Your time is too far in the past compared to mine. Closing connection.")) logger.info("%s's time is too far in the past (timeOffset %s seconds). Closing connection to it.", - self.destination, self.timeOffset) + self.destination, self.timeOffset) shared.timeOffsetWrongCount += 1 return False else: shared.timeOffsetWrongCount = 0 if not self.streams: - self.append_write_buf(protocol.assembleErrorMessage(fatal=2, - errorText="We don't have shared stream interests. Closing connection.")) - logger.debug ('Closed connection to %s because there is no overlapping interest in streams.', - str(self.destination)) + self.append_write_buf(protocol.assembleErrorMessage( + fatal=2, errorText="We don't have shared stream interests. Closing connection.")) + logger.debug('Closed connection to %s because there is no overlapping interest in streams.', + str(self.destination)) return False if self.destination in network.connectionpool.BMConnectionPool().inboundConnections: try: if not protocol.checkSocksIP(self.destination.host): - self.append_write_buf(protocol.assembleErrorMessage(fatal=2, - errorText="Too many connections from your IP. Closing connection.")) - logger.debug ('Closed connection to %s because we are already connected to that IP.', - str(self.destination)) + self.append_write_buf( + protocol.assembleErrorMessage( + fatal=2, errorText="Too many connections from your IP. Closing connection.")) + logger.debug('Closed connection to %s because we are already connected to that IP.', + str(self.destination)) return False - except: + except BaseException: pass if not self.isOutbound: # incoming from a peer we're connected to as outbound, or server full # report the same error to counter deanonymisation if state.Peer(self.destination.host, self.peerNode.port) in \ - network.connectionpool.BMConnectionPool().inboundConnections or \ - len(network.connectionpool.BMConnectionPool().inboundConnections) + \ - len(network.connectionpool.BMConnectionPool().outboundConnections) > \ - BMConfigParser().safeGetInt("bitmessagesettings", "maxtotalconnections") + \ - BMConfigParser().safeGetInt("bitmessagesettings", "maxbootstrapconnections"): + network.connectionpool.BMConnectionPool().inboundConnections or \ + len(network.connectionpool.BMConnectionPool().inboundConnections) + \ + len(network.connectionpool.BMConnectionPool().outboundConnections) > \ + BMConfigParser().safeGetInt("bitmessagesettings", "maxtotalconnections") + \ + BMConfigParser().safeGetInt("bitmessagesettings", "maxbootstrapconnections"): self.append_write_buf(protocol.assembleErrorMessage(fatal=2, - errorText="Server full, please try again later.")) - logger.debug ("Closed connection to %s due to server full or duplicate inbound/outbound.", - str(self.destination)) + errorText="Server full, please try again later.")) + logger.debug("Closed connection to %s due to server full or duplicate inbound/outbound.", + str(self.destination)) return False if network.connectionpool.BMConnectionPool().isAlreadyConnected(self.nonce): - self.append_write_buf(protocol.assembleErrorMessage(fatal=2, - errorText="I'm connected to myself. Closing connection.")) - logger.debug ("Closed connection to %s because I'm connected to myself.", - str(self.destination)) + self.append_write_buf( + protocol.assembleErrorMessage( + fatal=2, + errorText="I'm connected to myself. Closing connection.")) + logger.debug("Closed connection to %s because I'm connected to myself.", + str(self.destination)) return False return True @staticmethod def assembleAddr(peerList): + """Build up a packed address""" if isinstance(peerList, state.Peer): peerList = (peerList) if not peerList: @@ -546,6 +608,7 @@ class BMProto(AdvancedDispatcher, ObjectTracker): @staticmethod def stopDownloadingObject(hashId, forwardAnyway=False): + """Stop downloading an object""" for connection in network.connectionpool.BMConnectionPool().inboundConnections.values() + \ network.connectionpool.BMConnectionPool().outboundConnections.values(): try: @@ -564,6 +627,7 @@ class BMProto(AdvancedDispatcher, ObjectTracker): pass def handle_close(self): + """Handle close""" self.set_state("close") if not (self.accepting or self.connecting or self.connected): # already disconnected diff --git a/src/socks/__init__.py b/src/socks/__init__.py index 7fd2cba3..16a3005d 100644 --- a/src/socks/__init__.py +++ b/src/socks/__init__.py @@ -1,4 +1,9 @@ -"""SocksiPy - Python SOCKS module. +# pylint: disable=too-many-arguments,global-statement,too-many-branches +""" +src/socks/__init__.py +===================== + +SocksiPy - Python SOCKS module. Version 1.00 Copyright 2006 Dan-Haim. All rights reserved. @@ -29,10 +34,6 @@ OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMANGE. This module provides a standard socket-like interface for Python for tunneling connections through SOCKS proxies. -""" - -""" - Minor modifications made by Christopher Gilbert (http://motomastyle.com/) for use in PyLoris (http://pyloris.sourceforge.net/) @@ -43,7 +44,7 @@ mainly to merge bug fixes found in Sourceforge import socket import struct -import sys + PROXY_TYPE_SOCKS4 = 1 PROXY_TYPE_SOCKS5 = 2 @@ -52,46 +53,71 @@ PROXY_TYPE_HTTP = 3 _defaultproxy = None _orgsocket = socket.socket -class ProxyError(Exception): pass -class GeneralProxyError(ProxyError): pass -class Socks5AuthError(ProxyError): pass -class Socks5Error(ProxyError): pass -class Socks4Error(ProxyError): pass -class HTTPError(ProxyError): pass + +class ProxyError(Exception): + """Base class for other ProxyErrors""" + pass + + +class GeneralProxyError(ProxyError): + """Handle a general proxy error""" + pass + + +class Socks5AuthError(ProxyError): + """Handle a SOCKS5 auth error""" + pass + + +class Socks5Error(ProxyError): + """Handle a SOCKS5 non-auth error""" + pass + + +class Socks4Error(ProxyError): + """Handle a SOCKS4 error""" + pass + + +class HTTPError(ProxyError): + """Handle a HTTP error""" + pass + _generalerrors = ("success", - "invalid data", - "not connected", - "not available", - "bad proxy type", - "bad input", - "timed out", - "network unreachable", - "connection refused", - "host unreachable") + "invalid data", + "not connected", + "not available", + "bad proxy type", + "bad input", + "timed out", + "network unreachable", + "connection refused", + "host unreachable") _socks5errors = ("succeeded", - "general SOCKS server failure", - "connection not allowed by ruleset", - "Network unreachable", - "Host unreachable", - "Connection refused", - "TTL expired", - "Command not supported", - "Address type not supported", - "Unknown error") + "general SOCKS server failure", + "connection not allowed by ruleset", + "Network unreachable", + "Host unreachable", + "Connection refused", + "TTL expired", + "Command not supported", + "Address type not supported", + "Unknown error") _socks5autherrors = ("succeeded", - "authentication is required", - "all offered authentication methods were rejected", - "unknown username or invalid password", - "unknown error") + "authentication is required", + "all offered authentication methods were rejected", + "unknown username or invalid password", + "unknown error") _socks4errors = ("request granted", - "request rejected or failed", - "request rejected because SOCKS server cannot connect to identd on the client", - "request rejected because the client program and identd report different user-ids", - "unknown error") + "request rejected or failed", + "request rejected because SOCKS server cannot connect to identd on the client", + "request rejected because the client program and identd report different user-ids", + "unknown error") + def setdefaultproxy(proxytype=None, addr=None, port=None, rdns=True, username=None, password=None): """setdefaultproxy(proxytype, addr[, port[, rdns[, username[, password]]]]) @@ -101,6 +127,7 @@ def setdefaultproxy(proxytype=None, addr=None, port=None, rdns=True, username=No global _defaultproxy _defaultproxy = (proxytype, addr, port, rdns, username, password) + def wrapmodule(module): """wrapmodule(module) Attempts to replace a module's socket library with a SOCKS socket. Must set @@ -108,11 +135,12 @@ def wrapmodule(module): This will only work on modules that import socket directly into the namespace; most of the Python Standard Library falls into this category. """ - if _defaultproxy != None: + if _defaultproxy is not None: module.socket.socket = socksocket else: raise GeneralProxyError((4, "no proxy specified")) + class socksocket(socket.socket): """socksocket([family[, type[, proto]]]) -> socket object Open a SOCKS enabled socket. The parameters are the same as @@ -121,8 +149,9 @@ class socksocket(socket.socket): """ def __init__(self, family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0, _sock=None): + # pylint: disable=redefined-builtin _orgsocket.__init__(self, family, type, proto, _sock) - if _defaultproxy != None: + if _defaultproxy is not None: self.__proxy = _defaultproxy else: self.__proxy = (None, None, None, None, None, None) @@ -139,8 +168,9 @@ class socksocket(socket.socket): except socket.timeout: raise GeneralProxyError((6, "timed out")) while len(data) < count: - d = self.recv(count-len(data)) - if not d: raise GeneralProxyError((0, "connection closed unexpectedly")) + d = self.recv(count - len(data)) + if not d: + raise GeneralProxyError((0, "connection closed unexpectedly")) data = data + d return data @@ -181,7 +211,7 @@ class socksocket(socket.socket): Negotiates a connection through a SOCKS5 server. """ # First we'll send the authentication packages we support. - if (self.__proxy[4]!=None) and (self.__proxy[5]!=None): + if (self.__proxy[4] is not None) and (self.__proxy[5] is not None): # The username/password details were supplied to the # setproxy method so we support the USERNAME/PASSWORD # authentication (in addition to the standard none). @@ -203,7 +233,11 @@ class socksocket(socket.socket): elif chosenauth[1:2] == chr(0x02).encode(): # Okay, we need to perform a basic username/password # authentication. - self.sendall(chr(0x01).encode() + chr(len(self.__proxy[4])) + self.__proxy[4] + chr(len(self.__proxy[5])) + self.__proxy[5]) + self.sendall(chr(0x01).encode() + + chr(len(self.__proxy[4])) + + self.__proxy[4] + + chr(len(self.__proxy[5])) + + self.__proxy[5]) authstat = self.__recvall(2) if authstat[0:1] != chr(0x01).encode(): # Bad response @@ -250,7 +284,7 @@ class socksocket(socket.socket): elif resp[1:2] != chr(0x00).encode(): # Connection failed self.close() - if ord(resp[1:2])<=8: + if ord(resp[1:2]) <= 8: raise Socks5Error((ord(resp[1:2]), _socks5errors[ord(resp[1:2])])) else: raise Socks5Error((9, _socks5errors[9])) @@ -262,10 +296,10 @@ class socksocket(socket.socket): boundaddr = self.__recvall(ord(resp[4:5])) else: self.close() - raise GeneralProxyError((1,_generalerrors[1])) + raise GeneralProxyError((1, _generalerrors[1])) boundport = struct.unpack(">H", self.__recvall(2))[0] self.__proxysockname = (boundaddr, boundport) - if ipaddr != None: + if ipaddr is not None: self.__proxypeername = (socket.inet_ntoa(ipaddr), destport) else: self.__proxypeername = (destaddr, destport) @@ -285,7 +319,7 @@ class socksocket(socket.socket): elif resp[1:2] != chr(0x00).encode(): # Connection failed self.close() - if ord(resp[1:2])<=8: + if ord(resp[1:2]) <= 8: raise Socks5Error((ord(resp[1:2]), _socks5errors[ord(resp[1:2])])) else: raise Socks5Error((9, _socks5errors[9])) @@ -297,8 +331,8 @@ class socksocket(socket.socket): ip = self.__recvall(ord(resp[4:5])) else: self.close() - raise GeneralProxyError((1,_generalerrors[1])) - boundport = struct.unpack(">H", self.__recvall(2))[0] + raise GeneralProxyError((1, _generalerrors[1])) + _ = struct.unpack(">H", self.__recvall(2))[0] return ip def getproxysockname(self): @@ -321,9 +355,10 @@ class socksocket(socket.socket): return self.__proxypeername def getproxytype(self): + """Get the proxy type""" return self.__proxy[0] - def __negotiatesocks4(self,destaddr,destport): + def __negotiatesocks4(self, destaddr, destport): """__negotiatesocks4(self,destaddr,destport) Negotiates a connection through a SOCKS4 server. """ @@ -341,7 +376,7 @@ class socksocket(socket.socket): # Construct the request packet req = struct.pack(">BBH", 0x04, 0x01, destport) + ipaddr # The username parameter is considered userid for SOCKS4 - if self.__proxy[4] != None: + if self.__proxy[4] is not None: req = req + self.__proxy[4] req = req + chr(0x00).encode() # DNS name if remote resolving is required @@ -355,7 +390,7 @@ class socksocket(socket.socket): if resp[0:1] != chr(0x00).encode(): # Bad data self.close() - raise GeneralProxyError((1,_generalerrors[1])) + raise GeneralProxyError((1, _generalerrors[1])) if resp[1:2] != chr(0x5A).encode(): # Server returned an error self.close() @@ -366,7 +401,7 @@ class socksocket(socket.socket): raise Socks4Error((94, _socks4errors[4])) # Get the bound address/port self.__proxysockname = (socket.inet_ntoa(resp[4:]), struct.unpack(">H", resp[2:4])[0]) - if rmtrslv != None: + if rmtrslv is not None: self.__proxypeername = (socket.inet_ntoa(ipaddr), destport) else: self.__proxypeername = (destaddr, destport) @@ -380,7 +415,16 @@ class socksocket(socket.socket): addr = socket.gethostbyname(destaddr) else: addr = destaddr - self.sendall(("CONNECT " + addr + ":" + str(destport) + " HTTP/1.1\r\n" + "Host: " + destaddr + "\r\n\r\n").encode()) + self.sendall(''.join([ + "CONNECT ", + addr, + ":", + str(destport), + " HTTP/1.1\r\n", + "Host: ", + destaddr, + "\r\n\r\n", + ]).encode()) # We read the response until we get the string "\r\n\r\n" resp = self.recv(1) while resp.find("\r\n\r\n".encode()) == -1: @@ -410,10 +454,15 @@ class socksocket(socket.socket): To select the proxy server use setproxy(). """ # Do a minimal input check first - if (not type(destpair) in (list,tuple)) or (len(destpair) < 2) or (type(destpair[0]) != type('')) or (type(destpair[1]) != int): + if any([ + not isinstance(destpair, (list, tuple)), + len(destpair) < 2, + not isinstance(destpair[0], type('')), + not isinstance(destpair[1], int), + ]): raise GeneralProxyError((5, _generalerrors[5])) if self.__proxy[0] == PROXY_TYPE_SOCKS5: - if self.__proxy[2] != None: + if self.__proxy[2] is not None: portnum = self.__proxy[2] else: portnum = 1080 @@ -433,19 +482,19 @@ class socksocket(socket.socket): self.__negotiatesocks5() self.__connectsocks5(destpair[0], destpair[1]) elif self.__proxy[0] == PROXY_TYPE_SOCKS4: - if self.__proxy[2] != None: + if self.__proxy[2] is not None: portnum = self.__proxy[2] else: portnum = 1080 - _orgsocket.connect(self,(self.__proxy[1], portnum)) + _orgsocket.connect(self, (self.__proxy[1], portnum)) self.__negotiatesocks4(destpair[0], destpair[1]) elif self.__proxy[0] == PROXY_TYPE_HTTP: - if self.__proxy[2] != None: + if self.__proxy[2] is not None: portnum = self.__proxy[2] else: portnum = 8080 try: - _orgsocket.connect(self,(self.__proxy[1], portnum)) + _orgsocket.connect(self, (self.__proxy[1], portnum)) except socket.error as e: # ENETUNREACH, WSAENETUNREACH if e[0] in [101, 10051]: @@ -458,14 +507,15 @@ class socksocket(socket.socket): raise GeneralProxyError((9, _generalerrors[9])) raise self.__negotiatehttp(destpair[0], destpair[1]) - elif self.__proxy[0] == None: + elif self.__proxy[0] is None: _orgsocket.connect(self, (destpair[0], destpair[1])) else: raise GeneralProxyError((4, _generalerrors[4])) def resolve(self, host): + """TBC""" if self.__proxy[0] == PROXY_TYPE_SOCKS5: - if self.__proxy[2] != None: + if self.__proxy[2] is not None: portnum = self.__proxy[2] else: portnum = 1080 |