Bitmessage/PyBitmessage-2024-12-03
Archived
2
0
Fork 0
Code Issues 168 Pull Requests 53 Actions Packages Projects Releases 5 Wiki Activity

Changes based on style and lint checks. (final_code_quality_4) #1362

Closed
coffeedogs wants to merge 1 commits from final_code_quality_4 into v0.6
pull from: final_code_quality_4
merge into: Bitmessage:v0.6
Conversation 4 Commits 1 Files Changed 4 +475 -597
4 changed files with 475 additions and 597 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

354
src/bitmessageqt/newaddresswizard.py
View File

@ -1,354 +0,0 @@
#!/usr/bin/env python2.7
from PyQt4 import QtCore, QtGui
class NewAddressWizardIntroPage(QtGui.QWizardPage):
def __init__(self):
super(QtGui.QWizardPage, self).__init__()
self.setTitle("Creating a new address")
label = QtGui.QLabel("This wizard will help you create as many addresses as you like. Indeed, creating and abandoning addresses is encouraged.\n\n"
"What type of address would you like? Would you like to send emails or not?\n"
"You can still change your mind later, and register/unregister with an email service provider.\n\n")
label.setWordWrap(True)
self.emailAsWell = QtGui.QRadioButton("Combined email and bitmessage address")
self.onlyBM = QtGui.QRadioButton("Bitmessage-only address (no email)")
self.emailAsWell.setChecked(True)
self.registerField("emailAsWell", self.emailAsWell)
self.registerField("onlyBM", self.onlyBM)
layout = QtGui.QVBoxLayout()
layout.addWidget(label)
layout.addWidget(self.emailAsWell)
layout.addWidget(self.onlyBM)
self.setLayout(layout)
def nextId(self):
if self.emailAsWell.isChecked():
return 4
else:
return 1
class NewAddressWizardRngPassphrasePage(QtGui.QWizardPage):
def __init__(self):
super(QtGui.QWizardPage, self).__init__()
self.setTitle("Random or Passphrase")
label = QtGui.QLabel("<html><head/><body><p>You may generate addresses by using either random numbers or by using a passphrase. "
"If you use a passphrase, the address is called a &quot;deterministic&quot; address. "
"The \'Random Number\' option is selected by default but deterministic addresses have several pros and cons:</p>"
"<table border=0><tr><td><span style=\" font-weight:600;\">Pros:</span></td><td><span style=\" font-weight:600;\">Cons:</span></td></tr>"
"<tr><td>You can recreate your addresses on any computer from memory. "
"You need-not worry about backing up your keys.dat file as long as you can remember your passphrase.</td>"
"<td>You must remember (or write down) your passphrase if you expect to be able "
"to recreate your keys if they are lost. "
# "You must remember the address version number and the stream number along with your passphrase. "
"If you choose a weak passphrase and someone on the Internet can brute-force it, they can read your messages and send messages as you."
"</p></body></html>")
label.setWordWrap(True)
self.randomAddress = QtGui.QRadioButton("Use a random number generator to make an address")
self.deterministicAddress = QtGui.QRadioButton("Use a passphrase to make an address")
self.randomAddress.setChecked(True)
layout = QtGui.QVBoxLayout()
layout.addWidget(label)
layout.addWidget(self.randomAddress)
layout.addWidget(self.deterministicAddress)
self.setLayout(layout)
def nextId(self):
if self.randomAddress.isChecked():
return 2
else:
return 3
class NewAddressWizardRandomPage(QtGui.QWizardPage):
def __init__(self, addresses):
super(QtGui.QWizardPage, self).__init__()
self.setTitle("Random")
label = QtGui.QLabel("Random address.")
label.setWordWrap(True)
labelLabel = QtGui.QLabel("Label (not shown to anyone except you):")
self.labelLineEdit = QtGui.QLineEdit()
self.radioButtonMostAvailable = QtGui.QRadioButton("Use the most available stream\n"
"(best if this is the first of many addresses you will create)")
self.radioButtonExisting = QtGui.QRadioButton("Use the same stream as an existing address\n"
"(saves you some bandwidth and processing power)")
self.radioButtonMostAvailable.setChecked(True)
self.comboBoxExisting = QtGui.QComboBox()
self.comboBoxExisting.setEnabled(False)
self.comboBoxExisting.setEditable(True)
for address in addresses:
self.comboBoxExisting.addItem(address)
# self.comboBoxExisting.setObjectName(_fromUtf8("comboBoxExisting"))
self.checkBoxEighteenByteRipe = QtGui.QCheckBox("Spend several minutes of extra computing time to make the address(es) 1 or 2 characters shorter")
layout = QtGui.QGridLayout()
layout.addWidget(label, 0, 0)
layout.addWidget(labelLabel, 1, 0)
layout.addWidget(self.labelLineEdit, 2, 0)
layout.addWidget(self.radioButtonMostAvailable, 3, 0)
layout.addWidget(self.radioButtonExisting, 4, 0)
layout.addWidget(self.comboBoxExisting, 5, 0)
layout.addWidget(self.checkBoxEighteenByteRipe, 6, 0)
self.setLayout(layout)
QtCore.QObject.connect(self.radioButtonExisting, QtCore.SIGNAL("toggled(bool)"), self.comboBoxExisting.setEnabled)
self.registerField("label", self.labelLineEdit)
self.registerField("radioButtonMostAvailable", self.radioButtonMostAvailable)
self.registerField("radioButtonExisting", self.radioButtonExisting)
self.registerField("comboBoxExisting", self.comboBoxExisting)
# self.emailAsWell = QtGui.QRadioButton("Combined email and bitmessage account")
# self.onlyBM = QtGui.QRadioButton("Bitmessage-only account (no email)")
# self.emailAsWell.setChecked(True)
def nextId(self):
return 6
class NewAddressWizardPassphrasePage(QtGui.QWizardPage):
def __init__(self):
super(QtGui.QWizardPage, self).__init__()
self.setTitle("Passphrase")
label = QtGui.QLabel("Deterministric address.")
label.setWordWrap(True)
passphraseLabel = QtGui.QLabel("Passphrase")
self.lineEditPassphrase = QtGui.QLineEdit()
self.lineEditPassphrase.setEchoMode(QtGui.QLineEdit.Password)
self.lineEditPassphrase.setInputMethodHints(QtCore.Qt.ImhHiddenText|QtCore.Qt.ImhNoAutoUppercase|QtCore.Qt.ImhNoPredictiveText)
retypePassphraseLabel = QtGui.QLabel("Retype passphrase")
self.lineEditPassphraseAgain = QtGui.QLineEdit()
self.lineEditPassphraseAgain.setEchoMode(QtGui.QLineEdit.Password)
numberLabel = QtGui.QLabel("Number of addresses to make based on your passphrase:")
self.spinBoxNumberOfAddressesToMake = QtGui.QSpinBox()
self.spinBoxNumberOfAddressesToMake.setMinimum(1)
self.spinBoxNumberOfAddressesToMake.setProperty("value", 8)
# self.spinBoxNumberOfAddressesToMake.setObjectName(_fromUtf8("spinBoxNumberOfAddressesToMake"))
label2 = QtGui.QLabel("In addition to your passphrase, you must remember these numbers:")
label3 = QtGui.QLabel("Address version number: 4")
label4 = QtGui.QLabel("Stream number: 1")
layout = QtGui.QGridLayout()
layout.addWidget(label, 0, 0, 1, 4)
layout.addWidget(passphraseLabel, 1, 0, 1, 4)
layout.addWidget(self.lineEditPassphrase, 2, 0, 1, 4)
layout.addWidget(retypePassphraseLabel, 3, 0, 1, 4)
layout.addWidget(self.lineEditPassphraseAgain, 4, 0, 1, 4)
layout.addWidget(numberLabel, 5, 0, 1, 3)
layout.addWidget(self.spinBoxNumberOfAddressesToMake, 5, 3)
layout.setColumnMinimumWidth(3, 1)
layout.addWidget(label2, 6, 0, 1, 4)
layout.addWidget(label3, 7, 0, 1, 2)
layout.addWidget(label4, 7, 2, 1, 2)
self.setLayout(layout)
def nextId(self):
return 6
class NewAddressWizardEmailProviderPage(QtGui.QWizardPage):
def __init__(self):
super(QtGui.QWizardPage, self).__init__()
self.setTitle("Choose email provider")
label = QtGui.QLabel("Currently only Mailchuck email gateway is available "
"(@mailchuck.com email address). In the future, maybe other gateways will be available. "
"Press Next.")
label.setWordWrap(True)
# self.mailchuck = QtGui.QRadioButton("Mailchuck email gateway (@mailchuck.com)")
# self.mailchuck.setChecked(True)
layout = QtGui.QVBoxLayout()
layout.addWidget(label)
# layout.addWidget(self.mailchuck)
self.setLayout(layout)
def nextId(self):
return 5
class NewAddressWizardEmailAddressPage(QtGui.QWizardPage):
def __init__(self):
super(QtGui.QWizardPage, self).__init__()
self.setTitle("Email address")
label = QtGui.QLabel("Choosing an email address. Address must end with @mailchuck.com")
label.setWordWrap(True)
self.specificEmail = QtGui.QRadioButton("Pick your own email address:")
self.specificEmail.setChecked(True)
self.emailLineEdit = QtGui.QLineEdit()
self.randomEmail = QtGui.QRadioButton("Generate a random email address")
QtCore.QObject.connect(self.specificEmail, QtCore.SIGNAL("toggled(bool)"), self.emailLineEdit.setEnabled)
layout = QtGui.QVBoxLayout()
layout.addWidget(label)
layout.addWidget(self.specificEmail)
layout.addWidget(self.emailLineEdit)
layout.addWidget(self.randomEmail)
self.setLayout(layout)
def nextId(self):
return 6
class NewAddressWizardWaitPage(QtGui.QWizardPage):
def __init__(self):
super(QtGui.QWizardPage, self).__init__()
self.setTitle("Wait")
self.label = QtGui.QLabel("Wait!")
self.label.setWordWrap(True)
self.progressBar = QtGui.QProgressBar()
self.progressBar.setMinimum(0)
self.progressBar.setMaximum(100)
self.progressBar.setValue(0)
# self.emailAsWell = QtGui.QRadioButton("Combined email and bitmessage account")
# self.onlyBM = QtGui.QRadioButton("Bitmessage-only account (no email)")
# self.emailAsWell.setChecked(True)
layout = QtGui.QVBoxLayout()
layout.addWidget(self.label)
layout.addWidget(self.progressBar)
# layout.addWidget(self.emailAsWell)
# layout.addWidget(self.onlyBM)
self.setLayout(layout)
def update(self, i):
if i == 101 and self.wizard().currentId() == 6:
self.wizard().button(QtGui.QWizard.NextButton).click()
return
elif i == 101:
print "haha"
return
self.progressBar.setValue(i)
if i == 50:
self.emit(QtCore.SIGNAL('completeChanged()'))
def isComplete(self):
# print "val = " + str(self.progressBar.value())
if self.progressBar.value() >= 50:
return True
else:
return False
def initializePage(self):
if self.field("emailAsWell").toBool():
val = "yes/"
else:
val = "no/"
if self.field("onlyBM").toBool():
val += "yes"
else:
val += "no"
self.label.setText("Wait! " + val)
# self.wizard().button(QtGui.QWizard.NextButton).setEnabled(False)
self.progressBar.setValue(0)
self.thread = NewAddressThread()
self.connect(self.thread, self.thread.signal, self.update)
self.thread.start()
def nextId(self):
return 10
class NewAddressWizardConclusionPage(QtGui.QWizardPage):
def __init__(self):
super(QtGui.QWizardPage, self).__init__()
self.setTitle("All done!")
label = QtGui.QLabel("You successfully created a new address.")
label.setWordWrap(True)
layout = QtGui.QVBoxLayout()
layout.addWidget(label)
self.setLayout(layout)
class Ui_NewAddressWizard(QtGui.QWizard):
def __init__(self, addresses):
super(QtGui.QWizard, self).__init__()
self.pages = {}
page = NewAddressWizardIntroPage()
self.setPage(0, page)
self.setStartId(0)
page = NewAddressWizardRngPassphrasePage()
self.setPage(1, page)
page = NewAddressWizardRandomPage(addresses)
self.setPage(2, page)
page = NewAddressWizardPassphrasePage()
self.setPage(3, page)
page = NewAddressWizardEmailProviderPage()
self.setPage(4, page)
page = NewAddressWizardEmailAddressPage()
self.setPage(5, page)
page = NewAddressWizardWaitPage()
self.setPage(6, page)
page = NewAddressWizardConclusionPage()
self.setPage(10, page)
self.setWindowTitle("New address wizard")
self.adjustSize()
self.show()
class NewAddressThread(QtCore.QThread):
def __init__(self):
QtCore.QThread.__init__(self)
self.signal = QtCore.SIGNAL("signal")
def __del__(self):
self.wait()
def createDeterministic(self):
pass
def createPassphrase(self):
pass
def broadcastAddress(self):
pass
def registerMailchuck(self):
pass
def waitRegistration(self):
pass
def run(self):
import time
for i in range(1, 101):
time.sleep(0.1) # artificial time delay
self.emit(self.signal, i)
self.emit(self.signal, 101)
# self.terminate()
if __name__ == '__main__':
import sys
app = QtGui.QApplication(sys.argv)
wizard = Ui_NewAddressWizard(["a", "b", "c", "d"])
if (wizard.exec_()):
print "Email: " + ("yes" if wizard.field("emailAsWell").toBool() else "no")
print "BM: " + ("yes" if wizard.field("onlyBM").toBool() else "no")
else:
print "Wizard cancelled"
sys.exit()

278
src/class_sqlThread.py
View File

@ -1,29 +1,45 @@
import threading """
from bmconfigparser import BMConfigParser src/class_sqlThread.py
import sqlite3 ======================
import time """
import shutil # used for moving the messages.dat file # pylint: disable=attribute-defined-outside-init,protected-access
import sys
import os import os
from debug import logger import shutil # used for moving the messages.dat file
import sqlite3
import sys
import threading
import time
import helper_sql import helper_sql
import helper_startup import helper_startup
import paths import paths
import queues import queues
import state import state
import tr import tr
from bmconfigparser import BMConfigParser
# This thread exists because SQLITE3 is so un-threadsafe that we must from debug import logger
# submit queries to it and it puts results back in a different queue. They
# won't let us just use locks.
class sqlThread(threading.Thread): class sqlThread(threading.Thread):
"""
Implement SQL operations in a dedicated thread.
This thread exists because SQLITE3 is so un-threadsafe that we must submit queries to it and it puts results back
in a different queue. They won't let us just use locks.
This actually only applies for certain deployments, and/or really old version of sqlite. I haven't actually seen it
anywhere. Current versions do have support for threading and multiprocessing. I don't see an urgent reason to
refactor this, but it should be noted in the comment that the problem is mostly not valid. Sadly, last time I
checked, there is no reliable way to check whether the library is or isn't thread-safe.
"""
PeterSurda commented 2018-10-27 13:21:29 +02:00 (Migrated from github.com)
Review
Copy Link

This actually only applies for certain deployments, and/or really old version of sqlite. I haven't actually seen it anywhere. Current versions do have support for threading and multiprocessing. I don't see an urgent reason to refactor this, but it should be noted in the comment that the problem is mostly not valid. Sadly, last time I checked, there is no reliable way to check whether the library is or isn't thread-safe.

This actually only applies for certain deployments, and/or really old version of sqlite. I haven't actually seen it anywhere. Current versions do have support for threading and multiprocessing. I don't see an urgent reason to refactor this, but it should be noted in the comment that the problem is mostly not valid. Sadly, last time I checked, there is no reliable way to check whether the library is or isn't thread-safe.
def __init__(self): def __init__(self):
threading.Thread.__init__(self, name="SQL") threading.Thread.__init__(self, name="SQL")
def run(self): def run(self):
# pylint: disable=too-many-locals,too-many-branches,too-many-locals,too-many-statements
self.conn = sqlite3.connect(state.appdata + 'messages.dat') self.conn = sqlite3.connect(state.appdata + 'messages.dat')
self.conn.text_factory = str self.conn.text_factory = str
self.cur = self.conn.cursor() self.cur = self.conn.cursor()
@ -32,30 +48,38 @@ class sqlThread(threading.Thread):
try: try:
self.cur.execute( self.cur.execute(
'''CREATE TABLE inbox (msgid blob, toaddress text, fromaddress text, subject text, received text, message text, folder text, encodingtype int, read bool, sighash blob, UNIQUE(msgid) ON CONFLICT REPLACE)''' ) '''CREATE TABLE inbox (msgid blob, toaddress text, fromaddress text, subject text, received text,'''
''' message text, folder text, encodingtype int, read bool, sighash blob, UNIQUE(msgid)'''
''' ON CONFLICT REPLACE)''')
self.cur.execute( self.cur.execute(
'''CREATE TABLE sent (msgid blob, toaddress text, toripe blob, fromaddress text, subject text, message text, ackdata blob, senttime integer, lastactiontime integer, sleeptill integer, status text, retrynumber integer, folder text, encodingtype int, ttl int)''' ) '''CREATE TABLE sent (msgid blob, toaddress text, toripe blob, fromaddress text, subject text,'''
''' message text, ackdata blob, senttime integer, lastactiontime integer, sleeptill integer,'''
''' status text, retrynumber integer, folder text, encodingtype int, ttl int)''')
self.cur.execute( self.cur.execute(
'''CREATE TABLE subscriptions (label text, address text, enabled bool)''' ) '''CREATE TABLE subscriptions (label text, address text, enabled bool)''')
self.cur.execute( self.cur.execute(
'''CREATE TABLE addressbook (label text, address text)''' ) '''CREATE TABLE addressbook (label text, address text)''')
self.cur.execute( self.cur.execute(
'''CREATE TABLE blacklist (label text, address text, enabled bool)''' ) '''CREATE TABLE blacklist (label text, address text, enabled bool)''')
self.cur.execute( self.cur.execute(
'''CREATE TABLE whitelist (label text, address text, enabled bool)''' ) '''CREATE TABLE whitelist (label text, address text, enabled bool)''')
self.cur.execute( self.cur.execute(
'''CREATE TABLE pubkeys (address text, addressversion int, transmitdata blob, time int, usedpersonally text, UNIQUE(address) ON CONFLICT REPLACE)''' ) '''CREATE TABLE pubkeys (address text, addressversion int, transmitdata blob, time int,'''
''' usedpersonally text, UNIQUE(address) ON CONFLICT REPLACE)''')
self.cur.execute( self.cur.execute(
'''CREATE TABLE inventory (hash blob, objecttype int, streamnumber int, payload blob, expirestime integer, tag blob, UNIQUE(hash) ON CONFLICT REPLACE)''' ) '''CREATE TABLE inventory (hash blob, objecttype int, streamnumber int, payload blob,'''
''' expirestime integer, tag blob, UNIQUE(hash) ON CONFLICT REPLACE)''')
self.cur.execute( self.cur.execute(
'''INSERT INTO subscriptions VALUES('Bitmessage new releases/announcements','BM-GtovgYdgs7qXPkoYaRgrLFuFKz1SFpsw',1)''') '''INSERT INTO subscriptions VALUES'''
'''('Bitmessage new releases/announcements','BM-GtovgYdgs7qXPkoYaRgrLFuFKz1SFpsw',1)''')
self.cur.execute( self.cur.execute(
'''CREATE TABLE settings (key blob, value blob, UNIQUE(key) ON CONFLICT REPLACE)''' ) '''CREATE TABLE settings (key blob, value blob, UNIQUE(key) ON CONFLICT REPLACE)''')
self.cur.execute( '''INSERT INTO settings VALUES('version','10')''') self.cur.execute('''INSERT INTO settings VALUES('version','10')''')
self.cur.execute( '''INSERT INTO settings VALUES('lastvacuumtime',?)''', ( self.cur.execute('''INSERT INTO settings VALUES('lastvacuumtime',?)''', (
int(time.time()),)) int(time.time()),))
self.cur.execute( self.cur.execute(
'''CREATE TABLE objectprocessorqueue (objecttype int, data blob, UNIQUE(objecttype, data) ON CONFLICT REPLACE)''' ) '''CREATE TABLE objectprocessorqueue'''
''' (objecttype int, data blob, UNIQUE(objecttype, data) ON CONFLICT REPLACE)''')
self.conn.commit() self.conn.commit()
logger.info('Created messages database file') logger.info('Created messages database file')
except Exception as err: except Exception as err:
@ -120,33 +144,37 @@ class sqlThread(threading.Thread):
logger.debug( logger.debug(
"In messages.dat database, creating new 'settings' table.") "In messages.dat database, creating new 'settings' table.")
self.cur.execute( self.cur.execute(
'''CREATE TABLE settings (key text, value blob, UNIQUE(key) ON CONFLICT REPLACE)''' ) '''CREATE TABLE settings (key text, value blob, UNIQUE(key) ON CONFLICT REPLACE)''')
self.cur.execute( '''INSERT INTO settings VALUES('version','1')''') self.cur.execute('''INSERT INTO settings VALUES('version','1')''')
self.cur.execute( '''INSERT INTO settings VALUES('lastvacuumtime',?)''', ( self.cur.execute('''INSERT INTO settings VALUES('lastvacuumtime',?)''', (
int(time.time()),)) int(time.time()),))
logger.debug('In messages.dat database, removing an obsolete field from the pubkeys table.') logger.debug('In messages.dat database, removing an obsolete field from the pubkeys table.')
self.cur.execute( self.cur.execute(
'''CREATE TEMPORARY TABLE pubkeys_backup(hash blob, transmitdata blob, time int, usedpersonally text, UNIQUE(hash) ON CONFLICT REPLACE);''') '''CREATE TEMPORARY TABLE pubkeys_backup(hash blob, transmitdata blob, time int,'''
''' usedpersonally text, UNIQUE(hash) ON CONFLICT REPLACE);''')
self.cur.execute( self.cur.execute(
'''INSERT INTO pubkeys_backup SELECT hash, transmitdata, time, usedpersonally FROM pubkeys;''') '''INSERT INTO pubkeys_backup SELECT hash, transmitdata, time, usedpersonally FROM pubkeys;''')
self.cur.execute( '''DROP TABLE pubkeys''') self.cur.execute('''DROP TABLE pubkeys''')
self.cur.execute( self.cur.execute(
'''CREATE TABLE pubkeys (hash blob, transmitdata blob, time int, usedpersonally text, UNIQUE(hash) ON CONFLICT REPLACE)''' ) '''CREATE TABLE pubkeys (hash blob, transmitdata blob, time int, usedpersonally text, UNIQUE(hash)'''
''' ON CONFLICT REPLACE)''')
self.cur.execute( self.cur.execute(
'''INSERT INTO pubkeys SELECT hash, transmitdata, time, usedpersonally FROM pubkeys_backup;''') '''INSERT INTO pubkeys SELECT hash, transmitdata, time, usedpersonally FROM pubkeys_backup;''')
self.cur.execute( '''DROP TABLE pubkeys_backup;''') self.cur.execute('''DROP TABLE pubkeys_backup;''')
logger.debug('Deleting all pubkeys from inventory. They will be redownloaded and then saved with the correct times.') logger.debug('Deleting all pubkeys from inventory. They will be redownloaded and then saved with the'
' correct times.')
self.cur.execute( self.cur.execute(
'''delete from inventory where objecttype = 'pubkey';''') '''delete from inventory where objecttype = 'pubkey';''')
logger.debug('replacing Bitmessage announcements mailing list with a new one.') logger.debug('replacing Bitmessage announcements mailing list with a new one.')
self.cur.execute( self.cur.execute(
'''delete from subscriptions where address='BM-BbkPSZbzPwpVcYZpU4yHwf9ZPEapN5Zx' ''') '''delete from subscriptions where address='BM-BbkPSZbzPwpVcYZpU4yHwf9ZPEapN5Zx' ''')
self.cur.execute( self.cur.execute(
'''INSERT INTO subscriptions VALUES('Bitmessage new releases/announcements','BM-GtovgYdgs7qXPkoYaRgrLFuFKz1SFpsw',1)''') '''INSERT INTO subscriptions'''
''' VALUES('Bitmessage new releases/announcements','BM-GtovgYdgs7qXPkoYaRgrLFuFKz1SFpsw',1)''')
logger.debug('Commiting.') logger.debug('Commiting.')
self.conn.commit() self.conn.commit()
logger.debug('Vacuuming message.dat. You might notice that the file size gets much smaller.') logger.debug('Vacuuming message.dat. You might notice that the file size gets much smaller.')
self.cur.execute( ''' VACUUM ''') self.cur.execute(''' VACUUM ''')
# After code refactoring, the possible status values for sent messages # After code refactoring, the possible status values for sent messages
# have changed. # have changed.
@ -170,15 +198,19 @@ class sqlThread(threading.Thread):
'In messages.dat database, removing an obsolete field from' 'In messages.dat database, removing an obsolete field from'
' the inventory table.') ' the inventory table.')
self.cur.execute( self.cur.execute(
'''CREATE TEMPORARY TABLE inventory_backup(hash blob, objecttype text, streamnumber int, payload blob, receivedtime integer, UNIQUE(hash) ON CONFLICT REPLACE);''') '''CREATE TEMPORARY TABLE inventory_backup(hash blob, objecttype text, streamnumber int,'''
''' payload blob, receivedtime integer, UNIQUE(hash) ON CONFLICT REPLACE);''')
self.cur.execute( self.cur.execute(
'''INSERT INTO inventory_backup SELECT hash, objecttype, streamnumber, payload, receivedtime FROM inventory;''') '''INSERT INTO inventory_backup SELECT hash, objecttype, streamnumber, payload,'''
self.cur.execute( '''DROP TABLE inventory''') ''' receivedtime FROM inventory;''')
self.cur.execute('''DROP TABLE inventory''')
self.cur.execute( self.cur.execute(
'''CREATE TABLE inventory (hash blob, objecttype text, streamnumber int, payload blob, receivedtime integer, UNIQUE(hash) ON CONFLICT REPLACE)''' ) '''CREATE TABLE inventory (hash blob, objecttype text, streamnumber int, payload blob,'''
''' receivedtime integer, UNIQUE(hash) ON CONFLICT REPLACE)''')
self.cur.execute( self.cur.execute(
'''INSERT INTO inventory SELECT hash, objecttype, streamnumber, payload, receivedtime FROM inventory_backup;''') '''INSERT INTO inventory SELECT hash, objecttype, streamnumber, payload,'''
self.cur.execute( '''DROP TABLE inventory_backup;''') ''' receivedtime FROM inventory_backup;''')
self.cur.execute('''DROP TABLE inventory_backup;''')
item = '''update settings set value=? WHERE key='version';''' item = '''update settings set value=? WHERE key='version';'''
parameters = (3,) parameters = (3,)
self.cur.execute(item, parameters) self.cur.execute(item, parameters)
@ -208,7 +240,8 @@ class sqlThread(threading.Thread):
if currentVersion == 4: if currentVersion == 4:
self.cur.execute('''DROP TABLE pubkeys''') self.cur.execute('''DROP TABLE pubkeys''')
self.cur.execute( self.cur.execute(
'''CREATE TABLE pubkeys (hash blob, addressversion int, transmitdata blob, time int, usedpersonally text, UNIQUE(hash, addressversion) ON CONFLICT REPLACE)''') '''CREATE TABLE pubkeys (hash blob, addressversion int, transmitdata blob, time int,'''
''' usedpersonally text, UNIQUE(hash, addressversion) ON CONFLICT REPLACE)''')
self.cur.execute( self.cur.execute(
'''delete from inventory where objecttype = 'pubkey';''') '''delete from inventory where objecttype = 'pubkey';''')
item = '''update settings set value=? WHERE key='version';''' item = '''update settings set value=? WHERE key='version';'''
@ -224,7 +257,8 @@ class sqlThread(threading.Thread):
if currentVersion == 5: if currentVersion == 5:
self.cur.execute('''DROP TABLE knownnodes''') self.cur.execute('''DROP TABLE knownnodes''')
self.cur.execute( self.cur.execute(
'''CREATE TABLE objectprocessorqueue (objecttype text, data blob, UNIQUE(objecttype, data) ON CONFLICT REPLACE)''') '''CREATE TABLE objectprocessorqueue (objecttype text, data blob, UNIQUE(objecttype, data)'''
''' ON CONFLICT REPLACE)''')
item = '''update settings set value=? WHERE key='version';''' item = '''update settings set value=? WHERE key='version';'''
parameters = (6,) parameters = (6,)
self.cur.execute(item, parameters) self.cur.execute(item, parameters)
@ -240,10 +274,14 @@ class sqlThread(threading.Thread):
logger.debug( logger.debug(
'In messages.dat database, dropping and recreating' 'In messages.dat database, dropping and recreating'
' the inventory table.') ' the inventory table.')
self.cur.execute( '''DROP TABLE inventory''') self.cur.execute('''DROP TABLE inventory''')
self.cur.execute( '''CREATE TABLE inventory (hash blob, objecttype int, streamnumber int, payload blob, expirestime integer, tag blob, UNIQUE(hash) ON CONFLICT REPLACE)''' ) self.cur.execute(
self.cur.execute( '''DROP TABLE objectprocessorqueue''') '''CREATE TABLE inventory (hash blob, objecttype int, streamnumber int, payload blob,'''
self.cur.execute( '''CREATE TABLE objectprocessorqueue (objecttype int, data blob, UNIQUE(objecttype, data) ON CONFLICT REPLACE)''' ) ''' expirestime integer, tag blob, UNIQUE(hash) ON CONFLICT REPLACE)''')
self.cur.execute('''DROP TABLE objectprocessorqueue''')
self.cur.execute(
'''CREATE TABLE objectprocessorqueue (objecttype int, data blob, UNIQUE(objecttype, data)'''
''' ON CONFLICT REPLACE)''')
item = '''update settings set value=? WHERE key='version';''' item = '''update settings set value=? WHERE key='version';'''
parameters = (7,) parameters = (7,)
self.cur.execute(item, parameters) self.cur.execute(item, parameters)
@ -305,15 +343,21 @@ class sqlThread(threading.Thread):
' fields into the retrynumber field and adding the' ' fields into the retrynumber field and adding the'
' sleeptill and ttl fields...') ' sleeptill and ttl fields...')
self.cur.execute( self.cur.execute(
'''CREATE TEMPORARY TABLE sent_backup (msgid blob, toaddress text, toripe blob, fromaddress text, subject text, message text, ackdata blob, lastactiontime integer, status text, retrynumber integer, folder text, encodingtype int)''' ) '''CREATE TEMPORARY TABLE sent_backup (msgid blob, toaddress text, toripe blob, fromaddress text,'''
''' subject text, message text, ackdata blob, lastactiontime integer, status text,'''
''' retrynumber integer, folder text, encodingtype int)''')
self.cur.execute( self.cur.execute(
'''INSERT INTO sent_backup SELECT msgid, toaddress, toripe, fromaddress, subject, message, ackdata, lastactiontime, status, 0, folder, encodingtype FROM sent;''') '''INSERT INTO sent_backup SELECT msgid, toaddress, toripe, fromaddress, subject, message, ackdata,'''
self.cur.execute( '''DROP TABLE sent''') ''' lastactiontime, status, 0, folder, encodingtype FROM sent;''')
self.cur.execute('''DROP TABLE sent''')
self.cur.execute( self.cur.execute(
'''CREATE TABLE sent (msgid blob, toaddress text, toripe blob, fromaddress text, subject text, message text, ackdata blob, senttime integer, lastactiontime integer, sleeptill int, status text, retrynumber integer, folder text, encodingtype int, ttl int)''' ) '''CREATE TABLE sent (msgid blob, toaddress text, toripe blob, fromaddress text, subject text,'''
''' message text, ackdata blob, senttime integer, lastactiontime integer, sleeptill int,'''
''' status text, retrynumber integer, folder text, encodingtype int, ttl int)''')
self.cur.execute( self.cur.execute(
'''INSERT INTO sent SELECT msgid, toaddress, toripe, fromaddress, subject, message, ackdata, lastactiontime, lastactiontime, 0, status, 0, folder, encodingtype, 216000 FROM sent_backup;''') '''INSERT INTO sent SELECT msgid, toaddress, toripe, fromaddress, subject, message, ackdata,'''
self.cur.execute( '''DROP TABLE sent_backup''') ''' lastactiontime, lastactiontime, 0, status, 0, folder, encodingtype, 216000 FROM sent_backup;''')
self.cur.execute('''DROP TABLE sent_backup''')
logger.info('In messages.dat database, finished making TTL-related changes.') logger.info('In messages.dat database, finished making TTL-related changes.')
logger.debug('In messages.dat database, adding address field to the pubkeys table.') logger.debug('In messages.dat database, adding address field to the pubkeys table.')
# We're going to have to calculate the address for each row in the pubkeys # We're going to have to calculate the address for each row in the pubkeys
@ -330,16 +374,22 @@ class sqlThread(threading.Thread):
self.cur.execute(item, parameters) self.cur.execute(item, parameters)
# Now we can remove the hash field from the pubkeys table. # Now we can remove the hash field from the pubkeys table.
self.cur.execute( self.cur.execute(
'''CREATE TEMPORARY TABLE pubkeys_backup (address text, addressversion int, transmitdata blob, time int, usedpersonally text, UNIQUE(address) ON CONFLICT REPLACE)''' ) '''CREATE TEMPORARY TABLE pubkeys_backup (address text, addressversion int, transmitdata blob,'''
''' time int, usedpersonally text, UNIQUE(address) ON CONFLICT REPLACE)''')
self.cur.execute( self.cur.execute(
'''INSERT INTO pubkeys_backup SELECT address, addressversion, transmitdata, time, usedpersonally FROM pubkeys;''') '''INSERT INTO pubkeys_backup SELECT address, addressversion, transmitdata, time,'''
self.cur.execute( '''DROP TABLE pubkeys''') ''' usedpersonally FROM pubkeys;''')
self.cur.execute('''DROP TABLE pubkeys''')
self.cur.execute( self.cur.execute(
'''CREATE TABLE pubkeys (address text, addressversion int, transmitdata blob, time int, usedpersonally text, UNIQUE(address) ON CONFLICT REPLACE)''' ) '''CREATE TABLE pubkeys (address text, addressversion int, transmitdata blob, time int,'''
''' usedpersonally text, UNIQUE(address) ON CONFLICT REPLACE)''')
self.cur.execute( self.cur.execute(
'''INSERT INTO pubkeys SELECT address, addressversion, transmitdata, time, usedpersonally FROM pubkeys_backup;''') '''INSERT INTO pubkeys SELECT address, addressversion, transmitdata, time, usedpersonally'''
self.cur.execute( '''DROP TABLE pubkeys_backup''') ''' FROM pubkeys_backup;''')
logger.debug('In messages.dat database, done adding address field to the pubkeys table and removing the hash field.') self.cur.execute('''DROP TABLE pubkeys_backup''')
logger.debug(
'In messages.dat database, done adding address field to the pubkeys table and removing the hash field.'
)
self.cur.execute('''update settings set value=10 WHERE key='version';''') self.cur.execute('''update settings set value=10 WHERE key='version';''')
# Are you hoping to add a new option to the keys.dat file of existing # Are you hoping to add a new option to the keys.dat file of existing
@ -349,7 +399,7 @@ class sqlThread(threading.Thread):
try: try:
testpayload = '\x00\x00' testpayload = '\x00\x00'
t = ('1234', 1, testpayload, '12345678', 'no') t = ('1234', 1, testpayload, '12345678', 'no')
self.cur.execute( '''INSERT INTO pubkeys VALUES(?,?,?,?,?)''', t) self.cur.execute('''INSERT INTO pubkeys VALUES(?,?,?,?,?)''', t)
self.conn.commit() self.conn.commit()
self.cur.execute( self.cur.execute(
'''SELECT transmitdata FROM pubkeys WHERE address='1234' ''') '''SELECT transmitdata FROM pubkeys WHERE address='1234' ''')
@ -359,13 +409,25 @@ class sqlThread(threading.Thread):
self.cur.execute('''DELETE FROM pubkeys WHERE address='1234' ''') self.cur.execute('''DELETE FROM pubkeys WHERE address='1234' ''')
self.conn.commit() self.conn.commit()
if transmitdata == '': if transmitdata == '':
logger.fatal('Problem: The version of SQLite you have cannot store Null values. Please download and install the latest revision of your version of Python (for example, the latest Python 2.7 revision) and try again.\n') logger.fatal(
logger.fatal('PyBitmessage will now exit very abruptly. You may now see threading errors related to this abrupt exit but the problem you need to solve is related to SQLite.\n\n') 'Problem: The version of SQLite you have cannot store Null values. Please download and install'
' the latest revision of your version of Python (for example, the latest Python 2.7 revision)'
' and try again.\n')
logger.fatal(
'PyBitmessage will now exit very abruptly. You may now see threading errors related to this'
' abrupt exit but the problem you need to solve is related to SQLite.\n\n')
os._exit(0) os._exit(0)
except Exception as err: except Exception as err:
if str(err) == 'database or disk is full': if str(err) == 'database or disk is full':
logger.fatal('(While null value test) Alert: Your disk or data storage volume is full. sqlThread will now exit.') logger.fatal(
queues.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True))) '(While null value test) Alert: Your disk or data storage volume is full. sqlThread will now exit.'
)
queues.UISignalQueue.put((
'alert', (
tr._translate("MainWindow", "Disk full"),
tr._translate(
"MainWindow",
'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True)))
os._exit(0) os._exit(0)
else: else:
logger.error(err) logger.error(err)
@ -379,13 +441,22 @@ class sqlThread(threading.Thread):
for row in queryreturn: for row in queryreturn:
value, = row value, = row
if int(value) < int(time.time()) - 86400: if int(value) < int(time.time()) - 86400:
logger.info('It has been a long time since the messages.dat file has been vacuumed. Vacuuming now...') logger.info(
'It has been a long time since the messages.dat file has been vacuumed. Vacuuming now...')
try: try:
self.cur.execute( ''' VACUUM ''') self.cur.execute(''' VACUUM ''')
except Exception as err: except Exception as err:
if str(err) == 'database or disk is full': if str(err) == 'database or disk is full':
logger.fatal('(While VACUUM) Alert: Your disk or data storage volume is full. sqlThread will now exit.') logger.fatal(
queues.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True))) '(While VACUUM) Alert: Your disk or data storage volume is full. sqlThread will now exit.'
)
queues.UISignalQueue.put((
'alert', (
tr._translate("MainWindow", "Disk full"),
tr._translate(
"MainWindow",
'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'
), True)))
os._exit(0) os._exit(0)
item = '''update settings set value=? WHERE key='lastvacuumtime';''' item = '''update settings set value=? WHERE key='lastvacuumtime';'''
parameters = (int(time.time()),) parameters = (int(time.time()),)
@ -400,8 +471,15 @@ class sqlThread(threading.Thread):
self.conn.commit() self.conn.commit()
except Exception as err: except Exception as err:
if str(err) == 'database or disk is full': if str(err) == 'database or disk is full':
logger.fatal('(While committing) Alert: Your disk or data storage volume is full. sqlThread will now exit.') logger.fatal(
queues.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True))) '(While committing) Alert: Your disk or data storage volume is full.'
' sqlThread will now exit.')
queues.UISignalQueue.put((
'alert', (
tr._translate("MainWindow", "Disk full"),
tr._translate(
"MainWindow", 'Alert: Your disk or data storage volume is full.'
' Bitmessage will now exit.'), True)))
os._exit(0) os._exit(0)
elif item == 'exit': elif item == 'exit':
self.conn.close() self.conn.close()
@ -415,8 +493,16 @@ class sqlThread(threading.Thread):
self.conn.commit() self.conn.commit()
except Exception as err: except Exception as err:
if str(err) == 'database or disk is full': if str(err) == 'database or disk is full':
logger.fatal('(while movemessagstoprog) Alert: Your disk or data storage volume is full. sqlThread will now exit.') logger.fatal(
queues.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True))) '(while movemessagstoprog) Alert: Your disk or data storage volume is full.'
' sqlThread will now exit.')
queues.UISignalQueue.put((
'alert', (
tr._translate("MainWindow", "Disk full"),
tr._translate(
"MainWindow",
'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'
), True)))
os._exit(0) os._exit(0)
self.conn.close() self.conn.close()
shutil.move( shutil.move(
@ -431,8 +517,16 @@ class sqlThread(threading.Thread):
self.conn.commit() self.conn.commit()
except Exception as err: except Exception as err:
if str(err) == 'database or disk is full': if str(err) == 'database or disk is full':
logger.fatal('(while movemessagstoappdata) Alert: Your disk or data storage volume is full. sqlThread will now exit.') logger.fatal(
queues.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True))) '(while movemessagstoappdata) Alert: Your disk or data storage volume is full.'
' sqlThread will now exit.')
queues.UISignalQueue.put((
'alert', (
tr._translate("MainWindow", "Disk full"),
tr._translate(
"MainWindow",
'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'
), True)))
os._exit(0) os._exit(0)
self.conn.close() self.conn.close()
shutil.move( shutil.move(
@ -445,11 +539,19 @@ class sqlThread(threading.Thread):
self.cur.execute('''delete from sent where folder='trash' ''') self.cur.execute('''delete from sent where folder='trash' ''')
self.conn.commit() self.conn.commit()
try: try:
self.cur.execute( ''' VACUUM ''') self.cur.execute(''' VACUUM ''')
except Exception as err: except Exception as err:
if str(err) == 'database or disk is full': if str(err) == 'database or disk is full':
logger.fatal('(while deleteandvacuume) Alert: Your disk or data storage volume is full. sqlThread will now exit.') logger.fatal(
queues.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True))) '(while deleteandvacuume) Alert: Your disk or data storage volume is full.'
' sqlThread will now exit.')
queues.UISignalQueue.put((
'alert', (
tr._translate("MainWindow", "Disk full"),
tr._translate(
"MainWindow",
'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'
), True)))
os._exit(0) os._exit(0)
else: else:
parameters = helper_sql.sqlSubmitQueue.get() parameters = helper_sql.sqlSubmitQueue.get()
@ -461,11 +563,27 @@ class sqlThread(threading.Thread):
rowcount = self.cur.rowcount rowcount = self.cur.rowcount
except Exception as err: except Exception as err:
if str(err) == 'database or disk is full': if str(err) == 'database or disk is full':
logger.fatal('(while cur.execute) Alert: Your disk or data storage volume is full. sqlThread will now exit.') logger.fatal(
queues.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True))) '(while cur.execute) Alert: Your disk or data storage volume is full.'
' sqlThread will now exit.')
queues.UISignalQueue.put((
'alert', (
tr._translate("MainWindow", "Disk full"),
tr._translate(
"MainWindow",
'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'
), True)))
os._exit(0) os._exit(0)
else: else:
logger.fatal('Major error occurred when trying to execute a SQL statement within the sqlThread. Please tell Atheros about this error message or post it in the forum! Error occurred while trying to execute statement: "%s" Here are the parameters; you might want to censor this data with asterisks (***) as it can contain private information: %s. Here is the actual error message thrown by the sqlThread: %s', str(item), str(repr(parameters)), str(err)) logger.fatal(
'Major error occurred when trying to execute a SQL statement within the sqlThread. Please'
' tell Atheros about this error message or post it in the forum! Error occurred while'
' trying to execute statement: "%s" Here are the parameters; you might want to censor'
' this data with asterisks (***) as it can contain private information: %s. Here is the'
' actual error message thrown by the sqlThread: %s',
str(item),
str(repr(parameters)),
str(err))
logger.fatal('This program shall now abruptly exit!') logger.fatal('This program shall now abruptly exit!')
os._exit(0) os._exit(0)

258
src/network/bmproto.py
View File

@ -1,44 +1,59 @@
# pylint: disable=too-many-return-statements,too-many-public-methods,attribute-defined-outside-init,too-many-branches
# pylint: disable=too-many-instance-attributes,too-many-statements
"""
src/network/bmproto.py
======================
"""
import base64 import base64
import hashlib import hashlib
import random
import socket import socket
import struct import struct
import time import time
from binascii import hexlify from binascii import hexlify
import addresses
import helper_random
import knownnodes
import network.connectionpool
import protocol
import shared
import state
from bmconfigparser import BMConfigParser from bmconfigparser import BMConfigParser
from debug import logger from debug import logger
from inventory import Inventory from inventory import Inventory
import knownnodes
from network.advanceddispatcher import AdvancedDispatcher from network.advanceddispatcher import AdvancedDispatcher
from network.bmobject import (
BMObject, BMObjectAlreadyHaveError, BMObjectExpiredError, BMObjectInsufficientPOWError, BMObjectInvalidDataError,
BMObjectInvalidError, BMObjectUnwantedStreamError
)
from network.dandelion import Dandelion from network.dandelion import Dandelion
from network.bmobject import BMObject, BMObjectInsufficientPOWError, BMObjectInvalidDataError, \
BMObjectExpiredError, BMObjectUnwantedStreamError, BMObjectInvalidError, BMObjectAlreadyHaveError
import network.connectionpool
from network.node import Node from network.node import Node
from network.objectracker import ObjectTracker from network.objectracker import ObjectTracker
from network.proxy import Proxy, ProxyError, GeneralProxyError from network.proxy import ProxyError
from queues import addrQueue, invQueue, objectProcessorQueue, portCheckerQueue
import addresses
from queues import objectProcessorQueue, portCheckerQueue, invQueue, addrQueue
import shared
import state
import protocol
import helper_random
class BMProtoError(ProxyError): class BMProtoError(ProxyError):
"""A Bitmessage Protocol Base Error"""
errorCodes = ("Protocol error") errorCodes = ("Protocol error")
class BMProtoInsufficientDataError(BMProtoError): class BMProtoInsufficientDataError(BMProtoError):
"""A Bitmessage Protocol Insufficient Data Error"""
errorCodes = ("Insufficient data") errorCodes = ("Insufficient data")
class BMProtoExcessiveDataError(BMProtoError): class BMProtoExcessiveDataError(BMProtoError):
"""A Bitmessage Protocol Excessive Data Error"""
errorCodes = ("Too much data") errorCodes = ("Too much data")
class BMProto(AdvancedDispatcher, ObjectTracker): class BMProto(AdvancedDispatcher, ObjectTracker):
"""A parser for the Bitmessage Protocol"""
# ~1.6 MB which is the maximum possible size of an inv message. # ~1.6 MB which is the maximum possible size of an inv message.
maxMessageSize = 1600100 maxMessageSize = 1600100
# 2**18 = 256kB is the maximum size of an object payload # 2**18 = 256kB is the maximum size of an object payload
@ -53,12 +68,15 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
maxTimeOffset = 3600 maxTimeOffset = 3600
def __init__(self, address=None, sock=None): def __init__(self, address=None, sock=None):
# pylint: disable=super-init-not-called,unused-argument
AdvancedDispatcher.__init__(self, sock) AdvancedDispatcher.__init__(self, sock)
self.isOutbound = False self.isOutbound = False
# packet/connection from a local IP # packet/connection from a local IP
self.local = False self.local = False
def bm_proto_reset(self): def bm_proto_reset(self):
"""Reset the bitmessage object parser"""
self.magic = None self.magic = None
self.command = None self.command = None
self.payloadLength = 0 self.payloadLength = 0
@ -70,7 +88,10 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
self.object = None self.object = None
def state_bm_header(self): def state_bm_header(self):
self.magic, self.command, self.payloadLength, self.checksum = protocol.Header.unpack(self.read_buf[:protocol.Header.size]) """Predicate (with logging) to indicate the prescence of a header"""
self.magic, self.command, self.payloadLength, self.checksum = protocol.Header.unpack(
self.read_buf[:protocol.Header.size])
self.command = self.command.rstrip('\x00') self.command = self.command.rstrip('\x00')
if self.magic != 0xE9BEB4D9: if self.magic != 0xE9BEB4D9:
# skip 1 byte in order to sync # skip 1 byte in order to sync
@ -87,6 +108,7 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
return True return True
def state_bm_command(self): def state_bm_command(self):
"""Predicate (with logging) to indicate the presence of a command"""
self.payload = self.read_buf[:self.payloadLength] self.payload = self.read_buf[:self.payloadLength]
if self.checksum != hashlib.sha512(self.payload).digest()[0:4]: if self.checksum != hashlib.sha512(self.payload).digest()[0:4]:
logger.debug("Bad checksum, ignoring") logger.debug("Bad checksum, ignoring")
@ -123,7 +145,7 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
# broken read, ignore # broken read, ignore
pass pass
else: else:
#print "Skipping command %s due to invalid data" % (self.command) # print "Skipping command %s due to invalid data" % (self.command)
logger.debug("Closing due to invalid command %s", self.command) logger.debug("Closing due to invalid command %s", self.command)
self.close_reason = "Invalid command %s" % (self.command) self.close_reason = "Invalid command %s" % (self.command)
self.set_state("close") self.set_state("close")
@ -135,16 +157,21 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
return True return True
def decode_payload_string(self, length): def decode_payload_string(self, length):
value = self.payload[self.payloadOffset:self.payloadOffset+length] """Read and return `length` bytes from payload"""
value = self.payload[self.payloadOffset:self.payloadOffset + length]
self.payloadOffset += length self.payloadOffset += length
return value return value
def decode_payload_varint(self): def decode_payload_varint(self):
"""Decode a varint from the payload"""
value, offset = addresses.decodeVarint(self.payload[self.payloadOffset:]) value, offset = addresses.decodeVarint(self.payload[self.payloadOffset:])
self.payloadOffset += offset self.payloadOffset += offset
return value return value
def decode_payload_node(self): def decode_payload_node(self):
"""Decode node details from the payload"""
services, host, port = self.decode_payload_content("Q16sH") services, host, port = self.decode_payload_content("Q16sH")
if host[0:12] == '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xFF\xFF': if host[0:12] == '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xFF\xFF':
host = socket.inet_ntop(socket.AF_INET, str(host[12:16])) host = socket.inet_ntop(socket.AF_INET, str(host[12:16]))
@ -160,32 +187,39 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
return Node(services, host, port) return Node(services, host, port)
def decode_payload_content(self, pattern = "v"): def decode_payload_content(self, pattern="v"):
# L = varint indicating the length of the next array """
# l = varint indicating the length of the next item Decode the payload
# v = varint (or array)
# H = uint16 L = varint indicating the length of the next array
# I = uint32 l = varint indicating the length of the next item
# Q = uint64 v = varint (or array)
# i = net_addr (without time and stream number) H = uint16
# s = string I = uint32
# 0-9 = length of the next item Q = uint64
# , = end of array i = net_addr (without time and stream number)
s = string
0-9 = length of the next item
, = end of array
"""
def decode_simple(self, char="v"): def decode_simple(self, char="v"):
"""Some expected objects can be decoded very straightforwardly"""
if char == "v": if char == "v":
return self.decode_payload_varint() return self.decode_payload_varint()
if char == "i": if char == "i":
return self.decode_payload_node() return self.decode_payload_node()
PeterSurda commented 2018-11-01 07:43:10 +01:00 (Migrated from github.com)
Review
Copy Link

Again, doesn't indicate, but expects, the presence of a command.

Again, doesn't indicate, but expects, the presence of a command.
if char == "H": if char == "H":
self.payloadOffset += 2 self.payloadOffset += 2
return struct.unpack(">H", self.payload[self.payloadOffset-2:self.payloadOffset])[0] return struct.unpack(">H", self.payload[self.payloadOffset - 2:self.payloadOffset])[0]
if char == "I": if char == "I":
self.payloadOffset += 4 self.payloadOffset += 4
return struct.unpack(">I", self.payload[self.payloadOffset-4:self.payloadOffset])[0] return struct.unpack(">I", self.payload[self.payloadOffset - 4:self.payloadOffset])[0]
if char == "Q": if char == "Q":
self.payloadOffset += 8 self.payloadOffset += 8
return struct.unpack(">Q", self.payload[self.payloadOffset-8:self.payloadOffset])[0] return struct.unpack(">Q", self.payload[self.payloadOffset - 8:self.payloadOffset])[0]
return None
size = None size = None
isArray = False isArray = False
@ -198,27 +232,19 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
# retval (array) # retval (array)
parserStack = [[1, 1, False, pattern, 0, []]] parserStack = [[1, 1, False, pattern, 0, []]]
#try:
# sys._getframe(200)
# logger.error("Stack depth warning, pattern: %s", pattern)
# return
#except ValueError:
# pass
while True: while True:
i = parserStack[-1][3][parserStack[-1][4]] i = parserStack[-1][3][parserStack[-1][4]]
if i in "0123456789" and (size is None or parserStack[-1][3][parserStack[-1][4]-1] not in "lL"): if i in "0123456789" and (size is None or parserStack[-1][3][parserStack[-1][4] - 1] not in "lL"):
try: try:
size = size * 10 + int(i) size = size * 10 + int(i)
except TypeError: except TypeError:
size = int(i) size = int(i)
isArray = False isArray = False
elif i in "Ll" and size is None: elif i in "Ll" and size is None:
size = self.decode_payload_varint() size = self.decode_payload_varint()
if i == "L": isArray = bool(i == "L")
isArray = True
else:
isArray = False
elif size is not None: elif size is not None:
if isArray: if isArray:
parserStack.append([size, size, isArray, parserStack[-1][3][parserStack[-1][4]:], 0, []]) parserStack.append([size, size, isArray, parserStack[-1][3][parserStack[-1][4]:], 0, []])
@ -227,25 +253,26 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
for j in range(parserStack[-1][4], len(parserStack[-1][3])): for j in range(parserStack[-1][4], len(parserStack[-1][3])):
if parserStack[-1][3][j] not in "lL0123456789": if parserStack[-1][3][j] not in "lL0123456789":
break break
parserStack.append([size, size, isArray, parserStack[-1][3][parserStack[-1][4]:j+1], 0, []]) # pylint: disable=undefined-loop-variable
parserStack.append([size, size, isArray, parserStack[-1][3][parserStack[-1][4]:j + 1], 0, []])
parserStack[-2][4] += len(parserStack[-1][3]) - 1 parserStack[-2][4] += len(parserStack[-1][3]) - 1
size = None size = None
continue continue
elif i == "s": elif i == "s":
#if parserStack[-2][2]:
# parserStack[-1][5].append(self.payload[self.payloadOffset:self.payloadOffset + parserStack[-1][0]])
#else:
parserStack[-1][5] = self.payload[self.payloadOffset:self.payloadOffset + parserStack[-1][0]] parserStack[-1][5] = self.payload[self.payloadOffset:self.payloadOffset + parserStack[-1][0]]
self.payloadOffset += parserStack[-1][0] self.payloadOffset += parserStack[-1][0]
parserStack[-1][1] = 0 parserStack[-1][1] = 0
parserStack[-1][2] = True parserStack[-1][2] = True
#del parserStack[-1]
size = None size = None
elif i in "viHIQ": elif i in "viHIQ":
parserStack[-1][5].append(decode_simple(self, parserStack[-1][3][parserStack[-1][4]])) parserStack[-1][5].append(decode_simple(self, parserStack[-1][3][parserStack[-1][4]]))
size = None size = None
else: else:
size = None size = None
for depth in range(len(parserStack) - 1, -1, -1): for depth in range(len(parserStack) - 1, -1, -1):
parserStack[depth][4] += 1 parserStack[depth][4] += 1
if parserStack[depth][4] >= len(parserStack[depth][3]): if parserStack[depth][4] >= len(parserStack[depth][3]):
@ -270,16 +297,22 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
raise BMProtoInsufficientDataError() raise BMProtoInsufficientDataError()
def bm_command_error(self): def bm_command_error(self):
"""Decode an error message and log it"""
# pylint: disable=unused-variable
fatalStatus, banTime, inventoryVector, errorText = self.decode_payload_content("vvlsls") fatalStatus, banTime, inventoryVector, errorText = self.decode_payload_content("vvlsls")
logger.error("%s:%i error: %i, %s", self.destination.host, self.destination.port, fatalStatus, errorText) logger.error("%s:%i error: %i, %s", self.destination.host, self.destination.port, fatalStatus, errorText)
return True return True
def bm_command_getdata(self): def bm_command_getdata(self):
"""
Incoming request for object(s), if we have them and some other conditions are fulfilled, append them to the
write queue.
"""
items = self.decode_payload_content("l32s") items = self.decode_payload_content("l32s")
# skip? # skip?
if time.time() < self.skipUntil: if time.time() < self.skipUntil:
return True return True
#TODO make this more asynchronous # .. todo:: make this more asynchronous
helper_random.randomshuffle(items) helper_random.randomshuffle(items)
for i in map(str, items): for i in map(str, items):
if Dandelion().hasHash(i) and \ if Dandelion().hasHash(i) and \
@ -321,21 +354,24 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
return True return True
def bm_command_inv(self): def bm_command_inv(self):
"""Non-dandelion announce"""
return self._command_inv(False) return self._command_inv(False)
def bm_command_dinv(self): def bm_command_dinv(self):
""" """Dandelion stem announce"""
Dandelion stem announce
"""
return self._command_inv(True) return self._command_inv(True)
def bm_command_object(self): def bm_command_object(self):
"""Incoming object, process it"""
objectOffset = self.payloadOffset objectOffset = self.payloadOffset
nonce, expiresTime, objectType, version, streamNumber = self.decode_payload_content("QQIvv") nonce, expiresTime, objectType, version, streamNumber = self.decode_payload_content("QQIvv")
self.object = BMObject(nonce, expiresTime, objectType, version, streamNumber, self.payload, self.payloadOffset) self.object = BMObject(nonce, expiresTime, objectType, version, streamNumber, self.payload, self.payloadOffset)
if len(self.payload) - self.payloadOffset > BMProto.maxObjectPayloadSize: if len(self.payload) - self.payloadOffset > BMProto.maxObjectPayloadSize:
logger.info('The payload length of this object is too large (%d bytes). Ignoring it.' % (len(self.payload) - self.payloadOffset)) logger.info(
'The payload length of this object is too large (%d bytes). Ignoring it.',
len(self.payload) - self.payloadOffset
)
raise BMProtoExcessiveDataError() raise BMProtoExcessiveDataError()
try: try:
@ -348,7 +384,9 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
try: try:
self.object.checkStream() self.object.checkStream()
except (BMObjectUnwantedStreamError,) as e: except (BMObjectUnwantedStreamError,) as e:
BMProto.stopDownloadingObject(self.object.inventoryHash, BMConfigParser().get("inventory", "acceptmismatch")) BMProto.stopDownloadingObject(
self.object.inventoryHash, BMConfigParser().get(
"inventory", "acceptmismatch"))
if not BMConfigParser().get("inventory", "acceptmismatch"): if not BMConfigParser().get("inventory", "acceptmismatch"):
raise e raise e
@ -367,7 +405,10 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
Dandelion().removeHash(self.object.inventoryHash, "cycle detection") Dandelion().removeHash(self.object.inventoryHash, "cycle detection")
Inventory()[self.object.inventoryHash] = ( Inventory()[self.object.inventoryHash] = (
self.object.objectType, self.object.streamNumber, buffer(self.payload[objectOffset:]), self.object.expiresTime, buffer(self.object.tag)) self.object.objectType, self.object.streamNumber,
buffer(self.payload[objectOffset:]), self.object.expiresTime,
buffer(self.object.tag)
)
self.handleReceivedObject(self.object.streamNumber, self.object.inventoryHash) self.handleReceivedObject(self.object.streamNumber, self.object.inventoryHash)
PeterSurda commented 2018-10-27 17:28:49 +02:00 (Migrated from github.com)
Review
Copy Link
        """Incoming address, process it"""
```suggestion """Incoming address, process it""" ```
invQueue.put((self.object.streamNumber, self.object.inventoryHash, self.destination)) invQueue.put((self.object.streamNumber, self.object.inventoryHash, self.destination))
return True return True
@ -376,16 +417,17 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
return self.decode_payload_content("LQIQ16sH") return self.decode_payload_content("LQIQ16sH")
def bm_command_addr(self): def bm_command_addr(self):
addresses = self._decode_addr() """Incoming address, process it"""
for i in addresses: these_addresses = self._decode_addr()
seenTime, stream, services, ip, port = i for i in these_addresses:
seenTime, stream, _, ip, port = i
decodedIP = protocol.checkIPAddress(str(ip)) decodedIP = protocol.checkIPAddress(str(ip))
if stream not in state.streamsInWhichIAmParticipating: if stream not in state.streamsInWhichIAmParticipating:
continue continue
if ( if (
decodedIP and time.time() - seenTime > 0 and decodedIP and time.time() - seenTime > 0 and
seenTime > time.time() - BMProto.addressAlive and seenTime > time.time() - BMProto.addressAlive and
port > 0 port > 0
): ):
peer = state.Peer(decodedIP, port) peer = state.Peer(decodedIP, port)
try: try:
@ -407,18 +449,27 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
return True return True
def bm_command_portcheck(self): def bm_command_portcheck(self):
"""Incoming port check request, queue it."""
portCheckerQueue.put(state.Peer(self.destination, self.peerNode.port)) portCheckerQueue.put(state.Peer(self.destination, self.peerNode.port))
return True return True
def bm_command_ping(self): def bm_command_ping(self):
"""Incoming ping, respond to it."""
self.append_write_buf(protocol.CreatePacket('pong')) self.append_write_buf(protocol.CreatePacket('pong'))
return True return True
def bm_command_pong(self): def bm_command_pong(self):
# nothing really """
Incoming pong, ignore it.
PyBitmessage pings connections after about 5 minutes of inactivity, and leaves it to the TCP stack to handle
actual timeouts. So there is no need to do anything when a pong arrives.
"""
# pylint: disable=no-self-use
return True return True
def bm_command_verack(self): def bm_command_verack(self):
"""Incoming version. Parse and log, remember important things, like streams, bitfields, etc."""
self.verackReceived = True self.verackReceived = True
if self.verackSent: if self.verackSent:
if self.isSSL: if self.isSSL:
@ -429,6 +480,8 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
return True return True
def bm_command_version(self): def bm_command_version(self):
PeterSurda commented 2018-10-27 17:54:25 +02:00 (Migrated from github.com)
Review
Copy Link

I think this isn't necessary. Returning here will prevent verack from being sent. Without that the handshake is considered incomplete, other commands are ignored and the connection will timeout after 20 seconds. The error message in the write buffer has this time to make it through. Probably a leftover from early stages, please remove this "todo".

I think this isn't necessary. Returning here will prevent verack from being sent. Without that the handshake is considered incomplete, other commands are ignored and the connection will timeout after 20 seconds. The error message in the write buffer has this time to make it through. Probably a leftover from early stages, please remove this "todo".
"""Determine and log protocol version and other details"""
self.remoteProtocolVersion, self.services, self.timestamp, self.sockNode, self.peerNode, self.nonce, \ self.remoteProtocolVersion, self.services, self.timestamp, self.sockNode, self.peerNode, self.nonce, \
self.userAgent, self.streams = self.decode_payload_content("IQQiiQlsLv") self.userAgent, self.streams = self.decode_payload_content("IQQiiQlsLv")
self.nonce = struct.pack('>Q', self.nonce) self.nonce = struct.pack('>Q', self.nonce)
@ -439,17 +492,19 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
logger.debug("my external IP: %s", self.sockNode.host) logger.debug("my external IP: %s", self.sockNode.host)
logger.debug("remote node incoming address: %s:%i", self.destination.host, self.peerNode.port) logger.debug("remote node incoming address: %s:%i", self.destination.host, self.peerNode.port)
logger.debug("user agent: %s", self.userAgent) logger.debug("user agent: %s", self.userAgent)
logger.debug("streams: [%s]", ",".join(map(str,self.streams))) logger.debug("streams: [%s]", ",".join(map(str, self.streams)))
if not self.peerValidityChecks(): if not self.peerValidityChecks():
# TODO ABORT
return True return True
#shared.connectedHostsList[self.destination] = self.streams[0]
self.append_write_buf(protocol.CreatePacket('verack')) self.append_write_buf(protocol.CreatePacket('verack'))
self.verackSent = True self.verackSent = True
if not self.isOutbound: if not self.isOutbound:
self.append_write_buf(protocol.assembleVersionMessage(self.destination.host, self.destination.port, \ self.append_write_buf(
network.connectionpool.BMConnectionPool().streams, True, nodeid=self.nodeid)) protocol.assembleVersionMessage(
#print "%s:%i: Sending version" % (self.destination.host, self.destination.port) self.destination.host,
self.destination.port,
network.connectionpool.BMConnectionPool().streams,
True,
nodeid=self.nodeid))
if ((self.services & protocol.NODE_SSL == protocol.NODE_SSL) and if ((self.services & protocol.NODE_SSL == protocol.NODE_SSL) and
PeterSurda commented 2018-10-27 17:55:49 +02:00 (Migrated from github.com)
Review
Copy Link
        """Check the validity of the peer"""
```suggestion """Check the validity of the peer""" ```
PeterSurda commented 2018-10-27 17:57:22 +02:00 (Migrated from github.com)
Review
Copy Link

Typo?

Typo?
protocol.haveSSL(not self.isOutbound)): protocol.haveSSL(not self.isOutbound)):
self.isSSL = True self.isSSL = True
@ -462,69 +517,76 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
return True return True
def peerValidityChecks(self): def peerValidityChecks(self):
"""Check the validity of the peer"""
if self.remoteProtocolVersion < 3: if self.remoteProtocolVersion < 3:
self.append_write_buf(protocol.assembleErrorMessage(fatal=2, self.append_write_buf(protocol.assembleErrorMessage(
errorText="Your is using an old protocol. Closing connection.")) fatal=2, errorText="Your is using an old protocol. Closing connection."))
logger.debug ('Closing connection to old protocol version %s, node: %s', logger.debug('Closing connection to old protocol version %s, node: %s',
str(self.remoteProtocolVersion), str(self.destination)) str(self.remoteProtocolVersion), str(self.destination))
return False return False
if self.timeOffset > BMProto.maxTimeOffset: if self.timeOffset > BMProto.maxTimeOffset:
self.append_write_buf(protocol.assembleErrorMessage(fatal=2, self.append_write_buf(
errorText="Your time is too far in the future compared to mine. Closing connection.")) protocol.assembleErrorMessage(
fatal=2,
errorText="Your time is too far in the future compared to mine. Closing connection."))
logger.info("%s's time is too far in the future (%s seconds). Closing connection to it.", logger.info("%s's time is too far in the future (%s seconds). Closing connection to it.",
self.destination, self.timeOffset) self.destination, self.timeOffset)
shared.timeOffsetWrongCount += 1 shared.timeOffsetWrongCount += 1
return False return False
elif self.timeOffset < -BMProto.maxTimeOffset: elif self.timeOffset < -BMProto.maxTimeOffset:
self.append_write_buf(protocol.assembleErrorMessage(fatal=2, self.append_write_buf(protocol.assembleErrorMessage(
errorText="Your time is too far in the past compared to mine. Closing connection.")) fatal=2, errorText="Your time is too far in the past compared to mine. Closing connection."))
logger.info("%s's time is too far in the past (timeOffset %s seconds). Closing connection to it.", logger.info("%s's time is too far in the past (timeOffset %s seconds). Closing connection to it.",
self.destination, self.timeOffset) self.destination, self.timeOffset)
shared.timeOffsetWrongCount += 1 shared.timeOffsetWrongCount += 1
return False return False
else: else:
shared.timeOffsetWrongCount = 0 shared.timeOffsetWrongCount = 0
if not self.streams: if not self.streams:
self.append_write_buf(protocol.assembleErrorMessage(fatal=2, self.append_write_buf(protocol.assembleErrorMessage(
errorText="We don't have shared stream interests. Closing connection.")) fatal=2, errorText="We don't have shared stream interests. Closing connection."))
logger.debug ('Closed connection to %s because there is no overlapping interest in streams.', logger.debug('Closed connection to %s because there is no overlapping interest in streams.',
str(self.destination)) str(self.destination))
return False return False
if self.destination in network.connectionpool.BMConnectionPool().inboundConnections: if self.destination in network.connectionpool.BMConnectionPool().inboundConnections:
try: try:
if not protocol.checkSocksIP(self.destination.host): if not protocol.checkSocksIP(self.destination.host):
self.append_write_buf(protocol.assembleErrorMessage(fatal=2, self.append_write_buf(
errorText="Too many connections from your IP. Closing connection.")) protocol.assembleErrorMessage(
logger.debug ('Closed connection to %s because we are already connected to that IP.', fatal=2, errorText="Too many connections from your IP. Closing connection."))
str(self.destination)) logger.debug('Closed connection to %s because we are already connected to that IP.',
str(self.destination))
return False return False
except: except BaseException:
pass pass
if not self.isOutbound: if not self.isOutbound:
# incoming from a peer we're connected to as outbound, or server full # incoming from a peer we're connected to as outbound, or server full
# report the same error to counter deanonymisation # report the same error to counter deanonymisation
if state.Peer(self.destination.host, self.peerNode.port) in \ if state.Peer(self.destination.host, self.peerNode.port) in \
network.connectionpool.BMConnectionPool().inboundConnections or \ network.connectionpool.BMConnectionPool().inboundConnections or \
len(network.connectionpool.BMConnectionPool().inboundConnections) + \ len(network.connectionpool.BMConnectionPool().inboundConnections) + \
len(network.connectionpool.BMConnectionPool().outboundConnections) > \ len(network.connectionpool.BMConnectionPool().outboundConnections) > \
BMConfigParser().safeGetInt("bitmessagesettings", "maxtotalconnections") + \ BMConfigParser().safeGetInt("bitmessagesettings", "maxtotalconnections") + \
BMConfigParser().safeGetInt("bitmessagesettings", "maxbootstrapconnections"): BMConfigParser().safeGetInt("bitmessagesettings", "maxbootstrapconnections"):
self.append_write_buf(protocol.assembleErrorMessage(fatal=2, self.append_write_buf(protocol.assembleErrorMessage(fatal=2,
errorText="Server full, please try again later.")) errorText="Server full, please try again later."))
logger.debug ("Closed connection to %s due to server full or duplicate inbound/outbound.", logger.debug("Closed connection to %s due to server full or duplicate inbound/outbound.",
str(self.destination)) str(self.destination))
return False return False
if network.connectionpool.BMConnectionPool().isAlreadyConnected(self.nonce): if network.connectionpool.BMConnectionPool().isAlreadyConnected(self.nonce):
self.append_write_buf(protocol.assembleErrorMessage(fatal=2, self.append_write_buf(
errorText="I'm connected to myself. Closing connection.")) protocol.assembleErrorMessage(
logger.debug ("Closed connection to %s because I'm connected to myself.", fatal=2,
str(self.destination)) errorText="I'm connected to myself. Closing connection."))
logger.debug("Closed connection to %s because I'm connected to myself.",
str(self.destination))
return False return False
PeterSurda commented 2018-10-27 17:38:37 +02:00 (Migrated from github.com)
Review
Copy Link

This however IS a todo. This loop may take a while, during which time a lot of other stuff is locked. Should be refactored in the future.

This however IS a todo. This loop may take a while, during which time a lot of other stuff is locked. Should be refactored in the future.
coffeedogs commented 2018-10-31 14:11:00 +01:00 (Migrated from github.com)
Review
Copy Link

BTW, the rST syntax so that the sphinx documentation can both format and collect a list of all todo items.

BTW, the rST syntax so that the sphinx documentation can both format and collect a list of all todo items.
return True return True
@staticmethod @staticmethod
def assembleAddr(peerList): def assembleAddr(peerList):
"""Build up a packed address"""
if isinstance(peerList, state.Peer): if isinstance(peerList, state.Peer):
peerList = (peerList) peerList = (peerList)
if not peerList: if not peerList:
@ -546,6 +608,7 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
@staticmethod @staticmethod
def stopDownloadingObject(hashId, forwardAnyway=False): def stopDownloadingObject(hashId, forwardAnyway=False):
"""Stop downloading an object"""
for connection in network.connectionpool.BMConnectionPool().inboundConnections.values() + \ for connection in network.connectionpool.BMConnectionPool().inboundConnections.values() + \
network.connectionpool.BMConnectionPool().outboundConnections.values(): network.connectionpool.BMConnectionPool().outboundConnections.values():
try: try:
@ -564,6 +627,7 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
pass pass
def handle_close(self): def handle_close(self):
"""Handle close"""
self.set_state("close") self.set_state("close")
if not (self.accepting or self.connecting or self.connected): if not (self.accepting or self.connecting or self.connected):
# already disconnected # already disconnected

176
src/socks/__init__.py
View File

@ -1,4 +1,9 @@
"""SocksiPy - Python SOCKS module. # pylint: disable=too-many-arguments,global-statement,too-many-branches
"""
PeterSurda commented 2018-10-27 17:59:49 +02:00 (Migrated from github.com)
Review
Copy Link

This file will be removed once asyncore Socks resolver is finished, so I don't care about it. For P2P connections over Socks we're already using my port of this to asyncore. The only thing missing is a callback mechansim and tests, the actual resolving over wire protocol is implemented, but I never tested it (or maybe I tested it very shortly).

This file will be removed once asyncore Socks resolver is finished, so I don't care about it. For P2P connections over Socks we're already using my port of this to asyncore. The only thing missing is a callback mechansim and tests, the actual resolving over wire protocol is implemented, but I never tested it (or maybe I tested it very shortly).
src/socks/__init__.py
=====================
SocksiPy - Python SOCKS module.
Version 1.00 Version 1.00
Copyright 2006 Dan-Haim. All rights reserved. Copyright 2006 Dan-Haim. All rights reserved.
@ -29,10 +34,6 @@ OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMANGE.
This module provides a standard socket-like interface for Python This module provides a standard socket-like interface for Python
for tunneling connections through SOCKS proxies. for tunneling connections through SOCKS proxies.
"""
"""
Minor modifications made by Christopher Gilbert (http://motomastyle.com/) Minor modifications made by Christopher Gilbert (http://motomastyle.com/)
for use in PyLoris (http://pyloris.sourceforge.net/) for use in PyLoris (http://pyloris.sourceforge.net/)
@ -43,7 +44,7 @@ mainly to merge bug fixes found in Sourceforge
import socket import socket
import struct import struct
import sys
PROXY_TYPE_SOCKS4 = 1 PROXY_TYPE_SOCKS4 = 1
PROXY_TYPE_SOCKS5 = 2 PROXY_TYPE_SOCKS5 = 2
@ -52,46 +53,71 @@ PROXY_TYPE_HTTP = 3
_defaultproxy = None _defaultproxy = None
_orgsocket = socket.socket _orgsocket = socket.socket
class ProxyError(Exception): pass
class GeneralProxyError(ProxyError): pass class ProxyError(Exception):
class Socks5AuthError(ProxyError): pass """Base class for other ProxyErrors"""
class Socks5Error(ProxyError): pass pass
class Socks4Error(ProxyError): pass
class HTTPError(ProxyError): pass
class GeneralProxyError(ProxyError):
"""Handle a general proxy error"""
pass
class Socks5AuthError(ProxyError):
"""Handle a SOCKS5 auth error"""
pass
class Socks5Error(ProxyError):
"""Handle a SOCKS5 non-auth error"""
pass
class Socks4Error(ProxyError):
"""Handle a SOCKS4 error"""
pass
class HTTPError(ProxyError):
"""Handle a HTTP error"""
pass
_generalerrors = ("success", _generalerrors = ("success",
"invalid data", "invalid data",
"not connected", "not connected",
"not available", "not available",
"bad proxy type", "bad proxy type",
"bad input", "bad input",
"timed out", "timed out",
"network unreachable", "network unreachable",
"connection refused", "connection refused",
"host unreachable") "host unreachable")
_socks5errors = ("succeeded", _socks5errors = ("succeeded",
"general SOCKS server failure", "general SOCKS server failure",
"connection not allowed by ruleset", "connection not allowed by ruleset",
"Network unreachable", "Network unreachable",
"Host unreachable", "Host unreachable",
"Connection refused", "Connection refused",
"TTL expired", "TTL expired",
"Command not supported", "Command not supported",
"Address type not supported", "Address type not supported",
"Unknown error") "Unknown error")
_socks5autherrors = ("succeeded", _socks5autherrors = ("succeeded",
"authentication is required", "authentication is required",
"all offered authentication methods were rejected", "all offered authentication methods were rejected",
"unknown username or invalid password", "unknown username or invalid password",
"unknown error") "unknown error")
_socks4errors = ("request granted", _socks4errors = ("request granted",
"request rejected or failed", "request rejected or failed",
"request rejected because SOCKS server cannot connect to identd on the client", "request rejected because SOCKS server cannot connect to identd on the client",
"request rejected because the client program and identd report different user-ids", "request rejected because the client program and identd report different user-ids",
"unknown error") "unknown error")
def setdefaultproxy(proxytype=None, addr=None, port=None, rdns=True, username=None, password=None): def setdefaultproxy(proxytype=None, addr=None, port=None, rdns=True, username=None, password=None):
"""setdefaultproxy(proxytype, addr[, port[, rdns[, username[, password]]]]) """setdefaultproxy(proxytype, addr[, port[, rdns[, username[, password]]]])
@ -101,6 +127,7 @@ def setdefaultproxy(proxytype=None, addr=None, port=None, rdns=True, username=No
global _defaultproxy global _defaultproxy
_defaultproxy = (proxytype, addr, port, rdns, username, password) _defaultproxy = (proxytype, addr, port, rdns, username, password)
def wrapmodule(module): def wrapmodule(module):
"""wrapmodule(module) """wrapmodule(module)
Attempts to replace a module's socket library with a SOCKS socket. Must set Attempts to replace a module's socket library with a SOCKS socket. Must set
@ -108,11 +135,12 @@ def wrapmodule(module):
This will only work on modules that import socket directly into the namespace; This will only work on modules that import socket directly into the namespace;
most of the Python Standard Library falls into this category. most of the Python Standard Library falls into this category.
""" """
if _defaultproxy != None: if _defaultproxy is not None:
module.socket.socket = socksocket module.socket.socket = socksocket
else: else:
raise GeneralProxyError((4, "no proxy specified")) raise GeneralProxyError((4, "no proxy specified"))
class socksocket(socket.socket): class socksocket(socket.socket):
"""socksocket([family[, type[, proto]]]) -> socket object """socksocket([family[, type[, proto]]]) -> socket object
Open a SOCKS enabled socket. The parameters are the same as Open a SOCKS enabled socket. The parameters are the same as
@ -121,8 +149,9 @@ class socksocket(socket.socket):
""" """
def __init__(self, family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0, _sock=None): def __init__(self, family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0, _sock=None):
# pylint: disable=redefined-builtin
_orgsocket.__init__(self, family, type, proto, _sock) _orgsocket.__init__(self, family, type, proto, _sock)
if _defaultproxy != None: if _defaultproxy is not None:
self.__proxy = _defaultproxy self.__proxy = _defaultproxy
else: else:
self.__proxy = (None, None, None, None, None, None) self.__proxy = (None, None, None, None, None, None)
@ -139,8 +168,9 @@ class socksocket(socket.socket):
except socket.timeout: except socket.timeout:
raise GeneralProxyError((6, "timed out")) raise GeneralProxyError((6, "timed out"))
while len(data) < count: while len(data) < count:
d = self.recv(count-len(data)) d = self.recv(count - len(data))
if not d: raise GeneralProxyError((0, "connection closed unexpectedly")) if not d:
raise GeneralProxyError((0, "connection closed unexpectedly"))
data = data + d data = data + d
return data return data
@ -181,7 +211,7 @@ class socksocket(socket.socket):
Negotiates a connection through a SOCKS5 server. Negotiates a connection through a SOCKS5 server.
""" """
# First we'll send the authentication packages we support. # First we'll send the authentication packages we support.
if (self.__proxy[4]!=None) and (self.__proxy[5]!=None): if (self.__proxy[4] is not None) and (self.__proxy[5] is not None):
# The username/password details were supplied to the # The username/password details were supplied to the
# setproxy method so we support the USERNAME/PASSWORD # setproxy method so we support the USERNAME/PASSWORD
# authentication (in addition to the standard none). # authentication (in addition to the standard none).
@ -203,7 +233,11 @@ class socksocket(socket.socket):
elif chosenauth[1:2] == chr(0x02).encode(): elif chosenauth[1:2] == chr(0x02).encode():
# Okay, we need to perform a basic username/password # Okay, we need to perform a basic username/password
# authentication. # authentication.
self.sendall(chr(0x01).encode() + chr(len(self.__proxy[4])) + self.__proxy[4] + chr(len(self.__proxy[5])) + self.__proxy[5]) self.sendall(chr(0x01).encode() +
chr(len(self.__proxy[4])) +
self.__proxy[4] +
chr(len(self.__proxy[5])) +
self.__proxy[5])
authstat = self.__recvall(2) authstat = self.__recvall(2)
if authstat[0:1] != chr(0x01).encode(): if authstat[0:1] != chr(0x01).encode():
# Bad response # Bad response
@ -250,7 +284,7 @@ class socksocket(socket.socket):
elif resp[1:2] != chr(0x00).encode(): elif resp[1:2] != chr(0x00).encode():
# Connection failed # Connection failed
self.close() self.close()
if ord(resp[1:2])<=8: if ord(resp[1:2]) <= 8:
raise Socks5Error((ord(resp[1:2]), _socks5errors[ord(resp[1:2])])) raise Socks5Error((ord(resp[1:2]), _socks5errors[ord(resp[1:2])]))
else: else:
raise Socks5Error((9, _socks5errors[9])) raise Socks5Error((9, _socks5errors[9]))
@ -262,10 +296,10 @@ class socksocket(socket.socket):
boundaddr = self.__recvall(ord(resp[4:5])) boundaddr = self.__recvall(ord(resp[4:5]))
else: else:
self.close() self.close()
raise GeneralProxyError((1,_generalerrors[1])) raise GeneralProxyError((1, _generalerrors[1]))
boundport = struct.unpack(">H", self.__recvall(2))[0] boundport = struct.unpack(">H", self.__recvall(2))[0]
self.__proxysockname = (boundaddr, boundport) self.__proxysockname = (boundaddr, boundport)
if ipaddr != None: if ipaddr is not None:
self.__proxypeername = (socket.inet_ntoa(ipaddr), destport) self.__proxypeername = (socket.inet_ntoa(ipaddr), destport)
else: else:
self.__proxypeername = (destaddr, destport) self.__proxypeername = (destaddr, destport)
@ -285,7 +319,7 @@ class socksocket(socket.socket):
elif resp[1:2] != chr(0x00).encode(): elif resp[1:2] != chr(0x00).encode():
# Connection failed # Connection failed
self.close() self.close()
if ord(resp[1:2])<=8: if ord(resp[1:2]) <= 8:
raise Socks5Error((ord(resp[1:2]), _socks5errors[ord(resp[1:2])])) raise Socks5Error((ord(resp[1:2]), _socks5errors[ord(resp[1:2])]))
else: else:
raise Socks5Error((9, _socks5errors[9])) raise Socks5Error((9, _socks5errors[9]))
@ -297,8 +331,8 @@ class socksocket(socket.socket):
ip = self.__recvall(ord(resp[4:5])) ip = self.__recvall(ord(resp[4:5]))
else: else:
self.close() self.close()
raise GeneralProxyError((1,_generalerrors[1])) raise GeneralProxyError((1, _generalerrors[1]))
boundport = struct.unpack(">H", self.__recvall(2))[0] _ = struct.unpack(">H", self.__recvall(2))[0]
return ip return ip
def getproxysockname(self): def getproxysockname(self):
@ -321,9 +355,10 @@ class socksocket(socket.socket):
return self.__proxypeername return self.__proxypeername
def getproxytype(self): def getproxytype(self):
"""Get the proxy type"""
return self.__proxy[0] return self.__proxy[0]
def __negotiatesocks4(self,destaddr,destport): def __negotiatesocks4(self, destaddr, destport):
"""__negotiatesocks4(self,destaddr,destport) """__negotiatesocks4(self,destaddr,destport)
Negotiates a connection through a SOCKS4 server. Negotiates a connection through a SOCKS4 server.
""" """
@ -341,7 +376,7 @@ class socksocket(socket.socket):
# Construct the request packet # Construct the request packet
req = struct.pack(">BBH", 0x04, 0x01, destport) + ipaddr req = struct.pack(">BBH", 0x04, 0x01, destport) + ipaddr
# The username parameter is considered userid for SOCKS4 # The username parameter is considered userid for SOCKS4
if self.__proxy[4] != None: if self.__proxy[4] is not None:
req = req + self.__proxy[4] req = req + self.__proxy[4]
req = req + chr(0x00).encode() req = req + chr(0x00).encode()
# DNS name if remote resolving is required # DNS name if remote resolving is required
@ -355,7 +390,7 @@ class socksocket(socket.socket):
if resp[0:1] != chr(0x00).encode(): if resp[0:1] != chr(0x00).encode():
# Bad data # Bad data
self.close() self.close()
raise GeneralProxyError((1,_generalerrors[1])) raise GeneralProxyError((1, _generalerrors[1]))
if resp[1:2] != chr(0x5A).encode(): if resp[1:2] != chr(0x5A).encode():
# Server returned an error # Server returned an error
self.close() self.close()
@ -366,7 +401,7 @@ class socksocket(socket.socket):
raise Socks4Error((94, _socks4errors[4])) raise Socks4Error((94, _socks4errors[4]))
# Get the bound address/port # Get the bound address/port
self.__proxysockname = (socket.inet_ntoa(resp[4:]), struct.unpack(">H", resp[2:4])[0]) self.__proxysockname = (socket.inet_ntoa(resp[4:]), struct.unpack(">H", resp[2:4])[0])
if rmtrslv != None: if rmtrslv is not None:
self.__proxypeername = (socket.inet_ntoa(ipaddr), destport) self.__proxypeername = (socket.inet_ntoa(ipaddr), destport)
else: else:
self.__proxypeername = (destaddr, destport) self.__proxypeername = (destaddr, destport)
@ -380,7 +415,16 @@ class socksocket(socket.socket):
addr = socket.gethostbyname(destaddr) addr = socket.gethostbyname(destaddr)
else: else:
addr = destaddr addr = destaddr
self.sendall(("CONNECT " + addr + ":" + str(destport) + " HTTP/1.1\r\n" + "Host: " + destaddr + "\r\n\r\n").encode()) self.sendall(''.join([
"CONNECT ",
addr,
":",
str(destport),
" HTTP/1.1\r\n",
"Host: ",
destaddr,
"\r\n\r\n",
]).encode())
# We read the response until we get the string "\r\n\r\n" # We read the response until we get the string "\r\n\r\n"
resp = self.recv(1) resp = self.recv(1)
while resp.find("\r\n\r\n".encode()) == -1: while resp.find("\r\n\r\n".encode()) == -1:
@ -410,10 +454,15 @@ class socksocket(socket.socket):
To select the proxy server use setproxy(). To select the proxy server use setproxy().
""" """
# Do a minimal input check first # Do a minimal input check first
if (not type(destpair) in (list,tuple)) or (len(destpair) < 2) or (type(destpair[0]) != type('')) or (type(destpair[1]) != int): if any([
not isinstance(destpair, (list, tuple)),
len(destpair) < 2,
not isinstance(destpair[0], type('')),
not isinstance(destpair[1], int),
]):
raise GeneralProxyError((5, _generalerrors[5])) raise GeneralProxyError((5, _generalerrors[5]))
if self.__proxy[0] == PROXY_TYPE_SOCKS5: if self.__proxy[0] == PROXY_TYPE_SOCKS5:
if self.__proxy[2] != None: if self.__proxy[2] is not None:
portnum = self.__proxy[2] portnum = self.__proxy[2]
else: else:
portnum = 1080 portnum = 1080
@ -433,19 +482,19 @@ class socksocket(socket.socket):
self.__negotiatesocks5() self.__negotiatesocks5()
self.__connectsocks5(destpair[0], destpair[1]) self.__connectsocks5(destpair[0], destpair[1])
elif self.__proxy[0] == PROXY_TYPE_SOCKS4: elif self.__proxy[0] == PROXY_TYPE_SOCKS4:
if self.__proxy[2] != None: if self.__proxy[2] is not None:
portnum = self.__proxy[2] portnum = self.__proxy[2]
else: else:
portnum = 1080 portnum = 1080
_orgsocket.connect(self,(self.__proxy[1], portnum)) _orgsocket.connect(self, (self.__proxy[1], portnum))
self.__negotiatesocks4(destpair[0], destpair[1]) self.__negotiatesocks4(destpair[0], destpair[1])
elif self.__proxy[0] == PROXY_TYPE_HTTP: elif self.__proxy[0] == PROXY_TYPE_HTTP:
if self.__proxy[2] != None: if self.__proxy[2] is not None:
portnum = self.__proxy[2] portnum = self.__proxy[2]
else: else:
portnum = 8080 portnum = 8080
try: try:
_orgsocket.connect(self,(self.__proxy[1], portnum)) _orgsocket.connect(self, (self.__proxy[1], portnum))
except socket.error as e: except socket.error as e:
# ENETUNREACH, WSAENETUNREACH # ENETUNREACH, WSAENETUNREACH
if e[0] in [101, 10051]: if e[0] in [101, 10051]:
@ -458,14 +507,15 @@ class socksocket(socket.socket):
raise GeneralProxyError((9, _generalerrors[9])) raise GeneralProxyError((9, _generalerrors[9]))
raise raise
self.__negotiatehttp(destpair[0], destpair[1]) self.__negotiatehttp(destpair[0], destpair[1])
elif self.__proxy[0] == None: elif self.__proxy[0] is None:
_orgsocket.connect(self, (destpair[0], destpair[1])) _orgsocket.connect(self, (destpair[0], destpair[1]))
else: else:
raise GeneralProxyError((4, _generalerrors[4])) raise GeneralProxyError((4, _generalerrors[4]))
def resolve(self, host): def resolve(self, host):
"""TBC"""
if self.__proxy[0] == PROXY_TYPE_SOCKS5: if self.__proxy[0] == PROXY_TYPE_SOCKS5:
if self.__proxy[2] != None: if self.__proxy[2] is not None:
portnum = self.__proxy[2] portnum = self.__proxy[2]
else: else:
portnum = 1080 portnum = 1080