Bitmessage/PyBitmessage-2024-12-19
Archived
2
0
Fork 0
Code Issues 168 Pull Requests 52 Actions Packages Projects Releases 5 Wiki Activity

fixed conflicts after merge

Browse Source
This commit is contained in:
navjot 2019-12-18 17:31:07 +05:30
commit c98d1a70f4
No known key found for this signature in database
GPG Key ID: 9EE70AFD71357F1C
96 changed files with 2015 additions and 1405 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -17,6 +17,7 @@ dist
*.egg-info *.egg-info
docs/_*/* docs/_*/*
docs/autodoc/ docs/autodoc/
build/sphinx/
pyan/ pyan/
.buildozer/ .buildozer/
bin/ bin/

9
.readthedocs.yml Normal file
View File

@ -0,0 +1,9 @@
version: 2
python:
version: 2.7
install:
- requirements: docs/requirements.txt
- method: setuptools
path: .
system_packages: true

8
README.md
View File

@ -14,12 +14,10 @@ Development
---------- ----------
Bitmessage is a collaborative project. You are welcome to submit pull requests Bitmessage is a collaborative project. You are welcome to submit pull requests
although if you plan to put a non-trivial amount of work into coding new although if you plan to put a non-trivial amount of work into coding new
features, it is recommended that you first solicit feedback on the DevTalk features, it is recommended that you first describe your ideas in the
pseudo-mailing list: separate issue.
BM-2D9QKN4teYRvoq2fyzpiftPh9WP9qggtzh
Feel welcome to join chan "bitmessage", BM-2cWy7cvHoq3f1rYMerRJp8PT653jjSuEdY Feel welcome to join chan "bitmessage", BM-2cWy7cvHoq3f1rYMerRJp8PT653jjSuEdY
which is on preview here: https://beamstat.com/chan/bitmessage
References References
---------- ----------

4
docs/_static/custom.css vendored Normal file
View File

@ -0,0 +1,4 @@
/* Hide "On GitHub" section from versions menu */
li.wy-breadcrumbs-aside > a.fa {
display: none;
}

119
docs/conf.py
View File

@ -2,35 +2,24 @@
""" """
Configuration file for the Sphinx documentation builder. Configuration file for the Sphinx documentation builder.
This file does only contain a selection of the most common options. For a For a full list of options see the documentation:
full list see the documentation:
http://www.sphinx-doc.org/en/master/config http://www.sphinx-doc.org/en/master/config
-- Path setup --------------------------------------------------------------
If extensions (or modules to document with autodoc) are in another directory,
add these directories to sys.path here. If the directory is relative to the
documentation root, use os.path.abspath to make it absolute, like shown here.
""" """
import os import os
import sys import sys
from sphinx.apidoc import main
from mock import Mock as MagicMock
sys.path.insert(0, os.path.abspath('.'))
sys.path.insert(0, os.path.abspath('..'))
sys.path.insert(0, os.path.abspath('../src')) sys.path.insert(0, os.path.abspath('../src'))
sys.path.insert(0, os.path.abspath('../src/pyelliptic'))
import version from importlib import import_module
import version # noqa:E402
# -- Project information ----------------------------------------------------- # -- Project information -----------------------------------------------------
project = u'PyBitmessage' project = u'PyBitmessage'
copyright = u'2018, The Bitmessage Team' # pylint: disable=redefined-builtin copyright = u'2019, The Bitmessage Team' # pylint: disable=redefined-builtin
author = u'The Bitmessage Team' author = u'The Bitmessage Team'
# The short X.Y version # The short X.Y version
@ -50,15 +39,18 @@ release = version
# ones. # ones.
extensions = [ extensions = [
'sphinx.ext.autodoc', 'sphinx.ext.autodoc',
# 'sphinx.ext.doctest', # Currently disabled due to bad doctests 'sphinx.ext.coverage', # FIXME: unused
'sphinx.ext.imgmath', # legacy unused
'sphinx.ext.intersphinx', 'sphinx.ext.intersphinx',
'sphinx.ext.linkcode',
'sphinx.ext.napoleon',
'sphinx.ext.todo', 'sphinx.ext.todo',
'sphinx.ext.coverage', 'sphinxcontrib.apidoc',
'sphinx.ext.imgmath',
'sphinx.ext.viewcode',
'm2r', 'm2r',
] ]
default_role = 'obj'
# Add any paths that contain templates here, relative to this directory. # Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates'] templates_path = ['_templates']
@ -75,23 +67,29 @@ master_doc = 'index'
# #
# This is also used if you do content translation via gettext catalogs. # This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases. # Usually you set "language" from the command line for these cases.
language = None # language = None
# List of patterns, relative to source directory, that match files and # List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files. # directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path . # This pattern also affects html_static_path and html_extra_path .
exclude_patterns = [u'_build', 'Thumbs.db', '.DS_Store'] exclude_patterns = ['_build']
# The name of the Pygments (syntax highlighting) style to use. # The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx' pygments_style = 'sphinx'
# Don't prepend every class or function name with full module path
add_module_names = False
# A list of ignored prefixes for module index sorting.
modindex_common_prefix = ['pybitmessage.']
# -- Options for HTML output ------------------------------------------------- # -- Options for HTML output -------------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for # The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes. # a list of builtin themes.
# #
html_theme = 'alabaster' html_theme = 'sphinx_rtd_theme'
# Theme options are theme-specific and customize the look and feel of a theme # Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the # further. For a list of options available for each theme, see the
@ -104,6 +102,10 @@ html_theme = 'alabaster'
# so a file named "default.css" will overwrite the builtin "default.css". # so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static'] html_static_path = ['_static']
html_css_files = [
'custom.css',
]
# Custom sidebar templates, must be a dictionary that maps document names # Custom sidebar templates, must be a dictionary that maps document names
# to template names. # to template names.
# #
@ -114,10 +116,7 @@ html_static_path = ['_static']
# #
# html_sidebars = {} # html_sidebars = {}
# Deal with long lines in source view html_show_sourcelink = False
html_theme_options = {
'page_width': '1366px',
}
# -- Options for HTMLHelp output --------------------------------------------- # -- Options for HTMLHelp output ---------------------------------------------
@ -199,10 +198,74 @@ epub_exclude_files = ['search.html']
# -- Extension configuration ------------------------------------------------- # -- Extension configuration -------------------------------------------------
autodoc_mock_imports = [
'debug',
'pybitmessage.bitmessagekivy',
'pybitmessage.bitmessageqt.addressvalidator',
'pybitmessage.helper_startup',
'pybitmessage.network.httpd',
'pybitmessage.network.https',
'ctypes',
'dialog',
'gi',
'kivy',
'logging',
'msgpack',
'numpy',
'pkg_resources',
'pycanberra',
'pyopencl',
'PyQt4',
'pyxdg',
'qrcode',
'stem',
]
autodoc_member_order = 'bysource'
# Apidoc settings
apidoc_module_dir = '../pybitmessage'
apidoc_output_dir = 'autodoc'
apidoc_excluded_paths = [
'bitmessagekivy', 'build_osx.py',
'bitmessageqt/addressvalidator.py', 'bitmessageqt/migrationwizard.py',
'bitmessageqt/newaddresswizard.py', 'helper_startup.py',
'kivymd', 'main.py', 'navigationdrawer', 'network/http*',
'pybitmessage', 'tests', 'version.py'
]
apidoc_module_first = True
apidoc_separate_modules = True
apidoc_toc_file = False
apidoc_extra_args = ['-a']
# Napoleon settings
napoleon_google_docstring = True
# linkcode function
def linkcode_resolve(domain, info):
"""This generates source URL's for sphinx.ext.linkcode"""
if domain != 'py' or not info['module']:
return
try:
home = os.path.abspath(import_module('pybitmessage').__path__[0])
mod = import_module(info['module']).__file__
except ImportError:
return
repo = 'https://github.com/Bitmessage/PyBitmessage/blob/v0.6/src%s'
path = mod.replace(home, '')
if path != mod:
# put the link only for top level definitions
if len(info['fullname'].split('.')) > 1:
return
if path.endswith('.pyc'):
path = path[:-1]
return repo % path
# -- Options for intersphinx extension --------------------------------------- # -- Options for intersphinx extension ---------------------------------------
# Example configuration for intersphinx: refer to the Python standard library. # Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {'https://docs.python.org/': None} intersphinx_mapping = {'https://docs.python.org/2.7/': None}
# -- Options for todo extension ---------------------------------------------- # -- Options for todo extension ----------------------------------------------

2
docs/contribute.dir/develop.dir/fabric.rst
View File

@ -1,2 +1,2 @@
.. mdinclude:: fabfile/README.md .. mdinclude:: ../../../fabfile/README.md

2
docs/contribute.dir/develop.dir/overview.rst
View File

@ -62,7 +62,7 @@ To re-build them, run `fab build_docs:dep_graphs=true`. Note that the dot graph
.. figure:: ../../../../_static/deps-sfdp.png .. figure:: ../../../../_static/deps-sfdp.png
:alt: SFDP graph of dependencies :alt: SFDP graph of dependencies
:width: 100 pc :width: 100 pc
:index:`SFDP` graph of dependencies :index:`SFDP` graph of dependencies
.. figure:: ../../../../_static/deps-dot.png .. figure:: ../../../../_static/deps-dot.png

6
docs/contribute.dir/processes.rst
View File

@ -1,8 +1,8 @@
Processes Processes
========= =========
In other to keep the Bitmessage project running the team run a number of systems and accounts that form the In order to keep the Bitmessage project running, the team runs a number of systems and accounts that form the
development pipeline and continuous delivery process. We are always striving to improve the process. Towards development pipeline and continuous delivery process. We are always striving to improve this process. Towards
that end it is documented here. that end it is documented here.
@ -20,7 +20,7 @@ Our official Github_ account is Bitmessage. Our issue tracker is here as well.
BitMessage BitMessage
---------- ----------
We eat our own dog food! You can send us bug reports via the Bitmessage chan at xxx We eat our own dog food! You can send us bug reports via the [chan] bitmessage BM-2cWy7cvHoq3f1rYMerRJp8PT653jjSuEdY
.. _website: https://bitmessage.org .. _website: https://bitmessage.org

12
docs/index.rst
View File

@ -1,12 +1,20 @@
.. mdinclude:: ../README.md .. mdinclude:: ../README.md
Documentation
-------------
.. toctree::
:maxdepth: 3
autodoc/pybitmessage
Legacy pages
------------
.. toctree:: .. toctree::
:maxdepth: 2 :maxdepth: 2
overview
usage usage
contribute contribute
Indices and tables Indices and tables
------------------ ------------------

2
docs/requirements.txt Normal file
View File

@ -0,0 +1,2 @@
m2r
sphinxcontrib-apidoc

1
pybitmessage Symbolic link
View File

@ -0,0 +1 @@
src

5
setup.cfg
View File

@ -1,14 +1,17 @@
# Since there is overlap in the violations that the different tools check for, it makes sense to quiesce some warnings # Since there is overlap in the violations that the different tools check for, it makes sense to quiesce some warnings
# in some tools if those warnings in other tools are preferred. This avoids the need to add duplicate lint warnings. # in some tools if those warnings in other tools are preferred. This avoids the need to add duplicate lint warnings.
# max-line-length should be removed ASAP!
[pycodestyle] [pycodestyle]
max-line-length = 119 max-line-length = 119
[flake8] [flake8]
max-line-length = 119 max-line-length = 119
ignore = E722,F841 ignore = E722,F841,W503
# E722: pylint is preferred for bare-except # E722: pylint is preferred for bare-except
# F841: pylint is preferred for unused-variable # F841: pylint is preferred for unused-variable
# W503: deprecated: https://bugs.python.org/issue26763 - https://www.python.org/dev/peps/pep-0008/#should-a-line-break-before-or-after-a-binary-operator
# pylint honours the [MESSAGES CONTROL] section # pylint honours the [MESSAGES CONTROL] section
# as well as [MASTER] section # as well as [MASTER] section

14
setup.py
View File

@ -17,13 +17,7 @@ EXTRAS_REQUIRE = {
'qrcode': ['qrcode'], 'qrcode': ['qrcode'],
'sound;platform_system=="Windows"': ['winsound'], 'sound;platform_system=="Windows"': ['winsound'],
'tor': ['stem'], 'tor': ['stem'],
'docs': [ 'docs': ['sphinx', 'sphinxcontrib-apidoc', 'm2r']
'sphinx', # fab build_docs
'graphviz', # fab build_docs
'curses', # src/depends.py
'python2-pythondialog', # src/depends.py
'm2r', # fab build_docs
]
} }
@ -155,5 +149,9 @@ if __name__ == "__main__":
# ] # ]
}, },
scripts=['src/pybitmessage'], scripts=['src/pybitmessage'],
cmdclass={'install': InstallCmd} cmdclass={'install': InstallCmd},
command_options={
'build_sphinx': {
'source_dir': ('setup.py', 'docs')}
}
) )

37
src/addresses.py
View File

@ -1,7 +1,5 @@
""" """
src/addresses.py Operations with addresses
================
""" """
# pylint: disable=redefined-outer-name,inconsistent-return-statements # pylint: disable=redefined-outer-name,inconsistent-return-statements
@ -18,8 +16,9 @@ ALPHABET = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"
def encodeBase58(num, alphabet=ALPHABET): def encodeBase58(num, alphabet=ALPHABET):
"""Encode a number in Base X """Encode a number in Base X
`num`: The number to encode Args:
`alphabet`: The alphabet to use for encoding num: The number to encode
alphabet: The alphabet to use for encoding
""" """
if num == 0: if num == 0:
return alphabet[0] return alphabet[0]
@ -27,7 +26,6 @@ def encodeBase58(num, alphabet=ALPHABET):
base = len(alphabet) base = len(alphabet)
while num: while num:
rem = num % base rem = num % base
# print 'num is:', num
num = num // base num = num // base
arr.append(alphabet[rem]) arr.append(alphabet[rem])
arr.reverse() arr.reverse()
@ -37,9 +35,9 @@ def encodeBase58(num, alphabet=ALPHABET):
def decodeBase58(string, alphabet=ALPHABET): def decodeBase58(string, alphabet=ALPHABET):
"""Decode a Base X encoded string into the number """Decode a Base X encoded string into the number
Arguments: Args:
- `string`: The encoded string string: The encoded string
- `alphabet`: The alphabet to use for encoding alphabet: The alphabet to use for encoding
""" """
base = len(alphabet) base = len(alphabet)
num = 0 num = 0
@ -54,11 +52,20 @@ def decodeBase58(string, alphabet=ALPHABET):
return num return num
class varintEncodeError(Exception):
"""Exception class for encoding varint"""
pass
class varintDecodeError(Exception):
"""Exception class for decoding varint data"""
pass
def encodeVarint(integer): def encodeVarint(integer):
"""Convert integer into varint bytes""" """Convert integer into varint bytes"""
if integer < 0: if integer < 0:
logger.error('varint cannot be < 0') raise varintEncodeError('varint cannot be < 0')
raise SystemExit
if integer < 253: if integer < 253:
return pack('>B', integer) return pack('>B', integer)
if integer >= 253 and integer < 65536: if integer >= 253 and integer < 65536:
@ -68,13 +75,7 @@ def encodeVarint(integer):
if integer >= 4294967296 and integer < 18446744073709551616: if integer >= 4294967296 and integer < 18446744073709551616:
return pack('>B', 255) + pack('>Q', integer) return pack('>B', 255) + pack('>Q', integer)
if integer >= 18446744073709551616: if integer >= 18446744073709551616:
logger.error('varint cannot be >= 18446744073709551616') raise varintEncodeError('varint cannot be >= 18446744073709551616')
raise SystemExit
class varintDecodeError(Exception):
"""Exception class for decoding varint data"""
pass
def decodeVarint(data): def decodeVarint(data):

13
src/api.py
View File

@ -1,19 +1,15 @@
# pylint: disable=too-many-locals,too-many-lines,no-self-use,too-many-public-methods,too-many-branches # pylint: disable=too-many-locals,too-many-lines,no-self-use,too-many-public-methods,too-many-branches
# pylint: disable=too-many-statements # pylint: disable=too-many-statements
"""
src/api.py
==========
# Copyright (c) 2012-2016 Jonathan Warren # Copyright (c) 2012-2016 Jonathan Warren
# Copyright (c) 2012-2019 The Bitmessage developers # Copyright (c) 2012-2019 The Bitmessage developers
"""
This is not what you run to run the Bitmessage API. Instead, enable the API This is not what you run to run the Bitmessage API. Instead, enable the API
( https://bitmessage.org/wiki/API ) and optionally enable daemon mode ( https://bitmessage.org/wiki/API ) and optionally enable daemon mode
( https://bitmessage.org/wiki/Daemon ) then run bitmessagemain.py. ( https://bitmessage.org/wiki/Daemon ) then run bitmessagemain.py.
""" """
from __future__ import absolute_import
import base64 import base64
import errno import errno
import hashlib import hashlib
@ -42,8 +38,8 @@ from bmconfigparser import BMConfigParser
from debug import logger from debug import logger
from helper_ackPayload import genAckPayload from helper_ackPayload import genAckPayload
from helper_sql import SqlBulkExecute, sqlExecute, sqlQuery, sqlStoredProcedure from helper_sql import SqlBulkExecute, sqlExecute, sqlQuery, sqlStoredProcedure
from helper_threading import StoppableThread
from inventory import Inventory from inventory import Inventory
from network.threads import StoppableThread
str_chan = '[chan]' str_chan = '[chan]'
@ -99,6 +95,8 @@ class singleAPI(StoppableThread):
for attempt in range(50): for attempt in range(50):
try: try:
if attempt > 0: if attempt > 0:
logger.warning(
'Failed to start API listener on port %s', port)
port = random.randint(32767, 65535) port = random.randint(32767, 65535)
se = StoppableXMLRPCServer( se = StoppableXMLRPCServer(
(BMConfigParser().get( (BMConfigParser().get(
@ -110,8 +108,9 @@ class singleAPI(StoppableThread):
continue continue
else: else:
if attempt > 0: if attempt > 0:
logger.warning('Setting apiport to %s', port)
BMConfigParser().set( BMConfigParser().set(
"bitmessagesettings", "apiport", str(port)) 'bitmessagesettings', 'apiport', str(port))
BMConfigParser().save() BMConfigParser().save()
break break
se.register_introspection_functions() se.register_introspection_functions()

115
src/bitmessagemain.py
View File

@ -1,8 +1,7 @@
#!/usr/bin/python2.7
""" """
src/bitmessagemain.py The PyBitmessage startup script
=================================
""" """
# !/usr/bin/python2.7
# Copyright (c) 2012-2016 Jonathan Warren # Copyright (c) 2012-2016 Jonathan Warren
# Copyright (c) 2012-2019 The Bitmessage developers # Copyright (c) 2012-2019 The Bitmessage developers
# Distributed under the MIT/X11 software license. See the accompanying # Distributed under the MIT/X11 software license. See the accompanying
@ -11,8 +10,6 @@ src/bitmessagemain.py
# Right now, PyBitmessage only support connecting to stream 1. It doesn't # Right now, PyBitmessage only support connecting to stream 1. It doesn't
# yet contain logic to expand into further streams. # yet contain logic to expand into further streams.
# The software version variable is now held in shared.py
import os import os
import sys import sys
import ctypes import ctypes
@ -26,41 +23,28 @@ import time
import traceback import traceback
from struct import pack from struct import pack
from helper_startup import (
isOurOperatingSystemLimitedToHavingVeryFewHalfOpenConnections
)
from singleinstance import singleinstance
import defaults import defaults
import depends import depends
import shared import shared
import knownnodes
import state import state
import shutdown import shutdown
from debug import logger
# Classes
from class_sqlThread import sqlThread
from class_singleCleaner import singleCleaner
from class_objectProcessor import objectProcessor
from class_singleWorker import singleWorker
from class_addressGenerator import addressGenerator
from bmconfigparser import BMConfigParser from bmconfigparser import BMConfigParser
from debug import logger # this should go before any threads
from helper_startup import (
isOurOperatingSystemLimitedToHavingVeryFewHalfOpenConnections
)
from inventory import Inventory from inventory import Inventory
from knownnodes import readKnownNodes
from network.connectionpool import BMConnectionPool # Network objects and threads
from network.dandelion import Dandelion from network import (
from network.networkthread import BMNetworkThread BMConnectionPool, Dandelion,
from network.receivequeuethread import ReceiveQueueThread AddrThread, AnnounceThread, BMNetworkThread, InvThread, ReceiveQueueThread,
from network.announcethread import AnnounceThread DownloadThread, UploadThread)
from network.invthread import InvThread from singleinstance import singleinstance
from network.addrthread import AddrThread # Synchronous threads
from network.downloadthread import DownloadThread from threads import (
from network.uploadthread import UploadThread set_thread_name,
addressGenerator, objectProcessor, singleCleaner, singleWorker, sqlThread)
# Helper Functions
import helper_threading
app_dir = os.path.dirname(os.path.abspath(__file__)) app_dir = os.path.dirname(os.path.abspath(__file__))
os.chdir(app_dir) os.chdir(app_dir)
@ -70,7 +54,7 @@ depends.check_dependencies()
def connectToStream(streamNumber): def connectToStream(streamNumber):
"""Method helps us to connect with the stream""" """Connect to a stream"""
state.streamsInWhichIAmParticipating.append(streamNumber) state.streamsInWhichIAmParticipating.append(streamNumber)
if isOurOperatingSystemLimitedToHavingVeryFewHalfOpenConnections(): if isOurOperatingSystemLimitedToHavingVeryFewHalfOpenConnections():
@ -87,14 +71,6 @@ def connectToStream(streamNumber):
except: except:
pass pass
with knownnodes.knownNodesLock:
if streamNumber not in knownnodes.knownNodes:
knownnodes.knownNodes[streamNumber] = {}
if streamNumber * 2 not in knownnodes.knownNodes:
knownnodes.knownNodes[streamNumber * 2] = {}
if streamNumber * 2 + 1 not in knownnodes.knownNodes:
knownnodes.knownNodes[streamNumber * 2 + 1] = {}
BMConnectionPool().connectToStream(streamNumber) BMConnectionPool().connectToStream(streamNumber)
@ -111,7 +87,7 @@ def _fixSocket():
addressToString = ctypes.windll.ws2_32.WSAAddressToStringA addressToString = ctypes.windll.ws2_32.WSAAddressToStringA
def inet_ntop(family, host): def inet_ntop(family, host):
"""Method converts an IP address in packed binary format to string format""" """Converting an IP address in packed binary format to string format"""
if family == socket.AF_INET: if family == socket.AF_INET:
if len(host) != 4: if len(host) != 4:
raise ValueError("invalid IPv4 host") raise ValueError("invalid IPv4 host")
@ -133,7 +109,7 @@ def _fixSocket():
stringToAddress = ctypes.windll.ws2_32.WSAStringToAddressA stringToAddress = ctypes.windll.ws2_32.WSAStringToAddressA
def inet_pton(family, host): def inet_pton(family, host):
"""Method converts an IP address in string format to a packed binary format""" """Converting an IP address in string format to a packed binary format"""
buf = "\0" * 28 buf = "\0" * 28
lengthBuf = pack("I", len(buf)) lengthBuf = pack("I", len(buf))
if stringToAddress(str(host), if stringToAddress(str(host),
@ -188,9 +164,8 @@ def signal_handler(signum, frame):
because the UI captures the signal.' because the UI captures the signal.'
class Main: # pylint: disable=no-init, old-style-class class Main(object):
"""Method starts the proxy config plugin""" """Main PyBitmessage class"""
@staticmethod @staticmethod
def start_proxyconfig(config): def start_proxyconfig(config):
"""Check socksproxytype and start any proxy configuration plugin""" """Check socksproxytype and start any proxy configuration plugin"""
@ -213,15 +188,15 @@ class Main: # pylint: disable=no-init, old-style-class
'Started proxy config plugin %s in %s sec', 'Started proxy config plugin %s in %s sec',
proxy_type, time.time() - proxyconfig_start) proxy_type, time.time() - proxyconfig_start)
def start(self): # pylint: disable=too-many-statements, too-many-branches, too-many-locals def start(self): # pylint: disable=too-many-statements, too-many-branches, too-many-locals
"""This method helps to start the daemon""" """Start main application"""
_fixSocket() _fixSocket()
config = BMConfigParser() config = BMConfigParser()
daemon = config.safeGetBoolean('bitmessagesettings', 'daemon') daemon = config.safeGetBoolean('bitmessagesettings', 'daemon')
try: try:
opts, args = getopt.getopt( opts, _ = getopt.getopt(
sys.argv[1:], "hcdt", sys.argv[1:], "hcdt",
["help", "curses", "daemon", "test"]) ["help", "curses", "daemon", "test"])
@ -229,7 +204,7 @@ class Main: # pylint: disable=no-init, old-style-class
self.usage() self.usage()
sys.exit(2) sys.exit(2)
for opt, arg in opts: for opt, _ in opts:
if opt in ("-h", "--help"): if opt in ("-h", "--help"):
self.usage() self.usage()
sys.exit() sys.exit()
@ -287,7 +262,7 @@ class Main: # pylint: disable=no-init, old-style-class
self.setSignalHandler() self.setSignalHandler()
helper_threading.set_thread_name("PyBitmessage") set_thread_name("PyBitmessage")
state.dandelion = config.safeGetInt('network', 'dandelion') state.dandelion = config.safeGetInt('network', 'dandelion')
# dandelion requires outbound connections, without them, # dandelion requires outbound connections, without them,
@ -302,7 +277,10 @@ class Main: # pylint: disable=no-init, old-style-class
defaults.networkDefaultProofOfWorkNonceTrialsPerByte / 100) defaults.networkDefaultProofOfWorkNonceTrialsPerByte / 100)
defaults.networkDefaultPayloadLengthExtraBytes = int( defaults.networkDefaultPayloadLengthExtraBytes = int(
defaults.networkDefaultPayloadLengthExtraBytes / 100) defaults.networkDefaultPayloadLengthExtraBytes / 100)
knownnodes.readKnownNodes()
readKnownNodes()
# Not needed if objproc is disabled # Not needed if objproc is disabled
if state.enableObjProc: if state.enableObjProc:
@ -449,7 +427,8 @@ class Main: # pylint: disable=no-init, old-style-class
# wait until grandchild ready # wait until grandchild ready
while True: while True:
time.sleep(1) time.sleep(1)
os._exit(0) # pylint: disable=protected-access
os._exit(0) # pylint: disable=protected-access
except AttributeError: except AttributeError:
# fork not implemented # fork not implemented
pass pass
@ -470,7 +449,8 @@ class Main: # pylint: disable=no-init, old-style-class
# wait until child ready # wait until child ready
while True: while True:
time.sleep(1) time.sleep(1)
os._exit(0) # pylint: disable=protected-access
os._exit(0) # pylint: disable=protected-access
except AttributeError: except AttributeError:
# fork not implemented # fork not implemented
pass pass
@ -491,7 +471,8 @@ class Main: # pylint: disable=no-init, old-style-class
os.kill(parentPid, signal.SIGTERM) os.kill(parentPid, signal.SIGTERM)
os.kill(grandfatherPid, signal.SIGTERM) os.kill(grandfatherPid, signal.SIGTERM)
def setSignalHandler(self): # pylint: disable=no-self-use @staticmethod
def setSignalHandler():
"""Setting the Signal Handler""" """Setting the Signal Handler"""
signal.signal(signal.SIGINT, signal_handler) signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler) signal.signal(signal.SIGTERM, signal_handler)
@ -499,9 +480,9 @@ class Main: # pylint: disable=no-init, old-style-class
@staticmethod @staticmethod
def usage(): def usage():
"""After passing argument, method displays the usages""" """Displaying the usages"""
print 'Usage: ' + sys.argv[0] + ' [OPTIONS]' print('Usage: ' + sys.argv[0] + ' [OPTIONS]')
print ''' print('''
Options: Options:
-h, --help show this help message and exit -h, --help show this help message and exit
-c, --curses use curses (text mode) interface -c, --curses use curses (text mode) interface
@ -509,17 +490,19 @@ Options:
-t, --test dryrun, make testing -t, --test dryrun, make testing
All parameters are optional. All parameters are optional.
''' ''')
def stop(self): # pylint: disable=no-self-use @staticmethod
"""Method helps to stop the Bitmessage Deamon""" def stop():
"""Stop main application"""
with shared.printLock: with shared.printLock:
print 'Stopping Bitmessage Deamon.' print 'Stopping Bitmessage Deamon.'
shutdown.doCleanShutdown() shutdown.doCleanShutdown()
# ..todo: nice function but no one is using this # .. todo:: nice function but no one is using this
def getApiAddress(self): # pylint: disable=no-self-use @staticmethod
"""This method returns the Api Addresses""" def getApiAddress():
"""This function returns API address and port"""
if not BMConfigParser().safeGetBoolean( if not BMConfigParser().safeGetBoolean(
'bitmessagesettings', 'apienabled'): 'bitmessagesettings', 'apienabled'):
return None return None
@ -529,7 +512,7 @@ All parameters are optional.
def main(): def main():
"""Start of the main thread""" """Triggers main module"""
mainprogram = Main() mainprogram = Main()
mainprogram.start() mainprogram.start()

2
src/bitmessageqt/networkstatus.py
View File

@ -14,7 +14,7 @@ import network.stats
import shared import shared
import widgets import widgets
from inventory import Inventory from inventory import Inventory
from network.connectionpool import BMConnectionPool from network import BMConnectionPool
from retranslateui import RetranslateMixin from retranslateui import RetranslateMixin
from tr import _translate from tr import _translate
from uisignaler import UISignaler from uisignaler import UISignaler

10
src/bitmessageqt/settings.py
View File

@ -99,6 +99,8 @@ class SettingsDialog(QtGui.QDialog):
config.getboolean('bitmessagesettings', 'socksauthentication')) config.getboolean('bitmessagesettings', 'socksauthentication'))
self.checkBoxSocksListen.setChecked( self.checkBoxSocksListen.setChecked(
config.getboolean('bitmessagesettings', 'sockslisten')) config.getboolean('bitmessagesettings', 'sockslisten'))
self.checkBoxOnionOnly.setChecked(
config.safeGetBoolean('bitmessagesettings', 'onionservicesonly'))
proxy_type = config.safeGet( proxy_type = config.safeGet(
'bitmessagesettings', 'socksproxytype', 'none') 'bitmessagesettings', 'socksproxytype', 'none')
@ -110,6 +112,7 @@ class SettingsDialog(QtGui.QDialog):
self.lineEditSocksPassword.setEnabled(False) self.lineEditSocksPassword.setEnabled(False)
self.checkBoxAuthentication.setEnabled(False) self.checkBoxAuthentication.setEnabled(False)
self.checkBoxSocksListen.setEnabled(False) self.checkBoxSocksListen.setEnabled(False)
self.checkBoxOnionOnly.setEnabled(False)
elif proxy_type == 'SOCKS4a': elif proxy_type == 'SOCKS4a':
self.comboBoxProxyType.setCurrentIndex(1) self.comboBoxProxyType.setCurrentIndex(1)
elif proxy_type == 'SOCKS5': elif proxy_type == 'SOCKS5':
@ -200,11 +203,13 @@ class SettingsDialog(QtGui.QDialog):
self.lineEditSocksPassword.setEnabled(False) self.lineEditSocksPassword.setEnabled(False)
self.checkBoxAuthentication.setEnabled(False) self.checkBoxAuthentication.setEnabled(False)
self.checkBoxSocksListen.setEnabled(False) self.checkBoxSocksListen.setEnabled(False)
self.checkBoxOnionOnly.setEnabled(False)
elif comboBoxIndex in (1, 2): elif comboBoxIndex in (1, 2):
self.lineEditSocksHostname.setEnabled(True) self.lineEditSocksHostname.setEnabled(True)
self.lineEditSocksPort.setEnabled(True) self.lineEditSocksPort.setEnabled(True)
self.checkBoxAuthentication.setEnabled(True) self.checkBoxAuthentication.setEnabled(True)
self.checkBoxSocksListen.setEnabled(True) self.checkBoxSocksListen.setEnabled(True)
self.checkBoxOnionOnly.setEnabled(True)
if self.checkBoxAuthentication.isChecked(): if self.checkBoxAuthentication.isChecked():
self.lineEditSocksUsername.setEnabled(True) self.lineEditSocksUsername.setEnabled(True)
self.lineEditSocksPassword.setEnabled(True) self.lineEditSocksPassword.setEnabled(True)
@ -334,6 +339,11 @@ class SettingsDialog(QtGui.QDialog):
self.lineEditSocksPassword.text())) self.lineEditSocksPassword.text()))
self.config.set('bitmessagesettings', 'sockslisten', str( self.config.set('bitmessagesettings', 'sockslisten', str(
self.checkBoxSocksListen.isChecked())) self.checkBoxSocksListen.isChecked()))
if self.checkBoxOnionOnly.isChecked() \
and not self.config.safeGetBoolean('bitmessagesettings', 'onionservicesonly'):
self.net_restart_needed = True
self.config.set('bitmessagesettings', 'onionservicesonly', str(
self.checkBoxOnionOnly.isChecked()))
try: try:
# Rounding to integers just for aesthetics # Rounding to integers just for aesthetics
self.config.set('bitmessagesettings', 'maxdownloadrate', str( self.config.set('bitmessagesettings', 'maxdownloadrate', str(

7
src/bitmessageqt/settings.ui
View File

@ -403,6 +403,13 @@
</property> </property>
</widget> </widget>
</item> </item>
<item row="4" column="1" colspan="4">
<widget class="QCheckBox" name="checkBoxOnionOnly">
<property name="text">
<string>Only connect to onion services (*.onion)</string>
</property>
</widget>
</item>
<item row="0" column="1"> <item row="0" column="1">
<widget class="QComboBox" name="comboBoxProxyType"> <widget class="QComboBox" name="comboBoxProxyType">
<item> <item>

28
src/bmconfigparser.py
View File

@ -43,8 +43,10 @@ BMConfigDefaults = {
@Singleton @Singleton
class BMConfigParser(ConfigParser.SafeConfigParser): class BMConfigParser(ConfigParser.SafeConfigParser):
"""Singleton class inherited from ConfigParser.SafeConfigParser """
with additional methods specific to bitmessage config.""" Singleton class inherited from :class:`ConfigParser.SafeConfigParser`
with additional methods specific to bitmessage config.
"""
_temp = {} _temp = {}
@ -56,7 +58,7 @@ class BMConfigParser(ConfigParser.SafeConfigParser):
raise ValueError("Invalid value %s" % value) raise ValueError("Invalid value %s" % value)
return ConfigParser.ConfigParser.set(self, section, option, value) return ConfigParser.ConfigParser.set(self, section, option, value)
def get(self, section, option, raw=False, variables=None): def get(self, section, option, raw=False, variables=None): # pylint: disable=arguments-differ
try: try:
if section == "bitmessagesettings" and option == "timeformat": if section == "bitmessagesettings" and option == "timeformat":
return ConfigParser.ConfigParser.get( return ConfigParser.ConfigParser.get(
@ -84,6 +86,7 @@ class BMConfigParser(ConfigParser.SafeConfigParser):
self._temp[section] = {option: value} self._temp[section] = {option: value}
def safeGetBoolean(self, section, field): def safeGetBoolean(self, section, field):
"""Return value as boolean, False on exceptions"""
try: try:
return self.getboolean(section, field) return self.getboolean(section, field)
except (ConfigParser.NoSectionError, ConfigParser.NoOptionError, except (ConfigParser.NoSectionError, ConfigParser.NoOptionError,
@ -91,6 +94,7 @@ class BMConfigParser(ConfigParser.SafeConfigParser):
return False return False
def safeGetInt(self, section, field, default=0): def safeGetInt(self, section, field, default=0):
"""Return value as integer, default on exceptions, 0 if default missing"""
try: try:
return self.getint(section, field) return self.getint(section, field)
except (ConfigParser.NoSectionError, ConfigParser.NoOptionError, except (ConfigParser.NoSectionError, ConfigParser.NoOptionError,
@ -98,18 +102,22 @@ class BMConfigParser(ConfigParser.SafeConfigParser):
return default return default
def safeGet(self, section, option, default=None): def safeGet(self, section, option, default=None):
"""Return value as is, default on exceptions, None if default missing"""
try: try:
return self.get(section, option) return self.get(section, option)
except (ConfigParser.NoSectionError, ConfigParser.NoOptionError, except (ConfigParser.NoSectionError, ConfigParser.NoOptionError,
ValueError, AttributeError): ValueError, AttributeError):
return default return default
def items(self, section, raw=False, variables=None): def items(self, section, raw=False, variables=None): # pylint: disable=arguments-differ
"""Return section variables as parent, but override the "raw" argument to always True"""
return ConfigParser.ConfigParser.items(self, section, True, variables) return ConfigParser.ConfigParser.items(self, section, True, variables)
def addresses(self): @staticmethod
return filter( def addresses():
lambda x: x.startswith('BM-'), BMConfigParser().sections()) """Return a list of local bitmessage addresses (from section labels)"""
return [
x for x in BMConfigParser().sections() if x.startswith('BM-')]
def read(self, filenames): def read(self, filenames):
ConfigParser.ConfigParser.read(self, filenames) ConfigParser.ConfigParser.read(self, filenames)
@ -130,6 +138,7 @@ class BMConfigParser(ConfigParser.SafeConfigParser):
continue continue
def save(self): def save(self):
"""Save the runtime config onto the filesystem"""
fileName = os.path.join(state.appdata, 'keys.dat') fileName = os.path.join(state.appdata, 'keys.dat')
fileNameBak = '.'.join([ fileNameBak = '.'.join([
fileName, datetime.now().strftime("%Y%j%H%M%S%f"), 'bak']) fileName, datetime.now().strftime("%Y%j%H%M%S%f"), 'bak'])
@ -151,12 +160,15 @@ class BMConfigParser(ConfigParser.SafeConfigParser):
os.remove(fileNameBak) os.remove(fileNameBak)
def validate(self, section, option, value): def validate(self, section, option, value):
"""Input validator interface (using factory pattern)"""
try: try:
return getattr(self, 'validate_%s_%s' % (section, option))(value) return getattr(self, 'validate_%s_%s' % (section, option))(value)
except AttributeError: except AttributeError:
return True return True
def validate_bitmessagesettings_maxoutboundconnections(self, value): @staticmethod
def validate_bitmessagesettings_maxoutboundconnections(value):
"""Reject maxoutboundconnections that are too high or too low"""
try: try:
value = int(value) value = int(value)
except ValueError: except ValueError:

36
src/class_addressGenerator.py
View File

@ -1,4 +1,6 @@
"""
A thread for creating addresses
"""
import time import time
import hashlib import hashlib
from binascii import hexlify from binascii import hexlify
@ -11,13 +13,13 @@ import shared
import defaults import defaults
import highlevelcrypto import highlevelcrypto
from bmconfigparser import BMConfigParser from bmconfigparser import BMConfigParser
from debug import logger
from addresses import decodeAddress, encodeAddress, encodeVarint from addresses import decodeAddress, encodeAddress, encodeVarint
from fallback import RIPEMD160Hash from fallback import RIPEMD160Hash
from helper_threading import StoppableThread from network import StoppableThread
class addressGenerator(StoppableThread): class addressGenerator(StoppableThread):
"""A thread for creating addresses"""
name = "addressGenerator" name = "addressGenerator"
@ -29,7 +31,11 @@ class addressGenerator(StoppableThread):
super(addressGenerator, self).stopThread() super(addressGenerator, self).stopThread()
def run(self): def run(self):
"""
Process the requests for addresses generation
from `.queues.addressGeneratorQueue`
"""
# pylint: disable=too-many-locals, too-many-branches, protected-access, too-many-statements
while state.shutdown == 0: while state.shutdown == 0:
queueValue = queues.addressGeneratorQueue.get() queueValue = queues.addressGeneratorQueue.get()
nonceTrialsPerByte = 0 nonceTrialsPerByte = 0
@ -85,12 +91,12 @@ class addressGenerator(StoppableThread):
elif queueValue[0] == 'stopThread': elif queueValue[0] == 'stopThread':
break break
else: else:
logger.error( self.logger.error(
'Programming error: A structure with the wrong number' 'Programming error: A structure with the wrong number'
' of values was passed into the addressGeneratorQueue.' ' of values was passed into the addressGeneratorQueue.'
' Here is the queueValue: %r\n', queueValue) ' Here is the queueValue: %r\n', queueValue)
if addressVersionNumber < 3 or addressVersionNumber > 4: if addressVersionNumber < 3 or addressVersionNumber > 4:
logger.error( self.logger.error(
'Program error: For some reason the address generator' 'Program error: For some reason the address generator'
' queue has been given a request to create at least' ' queue has been given a request to create at least'
' one version %s address which it cannot do.\n', ' one version %s address which it cannot do.\n',
@ -137,10 +143,10 @@ class addressGenerator(StoppableThread):
'\x00' * numberOfNullBytesDemandedOnFrontOfRipeHash '\x00' * numberOfNullBytesDemandedOnFrontOfRipeHash
): ):
break break
logger.info( self.logger.info(
'Generated address with ripe digest: %s', hexlify(ripe)) 'Generated address with ripe digest: %s', hexlify(ripe))
try: try:
logger.info( self.logger.info(
'Address generator calculated %s addresses at %s' 'Address generator calculated %s addresses at %s'
' addresses per second before finding one with' ' addresses per second before finding one with'
' the correct ripe-prefix.', ' the correct ripe-prefix.',
@ -202,8 +208,8 @@ class addressGenerator(StoppableThread):
elif command == 'createDeterministicAddresses' \ elif command == 'createDeterministicAddresses' \
or command == 'getDeterministicAddress' \ or command == 'getDeterministicAddress' \
or command == 'createChan' or command == 'joinChan': or command == 'createChan' or command == 'joinChan':
if len(deterministicPassphrase) == 0: if not deterministicPassphrase:
logger.warning( self.logger.warning(
'You are creating deterministic' 'You are creating deterministic'
' address(es) using a blank passphrase.' ' address(es) using a blank passphrase.'
' Bitmessage will do it but it is rather stupid.') ' Bitmessage will do it but it is rather stupid.')
@ -256,10 +262,10 @@ class addressGenerator(StoppableThread):
): ):
break break
logger.info( self.logger.info(
'Generated address with ripe digest: %s', hexlify(ripe)) 'Generated address with ripe digest: %s', hexlify(ripe))
try: try:
logger.info( self.logger.info(
'Address generator calculated %s addresses' 'Address generator calculated %s addresses'
' at %s addresses per second before finding' ' at %s addresses per second before finding'
' one with the correct ripe-prefix.', ' one with the correct ripe-prefix.',
@ -309,7 +315,7 @@ class addressGenerator(StoppableThread):
addressAlreadyExists = True addressAlreadyExists = True
if addressAlreadyExists: if addressAlreadyExists:
logger.info( self.logger.info(
'%s already exists. Not adding it again.', '%s already exists. Not adding it again.',
address address
) )
@ -322,7 +328,7 @@ class addressGenerator(StoppableThread):
).arg(address) ).arg(address)
)) ))
else: else:
logger.debug('label: %s', label) self.logger.debug('label: %s', label)
BMConfigParser().set(address, 'label', label) BMConfigParser().set(address, 'label', label)
BMConfigParser().set(address, 'enabled', 'true') BMConfigParser().set(address, 'enabled', 'true')
BMConfigParser().set(address, 'decoy', 'false') BMConfigParser().set(address, 'decoy', 'false')
@ -351,7 +357,7 @@ class addressGenerator(StoppableThread):
address) address)
shared.myECCryptorObjects[ripe] = \ shared.myECCryptorObjects[ripe] = \
highlevelcrypto.makeCryptor( highlevelcrypto.makeCryptor(
hexlify(potentialPrivEncryptionKey)) hexlify(potentialPrivEncryptionKey))
shared.myAddressesByHash[ripe] = address shared.myAddressesByHash[ripe] = address
tag = hashlib.sha512(hashlib.sha512( tag = hashlib.sha512(hashlib.sha512(
encodeVarint(addressVersionNumber) + encodeVarint(addressVersionNumber) +

106
src/class_objectProcessor.py
View File

@ -1,6 +1,9 @@
"""
The objectProcessor thread, of which there is only one, processes the network objects
"""
import hashlib import hashlib
import logging
import random import random
import shared
import threading import threading
import time import time
from binascii import hexlify from binascii import hexlify
@ -8,11 +11,13 @@ from subprocess import call # nosec
import highlevelcrypto import highlevelcrypto
import knownnodes import knownnodes
import shared
from addresses import ( from addresses import (
calculateInventoryHash, decodeAddress, decodeVarint, encodeAddress, calculateInventoryHash, decodeAddress, decodeVarint, encodeAddress,
encodeVarint, varintDecodeError encodeVarint, varintDecodeError
) )
from bmconfigparser import BMConfigParser from bmconfigparser import BMConfigParser
import helper_bitcoin import helper_bitcoin
import helper_inbox import helper_inbox
import helper_msgcoding import helper_msgcoding
@ -20,13 +25,18 @@ import helper_sent
from helper_sql import SqlBulkExecute, sqlExecute, sqlQuery from helper_sql import SqlBulkExecute, sqlExecute, sqlQuery
from helper_ackPayload import genAckPayload from helper_ackPayload import genAckPayload
from network import bmproto from network import bmproto
from network.node import Peer
import protocol import protocol
import queues import queues
import state import state
import tr import tr
from debug import logger
from fallback import RIPEMD160Hash from fallback import RIPEMD160Hash
import l10n import l10n
# pylint: disable=too-many-locals, too-many-return-statements, too-many-branches, too-many-statements
logger = logging.getLogger('default')
class objectProcessor(threading.Thread): class objectProcessor(threading.Thread):
@ -55,6 +65,7 @@ class objectProcessor(threading.Thread):
self.successfullyDecryptMessageTimings = [] self.successfullyDecryptMessageTimings = []
def run(self): def run(self):
"""Process the objects from `.queues.objectProcessorQueue`"""
while True: while True:
objectType, data = queues.objectProcessorQueue.get() objectType, data = queues.objectProcessorQueue.get()
@ -118,7 +129,10 @@ class objectProcessor(threading.Thread):
state.shutdown = 2 state.shutdown = 2
break break
def checkackdata(self, data): @staticmethod
def checkackdata(data):
"""Checking Acknowledgement of message received or not?"""
# pylint: disable=protected-access
# Let's check whether this is a message acknowledgement bound for us. # Let's check whether this is a message acknowledgement bound for us.
if len(data) < 32: if len(data) < 32:
return return
@ -158,7 +172,7 @@ class objectProcessor(threading.Thread):
if not host: if not host:
return return
peer = state.Peer(host, port) peer = Peer(host, port)
with knownnodes.knownNodesLock: with knownnodes.knownNodesLock:
knownnodes.addKnownNode( knownnodes.addKnownNode(
stream, peer, is_self=state.ownAddresses.get(peer)) stream, peer, is_self=state.ownAddresses.get(peer))
@ -268,6 +282,7 @@ class objectProcessor(threading.Thread):
queues.workerQueue.put(('sendOutOrStoreMyV4Pubkey', myAddress)) queues.workerQueue.put(('sendOutOrStoreMyV4Pubkey', myAddress))
def processpubkey(self, data): def processpubkey(self, data):
"""Process a pubkey object"""
pubkeyProcessingStartTime = time.time() pubkeyProcessingStartTime = time.time()
shared.numberOfPubkeysProcessed += 1 shared.numberOfPubkeysProcessed += 1
queues.UISignalQueue.put(( queues.UISignalQueue.put((
@ -316,13 +331,14 @@ class objectProcessor(threading.Thread):
'\x04' + publicSigningKey + '\x04' + publicEncryptionKey) '\x04' + publicSigningKey + '\x04' + publicEncryptionKey)
ripe = RIPEMD160Hash(sha.digest()).digest() ripe = RIPEMD160Hash(sha.digest()).digest()
logger.debug( if logger.isEnabledFor(logging.DEBUG):
'within recpubkey, addressVersion: %s, streamNumber: %s' logger.debug(
'\nripe %s\npublicSigningKey in hex: %s' 'within recpubkey, addressVersion: %s, streamNumber: %s'
'\npublicEncryptionKey in hex: %s', '\nripe %s\npublicSigningKey in hex: %s'
addressVersion, streamNumber, hexlify(ripe), '\npublicEncryptionKey in hex: %s',
hexlify(publicSigningKey), hexlify(publicEncryptionKey) addressVersion, streamNumber, hexlify(ripe),
) hexlify(publicSigningKey), hexlify(publicEncryptionKey)
)
address = encodeAddress(addressVersion, streamNumber, ripe) address = encodeAddress(addressVersion, streamNumber, ripe)
@ -380,13 +396,14 @@ class objectProcessor(threading.Thread):
sha.update(publicSigningKey + publicEncryptionKey) sha.update(publicSigningKey + publicEncryptionKey)
ripe = RIPEMD160Hash(sha.digest()).digest() ripe = RIPEMD160Hash(sha.digest()).digest()
logger.debug( if logger.isEnabledFor(logging.DEBUG):
'within recpubkey, addressVersion: %s, streamNumber: %s' logger.debug(
'\nripe %s\npublicSigningKey in hex: %s' 'within recpubkey, addressVersion: %s, streamNumber: %s'
'\npublicEncryptionKey in hex: %s', '\nripe %s\npublicSigningKey in hex: %s'
addressVersion, streamNumber, hexlify(ripe), '\npublicEncryptionKey in hex: %s',
hexlify(publicSigningKey), hexlify(publicEncryptionKey) addressVersion, streamNumber, hexlify(ripe),
) hexlify(publicSigningKey), hexlify(publicEncryptionKey)
)
address = encodeAddress(addressVersion, streamNumber, ripe) address = encodeAddress(addressVersion, streamNumber, ripe)
queryreturn = sqlQuery( queryreturn = sqlQuery(
@ -438,6 +455,7 @@ class objectProcessor(threading.Thread):
timeRequiredToProcessPubkey) timeRequiredToProcessPubkey)
def processmsg(self, data): def processmsg(self, data):
"""Process a message object"""
messageProcessingStartTime = time.time() messageProcessingStartTime = time.time()
shared.numberOfMessagesProcessed += 1 shared.numberOfMessagesProcessed += 1
queues.UISignalQueue.put(( queues.UISignalQueue.put((
@ -579,17 +597,18 @@ class objectProcessor(threading.Thread):
logger.debug('ECDSA verify failed') logger.debug('ECDSA verify failed')
return return
logger.debug('ECDSA verify passed') logger.debug('ECDSA verify passed')
logger.debug( if logger.isEnabledFor(logging.DEBUG):
'As a matter of intellectual curiosity, here is the Bitcoin' logger.debug(
' address associated with the keys owned by the other person:' 'As a matter of intellectual curiosity, here is the Bitcoin'
' %s ..and here is the testnet address: %s. The other person' ' address associated with the keys owned by the other person:'
' must take their private signing key from Bitmessage and' ' %s ..and here is the testnet address: %s. The other person'
' import it into Bitcoin (or a service like Blockchain.info)' ' must take their private signing key from Bitmessage and'
' for it to be of any use. Do not use this unless you know' ' import it into Bitcoin (or a service like Blockchain.info)'
' what you are doing.', ' for it to be of any use. Do not use this unless you know'
helper_bitcoin.calculateBitcoinAddressFromPubkey(pubSigningKey), ' what you are doing.',
helper_bitcoin.calculateTestnetAddressFromPubkey(pubSigningKey) helper_bitcoin.calculateBitcoinAddressFromPubkey(pubSigningKey),
) helper_bitcoin.calculateTestnetAddressFromPubkey(pubSigningKey)
)
# Used to detect and ignore duplicate messages in our inbox # Used to detect and ignore duplicate messages in our inbox
sigHash = hashlib.sha512( sigHash = hashlib.sha512(
hashlib.sha512(signature).digest()).digest()[32:] hashlib.sha512(signature).digest()).digest()[32:]
@ -732,7 +751,7 @@ class objectProcessor(threading.Thread):
# We really should have a discussion about how to # We really should have a discussion about how to
# set the TTL for mailing list broadcasts. This is obviously # set the TTL for mailing list broadcasts. This is obviously
# hard-coded. # hard-coded.
TTL = 2*7*24*60*60 # 2 weeks TTL = 2 * 7 * 24 * 60 * 60 # 2 weeks
t = ('', t = ('',
toAddress, toAddress,
ripe, ripe,
@ -784,6 +803,7 @@ class objectProcessor(threading.Thread):
) )
def processbroadcast(self, data): def processbroadcast(self, data):
"""Process a broadcast object"""
messageProcessingStartTime = time.time() messageProcessingStartTime = time.time()
shared.numberOfBroadcastsProcessed += 1 shared.numberOfBroadcastsProcessed += 1
queues.UISignalQueue.put(( queues.UISignalQueue.put((
@ -968,7 +988,7 @@ class objectProcessor(threading.Thread):
fromAddress = encodeAddress( fromAddress = encodeAddress(
sendersAddressVersion, sendersStream, calculatedRipe) sendersAddressVersion, sendersStream, calculatedRipe)
logger.info('fromAddress: %s' % fromAddress) logger.info('fromAddress: %s', fromAddress)
# Let's store the public key in case we want to reply to this person. # Let's store the public key in case we want to reply to this person.
sqlExecute('''INSERT INTO pubkeys VALUES (?,?,?,?,?)''', sqlExecute('''INSERT INTO pubkeys VALUES (?,?,?,?,?)''',
@ -985,7 +1005,7 @@ class objectProcessor(threading.Thread):
fromAddress = encodeAddress( fromAddress = encodeAddress(
sendersAddressVersion, sendersStream, calculatedRipe) sendersAddressVersion, sendersStream, calculatedRipe)
logger.debug('fromAddress: ' + fromAddress) logger.debug('fromAddress: %s', fromAddress)
try: try:
decodedMessage = helper_msgcoding.MsgDecode( decodedMessage = helper_msgcoding.MsgDecode(
@ -1046,17 +1066,18 @@ class objectProcessor(threading.Thread):
# for it. # for it.
elif addressVersion >= 4: elif addressVersion >= 4:
tag = hashlib.sha512(hashlib.sha512( tag = hashlib.sha512(hashlib.sha512(
encodeVarint(addressVersion) + encodeVarint(streamNumber) + ripe encodeVarint(addressVersion) + encodeVarint(streamNumber)
+ ripe
).digest()).digest()[32:] ).digest()).digest()[32:]
if tag in state.neededPubkeys: if tag in state.neededPubkeys:
del state.neededPubkeys[tag] del state.neededPubkeys[tag]
self.sendMessages(address) self.sendMessages(address)
def sendMessages(self, address): @staticmethod
def sendMessages(address):
""" """
This function is called by the possibleNewPubkey function when This method is called by the `possibleNewPubkey` when it sees
that function sees that we now have the necessary pubkey that we now have the necessary pubkey to send one or more messages.
to send one or more messages.
""" """
logger.info('We have been awaiting the arrival of this pubkey.') logger.info('We have been awaiting the arrival of this pubkey.')
sqlExecute( sqlExecute(
@ -1066,7 +1087,9 @@ class objectProcessor(threading.Thread):
" AND folder='sent'", address) " AND folder='sent'", address)
queues.workerQueue.put(('sendmessage', '')) queues.workerQueue.put(('sendmessage', ''))
def ackDataHasAValidHeader(self, ackData): @staticmethod
def ackDataHasAValidHeader(ackData):
"""Checking ackData with valid Header, not sending ackData when false"""
if len(ackData) < protocol.Header.size: if len(ackData) < protocol.Header.size:
logger.info( logger.info(
'The length of ackData is unreasonably short. Not sending' 'The length of ackData is unreasonably short. Not sending'
@ -1101,11 +1124,12 @@ class objectProcessor(threading.Thread):
return False return False
return True return True
def addMailingListNameToSubject(self, subject, mailingListName): @staticmethod
def addMailingListNameToSubject(subject, mailingListName):
"""Adding mailingListName to subject"""
subject = subject.strip() subject = subject.strip()
if subject[:3] == 'Re:' or subject[:3] == 'RE:': if subject[:3] == 'Re:' or subject[:3] == 'RE:':
subject = subject[3:].strip() subject = subject[3:].strip()
if '[' + mailingListName + ']' in subject: if '[' + mailingListName + ']' in subject:
return subject return subject
else: return '[' + mailingListName + '] ' + subject
return '[' + mailingListName + '] ' + subject

24
src/class_objectProcessorQueue.py
View File

@ -1,24 +0,0 @@
import Queue
import threading
import time
class ObjectProcessorQueue(Queue.Queue):
maxSize = 32000000
def __init__(self):
Queue.Queue.__init__(self)
self.sizeLock = threading.Lock()
self.curSize = 0 # in Bytes. We maintain this to prevent nodes from flooing us with objects which take up too much memory. If this gets too big we'll sleep before asking for further objects.
def put(self, item, block = True, timeout = None):
while self.curSize >= self.maxSize:
time.sleep(1)
with self.sizeLock:
self.curSize += len(item[1])
Queue.Queue.put(self, item, block, timeout)
def get(self, block = True, timeout = None):
item = Queue.Queue.get(self, block, timeout)
with self.sizeLock:
self.curSize -= len(item[1])
return item

149
src/class_singleCleaner.py
View File

@ -1,20 +1,21 @@
""" """
The singleCleaner class is a timer-driven thread that cleans data structures The `singleCleaner` class is a timer-driven thread that cleans data structures
to free memory, resends messages when a remote node doesn't respond, and to free memory, resends messages when a remote node doesn't respond, and
sends pong messages to keep connections alive if the network isn't busy. sends pong messages to keep connections alive if the network isn't busy.
It cleans these data structures in memory: It cleans these data structures in memory:
inventory (moves data to the on-disk sql database) - inventory (moves data to the on-disk sql database)
inventorySets (clears then reloads data out of sql database) - inventorySets (clears then reloads data out of sql database)
It cleans these tables on the disk: It cleans these tables on the disk:
inventory (clears expired objects) - inventory (clears expired objects)
pubkeys (clears pubkeys older than 4 weeks old which we have not used - pubkeys (clears pubkeys older than 4 weeks old which we have not used
personally) personally)
knownNodes (clears addresses which have not been online for over 3 days) - knownNodes (clears addresses which have not been online for over 3 days)
It resends messages when there has been no response: It resends messages when there has been no response:
resends getpubkey messages in 5 days (then 10 days, then 20 days, etc...) - resends getpubkey messages in 5 days (then 10 days, then 20 days, etc...)
resends msg messages in 5 days (then 10 days, then 20 days, etc...) - resends msg messages in 5 days (then 10 days, then 20 days, etc...)
""" """
# pylint: disable=relative-import, protected-access # pylint: disable=relative-import, protected-access
@ -23,36 +24,35 @@ import os
import time import time
import shared import shared
import knownnodes
import queues
import shared
import state
import tr import tr
from bmconfigparser import BMConfigParser from bmconfigparser import BMConfigParser
from helper_sql import sqlQuery, sqlExecute from helper_sql import sqlQuery, sqlExecute
from helper_threading import StoppableThread
from inventory import Inventory from inventory import Inventory
from network.connectionpool import BMConnectionPool from network import BMConnectionPool, StoppableThread
from debug import logger
from datetime import datetime, timedelta
import knownnodes
import queues
import state
class singleCleaner(StoppableThread): class singleCleaner(StoppableThread):
"""Base method that Cleanup knownnodes and handle possible severe exception""" """The singleCleaner thread class"""
name = "singleCleaner" name = "singleCleaner"
cycleLength = 300 cycleLength = 300
expireDiscoveredPeers = 300 expireDiscoveredPeers = 300
def run(self): # pylint: disable=too-many-branches def run(self): # pylint: disable=too-many-branches
gc.disable() gc.disable()
timeWeLastClearedInventoryAndPubkeysTables = 0 timeWeLastClearedInventoryAndPubkeysTables = 0
try: try:
shared.maximumLengthOfTimeToBotherResendingMessages = ( shared.maximumLengthOfTimeToBotherResendingMessages = (
float(BMConfigParser().get( float(BMConfigParser().get(
'bitmessagesettings', 'stopresendingafterxdays')) * 'bitmessagesettings', 'stopresendingafterxdays'))
24 * 60 * 60 * 24 * 60 * 60
) + ( ) + (
float(BMConfigParser().get( float(BMConfigParser().get(
'bitmessagesettings', 'stopresendingafterxmonths')) * 'bitmessagesettings', 'stopresendingafterxmonths'))
(60 * 60 * 24 * 365) / 12) * (60 * 60 * 24 * 365) / 12)
except: except:
# Either the user hasn't set stopresendingafterxdays and # Either the user hasn't set stopresendingafterxdays and
# stopresendingafterxmonths yet or the options are missing # stopresendingafterxmonths yet or the options are missing
@ -93,13 +93,13 @@ class singleCleaner(StoppableThread):
queryreturn = sqlQuery( queryreturn = sqlQuery(
"SELECT toaddress, ackdata, status FROM sent" "SELECT toaddress, ackdata, status FROM sent"
" WHERE ((status='awaitingpubkey' OR status='msgsent')" " WHERE ((status='awaitingpubkey' OR status='msgsent')"
" AND folder LIKE '%sent%' AND sleeptill<? AND senttime>?)", " AND folder='sent' AND sleeptill<? AND senttime>?)",
int(time.time()), int(time.time()) - int(time.time()), int(time.time())
shared.maximumLengthOfTimeToBotherResendingMessages - shared.maximumLengthOfTimeToBotherResendingMessages
) )
for row in queryreturn: for row in queryreturn:
if len(row) < 2: if len(row) < 2:
logger.error( self.logger.error(
'Something went wrong in the singleCleaner thread:' 'Something went wrong in the singleCleaner thread:'
' a query did not return the requested fields. %r', ' a query did not return the requested fields. %r',
row row
@ -108,17 +108,18 @@ class singleCleaner(StoppableThread):
break break
toAddress, ackData, status = row toAddress, ackData, status = row
if status == 'awaitingpubkey': if status == 'awaitingpubkey':
resendPubkeyRequest(toAddress) self.resendPubkeyRequest(toAddress)
elif status == 'msgsent': elif status == 'msgsent':
resendMsg(ackData) self.resendMsg(ackData)
deleteTrashMsgPermonantly() deleteTrashMsgPermonantly()
try: try:
# Cleanup knownnodes and handle possible severe exception # Cleanup knownnodes and handle possible severe exception
# while writing it to disk # while writing it to disk
knownnodes.cleanupKnownNodes() knownnodes.cleanupKnownNodes()
except Exception as err: except Exception as err:
# pylint: disable=protected-access
if "Errno 28" in str(err): if "Errno 28" in str(err):
logger.fatal( self.logger.fatal(
'(while writing knownnodes to disk)' '(while writing knownnodes to disk)'
' Alert: Your disk or data storage volume is full.' ' Alert: Your disk or data storage volume is full.'
) )
@ -131,20 +132,11 @@ class singleCleaner(StoppableThread):
' is full. Bitmessage will now exit.'), ' is full. Bitmessage will now exit.'),
True) True)
)) ))
# ..FIXME redundant? if shared.thisapp.daemon or not state.enableGUI:
# pylint: disable=no-member
if shared.daemon or not state.enableGUI:
os._exit(1) os._exit(1)
# # clear download queues
# for thread in threading.enumerate():
# if thread.isAlive() and hasattr(thread, 'downloadQueue'):
# thread.downloadQueue.clear()
# inv/object tracking # inv/object tracking
for connection in \ for connection in BMConnectionPool().connections():
BMConnectionPool().inboundConnections.values() + \
BMConnectionPool().outboundConnections.values():
connection.clean() connection.clean()
# discovery tracking # discovery tracking
@ -155,53 +147,52 @@ class singleCleaner(StoppableThread):
del state.discoveredPeers[k] del state.discoveredPeers[k]
except KeyError: except KeyError:
pass pass
# ..TODO: cleanup pending upload / download
# ..todo:: cleanup pending upload / download
gc.collect() gc.collect()
if state.shutdown == 0: if state.shutdown == 0:
self.stop.wait(singleCleaner.cycleLength) self.stop.wait(singleCleaner.cycleLength)
def resendPubkeyRequest(self, address):
"""Resend pubkey request for address"""
self.logger.debug(
'It has been a long time and we haven\'t heard a response to our'
' getpubkey request. Sending again.'
)
try:
# We need to take this entry out of the neededPubkeys structure
# because the queues.workerQueue checks to see whether the entry
# is already present and will not do the POW and send the message
# because it assumes that it has already done it recently.
del state.neededPubkeys[address]
except:
pass
def resendPubkeyRequest(address): queues.UISignalQueue.put((
"""After a long time, method send getpubkey request""" 'updateStatusBar',
logger.debug( 'Doing work necessary to again attempt to request a public key...'
'It has been a long time and we haven\'t heard a response to our' ))
' getpubkey request. Sending again.' sqlExecute(
) '''UPDATE sent SET status='msgqueued' WHERE toaddress=?''',
try: address)
# We need to take this entry out of the neededPubkeys structure queues.workerQueue.put(('sendmessage', ''))
# because the queues.workerQueue checks to see whether the entry
# is already present and will not do the POW and send the message
# because it assumes that it has already done it recently.
del state.neededPubkeys[address]
except:
pass
queues.UISignalQueue.put(( def resendMsg(self, ackdata):
'updateStatusBar', """Resend message by ackdata"""
'Doing work necessary to again attempt to request a public key...' self.logger.debug(
)) 'It has been a long time and we haven\'t heard an acknowledgement'
sqlExecute( ' to our msg. Sending again.'
'''UPDATE sent SET status='msgqueued' WHERE toaddress=?''', )
address) sqlExecute(
queues.workerQueue.put(('sendmessage', '')) '''UPDATE sent SET status='msgqueued' WHERE ackdata=?''',
ackdata)
queues.workerQueue.put(('sendmessage', ''))
def resendMsg(ackdata): queues.UISignalQueue.put((
"""After a long time, method send acknowledgement msg""" 'updateStatusBar',
logger.debug( 'Doing work necessary to again attempt to deliver a message...'
'It has been a long time and we haven\'t heard an acknowledgement' ))
' to our msg. Sending again.'
)
sqlExecute(
'''UPDATE sent SET status='msgqueued' WHERE ackdata=?''',
ackdata)
queues.workerQueue.put(('sendmessage', ''))
queues.UISignalQueue.put((
'updateStatusBar',
'Doing work necessary to again attempt to deliver a message...'
))
def deleteTrashMsgPermonantly(): def deleteTrashMsgPermonantly():

91
src/class_singleWorker.py
View File

@ -1,6 +1,5 @@
""" """
src/class_singleWorker.py Thread for performing PoW
=========================
""" """
# pylint: disable=protected-access,too-many-branches,too-many-statements # pylint: disable=protected-access,too-many-branches,too-many-statements
# pylint: disable=no-self-use,too-many-lines,too-many-locals,relative-import # pylint: disable=no-self-use,too-many-lines,too-many-locals,relative-import
@ -26,10 +25,9 @@ import state
import tr import tr
from addresses import calculateInventoryHash, decodeAddress, decodeVarint, encodeVarint from addresses import calculateInventoryHash, decodeAddress, decodeVarint, encodeVarint
from bmconfigparser import BMConfigParser from bmconfigparser import BMConfigParser
from debug import logger
from helper_sql import sqlExecute, sqlQuery from helper_sql import sqlExecute, sqlQuery
from helper_threading import StoppableThread
from inventory import Inventory from inventory import Inventory
from network import StoppableThread
# This thread, of which there is only one, does the heavy lifting: # This thread, of which there is only one, does the heavy lifting:
# calculating POWs. # calculating POWs.
@ -101,7 +99,7 @@ class singleWorker(StoppableThread):
'''SELECT ackdata FROM sent WHERE status = 'msgsent' ''') '''SELECT ackdata FROM sent WHERE status = 'msgsent' ''')
for row in queryreturn: for row in queryreturn:
ackdata, = row ackdata, = row
logger.info('Watching for ackdata %s', hexlify(ackdata)) self.logger.info('Watching for ackdata %s', hexlify(ackdata))
shared.ackdataForWhichImWatching[ackdata] = 0 shared.ackdataForWhichImWatching[ackdata] = 0
# Fix legacy (headerless) watched ackdata to include header # Fix legacy (headerless) watched ackdata to include header
@ -176,14 +174,14 @@ class singleWorker(StoppableThread):
self.busy = 0 self.busy = 0
return return
else: else:
logger.error( self.logger.error(
'Probable programming error: The command sent' 'Probable programming error: The command sent'
' to the workerThread is weird. It is: %s\n', ' to the workerThread is weird. It is: %s\n',
command command
) )
queues.workerQueue.task_done() queues.workerQueue.task_done()
logger.info("Quitting...") self.logger.info("Quitting...")
def _getKeysForAddress(self, address): def _getKeysForAddress(self, address):
privSigningKeyBase58 = BMConfigParser().get( privSigningKeyBase58 = BMConfigParser().get(
@ -220,25 +218,24 @@ class singleWorker(StoppableThread):
)) / (2 ** 16)) )) / (2 ** 16))
)) ))
initialHash = hashlib.sha512(payload).digest() initialHash = hashlib.sha512(payload).digest()
logger.info( self.logger.info(
'%s Doing proof of work... TTL set to %s', log_prefix, TTL) '%s Doing proof of work... TTL set to %s', log_prefix, TTL)
if log_time: if log_time:
start_time = time.time() start_time = time.time()
trialValue, nonce = proofofwork.run(target, initialHash) trialValue, nonce = proofofwork.run(target, initialHash)
logger.info( self.logger.info(
'%s Found proof of work %s Nonce: %s', '%s Found proof of work %s Nonce: %s',
log_prefix, trialValue, nonce log_prefix, trialValue, nonce
) )
try: try:
delta = time.time() - start_time delta = time.time() - start_time
logger.info( self.logger.info(
'PoW took %.1f seconds, speed %s.', 'PoW took %.1f seconds, speed %s.',
delta, sizeof_fmt(nonce / delta) delta, sizeof_fmt(nonce / delta)
) )
except: # NameError except: # NameError
pass pass
payload = pack('>Q', nonce) + payload payload = pack('>Q', nonce) + payload
# inventoryHash = calculateInventoryHash(payload)
return payload return payload
def doPOWForMyV2Pubkey(self, adressHash): def doPOWForMyV2Pubkey(self, adressHash):
@ -263,7 +260,7 @@ class singleWorker(StoppableThread):
_, _, pubSigningKey, pubEncryptionKey = \ _, _, pubSigningKey, pubEncryptionKey = \
self._getKeysForAddress(myAddress) self._getKeysForAddress(myAddress)
except Exception as err: except Exception as err:
logger.error( self.logger.error(
'Error within doPOWForMyV2Pubkey. Could not read' 'Error within doPOWForMyV2Pubkey. Could not read'
' the keys from the keys.dat file for a requested' ' the keys from the keys.dat file for a requested'
' address. %s\n', err ' address. %s\n', err
@ -281,7 +278,8 @@ class singleWorker(StoppableThread):
Inventory()[inventoryHash] = ( Inventory()[inventoryHash] = (
objectType, streamNumber, payload, embeddedTime, '') objectType, streamNumber, payload, embeddedTime, '')
logger.info('broadcasting inv with hash: %s', hexlify(inventoryHash)) self.logger.info(
'broadcasting inv with hash: %s', hexlify(inventoryHash))
queues.invQueue.put((streamNumber, inventoryHash)) queues.invQueue.put((streamNumber, inventoryHash))
queues.UISignalQueue.put(('updateStatusBar', '')) queues.UISignalQueue.put(('updateStatusBar', ''))
@ -306,7 +304,7 @@ class singleWorker(StoppableThread):
# The address has been deleted. # The address has been deleted.
return return
if BMConfigParser().safeGetBoolean(myAddress, 'chan'): if BMConfigParser().safeGetBoolean(myAddress, 'chan'):
logger.info('This is a chan address. Not sending pubkey.') self.logger.info('This is a chan address. Not sending pubkey.')
return return
_, addressVersionNumber, streamNumber, adressHash = decodeAddress( _, addressVersionNumber, streamNumber, adressHash = decodeAddress(
myAddress) myAddress)
@ -336,7 +334,7 @@ class singleWorker(StoppableThread):
privSigningKeyHex, _, pubSigningKey, pubEncryptionKey = \ privSigningKeyHex, _, pubSigningKey, pubEncryptionKey = \
self._getKeysForAddress(myAddress) self._getKeysForAddress(myAddress)
except Exception as err: except Exception as err:
logger.error( self.logger.error(
'Error within sendOutOrStoreMyV3Pubkey. Could not read' 'Error within sendOutOrStoreMyV3Pubkey. Could not read'
' the keys from the keys.dat file for a requested' ' the keys from the keys.dat file for a requested'
' address. %s\n', err ' address. %s\n', err
@ -363,7 +361,8 @@ class singleWorker(StoppableThread):
Inventory()[inventoryHash] = ( Inventory()[inventoryHash] = (
objectType, streamNumber, payload, embeddedTime, '') objectType, streamNumber, payload, embeddedTime, '')
logger.info('broadcasting inv with hash: %s', hexlify(inventoryHash)) self.logger.info(
'broadcasting inv with hash: %s', hexlify(inventoryHash))
queues.invQueue.put((streamNumber, inventoryHash)) queues.invQueue.put((streamNumber, inventoryHash))
queues.UISignalQueue.put(('updateStatusBar', '')) queues.UISignalQueue.put(('updateStatusBar', ''))
@ -386,7 +385,7 @@ class singleWorker(StoppableThread):
# The address has been deleted. # The address has been deleted.
return return
if shared.BMConfigParser().safeGetBoolean(myAddress, 'chan'): if shared.BMConfigParser().safeGetBoolean(myAddress, 'chan'):
logger.info('This is a chan address. Not sending pubkey.') self.logger.info('This is a chan address. Not sending pubkey.')
return return
_, addressVersionNumber, streamNumber, addressHash = decodeAddress( _, addressVersionNumber, streamNumber, addressHash = decodeAddress(
myAddress) myAddress)
@ -405,7 +404,7 @@ class singleWorker(StoppableThread):
privSigningKeyHex, _, pubSigningKey, pubEncryptionKey = \ privSigningKeyHex, _, pubSigningKey, pubEncryptionKey = \
self._getKeysForAddress(myAddress) self._getKeysForAddress(myAddress)
except Exception as err: except Exception as err:
logger.error( self.logger.error(
'Error within sendOutOrStoreMyV4Pubkey. Could not read' 'Error within sendOutOrStoreMyV4Pubkey. Could not read'
' the keys from the keys.dat file for a requested' ' the keys from the keys.dat file for a requested'
' address. %s\n', err ' address. %s\n', err
@ -453,7 +452,8 @@ class singleWorker(StoppableThread):
doubleHashOfAddressData[32:] doubleHashOfAddressData[32:]
) )
logger.info('broadcasting inv with hash: %s', hexlify(inventoryHash)) self.logger.info(
'broadcasting inv with hash: %s', hexlify(inventoryHash))
queues.invQueue.put((streamNumber, inventoryHash)) queues.invQueue.put((streamNumber, inventoryHash))
queues.UISignalQueue.put(('updateStatusBar', '')) queues.UISignalQueue.put(('updateStatusBar', ''))
@ -462,7 +462,7 @@ class singleWorker(StoppableThread):
myAddress, 'lastpubkeysendtime', str(int(time.time()))) myAddress, 'lastpubkeysendtime', str(int(time.time())))
BMConfigParser().save() BMConfigParser().save()
except Exception as err: except Exception as err:
logger.error( self.logger.error(
'Error: Couldn\'t add the lastpubkeysendtime' 'Error: Couldn\'t add the lastpubkeysendtime'
' to the keys.dat file. Error message: %s', err ' to the keys.dat file. Error message: %s', err
) )
@ -470,8 +470,8 @@ class singleWorker(StoppableThread):
def sendOnionPeerObj(self, peer=None): def sendOnionPeerObj(self, peer=None):
"""Send onionpeer object representing peer""" """Send onionpeer object representing peer"""
if not peer: # find own onionhostname if not peer: # find own onionhostname
for peer in state.ownAddresses: # pylint: disable=redefined-argument-from-local for peer_ in state.ownAddresses:
if peer.host.endswith('.onion'): if peer_.host.endswith('.onion'):
break break
else: else:
return return
@ -500,7 +500,7 @@ class singleWorker(StoppableThread):
objectType, streamNumber, buffer(payload), objectType, streamNumber, buffer(payload),
embeddedTime, buffer(tag) embeddedTime, buffer(tag)
) )
logger.info( self.logger.info(
'sending inv (within sendOnionPeerObj function) for object: %s', 'sending inv (within sendOnionPeerObj function) for object: %s',
hexlify(inventoryHash)) hexlify(inventoryHash))
queues.invQueue.put((streamNumber, inventoryHash)) queues.invQueue.put((streamNumber, inventoryHash))
@ -523,7 +523,7 @@ class singleWorker(StoppableThread):
_, addressVersionNumber, streamNumber, ripe = \ _, addressVersionNumber, streamNumber, ripe = \
decodeAddress(fromaddress) decodeAddress(fromaddress)
if addressVersionNumber <= 1: if addressVersionNumber <= 1:
logger.error( self.logger.error(
'Error: In the singleWorker thread, the ' 'Error: In the singleWorker thread, the '
' sendBroadcast function doesn\'t understand' ' sendBroadcast function doesn\'t understand'
' the address version.\n') ' the address version.\n')
@ -639,7 +639,7 @@ class singleWorker(StoppableThread):
# to not let the user try to send a message this large # to not let the user try to send a message this large
# until we implement message continuation. # until we implement message continuation.
if len(payload) > 2 ** 18: # 256 KiB if len(payload) > 2 ** 18: # 256 KiB
logger.critical( self.logger.critical(
'This broadcast object is too large to send.' 'This broadcast object is too large to send.'
' This should never happen. Object size: %s', ' This should never happen. Object size: %s',
len(payload) len(payload)
@ -650,7 +650,7 @@ class singleWorker(StoppableThread):
objectType = 3 objectType = 3
Inventory()[inventoryHash] = ( Inventory()[inventoryHash] = (
objectType, streamNumber, payload, embeddedTime, tag) objectType, streamNumber, payload, embeddedTime, tag)
logger.info( self.logger.info(
'sending inv (within sendBroadcast function)' 'sending inv (within sendBroadcast function)'
' for object: %s', ' for object: %s',
hexlify(inventoryHash) hexlify(inventoryHash)
@ -870,8 +870,8 @@ class singleWorker(StoppableThread):
"MainWindow", "MainWindow",
"Looking up the receiver\'s public key")) "Looking up the receiver\'s public key"))
)) ))
logger.info('Sending a message.') self.logger.info('Sending a message.')
logger.debug( self.logger.debug(
'First 150 characters of message: %s', 'First 150 characters of message: %s',
repr(message[:150]) repr(message[:150])
) )
@ -915,7 +915,7 @@ class singleWorker(StoppableThread):
if not shared.BMConfigParser().safeGetBoolean( if not shared.BMConfigParser().safeGetBoolean(
'bitmessagesettings', 'willinglysendtomobile' 'bitmessagesettings', 'willinglysendtomobile'
): ):
logger.info( self.logger.info(
'The receiver is a mobile user but the' 'The receiver is a mobile user but the'
' sender (you) has not selected that you' ' sender (you) has not selected that you'
' are willing to send to mobiles. Aborting' ' are willing to send to mobiles. Aborting'
@ -981,7 +981,7 @@ class singleWorker(StoppableThread):
defaults.networkDefaultPayloadLengthExtraBytes: defaults.networkDefaultPayloadLengthExtraBytes:
requiredPayloadLengthExtraBytes = \ requiredPayloadLengthExtraBytes = \
defaults.networkDefaultPayloadLengthExtraBytes defaults.networkDefaultPayloadLengthExtraBytes
logger.debug( self.logger.debug(
'Using averageProofOfWorkNonceTrialsPerByte: %s' 'Using averageProofOfWorkNonceTrialsPerByte: %s'
' and payloadLengthExtraBytes: %s.', ' and payloadLengthExtraBytes: %s.',
requiredAverageProofOfWorkNonceTrialsPerByte, requiredAverageProofOfWorkNonceTrialsPerByte,
@ -1046,8 +1046,9 @@ class singleWorker(StoppableThread):
l10n.formatTimestamp())))) l10n.formatTimestamp()))))
continue continue
else: # if we are sending a message to ourselves or a chan.. else: # if we are sending a message to ourselves or a chan..
logger.info('Sending a message.') self.logger.info('Sending a message.')
logger.debug('First 150 characters of message: %r', message[:150]) self.logger.debug(
'First 150 characters of message: %r', message[:150])
behaviorBitfield = protocol.getBitfield(fromaddress) behaviorBitfield = protocol.getBitfield(fromaddress)
try: try:
@ -1066,7 +1067,7 @@ class singleWorker(StoppableThread):
" message. %1" " message. %1"
).arg(l10n.formatTimestamp())) ).arg(l10n.formatTimestamp()))
)) ))
logger.error( self.logger.error(
'Error within sendMsg. Could not read the keys' 'Error within sendMsg. Could not read the keys'
' from the keys.dat file for our own address. %s\n', ' from the keys.dat file for our own address. %s\n',
err) err)
@ -1142,14 +1143,14 @@ class singleWorker(StoppableThread):
payload += encodeVarint(encodedMessage.length) payload += encodeVarint(encodedMessage.length)
payload += encodedMessage.data payload += encodedMessage.data
if BMConfigParser().has_section(toaddress): if BMConfigParser().has_section(toaddress):
logger.info( self.logger.info(
'Not bothering to include ackdata because we are' 'Not bothering to include ackdata because we are'
' sending to ourselves or a chan.' ' sending to ourselves or a chan.'
) )
fullAckPayload = '' fullAckPayload = ''
elif not protocol.checkBitfield( elif not protocol.checkBitfield(
behaviorBitfield, protocol.BITFIELD_DOESACK): behaviorBitfield, protocol.BITFIELD_DOESACK):
logger.info( self.logger.info(
'Not bothering to include ackdata because' 'Not bothering to include ackdata because'
' the receiver said that they won\'t relay it anyway.' ' the receiver said that they won\'t relay it anyway.'
) )
@ -1202,7 +1203,7 @@ class singleWorker(StoppableThread):
requiredPayloadLengthExtraBytes requiredPayloadLengthExtraBytes
)) / (2 ** 16)) )) / (2 ** 16))
)) ))
logger.info( self.logger.info(
'(For msg message) Doing proof of work. Total required' '(For msg message) Doing proof of work. Total required'
' difficulty: %f. Required small message difficulty: %f.', ' difficulty: %f. Required small message difficulty: %f.',
float(requiredAverageProofOfWorkNonceTrialsPerByte) / float(requiredAverageProofOfWorkNonceTrialsPerByte) /
@ -1215,12 +1216,12 @@ class singleWorker(StoppableThread):
initialHash = hashlib.sha512(encryptedPayload).digest() initialHash = hashlib.sha512(encryptedPayload).digest()
trialValue, nonce = proofofwork.run(target, initialHash) trialValue, nonce = proofofwork.run(target, initialHash)
print("nonce calculated value#############################", nonce) print("nonce calculated value#############################", nonce)
logger.info( self.logger.info(
'(For msg message) Found proof of work %s Nonce: %s', '(For msg message) Found proof of work %s Nonce: %s',
trialValue, nonce trialValue, nonce
) )
try: try:
logger.info( self.logger.info(
'PoW took %.1f seconds, speed %s.', 'PoW took %.1f seconds, speed %s.',
time.time() - powStartTime, time.time() - powStartTime,
sizeof_fmt(nonce / (time.time() - powStartTime)) sizeof_fmt(nonce / (time.time() - powStartTime))
@ -1235,7 +1236,7 @@ class singleWorker(StoppableThread):
# in the code to not let the user try to send a message # in the code to not let the user try to send a message
# this large until we implement message continuation. # this large until we implement message continuation.
if len(encryptedPayload) > 2 ** 18: # 256 KiB if len(encryptedPayload) > 2 ** 18: # 256 KiB
logger.critical( self.logger.critical(
'This msg object is too large to send. This should' 'This msg object is too large to send. This should'
' never happen. Object size: %i', ' never happen. Object size: %i',
len(encryptedPayload) len(encryptedPayload)
@ -1266,7 +1267,7 @@ class singleWorker(StoppableThread):
" Sent on %1" " Sent on %1"
).arg(l10n.formatTimestamp())) ).arg(l10n.formatTimestamp()))
)) ))
logger.info( self.logger.info(
'Broadcasting inv for my msg(within sendmsg function): %s', 'Broadcasting inv for my msg(within sendmsg function): %s',
hexlify(inventoryHash) hexlify(inventoryHash)
) )
@ -1319,7 +1320,7 @@ class singleWorker(StoppableThread):
toStatus, addressVersionNumber, streamNumber, ripe = decodeAddress( toStatus, addressVersionNumber, streamNumber, ripe = decodeAddress(
toAddress) toAddress)
if toStatus != 'success': if toStatus != 'success':
logger.error( self.logger.error(
'Very abnormal error occurred in requestPubKey.' 'Very abnormal error occurred in requestPubKey.'
' toAddress is: %r. Please report this error to Atheros.', ' toAddress is: %r. Please report this error to Atheros.',
toAddress toAddress
@ -1333,7 +1334,7 @@ class singleWorker(StoppableThread):
toAddress toAddress
) )
if not queryReturn: if not queryReturn:
logger.critical( self.logger.critical(
'BUG: Why are we requesting the pubkey for %s' 'BUG: Why are we requesting the pubkey for %s'
' if there are no messages in the sent folder' ' if there are no messages in the sent folder'
' to that address?', toAddress ' to that address?', toAddress
@ -1381,11 +1382,11 @@ class singleWorker(StoppableThread):
payload += encodeVarint(streamNumber) payload += encodeVarint(streamNumber)
if addressVersionNumber <= 3: if addressVersionNumber <= 3:
payload += ripe payload += ripe
logger.info( self.logger.info(
'making request for pubkey with ripe: %s', hexlify(ripe)) 'making request for pubkey with ripe: %s', hexlify(ripe))
else: else:
payload += tag payload += tag
logger.info( self.logger.info(
'making request for v4 pubkey with tag: %s', hexlify(tag)) 'making request for v4 pubkey with tag: %s', hexlify(tag))
# print 'trial value', trialValue # print 'trial value', trialValue
@ -1406,7 +1407,7 @@ class singleWorker(StoppableThread):
objectType = 1 objectType = 1
Inventory()[inventoryHash] = ( Inventory()[inventoryHash] = (
objectType, streamNumber, payload, embeddedTime, '') objectType, streamNumber, payload, embeddedTime, '')
logger.info('sending inv (for the getpubkey message)') self.logger.info('sending inv (for the getpubkey message)')
queues.invQueue.put((streamNumber, inventoryHash)) queues.invQueue.put((streamNumber, inventoryHash))
# wait 10% past expiration # wait 10% past expiration

14
src/class_smtpDeliver.py
View File

@ -5,7 +5,6 @@ src/class_smtpDeliver.py
# pylint: disable=unused-variable # pylint: disable=unused-variable
import smtplib import smtplib
import sys
import urlparse import urlparse
from email.header import Header from email.header import Header
from email.mime.text import MIMEText from email.mime.text import MIMEText
@ -13,8 +12,7 @@ from email.mime.text import MIMEText
import queues import queues
import state import state
from bmconfigparser import BMConfigParser from bmconfigparser import BMConfigParser
from debug import logger from network.threads import StoppableThread
from helper_threading import StoppableThread
SMTPDOMAIN = "bmaddr.lan" SMTPDOMAIN = "bmaddr.lan"
@ -75,10 +73,12 @@ class smtpDeliver(StoppableThread):
client.starttls() client.starttls()
client.ehlo() client.ehlo()
client.sendmail(msg['From'], [to], msg.as_string()) client.sendmail(msg['From'], [to], msg.as_string())
logger.info("Delivered via SMTP to %s through %s:%i ...", to, u.hostname, u.port) self.logger.info(
'Delivered via SMTP to %s through %s:%i ...',
to, u.hostname, u.port)
client.quit() client.quit()
except: except:
logger.error("smtp delivery error", exc_info=True) self.logger.error('smtp delivery error', exc_info=True)
elif command == 'displayNewSentMessage': elif command == 'displayNewSentMessage':
toAddress, fromLabel, fromAddress, subject, message, ackdata = data toAddress, fromLabel, fromAddress, subject, message, ackdata = data
elif command == 'updateNetworkStatusTab': elif command == 'updateNetworkStatusTab':
@ -112,5 +112,5 @@ class smtpDeliver(StoppableThread):
elif command == 'stopThread': elif command == 'stopThread':
break break
else: else:
sys.stderr.write( self.logger.warning(
'Command sent to smtpDeliver not recognized: %s\n' % command) 'Command sent to smtpDeliver not recognized: %s', command)

104
src/class_smtpServer.py
View File

@ -1,28 +1,37 @@
"""
SMTP server thread
"""
import asyncore import asyncore
import base64 import base64
import email import email
from email.parser import Parser import logging
from email.header import decode_header
import re import re
import signal import signal
import smtpd import smtpd
import threading import threading
import time import time
from email.header import decode_header
from email.parser import Parser
import queues
from addresses import decodeAddress from addresses import decodeAddress
from bmconfigparser import BMConfigParser from bmconfigparser import BMConfigParser
from debug import logger
from helper_sql import sqlExecute
from helper_ackPayload import genAckPayload from helper_ackPayload import genAckPayload
from helper_threading import StoppableThread from helper_sql import sqlExecute
import queues from network.threads import StoppableThread
from version import softwareVersion from version import softwareVersion
SMTPDOMAIN = "bmaddr.lan" SMTPDOMAIN = "bmaddr.lan"
LISTENPORT = 8425 LISTENPORT = 8425
logger = logging.getLogger('default')
# pylint: disable=attribute-defined-outside-init
class smtpServerChannel(smtpd.SMTPChannel): class smtpServerChannel(smtpd.SMTPChannel):
"""Asyncore channel for SMTP protocol (server)"""
def smtp_EHLO(self, arg): def smtp_EHLO(self, arg):
"""Process an EHLO"""
if not arg: if not arg:
self.push('501 Syntax: HELO hostname') self.push('501 Syntax: HELO hostname')
return return
@ -30,15 +39,17 @@ class smtpServerChannel(smtpd.SMTPChannel):
self.push('250 AUTH PLAIN') self.push('250 AUTH PLAIN')
def smtp_AUTH(self, arg): def smtp_AUTH(self, arg):
"""Process AUTH"""
if not arg or arg[0:5] not in ["PLAIN"]: if not arg or arg[0:5] not in ["PLAIN"]:
self.push('501 Syntax: AUTH PLAIN') self.push('501 Syntax: AUTH PLAIN')
return return
authstring = arg[6:] authstring = arg[6:]
try: try:
decoded = base64.b64decode(authstring) decoded = base64.b64decode(authstring)
correctauth = "\x00" + BMConfigParser().safeGet("bitmessagesettings", "smtpdusername", "") + \ correctauth = "\x00" + BMConfigParser().safeGet(
"\x00" + BMConfigParser().safeGet("bitmessagesettings", "smtpdpassword", "") "bitmessagesettings", "smtpdusername", "") + "\x00" + BMConfigParser().safeGet(
logger.debug("authstring: %s / %s", correctauth, decoded) "bitmessagesettings", "smtpdpassword", "")
logger.debug('authstring: %s / %s', correctauth, decoded)
if correctauth == decoded: if correctauth == decoded:
self.auth = True self.auth = True
self.push('235 2.7.0 Authentication successful') self.push('235 2.7.0 Authentication successful')
@ -48,22 +59,26 @@ class smtpServerChannel(smtpd.SMTPChannel):
self.push('501 Authentication fail') self.push('501 Authentication fail')
def smtp_DATA(self, arg): def smtp_DATA(self, arg):
"""Process DATA"""
if not hasattr(self, "auth") or not self.auth: if not hasattr(self, "auth") or not self.auth:
self.push ("530 Authentication required") self.push('530 Authentication required')
return return
smtpd.SMTPChannel.smtp_DATA(self, arg) smtpd.SMTPChannel.smtp_DATA(self, arg)
class smtpServerPyBitmessage(smtpd.SMTPServer): class smtpServerPyBitmessage(smtpd.SMTPServer):
"""Asyncore SMTP server class"""
def handle_accept(self): def handle_accept(self):
"""Accept a connection"""
pair = self.accept() pair = self.accept()
if pair is not None: if pair is not None:
conn, addr = pair conn, addr = pair
# print >> DEBUGSTREAM, 'Incoming connection from %s' % repr(addr) # print >> DEBUGSTREAM, 'Incoming connection from %s' % repr(addr)
self.channel = smtpServerChannel(self, conn, addr) self.channel = smtpServerChannel(self, conn, addr)
def send(self, fromAddress, toAddress, subject, message): def send(self, fromAddress, toAddress, subject, message): # pylint: disable=arguments-differ
status, addressVersionNumber, streamNumber, ripe = decodeAddress(toAddress) """Send a bitmessage"""
streamNumber, ripe = decodeAddress(toAddress)[2:]
stealthLevel = BMConfigParser().safeGetInt('bitmessagesettings', 'ackstealthlevel') stealthLevel = BMConfigParser().safeGetInt('bitmessagesettings', 'ackstealthlevel')
ackdata = genAckPayload(streamNumber, stealthLevel) ackdata = genAckPayload(streamNumber, stealthLevel)
sqlExecute( sqlExecute(
@ -75,19 +90,21 @@ class smtpServerPyBitmessage(smtpd.SMTPServer):
subject, subject,
message, message,
ackdata, ackdata,
int(time.time()), # sentTime (this will never change) int(time.time()), # sentTime (this will never change)
int(time.time()), # lastActionTime int(time.time()), # lastActionTime
0, # sleepTill time. This will get set when the POW gets done. 0, # sleepTill time. This will get set when the POW gets done.
'msgqueued', 'msgqueued',
0, # retryNumber 0, # retryNumber
'sent', # folder 'sent', # folder
2, # encodingtype 2, # encodingtype
min(BMConfigParser().getint('bitmessagesettings', 'ttl'), 86400 * 2) # not necessary to have a TTL higher than 2 days # not necessary to have a TTL higher than 2 days
min(BMConfigParser().getint('bitmessagesettings', 'ttl'), 86400 * 2)
) )
queues.workerQueue.put(('sendmessage', toAddress)) queues.workerQueue.put(('sendmessage', toAddress))
def decode_header(self, hdr): def decode_header(self, hdr):
"""Email header decoding"""
ret = [] ret = []
for h in decode_header(self.msg_headers[hdr]): for h in decode_header(self.msg_headers[hdr]):
if h[1]: if h[1]:
@ -97,37 +114,37 @@ class smtpServerPyBitmessage(smtpd.SMTPServer):
return ret return ret
def process_message(self, peer, mailfrom, rcpttos, data): # pylint: disable=too-many-locals, too-many-branches
def process_message(self, peer, mailfrom, rcpttos, data): """Process an email"""
# print 'Receiving message from:', peer # print 'Receiving message from:', peer
p = re.compile(".*<([^>]+)>") p = re.compile(".*<([^>]+)>")
if not hasattr(self.channel, "auth") or not self.channel.auth: if not hasattr(self.channel, "auth") or not self.channel.auth:
logger.error("Missing or invalid auth") logger.error('Missing or invalid auth')
return return
try: try:
self.msg_headers = Parser().parsestr(data) self.msg_headers = Parser().parsestr(data)
except: except:
logger.error("Invalid headers") logger.error('Invalid headers')
return return
try: try:
sender, domain = p.sub(r'\1', mailfrom).split("@") sender, domain = p.sub(r'\1', mailfrom).split("@")
if domain != SMTPDOMAIN: if domain != SMTPDOMAIN:
raise Exception("Bad domain %s", domain) raise Exception("Bad domain %s" % domain)
if sender not in BMConfigParser().addresses(): if sender not in BMConfigParser().addresses():
raise Exception("Nonexisting user %s", sender) raise Exception("Nonexisting user %s" % sender)
except Exception as err: except Exception as err:
logger.debug("Bad envelope from %s: %s", mailfrom, repr(err)) logger.debug('Bad envelope from %s: %r', mailfrom, err)
msg_from = self.decode_header("from") msg_from = self.decode_header("from")
try: try:
msg_from = p.sub(r'\1', self.decode_header("from")[0]) msg_from = p.sub(r'\1', self.decode_header("from")[0])
sender, domain = msg_from.split("@") sender, domain = msg_from.split("@")
if domain != SMTPDOMAIN: if domain != SMTPDOMAIN:
raise Exception("Bad domain %s", domain) raise Exception("Bad domain %s" % domain)
if sender not in BMConfigParser().addresses(): if sender not in BMConfigParser().addresses():
raise Exception("Nonexisting user %s", sender) raise Exception("Nonexisting user %s" % sender)
except Exception as err: except Exception as err:
logger.error("Bad headers from %s: %s", msg_from, repr(err)) logger.error('Bad headers from %s: %r', msg_from, err)
return return
try: try:
@ -145,18 +162,20 @@ class smtpServerPyBitmessage(smtpd.SMTPServer):
try: try:
rcpt, domain = p.sub(r'\1', to).split("@") rcpt, domain = p.sub(r'\1', to).split("@")
if domain != SMTPDOMAIN: if domain != SMTPDOMAIN:
raise Exception("Bad domain %s", domain) raise Exception("Bad domain %s" % domain)
logger.debug("Sending %s to %s about %s", sender, rcpt, msg_subject) logger.debug(
'Sending %s to %s about %s', sender, rcpt, msg_subject)
self.send(sender, rcpt, msg_subject, body) self.send(sender, rcpt, msg_subject, body)
logger.info("Relayed %s to %s", sender, rcpt) logger.info('Relayed %s to %s', sender, rcpt)
except Exception as err: except Exception as err:
logger.error( "Bad to %s: %s", to, repr(err)) logger.error('Bad to %s: %r', to, err)
continue continue
return return
class smtpServer(StoppableThread): class smtpServer(StoppableThread):
def __init__(self, parent=None): """SMTP server thread"""
def __init__(self, _=None):
super(smtpServer, self).__init__(name="smtpServerThread") super(smtpServer, self).__init__(name="smtpServerThread")
self.server = smtpServerPyBitmessage(('127.0.0.1', LISTENPORT), None) self.server = smtpServerPyBitmessage(('127.0.0.1', LISTENPORT), None)
@ -168,21 +187,26 @@ class smtpServer(StoppableThread):
def run(self): def run(self):
asyncore.loop(1) asyncore.loop(1)
def signals(signal, frame):
print "Got signal, terminating" def signals(_, __):
"""Signal handler"""
logger.warning('Got signal, terminating')
for thread in threading.enumerate(): for thread in threading.enumerate():
if thread.isAlive() and isinstance(thread, StoppableThread): if thread.isAlive() and isinstance(thread, StoppableThread):
thread.stopThread() thread.stopThread()
def runServer(): def runServer():
print "Running SMTPd thread" """Run SMTP server as a standalone python process"""
logger.warning('Running SMTPd thread')
smtpThread = smtpServer() smtpThread = smtpServer()
smtpThread.start() smtpThread.start()
signal.signal(signal.SIGINT, signals) signal.signal(signal.SIGINT, signals)
signal.signal(signal.SIGTERM, signals) signal.signal(signal.SIGTERM, signals)
print "Processing" logger.warning('Processing')
smtpThread.join() smtpThread.join()
print "The end" logger.warning('The end')
if __name__ == "__main__": if __name__ == "__main__":
runServer() runServer()

279
src/class_sqlThread.py
View File

@ -1,29 +1,34 @@
"""
sqlThread is defined here
"""
import threading import threading
from bmconfigparser import BMConfigParser from bmconfigparser import BMConfigParser
import sqlite3 import sqlite3
import time import time
import shutil # used for moving the messages.dat file import shutil # used for moving the messages.dat file
import sys import sys
import os import os
from debug import logger from debug import logger
import helper_sql import helper_sql
import helper_startup import helper_startup
import paths import paths
import queues import queues
import state import state
import tr import tr
# pylint: disable=attribute-defined-outside-init,protected-access
# This thread exists because SQLITE3 is so un-threadsafe that we must
# submit queries to it and it puts results back in a different queue. They
# won't let us just use locks.
class sqlThread(threading.Thread): class sqlThread(threading.Thread):
"""A thread for all SQL operations"""
def __init__(self): def __init__(self):
threading.Thread.__init__(self, name="SQL") threading.Thread.__init__(self, name="SQL")
def run(self): def run(self): # pylint: disable=too-many-locals, too-many-branches, too-many-statements
"""Process SQL queries from `.helper_sql.sqlSubmitQueue`"""
self.conn = sqlite3.connect(state.appdata + 'messages.dat') self.conn = sqlite3.connect(state.appdata + 'messages.dat')
self.conn.text_factory = str self.conn.text_factory = str
self.cur = self.conn.cursor() self.cur = self.conn.cursor()
@ -32,30 +37,38 @@ class sqlThread(threading.Thread):
try: try:
self.cur.execute( self.cur.execute(
'''CREATE TABLE inbox (msgid blob, toaddress text, fromaddress text, subject text, received text, message text, folder text, encodingtype int, read bool, sighash blob, UNIQUE(msgid) ON CONFLICT REPLACE)''' ) '''CREATE TABLE inbox (msgid blob, toaddress text, fromaddress text, subject text,'''
''' received text, message text, folder text, encodingtype int, read bool, sighash blob,'''
''' UNIQUE(msgid) ON CONFLICT REPLACE)''')
self.cur.execute( self.cur.execute(
'''CREATE TABLE sent (msgid blob, toaddress text, toripe blob, fromaddress text, subject text, message text, ackdata blob, senttime integer, lastactiontime integer, sleeptill integer, status text, retrynumber integer, folder text, encodingtype int, ttl int)''' ) '''CREATE TABLE sent (msgid blob, toaddress text, toripe blob, fromaddress text, subject text,'''
''' message text, ackdata blob, senttime integer, lastactiontime integer,'''
''' sleeptill integer, status text, retrynumber integer, folder text, encodingtype int, ttl int)''')
self.cur.execute( self.cur.execute(
'''CREATE TABLE subscriptions (label text, address text, enabled bool)''' ) '''CREATE TABLE subscriptions (label text, address text, enabled bool)''')
self.cur.execute( self.cur.execute(
'''CREATE TABLE addressbook (label text, address text)''' ) '''CREATE TABLE addressbook (label text, address text)''')
self.cur.execute( self.cur.execute(
'''CREATE TABLE blacklist (label text, address text, enabled bool)''' ) '''CREATE TABLE blacklist (label text, address text, enabled bool)''')
self.cur.execute( self.cur.execute(
'''CREATE TABLE whitelist (label text, address text, enabled bool)''' ) '''CREATE TABLE whitelist (label text, address text, enabled bool)''')
self.cur.execute( self.cur.execute(
'''CREATE TABLE pubkeys (address text, addressversion int, transmitdata blob, time int, usedpersonally text, UNIQUE(address) ON CONFLICT REPLACE)''' ) '''CREATE TABLE pubkeys (address text, addressversion int, transmitdata blob, time int,'''
''' usedpersonally text, UNIQUE(address) ON CONFLICT REPLACE)''')
self.cur.execute( self.cur.execute(
'''CREATE TABLE inventory (hash blob, objecttype int, streamnumber int, payload blob, expirestime integer, tag blob, UNIQUE(hash) ON CONFLICT REPLACE)''' ) '''CREATE TABLE inventory (hash blob, objecttype int, streamnumber int, payload blob,'''
''' expirestime integer, tag blob, UNIQUE(hash) ON CONFLICT REPLACE)''')
self.cur.execute( self.cur.execute(
'''INSERT INTO subscriptions VALUES('Bitmessage new releases/announcements','BM-GtovgYdgs7qXPkoYaRgrLFuFKz1SFpsw',1)''') '''INSERT INTO subscriptions VALUES'''
'''('Bitmessage new releases/announcements','BM-GtovgYdgs7qXPkoYaRgrLFuFKz1SFpsw',1)''')
self.cur.execute( self.cur.execute(
'''CREATE TABLE settings (key blob, value blob, UNIQUE(key) ON CONFLICT REPLACE)''' ) '''CREATE TABLE settings (key blob, value blob, UNIQUE(key) ON CONFLICT REPLACE)''')
self.cur.execute( '''INSERT INTO settings VALUES('version','10')''') self.cur.execute('''INSERT INTO settings VALUES('version','10')''')
self.cur.execute( '''INSERT INTO settings VALUES('lastvacuumtime',?)''', ( self.cur.execute('''INSERT INTO settings VALUES('lastvacuumtime',?)''', (
int(time.time()),)) int(time.time()),))
self.cur.execute( self.cur.execute(
'''CREATE TABLE objectprocessorqueue (objecttype int, data blob, UNIQUE(objecttype, data) ON CONFLICT REPLACE)''' ) '''CREATE TABLE objectprocessorqueue'''
''' (objecttype int, data blob, UNIQUE(objecttype, data) ON CONFLICT REPLACE)''')
self.conn.commit() self.conn.commit()
logger.info('Created messages database file') logger.info('Created messages database file')
except Exception as err: except Exception as err:
@ -120,33 +133,38 @@ class sqlThread(threading.Thread):
logger.debug( logger.debug(
"In messages.dat database, creating new 'settings' table.") "In messages.dat database, creating new 'settings' table.")
self.cur.execute( self.cur.execute(
'''CREATE TABLE settings (key text, value blob, UNIQUE(key) ON CONFLICT REPLACE)''' ) '''CREATE TABLE settings (key text, value blob, UNIQUE(key) ON CONFLICT REPLACE)''')
self.cur.execute( '''INSERT INTO settings VALUES('version','1')''') self.cur.execute('''INSERT INTO settings VALUES('version','1')''')
self.cur.execute( '''INSERT INTO settings VALUES('lastvacuumtime',?)''', ( self.cur.execute('''INSERT INTO settings VALUES('lastvacuumtime',?)''', (
int(time.time()),)) int(time.time()),))
logger.debug('In messages.dat database, removing an obsolete field from the pubkeys table.') logger.debug('In messages.dat database, removing an obsolete field from the pubkeys table.')
self.cur.execute( self.cur.execute(
'''CREATE TEMPORARY TABLE pubkeys_backup(hash blob, transmitdata blob, time int, usedpersonally text, UNIQUE(hash) ON CONFLICT REPLACE);''') '''CREATE TEMPORARY TABLE pubkeys_backup(hash blob, transmitdata blob, time int,'''
''' usedpersonally text, UNIQUE(hash) ON CONFLICT REPLACE);''')
self.cur.execute( self.cur.execute(
'''INSERT INTO pubkeys_backup SELECT hash, transmitdata, time, usedpersonally FROM pubkeys;''') '''INSERT INTO pubkeys_backup SELECT hash, transmitdata, time, usedpersonally FROM pubkeys;''')
self.cur.execute( '''DROP TABLE pubkeys''') self.cur.execute('''DROP TABLE pubkeys''')
self.cur.execute( self.cur.execute(
'''CREATE TABLE pubkeys (hash blob, transmitdata blob, time int, usedpersonally text, UNIQUE(hash) ON CONFLICT REPLACE)''' ) '''CREATE TABLE pubkeys'''
''' (hash blob, transmitdata blob, time int, usedpersonally text, UNIQUE(hash) ON CONFLICT REPLACE)''')
self.cur.execute( self.cur.execute(
'''INSERT INTO pubkeys SELECT hash, transmitdata, time, usedpersonally FROM pubkeys_backup;''') '''INSERT INTO pubkeys SELECT hash, transmitdata, time, usedpersonally FROM pubkeys_backup;''')
self.cur.execute( '''DROP TABLE pubkeys_backup;''') self.cur.execute('''DROP TABLE pubkeys_backup;''')
logger.debug('Deleting all pubkeys from inventory. They will be redownloaded and then saved with the correct times.') logger.debug(
'Deleting all pubkeys from inventory.'
' They will be redownloaded and then saved with the correct times.')
self.cur.execute( self.cur.execute(
'''delete from inventory where objecttype = 'pubkey';''') '''delete from inventory where objecttype = 'pubkey';''')
logger.debug('replacing Bitmessage announcements mailing list with a new one.') logger.debug('replacing Bitmessage announcements mailing list with a new one.')
self.cur.execute( self.cur.execute(
'''delete from subscriptions where address='BM-BbkPSZbzPwpVcYZpU4yHwf9ZPEapN5Zx' ''') '''delete from subscriptions where address='BM-BbkPSZbzPwpVcYZpU4yHwf9ZPEapN5Zx' ''')
self.cur.execute( self.cur.execute(
'''INSERT INTO subscriptions VALUES('Bitmessage new releases/announcements','BM-GtovgYdgs7qXPkoYaRgrLFuFKz1SFpsw',1)''') '''INSERT INTO subscriptions VALUES'''
'''('Bitmessage new releases/announcements','BM-GtovgYdgs7qXPkoYaRgrLFuFKz1SFpsw',1)''')
logger.debug('Commiting.') logger.debug('Commiting.')
self.conn.commit() self.conn.commit()
logger.debug('Vacuuming message.dat. You might notice that the file size gets much smaller.') logger.debug('Vacuuming message.dat. You might notice that the file size gets much smaller.')
self.cur.execute( ''' VACUUM ''') self.cur.execute(''' VACUUM ''')
# After code refactoring, the possible status values for sent messages # After code refactoring, the possible status values for sent messages
# have changed. # have changed.
@ -170,15 +188,21 @@ class sqlThread(threading.Thread):
'In messages.dat database, removing an obsolete field from' 'In messages.dat database, removing an obsolete field from'
' the inventory table.') ' the inventory table.')
self.cur.execute( self.cur.execute(
'''CREATE TEMPORARY TABLE inventory_backup(hash blob, objecttype text, streamnumber int, payload blob, receivedtime integer, UNIQUE(hash) ON CONFLICT REPLACE);''') '''CREATE TEMPORARY TABLE inventory_backup'''
'''(hash blob, objecttype text, streamnumber int, payload blob,'''
''' receivedtime integer, UNIQUE(hash) ON CONFLICT REPLACE);''')
self.cur.execute( self.cur.execute(
'''INSERT INTO inventory_backup SELECT hash, objecttype, streamnumber, payload, receivedtime FROM inventory;''') '''INSERT INTO inventory_backup SELECT hash, objecttype, streamnumber, payload, receivedtime'''
self.cur.execute( '''DROP TABLE inventory''') ''' FROM inventory;''')
self.cur.execute('''DROP TABLE inventory''')
self.cur.execute( self.cur.execute(
'''CREATE TABLE inventory (hash blob, objecttype text, streamnumber int, payload blob, receivedtime integer, UNIQUE(hash) ON CONFLICT REPLACE)''' ) '''CREATE TABLE inventory'''
''' (hash blob, objecttype text, streamnumber int, payload blob, receivedtime integer,'''
''' UNIQUE(hash) ON CONFLICT REPLACE)''')
self.cur.execute( self.cur.execute(
'''INSERT INTO inventory SELECT hash, objecttype, streamnumber, payload, receivedtime FROM inventory_backup;''') '''INSERT INTO inventory SELECT hash, objecttype, streamnumber, payload, receivedtime'''
self.cur.execute( '''DROP TABLE inventory_backup;''') ''' FROM inventory_backup;''')
self.cur.execute('''DROP TABLE inventory_backup;''')
item = '''update settings set value=? WHERE key='version';''' item = '''update settings set value=? WHERE key='version';'''
parameters = (3,) parameters = (3,)
self.cur.execute(item, parameters) self.cur.execute(item, parameters)
@ -208,7 +232,8 @@ class sqlThread(threading.Thread):
if currentVersion == 4: if currentVersion == 4:
self.cur.execute('''DROP TABLE pubkeys''') self.cur.execute('''DROP TABLE pubkeys''')
self.cur.execute( self.cur.execute(
'''CREATE TABLE pubkeys (hash blob, addressversion int, transmitdata blob, time int, usedpersonally text, UNIQUE(hash, addressversion) ON CONFLICT REPLACE)''') '''CREATE TABLE pubkeys (hash blob, addressversion int, transmitdata blob, time int,'''
'''usedpersonally text, UNIQUE(hash, addressversion) ON CONFLICT REPLACE)''')
self.cur.execute( self.cur.execute(
'''delete from inventory where objecttype = 'pubkey';''') '''delete from inventory where objecttype = 'pubkey';''')
item = '''update settings set value=? WHERE key='version';''' item = '''update settings set value=? WHERE key='version';'''
@ -224,7 +249,8 @@ class sqlThread(threading.Thread):
if currentVersion == 5: if currentVersion == 5:
self.cur.execute('''DROP TABLE knownnodes''') self.cur.execute('''DROP TABLE knownnodes''')
self.cur.execute( self.cur.execute(
'''CREATE TABLE objectprocessorqueue (objecttype text, data blob, UNIQUE(objecttype, data) ON CONFLICT REPLACE)''') '''CREATE TABLE objectprocessorqueue'''
''' (objecttype text, data blob, UNIQUE(objecttype, data) ON CONFLICT REPLACE)''')
item = '''update settings set value=? WHERE key='version';''' item = '''update settings set value=? WHERE key='version';'''
parameters = (6,) parameters = (6,)
self.cur.execute(item, parameters) self.cur.execute(item, parameters)
@ -240,10 +266,15 @@ class sqlThread(threading.Thread):
logger.debug( logger.debug(
'In messages.dat database, dropping and recreating' 'In messages.dat database, dropping and recreating'
' the inventory table.') ' the inventory table.')
self.cur.execute( '''DROP TABLE inventory''') self.cur.execute('''DROP TABLE inventory''')
self.cur.execute( '''CREATE TABLE inventory (hash blob, objecttype int, streamnumber int, payload blob, expirestime integer, tag blob, UNIQUE(hash) ON CONFLICT REPLACE)''' ) self.cur.execute(
self.cur.execute( '''DROP TABLE objectprocessorqueue''') '''CREATE TABLE inventory'''
self.cur.execute( '''CREATE TABLE objectprocessorqueue (objecttype int, data blob, UNIQUE(objecttype, data) ON CONFLICT REPLACE)''' ) ''' (hash blob, objecttype int, streamnumber int, payload blob, expirestime integer,'''
''' tag blob, UNIQUE(hash) ON CONFLICT REPLACE)''')
self.cur.execute('''DROP TABLE objectprocessorqueue''')
self.cur.execute(
'''CREATE TABLE objectprocessorqueue'''
''' (objecttype int, data blob, UNIQUE(objecttype, data) ON CONFLICT REPLACE)''')
item = '''update settings set value=? WHERE key='version';''' item = '''update settings set value=? WHERE key='version';'''
parameters = (7,) parameters = (7,)
self.cur.execute(item, parameters) self.cur.execute(item, parameters)
@ -305,15 +336,24 @@ class sqlThread(threading.Thread):
' fields into the retrynumber field and adding the' ' fields into the retrynumber field and adding the'
' sleeptill and ttl fields...') ' sleeptill and ttl fields...')
self.cur.execute( self.cur.execute(
'''CREATE TEMPORARY TABLE sent_backup (msgid blob, toaddress text, toripe blob, fromaddress text, subject text, message text, ackdata blob, lastactiontime integer, status text, retrynumber integer, folder text, encodingtype int)''' ) '''CREATE TEMPORARY TABLE sent_backup'''
''' (msgid blob, toaddress text, toripe blob, fromaddress text, subject text, message text,'''
''' ackdata blob, lastactiontime integer, status text, retrynumber integer,'''
''' folder text, encodingtype int)''')
self.cur.execute( self.cur.execute(
'''INSERT INTO sent_backup SELECT msgid, toaddress, toripe, fromaddress, subject, message, ackdata, lastactiontime, status, 0, folder, encodingtype FROM sent;''') '''INSERT INTO sent_backup SELECT msgid, toaddress, toripe, fromaddress,'''
self.cur.execute( '''DROP TABLE sent''') ''' subject, message, ackdata, lastactiontime,'''
''' status, 0, folder, encodingtype FROM sent;''')
self.cur.execute('''DROP TABLE sent''')
self.cur.execute( self.cur.execute(
'''CREATE TABLE sent (msgid blob, toaddress text, toripe blob, fromaddress text, subject text, message text, ackdata blob, senttime integer, lastactiontime integer, sleeptill int, status text, retrynumber integer, folder text, encodingtype int, ttl int)''' ) '''CREATE TABLE sent'''
''' (msgid blob, toaddress text, toripe blob, fromaddress text, subject text, message text,'''
''' ackdata blob, senttime integer, lastactiontime integer, sleeptill int, status text,'''
''' retrynumber integer, folder text, encodingtype int, ttl int)''')
self.cur.execute( self.cur.execute(
'''INSERT INTO sent SELECT msgid, toaddress, toripe, fromaddress, subject, message, ackdata, lastactiontime, lastactiontime, 0, status, 0, folder, encodingtype, 216000 FROM sent_backup;''') '''INSERT INTO sent SELECT msgid, toaddress, toripe, fromaddress, subject, message, ackdata,'''
self.cur.execute( '''DROP TABLE sent_backup''') ''' lastactiontime, lastactiontime, 0, status, 0, folder, encodingtype, 216000 FROM sent_backup;''')
self.cur.execute('''DROP TABLE sent_backup''')
logger.info('In messages.dat database, finished making TTL-related changes.') logger.info('In messages.dat database, finished making TTL-related changes.')
logger.debug('In messages.dat database, adding address field to the pubkeys table.') logger.debug('In messages.dat database, adding address field to the pubkeys table.')
# We're going to have to calculate the address for each row in the pubkeys # We're going to have to calculate the address for each row in the pubkeys
@ -330,16 +370,24 @@ class sqlThread(threading.Thread):
self.cur.execute(item, parameters) self.cur.execute(item, parameters)
# Now we can remove the hash field from the pubkeys table. # Now we can remove the hash field from the pubkeys table.
self.cur.execute( self.cur.execute(
'''CREATE TEMPORARY TABLE pubkeys_backup (address text, addressversion int, transmitdata blob, time int, usedpersonally text, UNIQUE(address) ON CONFLICT REPLACE)''' ) '''CREATE TEMPORARY TABLE pubkeys_backup'''
''' (address text, addressversion int, transmitdata blob, time int,'''
''' usedpersonally text, UNIQUE(address) ON CONFLICT REPLACE)''')
self.cur.execute( self.cur.execute(
'''INSERT INTO pubkeys_backup SELECT address, addressversion, transmitdata, time, usedpersonally FROM pubkeys;''') '''INSERT INTO pubkeys_backup'''
self.cur.execute( '''DROP TABLE pubkeys''') ''' SELECT address, addressversion, transmitdata, time, usedpersonally FROM pubkeys;''')
self.cur.execute('''DROP TABLE pubkeys''')
self.cur.execute( self.cur.execute(
'''CREATE TABLE pubkeys (address text, addressversion int, transmitdata blob, time int, usedpersonally text, UNIQUE(address) ON CONFLICT REPLACE)''' ) '''CREATE TABLE pubkeys'''
''' (address text, addressversion int, transmitdata blob, time int, usedpersonally text,'''
''' UNIQUE(address) ON CONFLICT REPLACE)''')
self.cur.execute( self.cur.execute(
'''INSERT INTO pubkeys SELECT address, addressversion, transmitdata, time, usedpersonally FROM pubkeys_backup;''') '''INSERT INTO pubkeys SELECT'''
self.cur.execute( '''DROP TABLE pubkeys_backup''') ''' address, addressversion, transmitdata, time, usedpersonally FROM pubkeys_backup;''')
logger.debug('In messages.dat database, done adding address field to the pubkeys table and removing the hash field.') self.cur.execute('''DROP TABLE pubkeys_backup''')
logger.debug(
'In messages.dat database, done adding address field to the pubkeys table'
' and removing the hash field.')
self.cur.execute('''update settings set value=10 WHERE key='version';''') self.cur.execute('''update settings set value=10 WHERE key='version';''')
# Are you hoping to add a new option to the keys.dat file of existing # Are you hoping to add a new option to the keys.dat file of existing
@ -349,7 +397,7 @@ class sqlThread(threading.Thread):
try: try:
testpayload = '\x00\x00' testpayload = '\x00\x00'
t = ('1234', 1, testpayload, '12345678', 'no') t = ('1234', 1, testpayload, '12345678', 'no')
self.cur.execute( '''INSERT INTO pubkeys VALUES(?,?,?,?,?)''', t) self.cur.execute('''INSERT INTO pubkeys VALUES(?,?,?,?,?)''', t)
self.conn.commit() self.conn.commit()
self.cur.execute( self.cur.execute(
'''SELECT transmitdata FROM pubkeys WHERE address='1234' ''') '''SELECT transmitdata FROM pubkeys WHERE address='1234' ''')
@ -359,13 +407,29 @@ class sqlThread(threading.Thread):
self.cur.execute('''DELETE FROM pubkeys WHERE address='1234' ''') self.cur.execute('''DELETE FROM pubkeys WHERE address='1234' ''')
self.conn.commit() self.conn.commit()
if transmitdata == '': if transmitdata == '':
logger.fatal('Problem: The version of SQLite you have cannot store Null values. Please download and install the latest revision of your version of Python (for example, the latest Python 2.7 revision) and try again.\n') logger.fatal(
logger.fatal('PyBitmessage will now exit very abruptly. You may now see threading errors related to this abrupt exit but the problem you need to solve is related to SQLite.\n\n') 'Problem: The version of SQLite you have cannot store Null values.'
' Please download and install the latest revision of your version of Python'
' (for example, the latest Python 2.7 revision) and try again.\n')
logger.fatal(
'PyBitmessage will now exit very abruptly.'
' You may now see threading errors related to this abrupt exit'
' but the problem you need to solve is related to SQLite.\n\n')
os._exit(0) os._exit(0)
except Exception as err: except Exception as err:
if str(err) == 'database or disk is full': if str(err) == 'database or disk is full':
logger.fatal('(While null value test) Alert: Your disk or data storage volume is full. sqlThread will now exit.') logger.fatal(
queues.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True))) '(While null value test) Alert: Your disk or data storage volume is full.'
' sqlThread will now exit.')
queues.UISignalQueue.put((
'alert', (
tr._translate(
"MainWindow",
"Disk full"),
tr._translate(
"MainWindow",
'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'),
True)))
os._exit(0) os._exit(0)
else: else:
logger.error(err) logger.error(err)
@ -381,11 +445,21 @@ class sqlThread(threading.Thread):
if int(value) < int(time.time()) - 86400: if int(value) < int(time.time()) - 86400:
logger.info('It has been a long time since the messages.dat file has been vacuumed. Vacuuming now...') logger.info('It has been a long time since the messages.dat file has been vacuumed. Vacuuming now...')
try: try:
self.cur.execute( ''' VACUUM ''') self.cur.execute(''' VACUUM ''')
except Exception as err: except Exception as err:
if str(err) == 'database or disk is full': if str(err) == 'database or disk is full':
logger.fatal('(While VACUUM) Alert: Your disk or data storage volume is full. sqlThread will now exit.') logger.fatal(
queues.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True))) '(While VACUUM) Alert: Your disk or data storage volume is full.'
' sqlThread will now exit.')
queues.UISignalQueue.put((
'alert', (
tr._translate(
"MainWindow",
"Disk full"),
tr._translate(
"MainWindow",
'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'),
True)))
os._exit(0) os._exit(0)
item = '''update settings set value=? WHERE key='lastvacuumtime';''' item = '''update settings set value=? WHERE key='lastvacuumtime';'''
parameters = (int(time.time()),) parameters = (int(time.time()),)
@ -400,8 +474,18 @@ class sqlThread(threading.Thread):
self.conn.commit() self.conn.commit()
except Exception as err: except Exception as err:
if str(err) == 'database or disk is full': if str(err) == 'database or disk is full':
logger.fatal('(While committing) Alert: Your disk or data storage volume is full. sqlThread will now exit.') logger.fatal(
queues.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True))) '(While committing) Alert: Your disk or data storage volume is full.'
' sqlThread will now exit.')
queues.UISignalQueue.put((
'alert', (
tr._translate(
"MainWindow",
"Disk full"),
tr._translate(
"MainWindow",
'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'),
True)))
os._exit(0) os._exit(0)
elif item == 'exit': elif item == 'exit':
self.conn.close() self.conn.close()
@ -415,8 +499,18 @@ class sqlThread(threading.Thread):
self.conn.commit() self.conn.commit()
except Exception as err: except Exception as err:
if str(err) == 'database or disk is full': if str(err) == 'database or disk is full':
logger.fatal('(while movemessagstoprog) Alert: Your disk or data storage volume is full. sqlThread will now exit.') logger.fatal(
queues.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True))) '(while movemessagstoprog) Alert: Your disk or data storage volume is full.'
' sqlThread will now exit.')
queues.UISignalQueue.put((
'alert', (
tr._translate(
"MainWindow",
"Disk full"),
tr._translate(
"MainWindow",
'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'),
True)))
os._exit(0) os._exit(0)
self.conn.close() self.conn.close()
shutil.move( shutil.move(
@ -431,8 +525,18 @@ class sqlThread(threading.Thread):
self.conn.commit() self.conn.commit()
except Exception as err: except Exception as err:
if str(err) == 'database or disk is full': if str(err) == 'database or disk is full':
logger.fatal('(while movemessagstoappdata) Alert: Your disk or data storage volume is full. sqlThread will now exit.') logger.fatal(
queues.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True))) '(while movemessagstoappdata) Alert: Your disk or data storage volume is full.'
' sqlThread will now exit.')
queues.UISignalQueue.put((
'alert', (
tr._translate(
"MainWindow",
"Disk full"),
tr._translate(
"MainWindow",
'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'),
True)))
os._exit(0) os._exit(0)
self.conn.close() self.conn.close()
shutil.move( shutil.move(
@ -445,11 +549,21 @@ class sqlThread(threading.Thread):
self.cur.execute('''delete from sent where folder='trash' ''') self.cur.execute('''delete from sent where folder='trash' ''')
self.conn.commit() self.conn.commit()
try: try:
self.cur.execute( ''' VACUUM ''') self.cur.execute(''' VACUUM ''')
except Exception as err: except Exception as err:
if str(err) == 'database or disk is full': if str(err) == 'database or disk is full':
logger.fatal('(while deleteandvacuume) Alert: Your disk or data storage volume is full. sqlThread will now exit.') logger.fatal(
queues.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True))) '(while deleteandvacuume) Alert: Your disk or data storage volume is full.'
' sqlThread will now exit.')
queues.UISignalQueue.put((
'alert', (
tr._translate(
"MainWindow",
"Disk full"),
tr._translate(
"MainWindow",
'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'),
True)))
os._exit(0) os._exit(0)
else: else:
parameters = helper_sql.sqlSubmitQueue.get() parameters = helper_sql.sqlSubmitQueue.get()
@ -461,11 +575,30 @@ class sqlThread(threading.Thread):
rowcount = self.cur.rowcount rowcount = self.cur.rowcount
except Exception as err: except Exception as err:
if str(err) == 'database or disk is full': if str(err) == 'database or disk is full':
logger.fatal('(while cur.execute) Alert: Your disk or data storage volume is full. sqlThread will now exit.') logger.fatal(
queues.UISignalQueue.put(('alert', (tr._translate("MainWindow", "Disk full"), tr._translate("MainWindow", 'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'), True))) '(while cur.execute) Alert: Your disk or data storage volume is full.'
' sqlThread will now exit.')
queues.UISignalQueue.put((
'alert', (
tr._translate(
"MainWindow",
"Disk full"),
tr._translate(
"MainWindow",
'Alert: Your disk or data storage volume is full. Bitmessage will now exit.'),
True)))
os._exit(0) os._exit(0)
else: else:
logger.fatal('Major error occurred when trying to execute a SQL statement within the sqlThread. Please tell Atheros about this error message or post it in the forum! Error occurred while trying to execute statement: "%s" Here are the parameters; you might want to censor this data with asterisks (***) as it can contain private information: %s. Here is the actual error message thrown by the sqlThread: %s', str(item), str(repr(parameters)), str(err)) logger.fatal(
'Major error occurred when trying to execute a SQL statement within the sqlThread.'
' Please tell Atheros about this error message or post it in the forum!'
' Error occurred while trying to execute statement: "%s" Here are the parameters;'
' you might want to censor this data with asterisks (***)'
' as it can contain private information: %s.'
' Here is the actual error message thrown by the sqlThread: %s',
str(item),
str(repr(parameters)),
str(err))
logger.fatal('This program shall now abruptly exit!') logger.fatal('This program shall now abruptly exit!')
os._exit(0) os._exit(0)

93
src/debug.py
View File

@ -1,26 +1,38 @@
""" """
Logging and debuging facility Logging and debuging facility
============================= -----------------------------
Levels: Levels:
DEBUG DEBUG
Detailed information, typically of interest only when diagnosing problems. Detailed information, typically of interest only when diagnosing problems.
INFO INFO
Confirmation that things are working as expected. Confirmation that things are working as expected.
WARNING WARNING
An indication that something unexpected happened, or indicative of some problem in the An indication that something unexpected happened, or indicative of
near future (e.g. 'disk space low'). The software is still working as expected. some problem in the near future (e.g. 'disk space low'). The software
ERROR is still working as expected.
Due to a more serious problem, the software has not been able to perform some function. ERROR
CRITICAL Due to a more serious problem, the software has not been able to
A serious error, indicating that the program itself may be unable to continue running. perform some function.
CRITICAL
A serious error, indicating that the program itself may be unable to
continue running.
There are three loggers: `console_only`, `file_only` and `both`. There are three loggers by default: `console_only`, `file_only` and `both`.
You can configure logging in the logging.dat in the appdata dir.
It's format is described in the :func:`logging.config.fileConfig` doc.
Use: `from debug import logger` to import this facility into whatever module you wish to log messages from. Use:
Logging is thread-safe so you don't have to worry about locks, just import and log.
>>> import logging
>>> logger = logging.getLogger('default')
The old form: ``from debug import logger`` is also may be used,
but only in the top level modules.
Logging is thread-safe so you don't have to worry about locks,
just import and log.
""" """
import ConfigParser import ConfigParser
@ -28,6 +40,7 @@ import logging
import logging.config import logging.config
import os import os
import sys import sys
import helper_startup import helper_startup
import state import state
@ -40,14 +53,22 @@ log_level = 'WARNING'
def log_uncaught_exceptions(ex_cls, ex, tb): def log_uncaught_exceptions(ex_cls, ex, tb):
"""The last resort logging function used for sys.excepthook"""
logging.critical('Unhandled exception', exc_info=(ex_cls, ex, tb)) logging.critical('Unhandled exception', exc_info=(ex_cls, ex, tb))
def configureLogging(): def configureLogging():
"""
Configure logging,
using either logging.dat file in the state.appdata dir
or dictionary with hardcoded settings.
"""
sys.excepthook = log_uncaught_exceptions
fail_msg = '' fail_msg = ''
try: try:
logging_config = os.path.join(state.appdata, 'logging.dat') logging_config = os.path.join(state.appdata, 'logging.dat')
logging.config.fileConfig(logging_config) logging.config.fileConfig(
logging_config, disable_existing_loggers=False)
return ( return (
False, False,
'Loaded logger configuration from %s' % logging_config 'Loaded logger configuration from %s' % logging_config
@ -59,12 +80,11 @@ def configureLogging():
' logging config\n%s' % \ ' logging config\n%s' % \
(logging_config, sys.exc_info()) (logging_config, sys.exc_info())
else: else:
# no need to confuse the user if the logger config is missing entirely # no need to confuse the user if the logger config
# is missing entirely
fail_msg = 'Using default logger configuration' fail_msg = 'Using default logger configuration'
sys.excepthook = log_uncaught_exceptions logging_config = {
logging.config.dictConfig({
'version': 1, 'version': 1,
'formatters': { 'formatters': {
'default': { 'default': {
@ -106,34 +126,29 @@ def configureLogging():
'level': log_level, 'level': log_level,
'handlers': ['console'], 'handlers': ['console'],
}, },
}) }
logging_config['loggers']['default'] = logging_config['loggers'][
'file_only' if '-c' in sys.argv else 'both']
logging.config.dictConfig(logging_config)
return True, fail_msg return True, fail_msg
def initLogging():
preconfigured, msg = configureLogging()
if preconfigured:
if '-c' in sys.argv:
logger = logging.getLogger('file_only')
else:
logger = logging.getLogger('both')
else:
logger = logging.getLogger('default')
if msg:
logger.log(logging.WARNING if preconfigured else logging.INFO, msg)
return logger
def resetLogging(): def resetLogging():
"""Reconfigure logging in runtime when state.appdata dir changed"""
# pylint: disable=global-statement, used-before-assignment
global logger global logger
for i in logger.handlers.iterkeys(): for i in logger.handlers:
logger.removeHandler(i) logger.removeHandler(i)
i.flush() i.flush()
i.close() i.close()
logger = initLogging() configureLogging()
logger = logging.getLogger('default')
# ! # !
logger = initLogging() preconfigured, msg = configureLogging()
logger = logging.getLogger('default')
if msg:
logger.log(logging.WARNING if preconfigured else logging.INFO, msg)

22
src/defaults.py
View File

@ -1,24 +1,24 @@
""" """
src/defaults.py Common default values
===============
""" """
# sanity check, prevent doing ridiculous PoW #: sanity check, prevent doing ridiculous PoW
# 20 million PoWs equals approximately 2 days on dev's dual R9 290 #: 20 million PoWs equals approximately 2 days on dev's dual R9 290
ridiculousDifficulty = 20000000 ridiculousDifficulty = 20000000
# Remember here the RPC port read from namecoin.conf so we can restore to #: Remember here the RPC port read from namecoin.conf so we can restore to
# it as default whenever the user changes the "method" selection for #: it as default whenever the user changes the "method" selection for
# namecoin integration to "namecoind". #: namecoin integration to "namecoind".
namecoinDefaultRpcPort = "8336" namecoinDefaultRpcPort = "8336"
# If changed, these values will cause particularly unexpected behavior: # If changed, these values will cause particularly unexpected behavior:
# You won't be able to either send or receive messages because the proof # You won't be able to either send or receive messages because the proof
# of work you do (or demand) won't match that done or demanded by others. # of work you do (or demand) won't match that done or demanded by others.
# Don't change them! # Don't change them!
# The amount of work that should be performed (and demanded) per byte of the payload. #: The amount of work that should be performed (and demanded) per byte
#: of the payload.
networkDefaultProofOfWorkNonceTrialsPerByte = 1000 networkDefaultProofOfWorkNonceTrialsPerByte = 1000
# To make sending short messages a little more difficult, this value is #: To make sending short messages a little more difficult, this value is
# added to the payload length for use in calculating the proof of work #: added to the payload length for use in calculating the proof of work
# target. #: target.
networkDefaultPayloadLengthExtraBytes = 1000 networkDefaultPayloadLengthExtraBytes = 1000

23
src/depends.py
View File

@ -113,6 +113,7 @@ PACKAGES = {
def detectOS(): def detectOS():
"""Finding out what Operating System is running"""
if detectOS.result is not None: if detectOS.result is not None:
return detectOS.result return detectOS.result
if sys.platform.startswith('openbsd'): if sys.platform.startswith('openbsd'):
@ -132,6 +133,7 @@ detectOS.result = None
def detectOSRelease(): def detectOSRelease():
"""Detecting the release of OS"""
with open("/etc/os-release", 'r') as osRelease: with open("/etc/os-release", 'r') as osRelease:
version = None version = None
for line in osRelease: for line in osRelease:
@ -148,6 +150,7 @@ def detectOSRelease():
def try_import(module, log_extra=False): def try_import(module, log_extra=False):
"""Try to import the non imported packages"""
try: try:
return import_module(module) return import_module(module)
except ImportError: except ImportError:
@ -208,10 +211,8 @@ def check_sqlite():
).fetchone()[0] ).fetchone()[0]
logger.info('SQLite Library Source ID: %s', sqlite_source_id) logger.info('SQLite Library Source ID: %s', sqlite_source_id)
if sqlite_version_number >= 3006023: if sqlite_version_number >= 3006023:
compile_options = ', '.join(map( compile_options = ', '.join(
lambda row: row[0], [row[0] for row in conn.execute('PRAGMA compile_options;')])
conn.execute('PRAGMA compile_options;')
))
logger.info( logger.info(
'SQLite Library Compile Options: %s', compile_options) 'SQLite Library Compile Options: %s', compile_options)
# There is no specific version requirement as yet, so we just # There is no specific version requirement as yet, so we just
@ -230,13 +231,13 @@ def check_sqlite():
conn.close() conn.close()
def check_openssl(): def check_openssl(): # pylint: disable=too-many-branches, too-many-return-statements
"""Do openssl dependency check. """Do openssl dependency check.
Here we are checking for openssl with its all dependent libraries Here we are checking for openssl with its all dependent libraries
and version checking. and version checking.
""" """
# pylint: disable=protected-access, redefined-outer-name
ctypes = try_import('ctypes') ctypes = try_import('ctypes')
if not ctypes: if not ctypes:
logger.error('Unable to check OpenSSL.') logger.error('Unable to check OpenSSL.')
@ -303,7 +304,7 @@ def check_openssl():
' ECDH, and ECDSA enabled.') ' ECDH, and ECDSA enabled.')
return False return False
matches = cflags_regex.findall(openssl_cflags) matches = cflags_regex.findall(openssl_cflags)
if len(matches) > 0: if matches:
logger.error( logger.error(
'This OpenSSL library is missing the following required' 'This OpenSSL library is missing the following required'
' features: %s. PyBitmessage requires OpenSSL 0.9.8b' ' features: %s. PyBitmessage requires OpenSSL 0.9.8b'
@ -314,13 +315,13 @@ def check_openssl():
return False return False
# TODO: The minimum versions of pythondialog and dialog need to be determined # ..todo:: The minimum versions of pythondialog and dialog need to be determined
def check_curses(): def check_curses():
"""Do curses dependency check. """Do curses dependency check.
Here we are checking for curses if available or not with check Here we are checking for curses if available or not with check as interface
as interface requires the pythondialog\ package and the dialog requires the `pythondialog <https://pypi.org/project/pythondialog>`_ package
utility. and the dialog utility.
""" """
if sys.hexversion < 0x20600F0: if sys.hexversion < 0x20600F0:
logger.error( logger.error(

16
src/fallback/__init__.py
View File

@ -1,13 +1,19 @@
""" """
.. todo:: hello world Fallback expressions help PyBitmessage modules to run without some external
dependencies.
RIPEMD160Hash
-------------
We need to check :mod:`hashlib` for RIPEMD-160, as it won't be available
if OpenSSL is not linked against or the linked OpenSSL has RIPEMD disabled.
Try to use `pycryptodome <https://pypi.org/project/pycryptodome/>`_
in that case.
""" """
import hashlib import hashlib
# We need to check hashlib for RIPEMD-160, as it won't be available
# if OpenSSL is not linked against or the linked OpenSSL has RIPEMD
# disabled.
try: try:
hashlib.new('ripemd160') hashlib.new('ripemd160')
except ValueError: except ValueError:

32
src/helper_ackPayload.py
View File

@ -1,22 +1,28 @@
"""This module is for generating ack payload.""" """
This module is for generating ack payload
"""
from binascii import hexlify
from struct import pack
import highlevelcrypto import highlevelcrypto
import helper_random import helper_random
from binascii import hexlify
from struct import pack
from addresses import encodeVarint from addresses import encodeVarint
# This function generates payload objects for message acknowledgements
# Several stealth levels are available depending on the privacy needs;
# a higher level means better stealth, but also higher cost (size+POW)
# - level 0: a random 32-byte sequence with a message header appended
# - level 1: a getpubkey request for a (random) dummy key hash
# - level 2: a standard message, encrypted to a random pubkey
def genAckPayload(streamNumber=1, stealthLevel=0): def genAckPayload(streamNumber=1, stealthLevel=0):
"""Generate and return payload obj.""" """
if (stealthLevel == 2): # Generate privacy-enhanced payload Generate and return payload obj.
This function generates payload objects for message acknowledgements
Several stealth levels are available depending on the privacy needs;
a higher level means better stealth, but also higher cost (size+POW)
- level 0: a random 32-byte sequence with a message header appended
- level 1: a getpubkey request for a (random) dummy key hash
- level 2: a standard message, encrypted to a random pubkey
"""
if stealthLevel == 2: # Generate privacy-enhanced payload
# Generate a dummy privkey and derive the pubkey # Generate a dummy privkey and derive the pubkey
dummyPubKeyHex = highlevelcrypto.privToPub( dummyPubKeyHex = highlevelcrypto.privToPub(
hexlify(helper_random.randomBytes(32))) hexlify(helper_random.randomBytes(32)))
@ -29,7 +35,7 @@ def genAckPayload(streamNumber=1, stealthLevel=0):
acktype = 2 # message acktype = 2 # message
version = 1 version = 1
elif (stealthLevel == 1): # Basic privacy payload (random getpubkey) elif stealthLevel == 1: # Basic privacy payload (random getpubkey)
ackdata = helper_random.randomBytes(32) ackdata = helper_random.randomBytes(32)
acktype = 0 # getpubkey acktype = 0 # getpubkey
version = 4 version = 4

18
src/helper_bitcoin.py
View File

@ -1,10 +1,19 @@
"""
Calculates bitcoin and testnet address from pubkey
"""
import hashlib import hashlib
from debug import logger
from pyelliptic import arithmetic from pyelliptic import arithmetic
# This function expects that pubkey begin with \x04
def calculateBitcoinAddressFromPubkey(pubkey): def calculateBitcoinAddressFromPubkey(pubkey):
"""Calculate bitcoin address from given pubkey (65 bytes long hex string)"""
if len(pubkey) != 65: if len(pubkey) != 65:
print 'Could not calculate Bitcoin address from pubkey because function was passed a pubkey that was', len(pubkey), 'bytes long rather than 65.' logger.error('Could not calculate Bitcoin address from pubkey because'
' function was passed a pubkey that was'
' %i bytes long rather than 65.', len(pubkey))
return "error" return "error"
ripe = hashlib.new('ripemd160') ripe = hashlib.new('ripemd160')
sha = hashlib.new('sha256') sha = hashlib.new('sha256')
@ -24,8 +33,11 @@ def calculateBitcoinAddressFromPubkey(pubkey):
def calculateTestnetAddressFromPubkey(pubkey): def calculateTestnetAddressFromPubkey(pubkey):
"""This function expects that pubkey begin with the testnet prefix"""
if len(pubkey) != 65: if len(pubkey) != 65:
print 'Could not calculate Bitcoin address from pubkey because function was passed a pubkey that was', len(pubkey), 'bytes long rather than 65.' logger.error('Could not calculate Bitcoin address from pubkey because'
' function was passed a pubkey that was'
' %i bytes long rather than 65.', len(pubkey))
return "error" return "error"
ripe = hashlib.new('ripemd160') ripe = hashlib.new('ripemd160')
sha = hashlib.new('sha256') sha = hashlib.new('sha256')

7
src/helper_inbox.py
View File

@ -1,10 +1,11 @@
"""Helper Inbox performs inbox messagese related operations.""" """Helper Inbox performs inbox messages related operations"""
from helper_sql import sqlExecute, sqlQuery
import queues import queues
from helper_sql import sqlExecute, sqlQuery
def insert(t): def insert(t):
"""Perform an insert into the "inbox" table"""
sqlExecute('''INSERT INTO inbox VALUES (?,?,?,?,?,?,?,?,?,?)''', *t) sqlExecute('''INSERT INTO inbox VALUES (?,?,?,?,?,?,?,?,?,?)''', *t)
# shouldn't emit changedInboxUnread and displayNewInboxMessage # shouldn't emit changedInboxUnread and displayNewInboxMessage
# at the same time # at the same time
@ -12,11 +13,13 @@ def insert(t):
def trash(msgid): def trash(msgid):
"""Mark a message in the `inbox` as `trash`"""
sqlExecute('''UPDATE inbox SET folder='trash' WHERE msgid=?''', msgid) sqlExecute('''UPDATE inbox SET folder='trash' WHERE msgid=?''', msgid)
queues.UISignalQueue.put(('removeInboxRowByMsgid', msgid)) queues.UISignalQueue.put(('removeInboxRowByMsgid', msgid))
def isMessageAlreadyInInbox(sigHash): def isMessageAlreadyInInbox(sigHash):
"""Check for previous instances of this message"""
queryReturn = sqlQuery( queryReturn = sqlQuery(
'''SELECT COUNT(*) FROM inbox WHERE sighash=?''', sigHash) '''SELECT COUNT(*) FROM inbox WHERE sighash=?''', sigHash)
return queryReturn[0][0] != 0 return queryReturn[0][0] != 0

11
src/helper_msgcoding.py
View File

@ -25,19 +25,24 @@ BITMESSAGE_ENCODING_EXTENDED = 3
class MsgEncodeException(Exception): class MsgEncodeException(Exception):
"""Exception during message encoding"""
pass pass
class MsgDecodeException(Exception): class MsgDecodeException(Exception):
"""Exception during message decoding"""
pass pass
class DecompressionSizeException(MsgDecodeException): class DecompressionSizeException(MsgDecodeException):
# pylint: disable=super-init-not-called
"""Decompression resulted in too much data (attack protection)"""
def __init__(self, size): def __init__(self, size):
self.size = size self.size = size
class MsgEncode(object): class MsgEncode(object):
"""Message encoder class"""
def __init__(self, message, encoding=BITMESSAGE_ENCODING_SIMPLE): def __init__(self, message, encoding=BITMESSAGE_ENCODING_SIMPLE):
self.data = None self.data = None
self.encoding = encoding self.encoding = encoding
@ -52,6 +57,7 @@ class MsgEncode(object):
raise MsgEncodeException("Unknown encoding %i" % (encoding)) raise MsgEncodeException("Unknown encoding %i" % (encoding))
def encodeExtended(self, message): def encodeExtended(self, message):
"""Handle extended encoding"""
try: try:
msgObj = messagetypes.message.Message() msgObj = messagetypes.message.Message()
self.data = zlib.compress(msgpack.dumps(msgObj.encode(message)), 9) self.data = zlib.compress(msgpack.dumps(msgObj.encode(message)), 9)
@ -64,15 +70,18 @@ class MsgEncode(object):
self.length = len(self.data) self.length = len(self.data)
def encodeSimple(self, message): def encodeSimple(self, message):
"""Handle simple encoding"""
self.data = 'Subject:%(subject)s\nBody:%(body)s' % message self.data = 'Subject:%(subject)s\nBody:%(body)s' % message
self.length = len(self.data) self.length = len(self.data)
def encodeTrivial(self, message): def encodeTrivial(self, message):
"""Handle trivial encoding"""
self.data = message['body'] self.data = message['body']
self.length = len(self.data) self.length = len(self.data)
class MsgDecode(object): class MsgDecode(object):
"""Message decoder class"""
def __init__(self, encoding, data): def __init__(self, encoding, data):
self.encoding = encoding self.encoding = encoding
if self.encoding == BITMESSAGE_ENCODING_EXTENDED: if self.encoding == BITMESSAGE_ENCODING_EXTENDED:
@ -88,6 +97,7 @@ class MsgDecode(object):
self.subject = _translate("MsgDecode", "Unknown encoding") self.subject = _translate("MsgDecode", "Unknown encoding")
def decodeExtended(self, data): def decodeExtended(self, data):
"""Handle extended encoding"""
dc = zlib.decompressobj() dc = zlib.decompressobj()
tmp = "" tmp = ""
while len(tmp) <= BMConfigParser().safeGetInt("zlib", "maxsize"): while len(tmp) <= BMConfigParser().safeGetInt("zlib", "maxsize"):
@ -131,6 +141,7 @@ class MsgDecode(object):
self.body = msgObj.body self.body = msgObj.body
def decodeSimple(self, data): def decodeSimple(self, data):
"""Handle simple encoding"""
bodyPositionIndex = string.find(data, '\nBody:') bodyPositionIndex = string.find(data, '\nBody:')
if bodyPositionIndex > 1: if bodyPositionIndex > 1:
subject = data[8:bodyPositionIndex] subject = data[8:bodyPositionIndex]

3
src/helper_random.py
View File

@ -56,8 +56,7 @@ def randomrandrange(x, y=None):
""" """
if isinstance(y, NoneType): if isinstance(y, NoneType):
return random.randrange(x) # nosec return random.randrange(x) # nosec
else: return random.randrange(x, y) # nosec
return random.randrange(x, y) # nosec
def randomchoice(population): def randomchoice(population):

25
src/helper_search.py
View File

@ -1,6 +1,6 @@
#!/usr/bin/python2.7 """Additional SQL helper for searching messages"""
from helper_sql import * from helper_sql import sqlQuery
try: try:
from PyQt4 import QtGui from PyQt4 import QtGui
@ -8,13 +8,17 @@ try:
except ImportError: except ImportError:
haveQt = False haveQt = False
def search_translate (context, text):
def search_translate(context, text):
"""Translation wrapper"""
if haveQt: if haveQt:
return QtGui.QApplication.translate(context, text) return QtGui.QApplication.translate(context, text)
else: return text.lower()
return text.lower()
def search_sql(xAddress = "toaddress", account = None, folder = "inbox", where = None, what = None, unreadOnly = False):
def search_sql(xAddress="toaddress", account=None, folder="inbox", where=None, what=None, unreadOnly=False):
"""Perform a search in mailbox tables"""
# pylint: disable=too-many-arguments, too-many-branches
if what is not None and what != "": if what is not None and what != "":
what = "%" + what + "%" what = "%" + what + "%"
if where == search_translate("MainWindow", "To"): if where == search_translate("MainWindow", "To"):
@ -32,7 +36,7 @@ def search_sql(xAddress = "toaddress", account = None, folder = "inbox", where =
if folder == "sent": if folder == "sent":
sqlStatementBase = ''' sqlStatementBase = '''
SELECT toaddress, fromaddress, subject, status, ackdata, lastactiontime SELECT toaddress, fromaddress, subject, status, ackdata, lastactiontime
FROM sent ''' FROM sent '''
else: else:
sqlStatementBase = '''SELECT folder, msgid, toaddress, fromaddress, subject, received, read sqlStatementBase = '''SELECT folder, msgid, toaddress, fromaddress, subject, received, read
@ -62,13 +66,16 @@ def search_sql(xAddress = "toaddress", account = None, folder = "inbox", where =
sqlArguments.append(what) sqlArguments.append(what)
if unreadOnly: if unreadOnly:
sqlStatementParts.append("read = 0") sqlStatementParts.append("read = 0")
if len(sqlStatementParts) > 0: if sqlStatementParts:
sqlStatementBase += "WHERE " + " AND ".join(sqlStatementParts) sqlStatementBase += "WHERE " + " AND ".join(sqlStatementParts)
if folder == "sent": if folder == "sent":
sqlStatementBase += " ORDER BY lastactiontime" sqlStatementBase += " ORDER BY lastactiontime"
return sqlQuery(sqlStatementBase, sqlArguments) return sqlQuery(sqlStatementBase, sqlArguments)
def check_match(toAddress, fromAddress, subject, message, where = None, what = None):
def check_match(toAddress, fromAddress, subject, message, where=None, what=None):
"""Check if a single message matches a filter (used when new messages are added to messagelists)"""
# pylint: disable=too-many-arguments
if what is not None and what != "": if what is not None and what != "":
if where in (search_translate("MainWindow", "To"), search_translate("MainWindow", "All")): if where in (search_translate("MainWindow", "To"), search_translate("MainWindow", "All")):
if what.lower() not in toAddress.lower(): if what.lower() not in toAddress.lower():

5
src/helper_sent.py
View File

@ -1,4 +1,9 @@
"""
Insert values into sent table
"""
from helper_sql import * from helper_sql import *
def insert(t): def insert(t):
"""Perform an insert into the `sent` table"""
sqlExecute('''INSERT INTO sent VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)''', *t) sqlExecute('''INSERT INTO sent VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)''', *t)

36
src/helper_sql.py
View File

@ -1,17 +1,39 @@
"""Helper Sql performs sql operations.""" """
SQL-related functions defined here are really pass the queries (or other SQL
commands) to :class:`.threads.sqlThread` through `sqlSubmitQueue` queue and check
or return the result got from `sqlReturnQueue`.
This is done that way because :mod:`sqlite3` is so thread-unsafe that they
won't even let you call it from different threads using your own locks.
SQLite objects can only be used from one thread.
.. note:: This actually only applies for certain deployments, and/or
really old version of sqlite. I haven't actually seen it anywhere.
Current versions do have support for threading and multiprocessing.
I don't see an urgent reason to refactor this, but it should be noted
in the comment that the problem is mostly not valid. Sadly, last time
I checked, there is no reliable way to check whether the library is
or isn't thread-safe.
"""
import threading import threading
import Queue import Queue
sqlSubmitQueue = Queue.Queue() sqlSubmitQueue = Queue.Queue()
# SQLITE3 is so thread-unsafe that they won't even let you call it from different threads using your own locks. """the queue for SQL"""
# SQL objects #can only be called from one thread.
sqlReturnQueue = Queue.Queue() sqlReturnQueue = Queue.Queue()
"""the queue for results"""
sqlLock = threading.Lock() sqlLock = threading.Lock()
def sqlQuery(sqlStatement, *args): def sqlQuery(sqlStatement, *args):
"""SQLLITE execute statement and return query.""" """
Query sqlite and return results
:param str sqlStatement: SQL statement string
:param list args: SQL query parameters
:rtype: list
"""
sqlLock.acquire() sqlLock.acquire()
sqlSubmitQueue.put(sqlStatement) sqlSubmitQueue.put(sqlStatement)
@ -28,6 +50,7 @@ def sqlQuery(sqlStatement, *args):
def sqlExecuteChunked(sqlStatement, idCount, *args): def sqlExecuteChunked(sqlStatement, idCount, *args):
"""Execute chunked SQL statement to avoid argument limit"""
# SQLITE_MAX_VARIABLE_NUMBER, # SQLITE_MAX_VARIABLE_NUMBER,
# unfortunately getting/setting isn't exposed to python # unfortunately getting/setting isn't exposed to python
sqlExecuteChunked.chunkSize = 999 sqlExecuteChunked.chunkSize = 999
@ -58,6 +81,7 @@ def sqlExecuteChunked(sqlStatement, idCount, *args):
def sqlExecute(sqlStatement, *args): def sqlExecute(sqlStatement, *args):
"""Execute SQL statement (optionally with arguments)"""
sqlLock.acquire() sqlLock.acquire()
sqlSubmitQueue.put(sqlStatement) sqlSubmitQueue.put(sqlStatement)
@ -70,13 +94,15 @@ def sqlExecute(sqlStatement, *args):
sqlLock.release() sqlLock.release()
return rowcount return rowcount
def sqlStoredProcedure(procName): def sqlStoredProcedure(procName):
"""Schedule procName to be run"""
sqlLock.acquire() sqlLock.acquire()
sqlSubmitQueue.put(procName) sqlSubmitQueue.put(procName)
sqlLock.release() sqlLock.release()
class SqlBulkExecute: class SqlBulkExecute(object):
"""This is used when you have to execute the same statement in a cycle.""" """This is used when you have to execute the same statement in a cycle."""
def __enter__(self): def __enter__(self):

26
src/helper_startup.py
View File

@ -1,13 +1,9 @@
""" """
src/helper_startup.py Startup operations.
=====================
Helper Start performs all the startup operations.
""" """
# pylint: disable=too-many-branches,too-many-statements # pylint: disable=too-many-branches,too-many-statements
from __future__ import print_function from __future__ import print_function
import ConfigParser
import os import os
import platform import platform
import sys import sys
@ -19,28 +15,12 @@ import paths
import state import state
from bmconfigparser import BMConfigParser from bmconfigparser import BMConfigParser
# The user may de-select Portable Mode in the settings if they want # The user may de-select Portable Mode in the settings if they want
# the config files to stay in the application data folder. # the config files to stay in the application data folder.
StoreConfigFilesInSameDirectoryAsProgramByDefault = False StoreConfigFilesInSameDirectoryAsProgramByDefault = False
def _loadTrustedPeer():
try:
trustedPeer = BMConfigParser().get('bitmessagesettings', 'trustedpeer')
except ConfigParser.Error:
# This probably means the trusted peer wasn't specified so we
# can just leave it as None
return
try:
host, port = trustedPeer.split(':')
except ValueError:
sys.exit(
'Bad trustedpeer config setting! It should be set as'
' trustedpeer=<hostname>:<portnumber>'
)
state.trustedPeer = state.Peer(host, int(port))
def loadConfig(): def loadConfig():
"""Load the config""" """Load the config"""
config = BMConfigParser() config = BMConfigParser()
@ -135,8 +115,6 @@ def loadConfig():
else: else:
updateConfig() updateConfig()
_loadTrustedPeer()
def updateConfig(): def updateConfig():
"""Save the config""" """Save the config"""

58
src/helper_threading.py
View File

@ -1,58 +0,0 @@
"""Helper threading perform all the threading operations."""
import threading
from contextlib import contextmanager
import helper_random
try:
import prctl
except ImportError:
def set_thread_name(name):
"""Set the thread name for external use (visible from the OS)."""
threading.current_thread().name = name
else:
def set_thread_name(name):
"""Set a name for the thread for python internal use."""
prctl.set_name(name)
def _thread_name_hack(self):
set_thread_name(self.name)
threading.Thread.__bootstrap_original__(self)
threading.Thread.__bootstrap_original__ = threading.Thread._Thread__bootstrap
threading.Thread._Thread__bootstrap = _thread_name_hack
class StoppableThread(threading.Thread):
name = None
def __init__(self, name=None):
if name:
self.name = name
super(StoppableThread, self).__init__(name=self.name)
self.initStop()
helper_random.seed()
def initStop(self):
self.stop = threading.Event()
self._stopped = False
def stopThread(self):
self._stopped = True
self.stop.set()
class BusyError(threading.ThreadError):
pass
@contextmanager
def nonBlocking(lock):
locked = lock.acquire(False)
if not locked:
raise BusyError
try:
yield
finally:
lock.release()

54
src/highlevelcrypto.py
View File

@ -1,6 +1,10 @@
""" """
src/highlevelcrypto.py High level cryptographic functions based on `.pyelliptic` OpenSSL bindings.
======================
.. note::
Upstream pyelliptic was upgraded from SHA1 to SHA256 for signing.
We must upgrade PyBitmessage gracefully.
`More discussion. <https://github.com/yann2192/pyelliptic/issues/32>`_
""" """
from binascii import hexlify from binascii import hexlify
@ -12,12 +16,13 @@ from pyelliptic import arithmetic as a
def makeCryptor(privkey): def makeCryptor(privkey):
"""Return a private pyelliptic.ECC() instance""" """Return a private `.pyelliptic.ECC` instance"""
private_key = a.changebase(privkey, 16, 256, minlen=32) private_key = a.changebase(privkey, 16, 256, minlen=32)
public_key = pointMult(private_key) public_key = pointMult(private_key)
privkey_bin = '\x02\xca\x00\x20' + private_key privkey_bin = '\x02\xca\x00\x20' + private_key
pubkey_bin = '\x02\xca\x00\x20' + public_key[1:-32] + '\x00\x20' + public_key[-32:] pubkey_bin = '\x02\xca\x00\x20' + public_key[1:-32] + '\x00\x20' + public_key[-32:]
cryptor = pyelliptic.ECC(curve='secp256k1', privkey=privkey_bin, pubkey=pubkey_bin) cryptor = pyelliptic.ECC(
curve='secp256k1', privkey=privkey_bin, pubkey=pubkey_bin)
return cryptor return cryptor
@ -29,7 +34,7 @@ def hexToPubkey(pubkey):
def makePubCryptor(pubkey): def makePubCryptor(pubkey):
"""Return a public pyelliptic.ECC() instance""" """Return a public `.pyelliptic.ECC` instance"""
pubkey_bin = hexToPubkey(pubkey) pubkey_bin = hexToPubkey(pubkey)
return pyelliptic.ECC(curve='secp256k1', pubkey=pubkey_bin) return pyelliptic.ECC(curve='secp256k1', pubkey=pubkey_bin)
@ -43,7 +48,8 @@ def privToPub(privkey):
def encrypt(msg, hexPubkey): def encrypt(msg, hexPubkey):
"""Encrypts message with hex public key""" """Encrypts message with hex public key"""
return pyelliptic.ECC(curve='secp256k1').encrypt(msg, hexToPubkey(hexPubkey)) return pyelliptic.ECC(curve='secp256k1').encrypt(
msg, hexToPubkey(hexPubkey))
def decrypt(msg, hexPrivkey): def decrypt(msg, hexPrivkey):
@ -52,36 +58,38 @@ def decrypt(msg, hexPrivkey):
def decryptFast(msg, cryptor): def decryptFast(msg, cryptor):
"""Decrypts message with an existing pyelliptic.ECC.ECC object""" """Decrypts message with an existing `.pyelliptic.ECC` object"""
return cryptor.decrypt(msg) return cryptor.decrypt(msg)
def sign(msg, hexPrivkey): def sign(msg, hexPrivkey):
"""Signs with hex private key""" """
# pyelliptic is upgrading from SHA1 to SHA256 for signing. We must Signs with hex private key using SHA1 or SHA256 depending on
# upgrade PyBitmessage gracefully. "digestalg" setting
# https://github.com/yann2192/pyelliptic/pull/33 """
# More discussion: https://github.com/yann2192/pyelliptic/issues/32 digestAlg = BMConfigParser().safeGet(
digestAlg = BMConfigParser().safeGet('bitmessagesettings', 'digestalg', 'sha1') 'bitmessagesettings', 'digestalg', 'sha1')
if digestAlg == "sha1": if digestAlg == "sha1":
# SHA1, this will eventually be deprecated # SHA1, this will eventually be deprecated
return makeCryptor(hexPrivkey).sign(msg, digest_alg=OpenSSL.digest_ecdsa_sha1) return makeCryptor(hexPrivkey).sign(
msg, digest_alg=OpenSSL.digest_ecdsa_sha1)
elif digestAlg == "sha256": elif digestAlg == "sha256":
# SHA256. Eventually this will become the default # SHA256. Eventually this will become the default
return makeCryptor(hexPrivkey).sign(msg, digest_alg=OpenSSL.EVP_sha256) return makeCryptor(hexPrivkey).sign(msg, digest_alg=OpenSSL.EVP_sha256)
else: else:
raise ValueError("Unknown digest algorithm %s" % (digestAlg)) raise ValueError("Unknown digest algorithm %s" % digestAlg)
def verify(msg, sig, hexPubkey): def verify(msg, sig, hexPubkey):
"""Verifies with hex public key""" """Verifies with hex public key using SHA1 or SHA256"""
# As mentioned above, we must upgrade gracefully to use SHA256. So # As mentioned above, we must upgrade gracefully to use SHA256. So
# let us check the signature using both SHA1 and SHA256 and if one # let us check the signature using both SHA1 and SHA256 and if one
# of them passes then we will be satisfied. Eventually this can # of them passes then we will be satisfied. Eventually this can
# be simplified and we'll only check with SHA256. # be simplified and we'll only check with SHA256.
try: try:
# old SHA1 algorithm. # old SHA1 algorithm.
sigVerifyPassed = makePubCryptor(hexPubkey).verify(sig, msg, digest_alg=OpenSSL.digest_ecdsa_sha1) sigVerifyPassed = makePubCryptor(hexPubkey).verify(
sig, msg, digest_alg=OpenSSL.digest_ecdsa_sha1)
except: except:
sigVerifyPassed = False sigVerifyPassed = False
if sigVerifyPassed: if sigVerifyPassed:
@ -89,7 +97,8 @@ def verify(msg, sig, hexPubkey):
return True return True
# The signature check using SHA1 failed. Let us try it with SHA256. # The signature check using SHA1 failed. Let us try it with SHA256.
try: try:
return makePubCryptor(hexPubkey).verify(sig, msg, digest_alg=OpenSSL.EVP_sha256) return makePubCryptor(hexPubkey).verify(
sig, msg, digest_alg=OpenSSL.EVP_sha256)
except: except:
return False return False
@ -100,13 +109,14 @@ def pointMult(secret):
Evidently, this type of error can occur very rarely: Evidently, this type of error can occur very rarely:
File "highlevelcrypto.py", line 54, in pointMult >>> File "highlevelcrypto.py", line 54, in pointMult
group = OpenSSL.EC_KEY_get0_group(k) >>> group = OpenSSL.EC_KEY_get0_group(k)
WindowsError: exception: access violation reading 0x0000000000000008 >>> WindowsError: exception: access violation reading 0x0000000000000008
""" """
while True: while True:
try: try:
k = OpenSSL.EC_KEY_new_by_curve_name(OpenSSL.get_curve('secp256k1')) k = OpenSSL.EC_KEY_new_by_curve_name(
OpenSSL.get_curve('secp256k1'))
priv_key = OpenSSL.BN_bin2bn(secret, 32, None) priv_key = OpenSSL.BN_bin2bn(secret, 32, None)
group = OpenSSL.EC_KEY_get0_group(k) group = OpenSSL.EC_KEY_get0_group(k)
pub_key = OpenSSL.EC_POINT_new(group) pub_key = OpenSSL.EC_POINT_new(group)

53
src/knownnodes.py
View File

@ -3,6 +3,7 @@ Manipulations with knownNodes dictionary.
""" """
import json import json
import logging
import os import os
import pickle import pickle
import threading import threading
@ -10,28 +11,33 @@ import time
import state import state
from bmconfigparser import BMConfigParser from bmconfigparser import BMConfigParser
from debug import logger from network.node import Peer
knownNodesLock = threading.Lock() knownNodesLock = threading.Lock()
"""Thread lock for knownnodes modification"""
knownNodes = {stream: {} for stream in range(1, 4)} knownNodes = {stream: {} for stream in range(1, 4)}
"""The dict of known nodes for each stream"""
knownNodesTrimAmount = 2000 knownNodesTrimAmount = 2000
"""trim stream knownnodes dict to this length"""
# forget a node after rating is this low
knownNodesForgetRating = -0.5 knownNodesForgetRating = -0.5
"""forget a node after rating is this low"""
knownNodesActual = False knownNodesActual = False
logger = logging.getLogger('default')
DEFAULT_NODES = ( DEFAULT_NODES = (
state.Peer('5.45.99.75', 8444), Peer('5.45.99.75', 8444),
state.Peer('75.167.159.54', 8444), Peer('75.167.159.54', 8444),
state.Peer('95.165.168.168', 8444), Peer('95.165.168.168', 8444),
state.Peer('85.180.139.241', 8444), Peer('85.180.139.241', 8444),
state.Peer('158.222.217.190', 8080), Peer('158.222.217.190', 8080),
state.Peer('178.62.12.187', 8448), Peer('178.62.12.187', 8448),
state.Peer('24.188.198.204', 8111), Peer('24.188.198.204', 8111),
state.Peer('109.147.204.113', 1195), Peer('109.147.204.113', 1195),
state.Peer('178.11.46.221', 8444) Peer('178.11.46.221', 8444)
) )
@ -57,19 +63,17 @@ def json_deserialize_knownnodes(source):
for node in json.load(source): for node in json.load(source):
peer = node['peer'] peer = node['peer']
info = node['info'] info = node['info']
peer = state.Peer(str(peer['host']), peer.get('port', 8444)) peer = Peer(str(peer['host']), peer.get('port', 8444))
knownNodes[node['stream']][peer] = info knownNodes[node['stream']][peer] = info
if ( if not (knownNodesActual
not (knownNodesActual or info.get('self')) and or info.get('self')) and peer not in DEFAULT_NODES:
peer not in DEFAULT_NODES
):
knownNodesActual = True knownNodesActual = True
def pickle_deserialize_old_knownnodes(source): def pickle_deserialize_old_knownnodes(source):
""" """
Unpickle source and reorganize knownnodes dict if it's in old format Unpickle source and reorganize knownnodes dict if it has old format
the old format was {Peer:lastseen, ...} the old format was {Peer:lastseen, ...}
the new format is {Peer:{"lastseen":i, "rating":f}} the new format is {Peer:{"lastseen":i, "rating":f}}
""" """
@ -82,6 +86,7 @@ def pickle_deserialize_old_knownnodes(source):
def saveKnownNodes(dirName=None): def saveKnownNodes(dirName=None):
"""Save knownnodes to filesystem"""
if dirName is None: if dirName is None:
dirName = state.appdata dirName = state.appdata
with knownNodesLock: with knownNodesLock:
@ -90,6 +95,7 @@ def saveKnownNodes(dirName=None):
def addKnownNode(stream, peer, lastseen=None, is_self=False): def addKnownNode(stream, peer, lastseen=None, is_self=False):
"""Add a new node to the dict"""
knownNodes[stream][peer] = { knownNodes[stream][peer] = {
"lastseen": lastseen or time.time(), "lastseen": lastseen or time.time(),
"rating": 1 if is_self else 0, "rating": 1 if is_self else 0,
@ -98,6 +104,7 @@ def addKnownNode(stream, peer, lastseen=None, is_self=False):
def createDefaultKnownNodes(): def createDefaultKnownNodes():
"""Creating default Knownnodes"""
past = time.time() - 2418600 # 28 days - 10 min past = time.time() - 2418600 # 28 days - 10 min
for peer in DEFAULT_NODES: for peer in DEFAULT_NODES:
addKnownNode(1, peer, past) addKnownNode(1, peer, past)
@ -105,6 +112,7 @@ def createDefaultKnownNodes():
def readKnownNodes(): def readKnownNodes():
"""Load knownnodes from filesystem"""
try: try:
with open(state.appdata + 'knownnodes.dat', 'rb') as source: with open(state.appdata + 'knownnodes.dat', 'rb') as source:
with knownNodesLock: with knownNodesLock:
@ -125,12 +133,13 @@ def readKnownNodes():
if onionhostname and ".onion" in onionhostname: if onionhostname and ".onion" in onionhostname:
onionport = config.safeGetInt('bitmessagesettings', 'onionport') onionport = config.safeGetInt('bitmessagesettings', 'onionport')
if onionport: if onionport:
self_peer = state.Peer(onionhostname, onionport) self_peer = Peer(onionhostname, onionport)
addKnownNode(1, self_peer, is_self=True) addKnownNode(1, self_peer, is_self=True)
state.ownAddresses[self_peer] = True state.ownAddresses[self_peer] = True
def increaseRating(peer): def increaseRating(peer):
"""Increase rating of a peer node"""
increaseAmount = 0.1 increaseAmount = 0.1
maxRating = 1 maxRating = 1
with knownNodesLock: with knownNodesLock:
@ -145,6 +154,7 @@ def increaseRating(peer):
def decreaseRating(peer): def decreaseRating(peer):
"""Decrease rating of a peer node"""
decreaseAmount = 0.1 decreaseAmount = 0.1
minRating = -1 minRating = -1
with knownNodesLock: with knownNodesLock:
@ -159,6 +169,7 @@ def decreaseRating(peer):
def trimKnownNodes(recAddrStream=1): def trimKnownNodes(recAddrStream=1):
"""Triming Knownnodes"""
if len(knownNodes[recAddrStream]) < \ if len(knownNodes[recAddrStream]) < \
BMConfigParser().safeGetInt("knownnodes", "maxnodes"): BMConfigParser().safeGetInt("knownnodes", "maxnodes"):
return return
@ -175,7 +186,7 @@ def dns():
"""Add DNS names to knownnodes""" """Add DNS names to knownnodes"""
for port in [8080, 8444]: for port in [8080, 8444]:
addKnownNode( addKnownNode(
1, state.Peer('bootstrap%s.bitmessage.org' % port, port)) 1, Peer('bootstrap%s.bitmessage.org' % port, port))
def cleanupKnownNodes(): def cleanupKnownNodes():
@ -201,8 +212,8 @@ def cleanupKnownNodes():
del knownNodes[stream][node] del knownNodes[stream][node]
continue continue
# scrap old nodes (age > 3 hours) with low rating # scrap old nodes (age > 3 hours) with low rating
if (age > 10800 and knownNodes[stream][node]["rating"] <= if (age > 10800 and knownNodes[stream][node]["rating"]
knownNodesForgetRating): <= knownNodesForgetRating):
needToWriteKnownNodesToDisk = True needToWriteKnownNodesToDisk = True
del knownNodes[stream][node] del knownNodes[stream][node]
continue continue

51
src/l10n.py
View File

@ -1,4 +1,6 @@
"""
Localization
"""
import logging import logging
import os import os
import time import time
@ -6,8 +8,7 @@ import time
from bmconfigparser import BMConfigParser from bmconfigparser import BMConfigParser
#logger = logging.getLogger(__name__) logger = logging.getLogger('default')
logger = logging.getLogger('file_only')
DEFAULT_ENCODING = 'ISO8859-1' DEFAULT_ENCODING = 'ISO8859-1'
@ -50,7 +51,7 @@ except:
if BMConfigParser().has_option('bitmessagesettings', 'timeformat'): if BMConfigParser().has_option('bitmessagesettings', 'timeformat'):
time_format = BMConfigParser().get('bitmessagesettings', 'timeformat') time_format = BMConfigParser().get('bitmessagesettings', 'timeformat')
#Test the format string # Test the format string
try: try:
time.strftime(time_format) time.strftime(time_format)
except: except:
@ -59,48 +60,52 @@ if BMConfigParser().has_option('bitmessagesettings', 'timeformat'):
else: else:
time_format = DEFAULT_TIME_FORMAT time_format = DEFAULT_TIME_FORMAT
#It seems some systems lie about the encoding they use so we perform # It seems some systems lie about the encoding they use so we perform
#comprehensive decoding tests # comprehensive decoding tests
if time_format != DEFAULT_TIME_FORMAT: if time_format != DEFAULT_TIME_FORMAT:
try: try:
#Check day names # Check day names
for i in xrange(7): for i in xrange(7):
unicode(time.strftime(time_format, (0, 0, 0, 0, 0, 0, i, 0, 0)), encoding) unicode(time.strftime(time_format, (0, 0, 0, 0, 0, 0, i, 0, 0)), encoding)
#Check month names # Check month names
for i in xrange(1, 13): for i in xrange(1, 13):
unicode(time.strftime(time_format, (0, i, 0, 0, 0, 0, 0, 0, 0)), encoding) unicode(time.strftime(time_format, (0, i, 0, 0, 0, 0, 0, 0, 0)), encoding)
#Check AM/PM # Check AM/PM
unicode(time.strftime(time_format, (0, 0, 0, 11, 0, 0, 0, 0, 0)), encoding) unicode(time.strftime(time_format, (0, 0, 0, 11, 0, 0, 0, 0, 0)), encoding)
unicode(time.strftime(time_format, (0, 0, 0, 13, 0, 0, 0, 0, 0)), encoding) unicode(time.strftime(time_format, (0, 0, 0, 13, 0, 0, 0, 0, 0)), encoding)
#Check DST # Check DST
unicode(time.strftime(time_format, (0, 0, 0, 0, 0, 0, 0, 0, 1)), encoding) unicode(time.strftime(time_format, (0, 0, 0, 0, 0, 0, 0, 0, 1)), encoding)
except: except:
logger.exception('Could not decode locale formatted timestamp') logger.exception('Could not decode locale formatted timestamp')
time_format = DEFAULT_TIME_FORMAT time_format = DEFAULT_TIME_FORMAT
encoding = DEFAULT_ENCODING encoding = DEFAULT_ENCODING
def setlocale(category, newlocale): def setlocale(category, newlocale):
"""Set the locale"""
locale.setlocale(category, newlocale) locale.setlocale(category, newlocale)
# it looks like some stuff isn't initialised yet when this is called the # it looks like some stuff isn't initialised yet when this is called the
# first time and its init gets the locale settings from the environment # first time and its init gets the locale settings from the environment
os.environ["LC_ALL"] = newlocale os.environ["LC_ALL"] = newlocale
def formatTimestamp(timestamp = None, as_unicode = True):
#For some reason some timestamps are strings so we need to sanitize. def formatTimestamp(timestamp=None, as_unicode=True):
"""Return a formatted timestamp"""
# For some reason some timestamps are strings so we need to sanitize.
if timestamp is not None and not isinstance(timestamp, int): if timestamp is not None and not isinstance(timestamp, int):
try: try:
timestamp = int(timestamp) timestamp = int(timestamp)
except: except:
timestamp = None timestamp = None
#timestamp can't be less than 0. # timestamp can't be less than 0.
if timestamp is not None and timestamp < 0: if timestamp is not None and timestamp < 0:
timestamp = None timestamp = None
if timestamp is None: if timestamp is None:
timestring = time.strftime(time_format) timestring = time.strftime(time_format)
else: else:
#In case timestamp is too far in the future # In case timestamp is too far in the future
try: try:
timestring = time.strftime(time_format, time.localtime(timestamp)) timestring = time.strftime(time_format, time.localtime(timestamp))
except ValueError: except ValueError:
@ -110,17 +115,21 @@ def formatTimestamp(timestamp = None, as_unicode = True):
return unicode(timestring, encoding) return unicode(timestring, encoding)
return timestring return timestring
def getTranslationLanguage(): def getTranslationLanguage():
userlocale = None """Return the user's language choice"""
if BMConfigParser().has_option('bitmessagesettings', 'userlocale'): userlocale = BMConfigParser().safeGet(
userlocale = BMConfigParser().get('bitmessagesettings', 'userlocale') 'bitmessagesettings', 'userlocale', 'system')
return userlocale if userlocale and userlocale != 'system' else language
if userlocale in [None, '', 'system']:
return language
return userlocale
def getWindowsLocale(posixLocale): def getWindowsLocale(posixLocale):
"""
Get the Windows locale
Technically this converts the locale string from UNIX to Windows format,
because they use different ones in their
libraries. E.g. "en_EN.UTF-8" to "english".
"""
if posixLocale in windowsLanguageMap: if posixLocale in windowsLanguageMap:
return windowsLanguageMap[posixLocale] return windowsLanguageMap[posixLocale]
if "." in posixLocale: if "." in posixLocale:

136
src/message_data_reader.py
View File

@ -1,136 +0,0 @@
# pylint: disable=too-many-locals
"""
This program can be used to print out everything in your Inbox or Sent folders and also take things out of the trash.
Scroll down to the bottom to see the functions that you can uncomment. Save then run this file.
The functions which only read the database file seem to function just
fine even if you have Bitmessage running but you should definitly close
it before running the functions that make changes (like taking items out
of the trash).
"""
from __future__ import absolute_import
import sqlite3
from binascii import hexlify
from time import strftime, localtime
import paths
import queues
appdata = paths.lookupAppdataFolder()
conn = sqlite3.connect(appdata + 'messages.dat')
conn.text_factory = str
cur = conn.cursor()
def readInbox():
"""Print each row from inbox table"""
print 'Printing everything in inbox table:'
item = '''select * from inbox'''
parameters = ''
cur.execute(item, parameters)
output = cur.fetchall()
for row in output:
print row
def readSent():
"""Print each row from sent table"""
print 'Printing everything in Sent table:'
item = '''select * from sent where folder !='trash' '''
parameters = ''
cur.execute(item, parameters)
output = cur.fetchall()
for row in output:
(msgid, toaddress, toripe, fromaddress, subject, message, ackdata, lastactiontime,
sleeptill, status, retrynumber, folder, encodingtype, ttl) = row # pylint: disable=unused-variable
print(hexlify(msgid), toaddress, 'toripe:', hexlify(toripe), 'fromaddress:', fromaddress, 'ENCODING TYPE:',
encodingtype, 'SUBJECT:', repr(subject), 'MESSAGE:', repr(message), 'ACKDATA:', hexlify(ackdata),
lastactiontime, status, retrynumber, folder)
def readSubscriptions():
"""Print each row from subscriptions table"""
print 'Printing everything in subscriptions table:'
item = '''select * from subscriptions'''
parameters = ''
cur.execute(item, parameters)
output = cur.fetchall()
for row in output:
print row
def readPubkeys():
"""Print each row from pubkeys table"""
print 'Printing everything in pubkeys table:'
item = '''select address, transmitdata, time, usedpersonally from pubkeys'''
parameters = ''
cur.execute(item, parameters)
output = cur.fetchall()
for row in output:
address, transmitdata, time, usedpersonally = row
print(
'Address:', address, '\tTime first broadcast:', unicode(
strftime('%a, %d %b %Y %I:%M %p', localtime(time)), 'utf-8'),
'\tUsed by me personally:', usedpersonally, '\tFull pubkey message:', hexlify(transmitdata),
)
def readInventory():
"""Print each row from inventory table"""
print 'Printing everything in inventory table:'
item = '''select hash, objecttype, streamnumber, payload, expirestime from inventory'''
parameters = ''
cur.execute(item, parameters)
output = cur.fetchall()
for row in output:
obj_hash, objecttype, streamnumber, payload, expirestime = row
print 'Hash:', hexlify(obj_hash), objecttype, streamnumber, '\t', hexlify(payload), '\t', unicode(
strftime('%a, %d %b %Y %I:%M %p', localtime(expirestime)), 'utf-8')
def takeInboxMessagesOutOfTrash():
"""Update all inbox messages with folder=trash to have folder=inbox"""
item = '''update inbox set folder='inbox' where folder='trash' '''
parameters = ''
cur.execute(item, parameters)
_ = cur.fetchall()
conn.commit()
print 'done'
def takeSentMessagesOutOfTrash():
"""Update all sent messages with folder=trash to have folder=sent"""
item = '''update sent set folder='sent' where folder='trash' '''
parameters = ''
cur.execute(item, parameters)
_ = cur.fetchall()
conn.commit()
print 'done'
def markAllInboxMessagesAsUnread():
"""Update all messages in inbox to have read=0"""
item = '''update inbox set read='0' '''
parameters = ''
cur.execute(item, parameters)
_ = cur.fetchall()
conn.commit()
queues.UISignalQueue.put(('changedInboxUnread', None))
print 'done'
def vacuum():
"""Perform a vacuum on the database"""
item = '''VACUUM'''
parameters = ''
cur.execute(item, parameters)
_ = cur.fetchall()
conn.commit()
print 'done'
if __name__ == '__main__':
readInbox()

11
src/messagetypes/__init__.py
View File

@ -1,7 +1,4 @@
""" import logging
src/messagetypes/__init__.py
============================
"""
from importlib import import_module from importlib import import_module
from os import path, listdir from os import path, listdir
from string import lower from string import lower
@ -9,12 +6,14 @@ try:
from kivy.utils import platform from kivy.utils import platform
except: except:
platform = '' platform = ''
from debug import logger
import messagetypes import messagetypes
import paths import paths
logger = logging.getLogger('default')
class MsgBase(object): # pylint: disable=too-few-public-methods
class MsgBase(object): # pylint: disable=too-few-public-methods
"""Base class for message types""" """Base class for message types"""
def __init__(self): def __init__(self):
self.data = {"": lower(type(self).__name__)} self.data = {"": lower(type(self).__name__)}

9
src/messagetypes/message.py
View File

@ -1,11 +1,10 @@
""" import logging
src/messagetypes/message.py
===========================
"""
from debug import logger
from messagetypes import MsgBase from messagetypes import MsgBase
# pylint: disable=attribute-defined-outside-init # pylint: disable=attribute-defined-outside-init
logger = logging.getLogger('default')
class Message(MsgBase): class Message(MsgBase):
"""Encapsulate a message""" """Encapsulate a message"""

9
src/messagetypes/vote.py
View File

@ -1,11 +1,10 @@
""" import logging
src/messagetypes/vote.py
========================
"""
from debug import logger
from messagetypes import MsgBase from messagetypes import MsgBase
# pylint: disable=attribute-defined-outside-init # pylint: disable=attribute-defined-outside-init
logger = logging.getLogger('default')
class Vote(MsgBase): class Vote(MsgBase):
"""Module used to vote""" """Module used to vote"""

45
src/namecoin.py
View File

@ -1,31 +1,28 @@
# pylint: disable=too-many-branches,protected-access # pylint: disable=too-many-branches,protected-access
""" """
Copyright (C) 2013 by Daniel Kraft <d@domob.eu> Copyright (C) 2013 by Daniel Kraft <d@domob.eu>
This file is part of the Bitmessage project.
Permission is hereby granted, free of charge, to any person obtaining a copy Namecoin queries
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
.. todo:: from debug import logger crashes PyBitmessage due to a circular dependency. The debug module will also
override/disable logging.getLogger() # loggers so module level logging functions are used instead
""" """
# This file is part of the Bitmessage project.
from __future__ import absolute_import # Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
import base64 import base64
import httplib import httplib
@ -258,7 +255,7 @@ class namecoinConnection(object):
resp = self.con.getresponse() resp = self.con.getresponse()
result = resp.read() result = resp.read()
if resp.status != 200: if resp.status != 200:
raise Exception("Namecoin returned status %i: %s" % resp.status, resp.reason) raise Exception("Namecoin returned status %i: %s" % (resp.status, resp.reason))
except: except:
logger.info("HTTP receive error") logger.info("HTTP receive error")
except: except:
@ -288,7 +285,7 @@ class namecoinConnection(object):
return result return result
except socket.error as exc: except socket.error as exc:
raise Exception("Socket error in RPC connection: %s" % str(exc)) raise Exception("Socket error in RPC connection: %s" % exc)
def lookupNamecoinFolder(): def lookupNamecoinFolder():

17
src/network/__init__.py
View File

@ -0,0 +1,17 @@
from addrthread import AddrThread
from announcethread import AnnounceThread
from connectionpool import BMConnectionPool
from dandelion import Dandelion
from downloadthread import DownloadThread
from invthread import InvThread
from networkthread import BMNetworkThread
from receivequeuethread import ReceiveQueueThread
from threads import StoppableThread
from uploadthread import UploadThread
__all__ = [
"BMConnectionPool", "Dandelion",
"AddrThread", "AnnounceThread", "BMNetworkThread", "DownloadThread",
"InvThread", "ReceiveQueueThread", "UploadThread", "StoppableThread"
]

32
src/network/addrthread.py
View File

@ -1,9 +1,14 @@
"""
Announce addresses as they are received from other hosts
"""
import Queue import Queue
from helper_threading import StoppableThread import state
from helper_random import randomshuffle
from network.assemble import assemble_addr
from network.connectionpool import BMConnectionPool from network.connectionpool import BMConnectionPool
from queues import addrQueue from queues import addrQueue
import state from threads import StoppableThread
class AddrThread(StoppableThread): class AddrThread(StoppableThread):
@ -15,15 +20,26 @@ class AddrThread(StoppableThread):
while True: while True:
try: try:
data = addrQueue.get(False) data = addrQueue.get(False)
chunk.append((data[0], data[1])) chunk.append(data)
if len(data) > 2:
source = BMConnectionPool().getConnectionByAddr(data[2])
except Queue.Empty: except Queue.Empty:
break break
except KeyError:
continue
# finish if chunk:
# Choose peers randomly
connections = BMConnectionPool().establishedConnections()
randomshuffle(connections)
for i in connections:
randomshuffle(chunk)
filtered = []
for stream, peer, seen, destination in chunk:
# peer's own address or address received from peer
if i.destination in (peer, destination):
continue
if stream not in i.streams:
continue
filtered.append((stream, peer, seen))
if filtered:
i.append_write_buf(assemble_addr(filtered))
addrQueue.iterate() addrQueue.iterate()
for i in range(len(chunk)): for i in range(len(chunk)):

6
src/network/advanceddispatcher.py
View File

@ -10,8 +10,7 @@ import time
import network.asyncore_pollchoose as asyncore import network.asyncore_pollchoose as asyncore
import state import state
from debug import logger from threads import BusyError, nonBlocking
from helper_threading import BusyError, nonBlocking
class ProcessingError(Exception): class ProcessingError(Exception):
@ -84,7 +83,8 @@ class AdvancedDispatcher(asyncore.dispatcher):
try: try:
cmd = getattr(self, "state_" + str(self.state)) cmd = getattr(self, "state_" + str(self.state))
except AttributeError: except AttributeError:
logger.error("Unknown state %s", self.state, exc_info=True) self.logger.error(
'Unknown state %s', self.state, exc_info=True)
raise UnknownStateError(self.state) raise UnknownStateError(self.state)
if not cmd(): if not cmd():
break break

20
src/network/announcethread.py
View File

@ -2,22 +2,22 @@
src/network/announcethread.py src/network/announcethread.py
================================= =================================
""" """
import time import time
import state
from bmconfigparser import BMConfigParser from bmconfigparser import BMConfigParser
from debug import logger from network.assemble import assemble_addr
from helper_threading import StoppableThread
from network.bmproto import BMProto
from network.connectionpool import BMConnectionPool from network.connectionpool import BMConnectionPool
from network.udp import UDPSocket from network.udp import UDPSocket
import state from node import Peer
from threads import StoppableThread
class AnnounceThread(StoppableThread): class AnnounceThread(StoppableThread):
"""A thread to manage regular announcing of this node""" """A thread to manage regular announcing of this node"""
def __init__(self): name = "Announcer"
super(AnnounceThread, self).__init__(name="Announcer")
logger.info("init announce thread")
def run(self): def run(self):
lastSelfAnnounced = 0 lastSelfAnnounced = 0
@ -38,6 +38,8 @@ class AnnounceThread(StoppableThread):
for stream in state.streamsInWhichIAmParticipating: for stream in state.streamsInWhichIAmParticipating:
addr = ( addr = (
stream, stream,
state.Peer('127.0.0.1', BMConfigParser().safeGetInt("bitmessagesettings", "port")), Peer(
'127.0.0.1',
BMConfigParser().safeGetInt('bitmessagesettings', 'port')),
time.time()) time.time())
connection.append_write_buf(BMProto.assembleAddr([addr])) connection.append_write_buf(assemble_addr([addr]))

32
src/network/assemble.py Normal file
View File

@ -0,0 +1,32 @@
"""
Create bitmessage protocol command packets
"""
import struct
import addresses
from network.constants import MAX_ADDR_COUNT
from network.node import Peer
from protocol import CreatePacket, encodeHost
def assemble_addr(peerList):
"""Create address command"""
if isinstance(peerList, Peer):
peerList = (peerList)
if not peerList:
return b''
retval = b''
for i in range(0, len(peerList), MAX_ADDR_COUNT):
payload = addresses.encodeVarint(
len(peerList[i:i + MAX_ADDR_COUNT]))
for stream, peer, timestamp in peerList[i:i + MAX_ADDR_COUNT]:
payload += struct.pack(
'>Q', timestamp) # 64-bit time
payload += struct.pack('>I', stream)
payload += struct.pack(
'>q', 1) # service bit flags offered by this node
payload += encodeHost(peer.host)
payload += struct.pack('>H', peer.port) # remote port
retval += CreatePacket('addr', payload)
return retval

8
src/network/bmobject.py
View File

@ -1,18 +1,18 @@
""" """
src/network/bmobject.py BMObject and it's exceptions.
======================
""" """
import logging
import time import time
import protocol import protocol
import state import state
from addresses import calculateInventoryHash from addresses import calculateInventoryHash
from debug import logger
from inventory import Inventory from inventory import Inventory
from network.dandelion import Dandelion from network.dandelion import Dandelion
logger = logging.getLogger('default')
class BMObjectInsufficientPOWError(Exception): class BMObjectInsufficientPOWError(Exception):
"""Exception indicating the object doesn't have sufficient proof of work.""" """Exception indicating the object doesn't have sufficient proof of work."""

93
src/network/bmproto.py
View File

@ -5,6 +5,7 @@ src/network/bmproto.py
# pylint: disable=attribute-defined-outside-init # pylint: disable=attribute-defined-outside-init
import base64 import base64
import hashlib import hashlib
import logging
import socket import socket
import struct import struct
import time import time
@ -16,20 +17,27 @@ import knownnodes
import protocol import protocol
import state import state
from bmconfigparser import BMConfigParser from bmconfigparser import BMConfigParser
from debug import logger
from inventory import Inventory from inventory import Inventory
from network.advanceddispatcher import AdvancedDispatcher from network.advanceddispatcher import AdvancedDispatcher
from network.constants import (
ADDRESS_ALIVE,
MAX_MESSAGE_SIZE,
MAX_OBJECT_COUNT,
MAX_OBJECT_PAYLOAD_SIZE,
MAX_TIME_OFFSET)
from network.dandelion import Dandelion from network.dandelion import Dandelion
from network.bmobject import ( from network.bmobject import (
BMObject, BMObjectInsufficientPOWError, BMObjectInvalidDataError, BMObject, BMObjectInsufficientPOWError, BMObjectInvalidDataError,
BMObjectExpiredError, BMObjectUnwantedStreamError, BMObjectExpiredError, BMObjectUnwantedStreamError,
BMObjectInvalidError, BMObjectAlreadyHaveError) BMObjectInvalidError, BMObjectAlreadyHaveError)
from network.node import Node
from network.proxy import ProxyError from network.proxy import ProxyError
from node import Node, Peer
from objectracker import missingObjects, ObjectTracker from objectracker import missingObjects, ObjectTracker
from queues import objectProcessorQueue, portCheckerQueue, invQueue, addrQueue from queues import objectProcessorQueue, portCheckerQueue, invQueue, addrQueue
from randomtrackingdict import RandomTrackingDict from randomtrackingdict import RandomTrackingDict
logger = logging.getLogger('default')
class BMProtoError(ProxyError): class BMProtoError(ProxyError):
"""A Bitmessage Protocol Base Error""" """A Bitmessage Protocol Base Error"""
@ -49,18 +57,6 @@ class BMProtoExcessiveDataError(BMProtoError):
class BMProto(AdvancedDispatcher, ObjectTracker): class BMProto(AdvancedDispatcher, ObjectTracker):
"""A parser for the Bitmessage Protocol""" """A parser for the Bitmessage Protocol"""
# pylint: disable=too-many-instance-attributes, too-many-public-methods # pylint: disable=too-many-instance-attributes, too-many-public-methods
# ~1.6 MB which is the maximum possible size of an inv message.
maxMessageSize = 1600100
# 2**18 = 256kB is the maximum size of an object payload
maxObjectPayloadSize = 2**18
# protocol specification says max 1000 addresses in one addr command
maxAddrCount = 1000
# protocol specification says max 50000 objects in one inv command
maxObjectCount = 50000
# address is online if online less than this many seconds ago
addressAlive = 10800
# maximum time offset
maxTimeOffset = 3600
timeOffsetWrongCount = 0 timeOffsetWrongCount = 0
def __init__(self, address=None, sock=None): # pylint: disable=unused-argument, super-init-not-called def __init__(self, address=None, sock=None): # pylint: disable=unused-argument, super-init-not-called
@ -69,6 +65,8 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
# packet/connection from a local IP # packet/connection from a local IP
self.local = False self.local = False
self.pendingUpload = RandomTrackingDict() self.pendingUpload = RandomTrackingDict()
# canonical identifier of network group
self.network_group = None
def bm_proto_reset(self): def bm_proto_reset(self):
"""Reset the bitmessage object parser""" """Reset the bitmessage object parser"""
@ -96,7 +94,7 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
self.close_reason = "Bad magic" self.close_reason = "Bad magic"
self.set_state("close") self.set_state("close")
return False return False
if self.payloadLength > BMProto.maxMessageSize: if self.payloadLength > MAX_MESSAGE_SIZE:
self.invalid = True self.invalid = True
self.set_state( self.set_state(
"bm_command", "bm_command",
@ -339,7 +337,7 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
def _command_inv(self, dandelion=False): def _command_inv(self, dandelion=False):
items = self.decode_payload_content("l32s") items = self.decode_payload_content("l32s")
if len(items) > BMProto.maxObjectCount: if len(items) > MAX_OBJECT_COUNT:
logger.error( logger.error(
'Too many items in %sinv message!', 'd' if dandelion else '') 'Too many items in %sinv message!', 'd' if dandelion else '')
raise BMProtoExcessiveDataError() raise BMProtoExcessiveDataError()
@ -374,7 +372,7 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
nonce, expiresTime, objectType, version, streamNumber, nonce, expiresTime, objectType, version, streamNumber,
self.payload, self.payloadOffset) self.payload, self.payloadOffset)
if len(self.payload) - self.payloadOffset > BMProto.maxObjectPayloadSize: if len(self.payload) - self.payloadOffset > MAX_OBJECT_PAYLOAD_SIZE:
logger.info( logger.info(
'The payload length of this object is too large (%d bytes).' 'The payload length of this object is too large (%d bytes).'
' Ignoring it.', len(self.payload) - self.payloadOffset) ' Ignoring it.', len(self.payload) - self.payloadOffset)
@ -438,10 +436,10 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
continue continue
if ( if (
decodedIP and time.time() - seenTime > 0 and decodedIP and time.time() - seenTime > 0 and
seenTime > time.time() - BMProto.addressAlive and seenTime > time.time() - ADDRESS_ALIVE and
port > 0 port > 0
): ):
peer = state.Peer(decodedIP, port) peer = Peer(decodedIP, port)
try: try:
if knownnodes.knownNodes[stream][peer]["lastseen"] > seenTime: if knownnodes.knownNodes[stream][peer]["lastseen"] > seenTime:
continue continue
@ -457,12 +455,15 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
"rating": 0, "rating": 0,
"self": False, "self": False,
} }
addrQueue.put((stream, peer, self.destination)) # since we don't track peers outside of knownnodes,
# only spread if in knownnodes to prevent flood
addrQueue.put((stream, peer, seenTime,
self.destination))
return True return True
def bm_command_portcheck(self): def bm_command_portcheck(self):
"""Incoming port check request, queue it.""" """Incoming port check request, queue it."""
portCheckerQueue.put(state.Peer(self.destination, self.peerNode.port)) portCheckerQueue.put(Peer(self.destination, self.peerNode.port))
return True return True
def bm_command_ping(self): def bm_command_ping(self):
@ -508,7 +509,7 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
self.timeOffset = self.timestamp - int(time.time()) self.timeOffset = self.timestamp - int(time.time())
logger.debug('remoteProtocolVersion: %i', self.remoteProtocolVersion) logger.debug('remoteProtocolVersion: %i', self.remoteProtocolVersion)
logger.debug('services: 0x%08X', self.services) logger.debug('services: 0x%08X', self.services)
logger.debug('time offset: %i', self.timestamp - int(time.time())) logger.debug('time offset: %i', self.timeOffset)
logger.debug('my external IP: %s', self.sockNode.host) logger.debug('my external IP: %s', self.sockNode.host)
logger.debug( logger.debug(
'remote node incoming address: %s:%i', 'remote node incoming address: %s:%i',
@ -548,7 +549,7 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
'Closing connection to old protocol version %s, node: %s', 'Closing connection to old protocol version %s, node: %s',
self.remoteProtocolVersion, self.destination) self.remoteProtocolVersion, self.destination)
return False return False
if self.timeOffset > BMProto.maxTimeOffset: if self.timeOffset > MAX_TIME_OFFSET:
self.append_write_buf(protocol.assembleErrorMessage( self.append_write_buf(protocol.assembleErrorMessage(
errorText="Your time is too far in the future compared to mine." errorText="Your time is too far in the future compared to mine."
" Closing connection.", fatal=2)) " Closing connection.", fatal=2))
@ -557,7 +558,7 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
" Closing connection to it.", self.destination, self.timeOffset) " Closing connection to it.", self.destination, self.timeOffset)
BMProto.timeOffsetWrongCount += 1 BMProto.timeOffsetWrongCount += 1
return False return False
elif self.timeOffset < -BMProto.maxTimeOffset: elif self.timeOffset < -MAX_TIME_OFFSET:
self.append_write_buf(protocol.assembleErrorMessage( self.append_write_buf(protocol.assembleErrorMessage(
errorText="Your time is too far in the past compared to mine." errorText="Your time is too far in the past compared to mine."
" Closing connection.", fatal=2)) " Closing connection.", fatal=2))
@ -592,12 +593,14 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
# incoming from a peer we're connected to as outbound, # incoming from a peer we're connected to as outbound,
# or server full report the same error to counter deanonymisation # or server full report the same error to counter deanonymisation
if ( if (
state.Peer(self.destination.host, self.peerNode.port) in Peer(self.destination.host, self.peerNode.port)
connectionpool.BMConnectionPool().inboundConnections or in connectionpool.BMConnectionPool().inboundConnections
len(connectionpool.BMConnectionPool().inboundConnections) + or len(connectionpool.BMConnectionPool().inboundConnections)
len(connectionpool.BMConnectionPool().outboundConnections) > + len(connectionpool.BMConnectionPool().outboundConnections)
BMConfigParser().safeGetInt("bitmessagesettings", "maxtotalconnections") + > BMConfigParser().safeGetInt(
BMConfigParser().safeGetInt("bitmessagesettings", "maxbootstrapconnections") 'bitmessagesettings', 'maxtotalconnections')
+ BMConfigParser().safeGetInt(
'bitmessagesettings', 'maxbootstrapconnections')
): ):
self.append_write_buf(protocol.assembleErrorMessage( self.append_write_buf(protocol.assembleErrorMessage(
errorText="Server full, please try again later.", fatal=2)) errorText="Server full, please try again later.", fatal=2))
@ -617,36 +620,10 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
return True return True
@staticmethod
def assembleAddr(peerList):
"""Build up a packed address"""
if isinstance(peerList, state.Peer):
peerList = (peerList)
if not peerList:
return b''
retval = b''
for i in range(0, len(peerList), BMProto.maxAddrCount):
payload = addresses.encodeVarint(
len(peerList[i:i + BMProto.maxAddrCount]))
for address in peerList[i:i + BMProto.maxAddrCount]:
stream, peer, timestamp = address
payload += struct.pack(
'>Q', timestamp) # 64-bit time
payload += struct.pack('>I', stream)
payload += struct.pack(
'>q', 1) # service bit flags offered by this node
payload += protocol.encodeHost(peer.host)
payload += struct.pack('>H', peer.port) # remote port
retval += protocol.CreatePacket('addr', payload)
return retval
@staticmethod @staticmethod
def stopDownloadingObject(hashId, forwardAnyway=False): def stopDownloadingObject(hashId, forwardAnyway=False):
"""Stop downloading an object""" """Stop downloading an object"""
for connection in ( for connection in connectionpool.BMConnectionPool().connections():
connectionpool.BMConnectionPool().inboundConnections.values() +
connectionpool.BMConnectionPool().outboundConnections.values()
):
try: try:
del connection.objectsNewToMe[hashId] del connection.objectsNewToMe[hashId]
except KeyError: except KeyError:
@ -687,7 +664,7 @@ class BMStringParser(BMProto):
""" """
def __init__(self): def __init__(self):
super(BMStringParser, self).__init__() super(BMStringParser, self).__init__()
self.destination = state.Peer('127.0.0.1', 8444) self.destination = Peer('127.0.0.1', 8444)
self.payload = None self.payload = None
ObjectTracker.__init__(self) ObjectTracker.__init__(self)

11
src/network/connectionchooser.py
View File

@ -1,13 +1,15 @@
# pylint: disable=too-many-branches # pylint: disable=too-many-branches
import logging
import random # nosec import random # nosec
import knownnodes import knownnodes
import protocol import protocol
import state import state
from bmconfigparser import BMConfigParser from bmconfigparser import BMConfigParser
from debug import logger
from queues import Queue, portCheckerQueue from queues import Queue, portCheckerQueue
logger = logging.getLogger('default')
def getDiscoveredPeer(): def getDiscoveredPeer():
try: try:
@ -24,8 +26,8 @@ def getDiscoveredPeer():
def chooseConnection(stream): def chooseConnection(stream):
haveOnion = BMConfigParser().safeGet( haveOnion = BMConfigParser().safeGet(
"bitmessagesettings", "socksproxytype")[0:5] == 'SOCKS' "bitmessagesettings", "socksproxytype")[0:5] == 'SOCKS'
if state.trustedPeer: onionOnly = BMConfigParser().safeGetBoolean(
return state.trustedPeer "bitmessagesettings", "onionservicesonly")
try: try:
retval = portCheckerQueue.get(False) retval = portCheckerQueue.get(False)
portCheckerQueue.task_done() portCheckerQueue.task_done()
@ -47,6 +49,9 @@ def chooseConnection(stream):
logger.warning('Error in %s', peer) logger.warning('Error in %s', peer)
rating = 0 rating = 0
if haveOnion: if haveOnion:
# do not connect to raw IP addresses--keep all traffic within Tor overlay
if onionOnly and not peer.host.endswith('.onion'):
continue
# onion addresses have a higher priority when SOCKS # onion addresses have a higher priority when SOCKS
if peer.host.endswith('.onion') and rating > 0: if peer.host.endswith('.onion') and rating > 0:
rating = 1 rating = 1

136
src/network/connectionpool.py
View File

@ -1,10 +1,11 @@
""" """
src/network/connectionpool.py `BMConnectionPool` class definition
==================================
""" """
import errno import errno
import logging
import re import re
import socket import socket
import sys
import time import time
import asyncore_pollchoose as asyncore import asyncore_pollchoose as asyncore
@ -14,7 +15,7 @@ import protocol
import state import state
from bmconfigparser import BMConfigParser from bmconfigparser import BMConfigParser
from connectionchooser import chooseConnection from connectionchooser import chooseConnection
from debug import logger from node import Peer
from proxy import Proxy from proxy import Proxy
from singleton import Singleton from singleton import Singleton
from tcp import ( from tcp import (
@ -22,11 +23,27 @@ from tcp import (
TCPConnection, TCPServer) TCPConnection, TCPServer)
from udp import UDPSocket from udp import UDPSocket
logger = logging.getLogger('default')
@Singleton @Singleton
# pylint: disable=too-many-instance-attributes
class BMConnectionPool(object): class BMConnectionPool(object):
"""Pool of all existing connections""" """Pool of all existing connections"""
# pylint: disable=too-many-instance-attributes
trustedPeer = None
"""
If the trustedpeer option is specified in keys.dat then this will
contain a Peer which will be connected to instead of using the
addresses advertised by other peers.
The expected use case is where the user has a trusted server where
they run a Bitmessage daemon permanently. If they then run a second
instance of the client on a local machine periodically when they want
to check for messages it will sync with the network a lot faster
without compromising security.
"""
def __init__(self): def __init__(self):
asyncore.set_rates( asyncore.set_rates(
BMConfigParser().safeGetInt( BMConfigParser().safeGetInt(
@ -39,9 +56,33 @@ class BMConnectionPool(object):
self.listeningSockets = {} self.listeningSockets = {}
self.udpSockets = {} self.udpSockets = {}
self.streams = [] self.streams = []
self.lastSpawned = 0 self._lastSpawned = 0
self.spawnWait = 2 self._spawnWait = 2
self.bootstrapped = False self._bootstrapped = False
trustedPeer = BMConfigParser().safeGet(
'bitmessagesettings', 'trustedpeer')
try:
if trustedPeer:
host, port = trustedPeer.split(':')
self.trustedPeer = Peer(host, int(port))
except ValueError:
sys.exit(
'Bad trustedpeer config setting! It should be set as'
' trustedpeer=<hostname>:<portnumber>'
)
def connections(self):
"""
Shortcut for combined list of connections from
`inboundConnections` and `outboundConnections` dicts
"""
return self.inboundConnections.values() + self.outboundConnections.values()
def establishedConnections(self):
"""Shortcut for list of connections having fullyEstablished == True"""
return [
x for x in self.connections() if x.fullyEstablished]
def connectToStream(self, streamNumber): def connectToStream(self, streamNumber):
"""Connect to a bitmessage stream""" """Connect to a bitmessage stream"""
@ -72,10 +113,7 @@ class BMConnectionPool(object):
def isAlreadyConnected(self, nodeid): def isAlreadyConnected(self, nodeid):
"""Check if we're already connected to this peer""" """Check if we're already connected to this peer"""
for i in ( for i in self.connections():
self.inboundConnections.values() +
self.outboundConnections.values()
):
try: try:
if nodeid == i.nodeid: if nodeid == i.nodeid:
return True return True
@ -101,7 +139,7 @@ class BMConnectionPool(object):
if isinstance(connection, UDPSocket): if isinstance(connection, UDPSocket):
del self.udpSockets[connection.listening.host] del self.udpSockets[connection.listening.host]
elif isinstance(connection, TCPServer): elif isinstance(connection, TCPServer):
del self.listeningSockets[state.Peer( del self.listeningSockets[Peer(
connection.destination.host, connection.destination.port)] connection.destination.host, connection.destination.port)]
elif connection.isOutbound: elif connection.isOutbound:
try: try:
@ -127,10 +165,11 @@ class BMConnectionPool(object):
"bitmessagesettings", "onionbindip") "bitmessagesettings", "onionbindip")
else: else:
host = '127.0.0.1' host = '127.0.0.1'
if (BMConfigParser().safeGetBoolean( if (
"bitmessagesettings", "sockslisten") or BMConfigParser().safeGetBoolean("bitmessagesettings", "sockslisten")
BMConfigParser().safeGet( or BMConfigParser().safeGet("bitmessagesettings", "socksproxytype")
"bitmessagesettings", "socksproxytype") == "none"): == "none"
):
# python doesn't like bind + INADDR_ANY? # python doesn't like bind + INADDR_ANY?
# host = socket.INADDR_ANY # host = socket.INADDR_ANY
host = BMConfigParser().get("network", "bind") host = BMConfigParser().get("network", "bind")
@ -190,6 +229,7 @@ class BMConnectionPool(object):
def loop(self): # pylint: disable=too-many-branches,too-many-statements def loop(self): # pylint: disable=too-many-branches,too-many-statements
"""Main Connectionpool's loop""" """Main Connectionpool's loop"""
# pylint: disable=too-many-locals
# defaults to empty loop if outbound connections are maxed # defaults to empty loop if outbound connections are maxed
spawnConnections = False spawnConnections = False
acceptConnections = True acceptConnections = True
@ -203,11 +243,13 @@ class BMConnectionPool(object):
'bitmessagesettings', 'socksproxytype', '') 'bitmessagesettings', 'socksproxytype', '')
onionsocksproxytype = BMConfigParser().safeGet( onionsocksproxytype = BMConfigParser().safeGet(
'bitmessagesettings', 'onionsocksproxytype', '') 'bitmessagesettings', 'onionsocksproxytype', '')
if (socksproxytype[:5] == 'SOCKS' and if (
not BMConfigParser().safeGetBoolean( socksproxytype[:5] == 'SOCKS'
'bitmessagesettings', 'sockslisten') and and not BMConfigParser().safeGetBoolean(
'.onion' not in BMConfigParser().safeGet( 'bitmessagesettings', 'sockslisten')
'bitmessagesettings', 'onionhostname', '')): and '.onion' not in BMConfigParser().safeGet(
'bitmessagesettings', 'onionhostname', '')
):
acceptConnections = False acceptConnections = False
# pylint: disable=too-many-nested-blocks # pylint: disable=too-many-nested-blocks
@ -215,8 +257,8 @@ class BMConnectionPool(object):
if not knownnodes.knownNodesActual: if not knownnodes.knownNodesActual:
self.startBootstrappers() self.startBootstrappers()
knownnodes.knownNodesActual = True knownnodes.knownNodesActual = True
if not self.bootstrapped: if not self._bootstrapped:
self.bootstrapped = True self._bootstrapped = True
Proxy.proxy = ( Proxy.proxy = (
BMConfigParser().safeGet( BMConfigParser().safeGet(
'bitmessagesettings', 'sockshostname'), 'bitmessagesettings', 'sockshostname'),
@ -245,7 +287,7 @@ class BMConnectionPool(object):
for i in range( for i in range(
state.maximumNumberOfHalfOpenConnections - pending): state.maximumNumberOfHalfOpenConnections - pending):
try: try:
chosen = chooseConnection( chosen = self.trustedPeer or chooseConnection(
helper_random.randomchoice(self.streams)) helper_random.randomchoice(self.streams))
except ValueError: except ValueError:
continue continue
@ -256,10 +298,22 @@ class BMConnectionPool(object):
# don't connect to self # don't connect to self
if chosen in state.ownAddresses: if chosen in state.ownAddresses:
continue continue
# don't connect to the hosts from the same
# network group, defense against sibyl attacks
host_network_group = protocol.network_group(
chosen.host)
same_group = False
for j in self.outboundConnections.values():
if host_network_group == j.network_group:
same_group = True
if chosen.host == j.destination.host:
knownnodes.decreaseRating(chosen)
break
if same_group:
continue
try: try:
if (chosen.host.endswith(".onion") and if chosen.host.endswith(".onion") and Proxy.onion_proxy:
Proxy.onion_proxy is not None):
if onionsocksproxytype == "SOCKS5": if onionsocksproxytype == "SOCKS5":
self.addConnection(Socks5BMConnection(chosen)) self.addConnection(Socks5BMConnection(chosen))
elif onionsocksproxytype == "SOCKS4a": elif onionsocksproxytype == "SOCKS4a":
@ -274,12 +328,9 @@ class BMConnectionPool(object):
if e.errno == errno.ENETUNREACH: if e.errno == errno.ENETUNREACH:
continue continue
self.lastSpawned = time.time() self._lastSpawned = time.time()
else: else:
for i in ( for i in self.connections():
self.inboundConnections.values() +
self.outboundConnections.values()
):
# FIXME: rating will be increased after next connection # FIXME: rating will be increased after next connection
i.handle_close() i.handle_close()
@ -289,8 +340,8 @@ class BMConnectionPool(object):
self.startListening() self.startListening()
else: else:
for bind in re.sub( for bind in re.sub(
'[^\w.]+', ' ', # pylint: disable=anomalous-backslash-in-string r'[^\w.]+', ' ',
BMConfigParser().safeGet('network', 'bind') BMConfigParser().safeGet('network', 'bind')
).split(): ).split():
self.startListening(bind) self.startListening(bind)
logger.info('Listening for incoming connections.') logger.info('Listening for incoming connections.')
@ -299,8 +350,8 @@ class BMConnectionPool(object):
self.startUDPSocket() self.startUDPSocket()
else: else:
for bind in re.sub( for bind in re.sub(
'[^\w.]+', ' ', # pylint: disable=anomalous-backslash-in-string r'[^\w.]+', ' ',
BMConfigParser().safeGet('network', 'bind') BMConfigParser().safeGet('network', 'bind')
).split(): ).split():
self.startUDPSocket(bind) self.startUDPSocket(bind)
self.startUDPSocket(False) self.startUDPSocket(False)
@ -317,16 +368,13 @@ class BMConnectionPool(object):
i.accepting = i.connecting = i.connected = False i.accepting = i.connecting = i.connected = False
logger.info('Stopped udp sockets.') logger.info('Stopped udp sockets.')
loopTime = float(self.spawnWait) loopTime = float(self._spawnWait)
if self.lastSpawned < time.time() - self.spawnWait: if self._lastSpawned < time.time() - self._spawnWait:
loopTime = 2.0 loopTime = 2.0
asyncore.loop(timeout=loopTime, count=1000) asyncore.loop(timeout=loopTime, count=1000)
reaper = [] reaper = []
for i in ( for i in self.connections():
self.inboundConnections.values() +
self.outboundConnections.values()
):
minTx = time.time() - 20 minTx = time.time() - 20
if i.fullyEstablished: if i.fullyEstablished:
minTx -= 300 - 20 minTx -= 300 - 20
@ -338,10 +386,8 @@ class BMConnectionPool(object):
time.time() - i.lastTx) time.time() - i.lastTx)
i.set_state("close") i.set_state("close")
for i in ( for i in (
self.inboundConnections.values() + self.connections()
self.outboundConnections.values() + + self.listeningSockets.values() + self.udpSockets.values()
self.listeningSockets.values() +
self.udpSockets.values()
): ):
if not (i.accepting or i.connecting or i.connected): if not (i.accepting or i.connecting or i.connected):
reaper.append(i) reaper.append(i)

11
src/network/constants.py Normal file
View File

@ -0,0 +1,11 @@
"""
Network protocol constants
"""
ADDRESS_ALIVE = 10800 #: address is online if online less than this many seconds ago
MAX_ADDR_COUNT = 1000 #: protocol specification says max 1000 addresses in one addr command
MAX_MESSAGE_SIZE = 1600100 #: ~1.6 MB which is the maximum possible size of an inv message.
MAX_OBJECT_PAYLOAD_SIZE = 2**18 #: 2**18 = 256kB is the maximum size of an object payload
MAX_OBJECT_COUNT = 50000 #: protocol specification says max 50000 objects in one inv command
MAX_TIME_OFFSET = 3600 #: maximum time offset

11
src/network/dandelion.py
View File

@ -2,6 +2,7 @@
src/network/dandelion.py src/network/dandelion.py
======================== ========================
""" """
import logging
from collections import namedtuple from collections import namedtuple
from random import choice, sample, expovariate from random import choice, sample, expovariate
from threading import RLock from threading import RLock
@ -9,7 +10,6 @@ from time import time
import connectionpool import connectionpool
import state import state
from debug import logging
from queues import invQueue from queues import invQueue
from singleton import Singleton from singleton import Singleton
@ -24,6 +24,8 @@ MAX_STEMS = 2
Stem = namedtuple('Stem', ['child', 'stream', 'timeout']) Stem = namedtuple('Stem', ['child', 'stream', 'timeout'])
logger = logging.getLogger('default')
@Singleton @Singleton
class Dandelion(): # pylint: disable=old-style-class class Dandelion(): # pylint: disable=old-style-class
@ -72,9 +74,10 @@ class Dandelion(): # pylint: disable=old-style-class
def removeHash(self, hashId, reason="no reason specified"): def removeHash(self, hashId, reason="no reason specified"):
"""Switch inventory vector from stem to fluff mode""" """Switch inventory vector from stem to fluff mode"""
logging.debug( if logger.isEnabledFor(logging.DEBUG):
"%s entering fluff mode due to %s.", logger.debug(
''.join('%02x' % ord(i) for i in hashId), reason) '%s entering fluff mode due to %s.',
''.join('%02x' % ord(i) for i in hashId), reason)
with self.lock: with self.lock:
try: try:
del self.hashMap[hashId] del self.hashMap[hashId]

29
src/network/downloadthread.py
View File

@ -1,18 +1,17 @@
""" """
src/network/downloadthread.py `DownloadThread` class definition
=============================
""" """
import time import time
import addresses import addresses
import helper_random import helper_random
import protocol import protocol
from dandelion import Dandelion from dandelion import Dandelion
from debug import logger
from helper_threading import StoppableThread
from inventory import Inventory from inventory import Inventory
from network.connectionpool import BMConnectionPool from network.connectionpool import BMConnectionPool
from objectracker import missingObjects from objectracker import missingObjects
from threads import StoppableThread
class DownloadThread(StoppableThread): class DownloadThread(StoppableThread):
@ -25,12 +24,11 @@ class DownloadThread(StoppableThread):
def __init__(self): def __init__(self):
super(DownloadThread, self).__init__(name="Downloader") super(DownloadThread, self).__init__(name="Downloader")
logger.info("init download thread")
self.lastCleaned = time.time() self.lastCleaned = time.time()
def cleanPending(self): def cleanPending(self):
"""Expire pending downloads eventually""" """Expire pending downloads eventually"""
deadline = time.time() - DownloadThread.requestExpires deadline = time.time() - self.requestExpires
try: try:
toDelete = [k for k, v in missingObjects.iteritems() if v < deadline] toDelete = [k for k, v in missingObjects.iteritems() if v < deadline]
except RuntimeError: except RuntimeError:
@ -44,15 +42,12 @@ class DownloadThread(StoppableThread):
while not self._stopped: while not self._stopped:
requested = 0 requested = 0
# Choose downloading peers randomly # Choose downloading peers randomly
connections = [ connections = BMConnectionPool().establishedConnections()
x for x in
BMConnectionPool().inboundConnections.values() + BMConnectionPool().outboundConnections.values()
if x.fullyEstablished]
helper_random.randomshuffle(connections) helper_random.randomshuffle(connections)
try: requestChunk = max(int(
requestChunk = max(int(min(DownloadThread.maxRequestChunk, len(missingObjects)) / len(connections)), 1) min(self.maxRequestChunk, len(missingObjects))
except ZeroDivisionError: / len(connections)), 1) if connections else 1
requestChunk = 1
for i in connections: for i in connections:
now = time.time() now = time.time()
# avoid unnecessary delay # avoid unnecessary delay
@ -78,9 +73,11 @@ class DownloadThread(StoppableThread):
continue continue
payload[0:0] = addresses.encodeVarint(chunkCount) payload[0:0] = addresses.encodeVarint(chunkCount)
i.append_write_buf(protocol.CreatePacket('getdata', payload)) i.append_write_buf(protocol.CreatePacket('getdata', payload))
logger.debug("%s:%i Requesting %i objects", i.destination.host, i.destination.port, chunkCount) self.logger.debug(
'%s:%i Requesting %i objects',
i.destination.host, i.destination.port, chunkCount)
requested += chunkCount requested += chunkCount
if time.time() >= self.lastCleaned + DownloadThread.cleanInterval: if time.time() >= self.lastCleaned + self.cleanInterval:
self.cleanPending() self.cleanPending()
if not requested: if not requested:
self.stop.wait(1) self.stop.wait(1)

4
src/network/http-old.py → src/network/http_old.py
View File

@ -1,7 +1,3 @@
"""
src/network/http-old.py
=======================
"""
import asyncore import asyncore
import socket import socket
import time import time

21
src/network/invthread.py
View File

@ -9,10 +9,10 @@ from time import time
import addresses import addresses
import protocol import protocol
import state import state
from helper_threading import StoppableThread
from network.connectionpool import BMConnectionPool from network.connectionpool import BMConnectionPool
from network.dandelion import Dandelion from network.dandelion import Dandelion
from queues import invQueue from queues import invQueue
from threads import StoppableThread
def handleExpiredDandelion(expired): def handleExpiredDandelion(expired):
@ -20,9 +20,7 @@ def handleExpiredDandelion(expired):
the object""" the object"""
if not expired: if not expired:
return return
for i in \ for i in BMConnectionPool().connections():
BMConnectionPool().inboundConnections.values() + \
BMConnectionPool().outboundConnections.values():
if not i.fullyEstablished: if not i.fullyEstablished:
continue continue
for x in expired: for x in expired:
@ -44,9 +42,7 @@ class InvThread(StoppableThread):
def handleLocallyGenerated(stream, hashId): def handleLocallyGenerated(stream, hashId):
"""Locally generated inventory items require special handling""" """Locally generated inventory items require special handling"""
Dandelion().addHash(hashId, stream=stream) Dandelion().addHash(hashId, stream=stream)
for connection in \ for connection in BMConnectionPool().connections():
BMConnectionPool().inboundConnections.values() + \
BMConnectionPool().outboundConnections.values():
if state.dandelion and connection != Dandelion().objectChildStem(hashId): if state.dandelion and connection != Dandelion().objectChildStem(hashId):
continue continue
connection.objectsNewToThem[hashId] = time() connection.objectsNewToThem[hashId] = time()
@ -67,8 +63,7 @@ class InvThread(StoppableThread):
break break
if chunk: if chunk:
for connection in BMConnectionPool().inboundConnections.values() + \ for connection in BMConnectionPool().connections():
BMConnectionPool().outboundConnections.values():
fluffs = [] fluffs = []
stems = [] stems = []
for inv in chunk: for inv in chunk:
@ -96,13 +91,13 @@ class InvThread(StoppableThread):
if fluffs: if fluffs:
random.shuffle(fluffs) random.shuffle(fluffs)
connection.append_write_buf(protocol.CreatePacket( connection.append_write_buf(protocol.CreatePacket(
'inv', addresses.encodeVarint(len(fluffs)) + 'inv',
"".join(fluffs))) addresses.encodeVarint(len(fluffs)) + ''.join(fluffs)))
if stems: if stems:
random.shuffle(stems) random.shuffle(stems)
connection.append_write_buf(protocol.CreatePacket( connection.append_write_buf(protocol.CreatePacket(
'dinv', addresses.encodeVarint(len(stems)) + 'dinv',
"".join(stems))) addresses.encodeVarint(len(stems)) + ''.join(stems)))
invQueue.iterate() invQueue.iterate()
for i in range(len(chunk)): for i in range(len(chunk)):

11
src/network/networkthread.py
View File

@ -1,20 +1,13 @@
"""
src/network/networkthread.py
============================
"""
import network.asyncore_pollchoose as asyncore import network.asyncore_pollchoose as asyncore
import state import state
from debug import logger
from helper_threading import StoppableThread
from network.connectionpool import BMConnectionPool from network.connectionpool import BMConnectionPool
from queues import excQueue from queues import excQueue
from threads import StoppableThread
class BMNetworkThread(StoppableThread): class BMNetworkThread(StoppableThread):
"""A thread to handle network concerns""" """A thread to handle network concerns"""
def __init__(self): name = "Asyncore"
super(BMNetworkThread, self).__init__(name="Asyncore")
logger.info("init asyncore thread")
def run(self): def run(self):
try: try:

4
src/network/node.py
View File

@ -1,7 +1,7 @@
""" """
src/network/node.py Named tuples representing the network peers
===================
""" """
import collections import collections
Peer = collections.namedtuple('Peer', ['host', 'port'])
Node = collections.namedtuple('Node', ['services', 'host', 'port']) Node = collections.namedtuple('Node', ['services', 'host', 'port'])

3
src/network/objectracker.py
View File

@ -95,8 +95,7 @@ class ObjectTracker(object):
def handleReceivedObject(self, streamNumber, hashid): def handleReceivedObject(self, streamNumber, hashid):
"""Handling received object""" """Handling received object"""
for i in network.connectionpool.BMConnectionPool().inboundConnections.values( for i in network.connectionpool.BMConnectionPool().connections():
) + network.connectionpool.BMConnectionPool().outboundConnections.values():
if not i.fullyEstablished: if not i.fullyEstablished:
continue continue
try: try:

18
src/network/proxy.py
View File

@ -3,14 +3,16 @@ src/network/proxy.py
==================== ====================
""" """
# pylint: disable=protected-access # pylint: disable=protected-access
import logging
import socket import socket
import time import time
import asyncore_pollchoose as asyncore import asyncore_pollchoose as asyncore
import state
from advanceddispatcher import AdvancedDispatcher from advanceddispatcher import AdvancedDispatcher
from bmconfigparser import BMConfigParser from bmconfigparser import BMConfigParser
from debug import logger from node import Peer
logger = logging.getLogger('default')
class ProxyError(Exception): class ProxyError(Exception):
@ -88,9 +90,10 @@ class Proxy(AdvancedDispatcher):
def onion_proxy(self, address): def onion_proxy(self, address):
"""Set onion proxy address""" """Set onion proxy address"""
if address is not None and ( if address is not None and (
not isinstance(address, tuple) or len(address) < 2 or not isinstance(address, tuple) or len(address) < 2
not isinstance(address[0], str) or or not isinstance(address[0], str)
not isinstance(address[1], int)): or not isinstance(address[1], int)
):
raise ValueError raise ValueError
self.__class__._onion_proxy = address self.__class__._onion_proxy = address
@ -105,7 +108,7 @@ class Proxy(AdvancedDispatcher):
self.__class__._onion_auth = authTuple self.__class__._onion_auth = authTuple
def __init__(self, address): def __init__(self, address):
if not isinstance(address, state.Peer): if not isinstance(address, Peer):
raise ValueError raise ValueError
AdvancedDispatcher.__init__(self) AdvancedDispatcher.__init__(self)
self.destination = address self.destination = address
@ -144,5 +147,6 @@ class Proxy(AdvancedDispatcher):
def state_proxy_handshake_done(self): def state_proxy_handshake_done(self):
"""Handshake is complete at this point""" """Handshake is complete at this point"""
self.connectedAt = time.time() # pylint: disable=attribute-defined-outside-init # pylint: disable=attribute-defined-outside-init
self.connectedAt = time.time()
return False return False

23
src/network/receivequeuethread.py
View File

@ -2,18 +2,16 @@ import errno
import Queue import Queue
import socket import socket
from debug import logger import state
from helper_threading import StoppableThread
from network.connectionpool import BMConnectionPool from network.connectionpool import BMConnectionPool
from network.advanceddispatcher import UnknownStateError from network.advanceddispatcher import UnknownStateError
from queues import receiveDataQueue from queues import receiveDataQueue
import state from threads import StoppableThread
class ReceiveQueueThread(StoppableThread): class ReceiveQueueThread(StoppableThread):
def __init__(self, num=0): def __init__(self, num=0):
super(ReceiveQueueThread, self).__init__(name="ReceiveQueue_%i" % num) super(ReceiveQueueThread, self).__init__(name="ReceiveQueue_%i" % num)
logger.info("init receive queue thread %i", num)
def run(self): def run(self):
while not self._stopped and state.shutdown == 0: while not self._stopped and state.shutdown == 0:
@ -26,27 +24,26 @@ class ReceiveQueueThread(StoppableThread):
break break
# cycle as long as there is data # cycle as long as there is data
# methods should return False if there isn't enough data, or the connection is to be aborted # methods should return False if there isn't enough data,
# state_* methods should return False if there isn't enough data,
# or the connection is to be aborted # or the connection is to be aborted
# state_* methods should return False if there isn't
# enough data, or the connection is to be aborted
try: try:
connection = BMConnectionPool().getConnectionByAddr(dest) connection = BMConnectionPool().getConnectionByAddr(dest)
# KeyError = connection object not found except KeyError: # connection object not found
except KeyError:
receiveDataQueue.task_done() receiveDataQueue.task_done()
continue continue
try: try:
connection.process() connection.process()
# UnknownStateError = state isn't implemented except UnknownStateError: # state isn't implemented
except (UnknownStateError):
pass pass
except socket.error as err: except socket.error as err:
if err.errno == errno.EBADF: if err.errno == errno.EBADF:
connection.set_state("close", 0) connection.set_state("close", 0)
else: else:
logger.error("Socket error: %s", str(err)) self.logger.error('Socket error: %s', err)
except: except:
logger.error("Error processing", exc_info=True) self.logger.error('Error processing', exc_info=True)
receiveDataQueue.task_done() receiveDataQueue.task_done()

4
src/network/socks5.py
View File

@ -8,7 +8,7 @@ src/network/socks5.py
import socket import socket
import struct import struct
import state from node import Peer
from proxy import GeneralProxyError, Proxy, ProxyError from proxy import GeneralProxyError, Proxy, ProxyError
@ -200,7 +200,7 @@ class Socks5Resolver(Socks5):
def __init__(self, host): def __init__(self, host):
self.host = host self.host = host
self.port = 8444 self.port = 8444
Socks5.__init__(self, address=state.Peer(self.host, self.port)) Socks5.__init__(self, address=Peer(self.host, self.port))
def state_auth_done(self): def state_auth_done(self):
"""Perform resolving""" """Perform resolving"""

17
src/network/stats.py
View File

@ -19,16 +19,7 @@ currentSentSpeed = 0
def connectedHostsList(): def connectedHostsList():
"""List of all the connected hosts""" """List of all the connected hosts"""
retval = [] return BMConnectionPool().establishedConnections()
for i in BMConnectionPool().inboundConnections.values() + \
BMConnectionPool().outboundConnections.values():
if not i.fullyEstablished:
continue
try:
retval.append(i)
except AttributeError:
pass
return retval
def sentBytes(): def sentBytes():
@ -71,12 +62,6 @@ def downloadSpeed():
def pendingDownload(): def pendingDownload():
"""Getting pending downloads""" """Getting pending downloads"""
return len(missingObjects) return len(missingObjects)
# tmp = {}
# for connection in BMConnectionPool().inboundConnections.values() + \
# BMConnectionPool().outboundConnections.values():
# for k in connection.objectsNewToMe.keys():
# tmp[k] = True
# return len(tmp)
def pendingUpload(): def pendingUpload():

22
src/network/tcp.py
View File

@ -4,6 +4,7 @@ src/network/tcp.py
================== ==================
""" """
import logging
import math import math
import random import random
import socket import socket
@ -18,18 +19,22 @@ import protocol
import shared import shared
import state import state
from bmconfigparser import BMConfigParser from bmconfigparser import BMConfigParser
from debug import logger
from helper_random import randomBytes from helper_random import randomBytes
from inventory import Inventory from inventory import Inventory
from network.advanceddispatcher import AdvancedDispatcher from network.advanceddispatcher import AdvancedDispatcher
from network.assemble import assemble_addr
from network.bmproto import BMProto from network.bmproto import BMProto
from network.constants import MAX_OBJECT_COUNT
from network.dandelion import Dandelion from network.dandelion import Dandelion
from network.objectracker import ObjectTracker from network.objectracker import ObjectTracker
from network.socks4a import Socks4aConnection from network.socks4a import Socks4aConnection
from network.socks5 import Socks5Connection from network.socks5 import Socks5Connection
from network.tls import TLSDispatcher from network.tls import TLSDispatcher
from node import Peer
from queues import UISignalQueue, invQueue, receiveDataQueue from queues import UISignalQueue, invQueue, receiveDataQueue
logger = logging.getLogger('default')
class TCPConnection(BMProto, TLSDispatcher): class TCPConnection(BMProto, TLSDispatcher):
# pylint: disable=too-many-instance-attributes # pylint: disable=too-many-instance-attributes
@ -47,7 +52,7 @@ class TCPConnection(BMProto, TLSDispatcher):
self.connectedAt = 0 self.connectedAt = 0
self.skipUntil = 0 self.skipUntil = 0
if address is None and sock is not None: if address is None and sock is not None:
self.destination = state.Peer(*sock.getpeername()) self.destination = Peer(*sock.getpeername())
self.isOutbound = False self.isOutbound = False
TLSDispatcher.__init__(self, sock, server_side=True) TLSDispatcher.__init__(self, sock, server_side=True)
self.connectedAt = time.time() self.connectedAt = time.time()
@ -81,6 +86,7 @@ class TCPConnection(BMProto, TLSDispatcher):
) )
except socket.error: except socket.error:
pass # it's probably a hostname pass # it's probably a hostname
self.network_group = protocol.network_group(self.destination.host)
ObjectTracker.__init__(self) # pylint: disable=non-parent-init-called ObjectTracker.__init__(self) # pylint: disable=non-parent-init-called
self.bm_proto_reset() self.bm_proto_reset()
self.set_state("bm_header", expectBytes=protocol.Header.size) self.set_state("bm_header", expectBytes=protocol.Header.size)
@ -179,7 +185,7 @@ class TCPConnection(BMProto, TLSDispatcher):
for peer, params in addrs[substream]: for peer, params in addrs[substream]:
templist.append((substream, peer, params["lastseen"])) templist.append((substream, peer, params["lastseen"]))
if templist: if templist:
self.append_write_buf(BMProto.assembleAddr(templist)) self.append_write_buf(assemble_addr(templist))
def sendBigInv(self): def sendBigInv(self):
""" """
@ -218,7 +224,7 @@ class TCPConnection(BMProto, TLSDispatcher):
# Remove -1 below when sufficient time has passed for users to # Remove -1 below when sufficient time has passed for users to
# upgrade to versions of PyBitmessage that accept inv with 50,000 # upgrade to versions of PyBitmessage that accept inv with 50,000
# items # items
if objectCount >= BMProto.maxObjectCount - 1: if objectCount >= MAX_OBJECT_COUNT - 1:
sendChunk() sendChunk()
payload = b'' payload = b''
objectCount = 0 objectCount = 0
@ -332,7 +338,7 @@ def bootstrap(connection_class):
_connection_base = connection_class _connection_base = connection_class
def __init__(self, host, port): def __init__(self, host, port):
self._connection_base.__init__(self, state.Peer(host, port)) self._connection_base.__init__(self, Peer(host, port))
self.close_reason = self._succeed = False self.close_reason = self._succeed = False
def bm_command_addr(self): def bm_command_addr(self):
@ -369,6 +375,7 @@ class TCPServer(AdvancedDispatcher):
for attempt in range(50): for attempt in range(50):
try: try:
if attempt > 0: if attempt > 0:
logger.warning('Failed to bind on port %s', port)
port = random.randint(32767, 65535) port = random.randint(32767, 65535)
self.bind((host, port)) self.bind((host, port))
except socket.error as e: except socket.error as e:
@ -376,11 +383,12 @@ class TCPServer(AdvancedDispatcher):
continue continue
else: else:
if attempt > 0: if attempt > 0:
logger.warning('Setting port to %s', port)
BMConfigParser().set( BMConfigParser().set(
'bitmessagesettings', 'port', str(port)) 'bitmessagesettings', 'port', str(port))
BMConfigParser().save() BMConfigParser().save()
break break
self.destination = state.Peer(host, port) self.destination = Peer(host, port)
self.bound = True self.bound = True
self.listen(5) self.listen(5)
@ -398,7 +406,7 @@ class TCPServer(AdvancedDispatcher):
except (TypeError, IndexError): except (TypeError, IndexError):
return return
state.ownAddresses[state.Peer(*sock.getsockname())] = True state.ownAddresses[Peer(*sock.getsockname())] = True
if ( if (
len(connectionpool.BMConnectionPool().inboundConnections) + len(connectionpool.BMConnectionPool().inboundConnections) +
len(connectionpool.BMConnectionPool().outboundConnections) > len(connectionpool.BMConnectionPool().outboundConnections) >

49
src/network/threads.py Normal file
View File

@ -0,0 +1,49 @@
"""Threading primitives for the network package"""
import logging
import random
import threading
from contextlib import contextmanager
class StoppableThread(threading.Thread):
"""Base class for application threads with stopThread method"""
name = None
logger = logging.getLogger('default')
def __init__(self, name=None):
if name:
self.name = name
super(StoppableThread, self).__init__(name=self.name)
self.stop = threading.Event()
self._stopped = False
random.seed()
self.logger.info('Init thread %s', self.name)
def stopThread(self):
"""Stop the thread"""
self._stopped = True
self.stop.set()
class BusyError(threading.ThreadError):
"""
Thread error raised when another connection holds the lock
we are trying to acquire.
"""
pass
@contextmanager
def nonBlocking(lock):
"""
A context manager which acquires given lock non-blocking
and raises BusyError if failed to acquire.
"""
locked = lock.acquire(False)
if not locked:
raise BusyError
try:
yield
finally:
lock.release()

44
src/network/tls.py
View File

@ -2,17 +2,18 @@
SSL/TLS negotiation. SSL/TLS negotiation.
""" """
import logging
import os import os
import socket import socket
import ssl import ssl
import sys import sys
from debug import logger
from network.advanceddispatcher import AdvancedDispatcher from network.advanceddispatcher import AdvancedDispatcher
import network.asyncore_pollchoose as asyncore import network.asyncore_pollchoose as asyncore
from queues import receiveDataQueue from queues import receiveDataQueue
import paths import paths
logger = logging.getLogger('default')
_DISCONNECTED_SSL = frozenset((ssl.SSL_ERROR_EOF,)) _DISCONNECTED_SSL = frozenset((ssl.SSL_ERROR_EOF,))
@ -38,12 +39,13 @@ else:
sslProtocolCiphers = "AECDH-AES256-SHA" sslProtocolCiphers = "AECDH-AES256-SHA"
class TLSDispatcher(AdvancedDispatcher): # pylint: disable=too-many-instance-attributes class TLSDispatcher(AdvancedDispatcher):
"""TLS functionality for classes derived from AdvancedDispatcher""" """TLS functionality for classes derived from AdvancedDispatcher"""
# pylint: disable=too-many-arguments, super-init-not-called, unused-argument # pylint: disable=too-many-instance-attributes
# pylint: disable=too-many-arguments,super-init-not-called,unused-argument
def __init__( def __init__(
self, address=None, sock=None, certfile=None, keyfile=None, self, address=None, sock=None, certfile=None, keyfile=None,
server_side=False, ciphers=sslProtocolCiphers server_side=False, ciphers=sslProtocolCiphers
): ):
self.want_read = self.want_write = True self.want_read = self.want_write = True
if certfile is None: if certfile is None:
@ -95,7 +97,10 @@ class TLSDispatcher(AdvancedDispatcher): # pylint: disable=too-many-instanc
@staticmethod @staticmethod
def state_tls_handshake(): def state_tls_handshake():
"""Do nothing while TLS handshake is pending, as during this phase we need to react to callbacks instead""" """
Do nothing while TLS handshake is pending, as during this phase
we need to react to callbacks instead
"""
return False return False
def writable(self): def writable(self):
@ -121,10 +126,11 @@ class TLSDispatcher(AdvancedDispatcher): # pylint: disable=too-many-instanc
except AttributeError: except AttributeError:
return AdvancedDispatcher.readable(self) return AdvancedDispatcher.readable(self)
def handle_read(self): # pylint: disable=inconsistent-return-statements def handle_read(self): # pylint: disable=inconsistent-return-statements
""" """
Handle reads for sockets during TLS handshake. Requires special treatment as during the handshake, buffers must Handle reads for sockets during TLS handshake. Requires special
remain empty and normal reads must be ignored treatment as during the handshake, buffers must remain empty
and normal reads must be ignored.
""" """
try: try:
# wait for write buffer flush # wait for write buffer flush
@ -146,10 +152,11 @@ class TLSDispatcher(AdvancedDispatcher): # pylint: disable=too-many-instanc
self.handle_close() self.handle_close()
return return
def handle_write(self): # pylint: disable=inconsistent-return-statements def handle_write(self): # pylint: disable=inconsistent-return-statements
""" """
Handle writes for sockets during TLS handshake. Requires special treatment as during the handshake, buffers Handle writes for sockets during TLS handshake. Requires special
must remain empty and normal writes must be ignored treatment as during the handshake, buffers must remain empty
and normal writes must be ignored.
""" """
try: try:
# wait for write buffer flush # wait for write buffer flush
@ -192,18 +199,23 @@ class TLSDispatcher(AdvancedDispatcher): # pylint: disable=too-many-instanc
if not (self.want_write or self.want_read): if not (self.want_write or self.want_read):
raise raise
except socket.error as err: except socket.error as err:
if err.errno in asyncore._DISCONNECTED: # pylint: disable=protected-access # pylint: disable=protected-access
if err.errno in asyncore._DISCONNECTED:
self.handle_close() self.handle_close()
else: else:
raise raise
else: else:
if sys.version_info >= (2, 7, 9): if sys.version_info >= (2, 7, 9):
self.tlsVersion = self.sslSocket.version() self.tlsVersion = self.sslSocket.version()
logger.debug("%s:%i: TLS handshake success, TLS protocol version: %s", logger.debug(
self.destination.host, self.destination.port, self.sslSocket.version()) '%s:%i: TLS handshake success, TLS protocol version: %s',
self.destination.host, self.destination.port,
self.tlsVersion)
else: else:
self.tlsVersion = "TLSv1" self.tlsVersion = "TLSv1"
logger.debug("%s:%i: TLS handshake success", self.destination.host, self.destination.port) logger.debug(
'%s:%i: TLS handshake success',
self.destination.host, self.destination.port)
# The handshake has completed, so remove this channel and... # The handshake has completed, so remove this channel and...
self.del_channel() self.del_channel()
self.set_socket(self.sslSocket) self.set_socket(self.sslSocket)

21
src/network/udp.py
View File

@ -2,24 +2,28 @@
src/network/udp.py src/network/udp.py
================== ==================
""" """
import logging
import time import time
import socket import socket
import state import state
import protocol import protocol
from bmproto import BMProto from bmproto import BMProto
from debug import logger from node import Peer
from objectracker import ObjectTracker from objectracker import ObjectTracker
from queues import receiveDataQueue from queues import receiveDataQueue
logger = logging.getLogger('default')
class UDPSocket(BMProto): # pylint: disable=too-many-instance-attributes
class UDPSocket(BMProto): # pylint: disable=too-many-instance-attributes
"""Bitmessage protocol over UDP (class)""" """Bitmessage protocol over UDP (class)"""
port = 8444 port = 8444
announceInterval = 60 announceInterval = 60
def __init__(self, host=None, sock=None, announcing=False): def __init__(self, host=None, sock=None, announcing=False):
super(BMProto, self).__init__(sock=sock) # pylint: disable=bad-super-call # pylint: disable=bad-super-call
super(BMProto, self).__init__(sock=sock)
self.verackReceived = True self.verackReceived = True
self.verackSent = True self.verackSent = True
# .. todo:: sort out streams # .. todo:: sort out streams
@ -40,8 +44,8 @@ class UDPSocket(BMProto): # pylint: disable=too-many-instance-attribut
else: else:
self.socket = sock self.socket = sock
self.set_socket_reuse() self.set_socket_reuse()
self.listening = state.Peer(*self.socket.getsockname()) self.listening = Peer(*self.socket.getsockname())
self.destination = state.Peer(*self.socket.getsockname()) self.destination = Peer(*self.socket.getsockname())
ObjectTracker.__init__(self) ObjectTracker.__init__(self)
self.connecting = False self.connecting = False
self.connected = True self.connected = True
@ -79,7 +83,8 @@ class UDPSocket(BMProto): # pylint: disable=too-many-instance-attribut
decodedIP = protocol.checkIPAddress(str(ip)) decodedIP = protocol.checkIPAddress(str(ip))
if stream not in state.streamsInWhichIAmParticipating: if stream not in state.streamsInWhichIAmParticipating:
continue continue
if (seenTime < time.time() - self.maxTimeOffset or seenTime > time.time() + self.maxTimeOffset): if (seenTime < time.time() - self.maxTimeOffset
or seenTime > time.time() + self.maxTimeOffset):
continue continue
if decodedIP is False: if decodedIP is False:
# if the address isn't local, interpret it as # if the address isn't local, interpret it as
@ -92,7 +97,7 @@ class UDPSocket(BMProto): # pylint: disable=too-many-instance-attribut
self.destination.host, self.destination.port, remoteport) self.destination.host, self.destination.port, remoteport)
if self.local: if self.local:
state.discoveredPeers[ state.discoveredPeers[
state.Peer(self.destination.host, remoteport) Peer(self.destination.host, remoteport)
] = time.time() ] = time.time()
return True return True
@ -127,7 +132,7 @@ class UDPSocket(BMProto): # pylint: disable=too-many-instance-attribut
logger.error("socket error: %s", e) logger.error("socket error: %s", e)
return return
self.destination = state.Peer(*addr) self.destination = Peer(*addr)
encodedAddr = protocol.encodeHost(addr[0]) encodedAddr = protocol.encodeHost(addr[0])
self.local = bool(protocol.checkIPAddress(encodedAddr, True)) self.local = bool(protocol.checkIPAddress(encodedAddr, True))
# overwrite the old buffer to avoid mixing data and so that # overwrite the old buffer to avoid mixing data and so that

43
src/network/uploadthread.py
View File

@ -1,43 +1,40 @@
""" """
src/network/uploadthread.py `UploadThread` class definition
""" """
# pylint: disable=unsubscriptable-object
import time import time
import helper_random import helper_random
import protocol import protocol
from debug import logger
from helper_threading import StoppableThread
from inventory import Inventory from inventory import Inventory
from network.connectionpool import BMConnectionPool from network.connectionpool import BMConnectionPool
from network.dandelion import Dandelion from network.dandelion import Dandelion
from randomtrackingdict import RandomTrackingDict from randomtrackingdict import RandomTrackingDict
from threads import StoppableThread
class UploadThread(StoppableThread): class UploadThread(StoppableThread):
"""This is a thread that uploads the objects that the peers requested from me """ """
This is a thread that uploads the objects that the peers requested from me
"""
maxBufSize = 2097152 # 2MB maxBufSize = 2097152 # 2MB
name = "Uploader"
def __init__(self):
super(UploadThread, self).__init__(name="Uploader")
logger.info("init upload thread")
def run(self): def run(self):
while not self._stopped: while not self._stopped:
uploaded = 0 uploaded = 0
# Choose downloading peers randomly # Choose uploading peers randomly
connections = [x for x in BMConnectionPool().inboundConnections.values() + connections = BMConnectionPool().establishedConnections()
BMConnectionPool().outboundConnections.values() if x.fullyEstablished]
helper_random.randomshuffle(connections) helper_random.randomshuffle(connections)
for i in connections: for i in connections:
now = time.time() now = time.time()
# avoid unnecessary delay # avoid unnecessary delay
if i.skipUntil >= now: if i.skipUntil >= now:
continue continue
if len(i.write_buf) > UploadThread.maxBufSize: if len(i.write_buf) > self.maxBufSize:
continue continue
try: try:
request = i.pendingUpload.randomKeys(RandomTrackingDict.maxPending) request = i.pendingUpload.randomKeys(
RandomTrackingDict.maxPending)
except KeyError: except KeyError:
continue continue
payload = bytearray() payload = bytearray()
@ -47,22 +44,26 @@ class UploadThread(StoppableThread):
if Dandelion().hasHash(chunk) and \ if Dandelion().hasHash(chunk) and \
i != Dandelion().objectChildStem(chunk): i != Dandelion().objectChildStem(chunk):
i.antiIntersectionDelay() i.antiIntersectionDelay()
logger.info('%s asked for a stem object we didn\'t offer to it.', self.logger.info(
i.destination) '%s asked for a stem object we didn\'t offer to it.',
i.destination)
break break
try: try:
payload.extend(protocol.CreatePacket('object', payload.extend(protocol.CreatePacket(
Inventory()[chunk].payload)) 'object', Inventory()[chunk].payload))
chunk_count += 1 chunk_count += 1
except KeyError: except KeyError:
i.antiIntersectionDelay() i.antiIntersectionDelay()
logger.info('%s asked for an object we don\'t have.', i.destination) self.logger.info(
'%s asked for an object we don\'t have.',
i.destination)
break break
if not chunk_count: if not chunk_count:
continue continue
i.append_write_buf(payload) i.append_write_buf(payload)
logger.debug("%s:%i Uploading %i objects", self.logger.debug(
i.destination.host, i.destination.port, chunk_count) '%s:%i Uploading %i objects',
i.destination.host, i.destination.port, chunk_count)
uploaded += chunk_count uploaded += chunk_count
if not uploaded: if not uploaded:
self.stop.wait(1) self.stop.wait(1)

53
src/openclpow.py
View File

@ -1,8 +1,9 @@
#!/usr/bin/env python2.7 #!/usr/bin/env python2.7
"""
Module for Proof of Work using OpenCL
"""
from struct import pack, unpack from struct import pack, unpack
import time
import hashlib import hashlib
import random
import os import os
from bmconfigparser import BMConfigParser from bmconfigparser import BMConfigParser
@ -27,6 +28,8 @@ except ImportError:
def initCL(): def initCL():
"""Initlialise OpenCL engine"""
# pylint: disable=global-statement
global ctx, queue, program, hash_dt, libAvailable global ctx, queue, program, hash_dt, libAvailable
if libAvailable is False: if libAvailable is False:
return return
@ -40,12 +43,13 @@ def initCL():
for platform in cl.get_platforms(): for platform in cl.get_platforms():
gpus.extend(platform.get_devices(device_type=cl.device_type.GPU)) gpus.extend(platform.get_devices(device_type=cl.device_type.GPU))
if BMConfigParser().safeGet("bitmessagesettings", "opencl") == platform.vendor: if BMConfigParser().safeGet("bitmessagesettings", "opencl") == platform.vendor:
enabledGpus.extend(platform.get_devices(device_type=cl.device_type.GPU)) enabledGpus.extend(platform.get_devices(
device_type=cl.device_type.GPU))
if platform.vendor not in vendors: if platform.vendor not in vendors:
vendors.append(platform.vendor) vendors.append(platform.vendor)
except: except:
pass pass
if (len(enabledGpus) > 0): if enabledGpus:
ctx = cl.Context(devices=enabledGpus) ctx = cl.Context(devices=enabledGpus)
queue = cl.CommandQueue(ctx) queue = cl.CommandQueue(ctx)
f = open(os.path.join(paths.codePath(), "bitmsghash", 'bitmsghash.cl'), 'r') f = open(os.path.join(paths.codePath(), "bitmsghash", 'bitmsghash.cl'), 'r')
@ -55,23 +59,29 @@ def initCL():
else: else:
logger.info("No OpenCL GPUs found") logger.info("No OpenCL GPUs found")
del enabledGpus[:] del enabledGpus[:]
except Exception as e: except Exception:
logger.error("OpenCL fail: ", exc_info=True) logger.error("OpenCL fail: ", exc_info=True)
del enabledGpus[:] del enabledGpus[:]
def openclAvailable(): def openclAvailable():
return (len(gpus) > 0) """Are there any OpenCL GPUs available?"""
return bool(gpus)
def openclEnabled(): def openclEnabled():
return (len(enabledGpus) > 0) """Is OpenCL enabled (and available)?"""
return bool(enabledGpus)
def do_opencl_pow(hash, target):
def do_opencl_pow(hash_, target):
"""Perform PoW using OpenCL"""
output = numpy.zeros(1, dtype=[('v', numpy.uint64, 1)]) output = numpy.zeros(1, dtype=[('v', numpy.uint64, 1)])
if (len(enabledGpus) == 0): if not enabledGpus:
return output[0][0] return output[0][0]
data = numpy.zeros(1, dtype=hash_dt, order='C') data = numpy.zeros(1, dtype=hash_dt, order='C')
data[0]['v'] = ("0000000000000000" + hash).decode("hex") data[0]['v'] = ("0000000000000000" + hash_).decode("hex")
data[0]['target'] = target data[0]['target'] = target
hash_buf = cl.Buffer(ctx, cl.mem_flags.READ_ONLY | cl.mem_flags.COPY_HOST_PTR, hostbuf=data) hash_buf = cl.Buffer(ctx, cl.mem_flags.READ_ONLY | cl.mem_flags.COPY_HOST_PTR, hostbuf=data)
@ -83,9 +93,8 @@ def do_opencl_pow(hash, target):
kernel.set_arg(0, hash_buf) kernel.set_arg(0, hash_buf)
kernel.set_arg(1, dest_buf) kernel.set_arg(1, dest_buf)
start = time.time()
progress = 0 progress = 0
globamt = worksize*2000 globamt = worksize * 2000
while output[0][0] == 0 and shutdown == 0: while output[0][0] == 0 and shutdown == 0:
kernel.set_arg(2, pack("<Q", progress)) kernel.set_arg(2, pack("<Q", progress))
@ -96,20 +105,18 @@ def do_opencl_pow(hash, target):
cl.enqueue_copy(queue, output, dest_buf) cl.enqueue_copy(queue, output, dest_buf)
queue.finish() queue.finish()
progress += globamt progress += globamt
sofar = time.time() - start
# logger.debug("Working for %.3fs, %.2f Mh/s", sofar, (progress / sofar) / 1000000)
if shutdown != 0: if shutdown != 0:
raise Exception ("Interrupted") raise Exception("Interrupted")
taken = time.time() - start
# logger.debug("Took %d tries.", progress) # logger.debug("Took %d tries.", progress)
return output[0][0] return output[0][0]
#initCL()
if __name__ == "__main__": if __name__ == "__main__":
target = 54227212183L initCL()
initialHash = "3758f55b5a8d902fd3597e4ce6a2d3f23daff735f65d9698c270987f4e67ad590b93f3ffeba0ef2fd08a8dc2f87b68ae5a0dc819ab57f22ad2c4c9c8618a43b3".decode("hex") target_ = 54227212183
nonce = do_opencl_pow(initialHash.encode("hex"), target) initialHash = ("3758f55b5a8d902fd3597e4ce6a2d3f23daff735f65d9698c270987f4e67ad590"
trialValue, = unpack('>Q',hashlib.sha512(hashlib.sha512(pack('>Q',nonce) + initialHash).digest()).digest()[0:8]) "b93f3ffeba0ef2fd08a8dc2f87b68ae5a0dc819ab57f22ad2c4c9c8618a43b3").decode("hex")
print "{} - value {} < {}".format(nonce, trialValue, target) nonce = do_opencl_pow(initialHash.encode("hex"), target_)
trialValue, = unpack(
'>Q', hashlib.sha512(hashlib.sha512(pack('>Q', nonce) + initialHash).digest()).digest()[0:8])
print "{} - value {} < {}".format(nonce, trialValue, target_)

8
src/paths.py
View File

@ -1,6 +1,5 @@
""" """
src/paths.py Path related functions
============
""" """
# pylint: disable=import-error # pylint: disable=import-error
import logging import logging
@ -11,6 +10,7 @@ from datetime import datetime
from shutil import move from shutil import move
from kivy.utils import platform from kivy.utils import platform
logger = logging.getLogger('default') logger = logging.getLogger('default')
# When using py2exe or py2app, the variable frozen is added to the sys # When using py2exe or py2app, the variable frozen is added to the sys
@ -46,7 +46,8 @@ def lookupAppdataFolder():
dataFolder = os.path.join( dataFolder = os.path.join(
os.environ['HOME'], os.environ['HOME'],
'Library/Application Support/', APPNAME 'Library/Application Support/', APPNAME
) + '/' # ..fixme:: should also be os.path.sep ) + '/'
except KeyError: except KeyError:
sys.exit( sys.exit(
'Could not find home folder, please report this message' 'Could not find home folder, please report this message'
@ -83,6 +84,7 @@ def codePath():
return os.path.dirname(__file__) return os.path.dirname(__file__)
return ( return (
os.environ.get('RESOURCEPATH') os.environ.get('RESOURCEPATH')
# pylint: disable=protected-access
if frozen == "macosx_app" else sys._MEIPASS) if frozen == "macosx_app" else sys._MEIPASS)

2
src/plugins/proxyconfig_stem.py
View File

@ -18,7 +18,7 @@ class DebugLogger(object):
"""Safe logger wrapper for tor and plugin's logs""" """Safe logger wrapper for tor and plugin's logs"""
# pylint: disable=too-few-public-methods # pylint: disable=too-few-public-methods
def __init__(self): def __init__(self):
self._logger = logging.getLogger(__name__.split('.', 1)[0]) self._logger = logging.getLogger('default')
self._levels = { self._levels = {
'err': 40, 'err': 40,
'warn': 30, 'warn': 30,

171
src/protocol.py
View File

@ -1,12 +1,8 @@
# pylint: disable=too-many-boolean-expressions,too-many-return-statements,too-many-locals,too-many-statements
""" """
protocol.py
===========
Low-level protocol-related functions. Low-level protocol-related functions.
""" """
# pylint: disable=too-many-boolean-expressions,too-many-return-statements
from __future__ import absolute_import # pylint: disable=too-many-locals,too-many-statements
import base64 import base64
import hashlib import hashlib
@ -14,7 +10,6 @@ import random
import socket import socket
import sys import sys
import time import time
import traceback
from binascii import hexlify from binascii import hexlify
from struct import pack, unpack, Struct from struct import pack, unpack, Struct
@ -29,10 +24,18 @@ from fallback import RIPEMD160Hash
from helper_sql import sqlExecute from helper_sql import sqlExecute
from version import softwareVersion from version import softwareVersion
# Service flags # Service flags
#: This is a normal network node
NODE_NETWORK = 1 NODE_NETWORK = 1
#: This node supports SSL/TLS in the current connect (python < 2.7.9
#: only supports an SSL client, so in that case it would only have this
#: on when the connection is a client).
NODE_SSL = 2 NODE_SSL = 2
# (Proposal) This node may do PoW on behalf of some its peers
# (PoW offloading/delegating), but it doesn't have to. Clients may have
# to meet additional requirements (e.g. TLS authentication)
# NODE_POW = 4
#: Node supports dandelion
NODE_DANDELION = 8 NODE_DANDELION = 8
# Bitfield flags # Bitfield flags
@ -94,7 +97,8 @@ def isBitSetWithinBitfield(fourByteString, n):
def encodeHost(host): def encodeHost(host):
"""Encode a given host to be used in low-level socket operations""" """Encode a given host to be used in low-level socket operations"""
if host.find('.onion') > -1: if host.find('.onion') > -1:
return '\xfd\x87\xd8\x7e\xeb\x43' + base64.b32decode(host.split(".")[0], True) return '\xfd\x87\xd8\x7e\xeb\x43' + base64.b32decode(
host.split(".")[0], True)
elif host.find(':') == -1: elif host.find(':') == -1:
return '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xFF\xFF' + \ return '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xFF\xFF' + \
socket.inet_aton(host) socket.inet_aton(host)
@ -110,8 +114,39 @@ def networkType(host):
return 'IPv6' return 'IPv6'
def network_group(host):
"""Canonical identifier of network group
simplified, borrowed from
GetGroup() in src/netaddresses.cpp in bitcoin core"""
if not isinstance(host, str):
return None
network_type = networkType(host)
try:
raw_host = encodeHost(host)
except socket.error:
return host
if network_type == 'IPv4':
decoded_host = checkIPv4Address(raw_host[12:], True)
if decoded_host:
# /16 subnet
return raw_host[12:14]
elif network_type == 'IPv6':
decoded_host = checkIPv6Address(raw_host, True)
if decoded_host:
# /32 subnet
return raw_host[0:12]
else:
# just host, e.g. for tor
return host
# global network type group for local, private, unroutable
return network_type
def checkIPAddress(host, private=False): def checkIPAddress(host, private=False):
"""Returns hostStandardFormat if it is a valid IP address, otherwise returns False""" """
Returns hostStandardFormat if it is a valid IP address,
otherwise returns False
"""
if host[0:12] == '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xFF\xFF': if host[0:12] == '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xFF\xFF':
hostStandardFormat = socket.inet_ntop(socket.AF_INET, host[12:]) hostStandardFormat = socket.inet_ntop(socket.AF_INET, host[12:])
return checkIPv4Address(host[12:], hostStandardFormat, private) return checkIPv4Address(host[12:], hostStandardFormat, private)
@ -127,35 +162,46 @@ def checkIPAddress(host, private=False):
except ValueError: except ValueError:
return False return False
if hostStandardFormat == "": if hostStandardFormat == "":
# This can happen on Windows systems which are not 64-bit compatible # This can happen on Windows systems which are
# so let us drop the IPv6 address. # not 64-bit compatible so let us drop the IPv6 address.
return False return False
return checkIPv6Address(host, hostStandardFormat, private) return checkIPv6Address(host, hostStandardFormat, private)
def checkIPv4Address(host, hostStandardFormat, private=False): def checkIPv4Address(host, hostStandardFormat, private=False):
"""Returns hostStandardFormat if it is an IPv4 address, otherwise returns False""" """
Returns hostStandardFormat if it is an IPv4 address,
otherwise returns False
"""
if host[0] == '\x7F': # 127/8 if host[0] == '\x7F': # 127/8
if not private: if not private:
logger.debug('Ignoring IP address in loopback range: %s', hostStandardFormat) logger.debug(
'Ignoring IP address in loopback range: %s',
hostStandardFormat)
return hostStandardFormat if private else False return hostStandardFormat if private else False
if host[0] == '\x0A': # 10/8 if host[0] == '\x0A': # 10/8
if not private: if not private:
logger.debug('Ignoring IP address in private range: %s', hostStandardFormat) logger.debug(
'Ignoring IP address in private range: %s', hostStandardFormat)
return hostStandardFormat if private else False return hostStandardFormat if private else False
if host[0:2] == '\xC0\xA8': # 192.168/16 if host[0:2] == '\xC0\xA8': # 192.168/16
if not private: if not private:
logger.debug('Ignoring IP address in private range: %s', hostStandardFormat) logger.debug(
'Ignoring IP address in private range: %s', hostStandardFormat)
return hostStandardFormat if private else False return hostStandardFormat if private else False
if host[0:2] >= '\xAC\x10' and host[0:2] < '\xAC\x20': # 172.16/12 if host[0:2] >= '\xAC\x10' and host[0:2] < '\xAC\x20': # 172.16/12
if not private: if not private:
logger.debug('Ignoring IP address in private range: %s', hostStandardFormat) logger.debug(
'Ignoring IP address in private range: %s', hostStandardFormat)
return hostStandardFormat if private else False return hostStandardFormat if private else False
return False if private else hostStandardFormat return False if private else hostStandardFormat
def checkIPv6Address(host, hostStandardFormat, private=False): def checkIPv6Address(host, hostStandardFormat, private=False):
"""Returns hostStandardFormat if it is an IPv6 address, otherwise returns False""" """
Returns hostStandardFormat if it is an IPv6 address,
otherwise returns False
"""
if host == ('\x00' * 15) + '\x01': if host == ('\x00' * 15) + '\x01':
if not private: if not private:
logger.debug('Ignoring loopback address: %s', hostStandardFormat) logger.debug('Ignoring loopback address: %s', hostStandardFormat)
@ -166,7 +212,8 @@ def checkIPv6Address(host, hostStandardFormat, private=False):
return hostStandardFormat if private else False return hostStandardFormat if private else False
if (ord(host[0]) & 0xfe) == 0xfc: if (ord(host[0]) & 0xfe) == 0xfc:
if not private: if not private:
logger.debug('Ignoring unique local address: %s', hostStandardFormat) logger.debug(
'Ignoring unique local address: %s', hostStandardFormat)
return hostStandardFormat if private else False return hostStandardFormat if private else False
return False if private else hostStandardFormat return False if private else hostStandardFormat
@ -187,28 +234,27 @@ def haveSSL(server=False):
def checkSocksIP(host): def checkSocksIP(host):
"""Predicate to check if we're using a SOCKS proxy""" """Predicate to check if we're using a SOCKS proxy"""
sockshostname = BMConfigParser().safeGet(
'bitmessagesettings', 'sockshostname')
try: try:
if state.socksIP is None or not state.socksIP: if not state.socksIP:
state.socksIP = socket.gethostbyname(BMConfigParser().get("bitmessagesettings", "sockshostname")) state.socksIP = socket.gethostbyname(sockshostname)
# uninitialised except NameError: # uninitialised
except NameError: state.socksIP = socket.gethostbyname(sockshostname)
state.socksIP = socket.gethostbyname(BMConfigParser().get("bitmessagesettings", "sockshostname")) except (TypeError, socket.gaierror): # None, resolving failure
# resolving failure state.socksIP = sockshostname
except socket.gaierror:
state.socksIP = BMConfigParser().get("bitmessagesettings", "sockshostname")
return state.socksIP == host return state.socksIP == host
def isProofOfWorkSufficient(data, def isProofOfWorkSufficient(
nonceTrialsPerByte=0, data, nonceTrialsPerByte=0, payloadLengthExtraBytes=0, recvTime=0):
payloadLengthExtraBytes=0,
recvTime=0):
""" """
Validate an object's Proof of Work using method described in: Validate an object's Proof of Work using method described
https://bitmessage.org/wiki/Proof_of_work `here <https://bitmessage.org/wiki/Proof_of_work>`_
Arguments: Arguments:
int nonceTrialsPerByte (default: from default.py) int nonceTrialsPerByte (default: from `.defaults`)
int payloadLengthExtraBytes (default: from default.py) int payloadLengthExtraBytes (default: from `.defaults`)
float recvTime (optional) UNIX epoch time when object was float recvTime (optional) UNIX epoch time when object was
received from the network (default: current system time) received from the network (default: current system time)
Returns: Returns:
@ -222,18 +268,20 @@ def isProofOfWorkSufficient(data,
TTL = endOfLifeTime - (int(recvTime) if recvTime else int(time.time())) TTL = endOfLifeTime - (int(recvTime) if recvTime else int(time.time()))
if TTL < 300: if TTL < 300:
TTL = 300 TTL = 300
POW, = unpack('>Q', hashlib.sha512(hashlib.sha512(data[ POW, = unpack('>Q', hashlib.sha512(hashlib.sha512(
:8] + hashlib.sha512(data[8:]).digest()).digest()).digest()[0:8]) data[:8] + hashlib.sha512(data[8:]).digest()
return POW <= 2 ** 64 / (nonceTrialsPerByte * ).digest()).digest()[0:8])
(len(data) + payloadLengthExtraBytes + return POW <= 2 ** 64 / (
((TTL * (len(data) + payloadLengthExtraBytes)) / (2 ** 16)))) nonceTrialsPerByte * (
len(data) + payloadLengthExtraBytes +
((TTL * (len(data) + payloadLengthExtraBytes)) / (2 ** 16))))
# Packet creation # Packet creation
def CreatePacket(command, payload=''): def CreatePacket(command, payload=''):
"""Construct and return a number of bytes from a payload""" """Construct and return a packet"""
payload_length = len(payload) payload_length = len(payload)
checksum = hashlib.sha512(payload).digest()[0:4] checksum = hashlib.sha512(payload).digest()[0:4]
@ -243,8 +291,13 @@ def CreatePacket(command, payload=''):
return bytes(b) return bytes(b)
def assembleVersionMessage(remoteHost, remotePort, participatingStreams, server=False, nodeid=None): def assembleVersionMessage(
"""Construct the payload of a version message, return the resultng bytes of running CreatePacket() on it""" remoteHost, remotePort, participatingStreams, server=False, nodeid=None
):
"""
Construct the payload of a version message,
return the resulting bytes of running `CreatePacket` on it
"""
payload = '' payload = ''
payload += pack('>L', 3) # protocol version. payload += pack('>L', 3) # protocol version.
# bitflags of the services I offer. # bitflags of the services I offer.
@ -256,9 +309,10 @@ def assembleVersionMessage(remoteHost, remotePort, participatingStreams, server=
) )
payload += pack('>q', int(time.time())) payload += pack('>q', int(time.time()))
payload += pack( # boolservices of remote connection; ignored by the remote host.
'>q', 1) # boolservices of remote connection; ignored by the remote host. payload += pack('>q', 1)
if checkSocksIP(remoteHost) and server: # prevent leaking of tor outbound IP if checkSocksIP(remoteHost) and server:
# prevent leaking of tor outbound IP
payload += encodeHost('127.0.0.1') payload += encodeHost('127.0.0.1')
payload += pack('>H', 8444) payload += pack('>H', 8444)
else: else:
@ -277,21 +331,25 @@ def assembleVersionMessage(remoteHost, remotePort, participatingStreams, server=
(NODE_SSL if haveSSL(server) else 0) | (NODE_SSL if haveSSL(server) else 0) |
(NODE_DANDELION if state.dandelion else 0) (NODE_DANDELION if state.dandelion else 0)
) )
# = 127.0.0.1. This will be ignored by the remote host. The actual remote connected IP will be used. # = 127.0.0.1. This will be ignored by the remote host.
payload += '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xFF\xFF' + pack('>L', 2130706433) # The actual remote connected IP will be used.
payload += '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xFF\xFF' + pack(
'>L', 2130706433)
# we have a separate extPort and incoming over clearnet # we have a separate extPort and incoming over clearnet
# or outgoing through clearnet # or outgoing through clearnet
extport = BMConfigParser().safeGetInt('bitmessagesettings', 'extport') extport = BMConfigParser().safeGetInt('bitmessagesettings', 'extport')
if ( if (
extport and ((server and not checkSocksIP(remoteHost)) or ( extport and ((server and not checkSocksIP(remoteHost)) or (
BMConfigParser().get('bitmessagesettings', 'socksproxytype') == BMConfigParser().get('bitmessagesettings', 'socksproxytype')
'none' and not server)) == 'none' and not server))
): ):
payload += pack('>H', extport) payload += pack('>H', extport)
elif checkSocksIP(remoteHost) and server: # incoming connection over Tor elif checkSocksIP(remoteHost) and server: # incoming connection over Tor
payload += pack('>H', BMConfigParser().getint('bitmessagesettings', 'onionport')) payload += pack(
'>H', BMConfigParser().getint('bitmessagesettings', 'onionport'))
else: # no extport and not incoming over Tor else: # no extport and not incoming over Tor
payload += pack('>H', BMConfigParser().getint('bitmessagesettings', 'port')) payload += pack(
'>H', BMConfigParser().getint('bitmessagesettings', 'port'))
if nodeid is not None: if nodeid is not None:
payload += nodeid[0:8] payload += nodeid[0:8]
@ -315,7 +373,10 @@ def assembleVersionMessage(remoteHost, remotePort, participatingStreams, server=
def assembleErrorMessage(fatal=0, banTime=0, inventoryVector='', errorText=''): def assembleErrorMessage(fatal=0, banTime=0, inventoryVector='', errorText=''):
"""Construct the payload of an error message, return the resultng bytes of running CreatePacket() on it""" """
Construct the payload of an error message,
return the resulting bytes of running `CreatePacket` on it
"""
payload = encodeVarint(fatal) payload = encodeVarint(fatal)
payload += encodeVarint(banTime) payload += encodeVarint(banTime)
payload += encodeVarint(len(inventoryVector)) payload += encodeVarint(len(inventoryVector))
@ -452,7 +513,7 @@ def decryptAndCheckPubkeyPayload(data, address):
except Exception: except Exception:
logger.critical( logger.critical(
'Pubkey decryption was UNsuccessful because of' 'Pubkey decryption was UNsuccessful because of'
' an unhandled exception! This is definitely a bug! \n%s', ' an unhandled exception! This is definitely a bug!',
traceback.format_exc() exc_info=True
) )
return 'failed' return 'failed'

13
src/pyelliptic/__init__.py
View File

@ -1,10 +1,13 @@
""" """
src/pyelliptic/__init__.py Copyright (C) 2010
===================================== Author: Yann GUIBET
Contact: <yannguibet@gmail.com>
Python OpenSSL wrapper.
For modern cryptography with ECC, AES, HMAC, Blowfish, ...
This is an abandoned package maintained inside of the PyBitmessage.
""" """
# Copyright (C) 2010
# Author: Yann GUIBET
# Contact: <yannguibet@gmail.com>
from .openssl import OpenSSL from .openssl import OpenSSL
from .ecc import ECC from .ecc import ECC

5
src/pyelliptic/hash.py
View File

@ -1,8 +1,5 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
""" """
src/pyelliptic/hash.py Wrappers for hash functions from OpenSSL.
=====================
""" """
# Copyright (C) 2011 Yann GUIBET <yannguibet@gmail.com> # Copyright (C) 2011 Yann GUIBET <yannguibet@gmail.com>
# See LICENSE for details. # See LICENSE for details.

23
src/pyelliptic/openssl.py
View File

@ -1,26 +1,19 @@
# Copyright (C) 2011 Yann GUIBET <yannguibet@gmail.com>
# See LICENSE for details.
#
# Software slightly changed by Jonathan Warren <bitmessage at-symbol jonwarren.org>
# pylint: disable=protected-access, import-error
""" """
src/pyelliptic/openssl.py This module loads openssl libs with ctypes and incapsulates
================================= needed openssl functionality in class _OpenSSL.
""" """
# pylint: disable=import-error
import sys import sys
import ctypes import ctypes
from kivy.utils import platform from kivy.utils import platform
OpenSSL = None OpenSSL = None
# !/usr/bin/env python
# -*- coding: utf-8 -*-
"""
src/pyelliptic/openssl.py
=====================
"""
# Copyright (C) 2011 Yann GUIBET <yannguibet@gmail.com>
# See LICENSE for details.
#
# Software slightly changed by Jonathan Warren <bitmessage at-symbol jonwarren.org>
# pylint: disable=protected-access
class CipherName: class CipherName:
"""Class returns cipher name, pointer and blocksize""" """Class returns cipher name, pointer and blocksize"""

20
src/qidenticon.py
View File

@ -1,23 +1,19 @@
#!/usr/bin/env python
# -*- coding:utf-8 -*-
# pylint: disable=too-many-locals,too-many-arguments,too-many-function-args # pylint: disable=too-many-locals,too-many-arguments,too-many-function-args
""" """
= usage = Usage
-----
== python ==
>>> import qtidenticon >>> import qtidenticon
>>> qtidenticon.render_identicon(code, size) >>> qtidenticon.render_identicon(code, size)
Return a PIL Image class instance which have generated identicon image. Return a PIL Image class instance which have generated identicon image.
```size``` specifies `patch size`. Generated image size is 3 * ```size```. ``size`` specifies `patch size`. Generated image size is 3 * ``size``.
""" """
from PyQt4 import QtGui from PyQt4 import QtGui
from PyQt4.QtCore import QSize, QPointF, Qt from PyQt4.QtCore import QSize, QPointF, Qt
from PyQt4.QtGui import QPixmap, QPainter, QPolygonF from PyQt4.QtGui import QPixmap, QPainter, QPolygonF
__all__ = ['render_identicon', 'IdenticonRendererBase']
class IdenticonRendererBase(object): class IdenticonRendererBase(object):
"""Encapsulate methods around rendering identicons""" """Encapsulate methods around rendering identicons"""
@ -26,7 +22,7 @@ class IdenticonRendererBase(object):
def __init__(self, code): def __init__(self, code):
""" """
@param code code for icon :param code: code for icon
""" """
if not isinstance(code, int): if not isinstance(code, int):
code = int(code) code = int(code)
@ -36,8 +32,8 @@ class IdenticonRendererBase(object):
""" """
render identicon to QPicture render identicon to QPicture
@param size identicon patchsize. (image size is 3 * [size]) :param size: identicon patchsize. (image size is 3 * [size])
@return QPicture :returns: :class:`QPicture`
""" """
# decode the code # decode the code
@ -79,7 +75,7 @@ class IdenticonRendererBase(object):
def drawPatchQt(self, pos, turn, invert, patch_type, image, size, foreColor, def drawPatchQt(self, pos, turn, invert, patch_type, image, size, foreColor,
backColor, penwidth): # pylint: disable=unused-argument backColor, penwidth): # pylint: disable=unused-argument
""" """
@param size patch size :param size: patch size
""" """
path = self.PATH_SET[patch_type] path = self.PATH_SET[patch_type]
if not path: if not path:
@ -134,7 +130,7 @@ class IdenticonRendererBase(object):
class DonRenderer(IdenticonRendererBase): class DonRenderer(IdenticonRendererBase):
""" """
Don Park's implementation of identicon Don Park's implementation of identicon
see : http://www.docuverse.com/blog/donpark/2007/01/19/identicon-updated-and-source-released see: http://www.docuverse.com/blog/donpark/2007/01/19/identicon-updated-and-source-released
""" """
PATH_SET = [ PATH_SET = [

45
src/queues.py
View File

@ -1,20 +1,51 @@
import Queue """Most of the queues used by bitmessage threads are defined here."""
import Queue
import threading
import time
from class_objectProcessorQueue import ObjectProcessorQueue
from multiqueue import MultiQueue from multiqueue import MultiQueue
class ObjectProcessorQueue(Queue.Queue):
"""Special queue class using lock for `.threads.objectProcessor`"""
maxSize = 32000000
def __init__(self):
Queue.Queue.__init__(self)
self.sizeLock = threading.Lock()
#: in Bytes. We maintain this to prevent nodes from flooding us
#: with objects which take up too much memory. If this gets
#: too big we'll sleep before asking for further objects.
self.curSize = 0
def put(self, item, block=True, timeout=None):
while self.curSize >= self.maxSize:
time.sleep(1)
with self.sizeLock:
self.curSize += len(item[1])
Queue.Queue.put(self, item, block, timeout)
def get(self, block=True, timeout=None):
item = Queue.Queue.get(self, block, timeout)
with self.sizeLock:
self.curSize -= len(item[1])
return item
workerQueue = Queue.Queue() workerQueue = Queue.Queue()
UISignalQueue = Queue.Queue() UISignalQueue = Queue.Queue()
addressGeneratorQueue = Queue.Queue() addressGeneratorQueue = Queue.Queue()
# receiveDataThreads dump objects they hear on the network into this #: `.network.ReceiveQueueThread` instances dump objects they hear
# queue to be processed. #: on the network into this queue to be processed.
objectProcessorQueue = ObjectProcessorQueue() objectProcessorQueue = ObjectProcessorQueue()
invQueue = MultiQueue() invQueue = MultiQueue()
addrQueue = MultiQueue() addrQueue = MultiQueue()
portCheckerQueue = Queue.Queue() portCheckerQueue = Queue.Queue()
receiveDataQueue = Queue.Queue() receiveDataQueue = Queue.Queue()
# The address generator thread uses this queue to get information back #: The address generator thread uses this queue to get information back
# to the API thread. #: to the API thread.
apiAddressGeneratorReturnQueue = Queue.Queue() apiAddressGeneratorReturnQueue = Queue.Queue()
# Exceptions #: for exceptions
excQueue = Queue.Queue() excQueue = Queue.Queue()

116
src/shared.py
View File

@ -1,21 +1,28 @@
from __future__ import division """
Some shared functions
.. deprecated:: 0.6.3
Should be moved to different places and this file removed,
but it needs refactoring.
"""
from __future__ import division
# Libraries. # Libraries.
import hashlib
import os import os
import sys import sys
import stat import stat
import threading import threading
import hashlib
import subprocess import subprocess
from binascii import hexlify from binascii import hexlify
from pyelliptic import arithmetic from pyelliptic import arithmetic
from kivy.utils import platform from kivy.utils import platform
# Project imports. # Project imports.
import state
import highlevelcrypto import highlevelcrypto
import state
from addresses import decodeAddress, encodeVarint
from bmconfigparser import BMConfigParser from bmconfigparser import BMConfigParser
from debug import logger from debug import logger
from addresses import decodeAddress, encodeVarint
from helper_sql import sqlQuery from helper_sql import sqlQuery
@ -56,6 +63,7 @@ maximumLengthOfTimeToBotherResendingMessages = 0
def isAddressInMyAddressBook(address): def isAddressInMyAddressBook(address):
"""Is address in my addressbook?"""
queryreturn = sqlQuery( queryreturn = sqlQuery(
'''select address from addressbook where address=?''', '''select address from addressbook where address=?''',
address) address)
@ -64,6 +72,7 @@ def isAddressInMyAddressBook(address):
# At this point we should really just have a isAddressInMy(book, address)... # At this point we should really just have a isAddressInMy(book, address)...
def isAddressInMySubscriptionsList(address): def isAddressInMySubscriptionsList(address):
"""Am I subscribed to this address?"""
queryreturn = sqlQuery( queryreturn = sqlQuery(
'''select * from subscriptions where address=?''', '''select * from subscriptions where address=?''',
str(address)) str(address))
@ -71,6 +80,9 @@ def isAddressInMySubscriptionsList(address):
def isAddressInMyAddressBookSubscriptionsListOrWhitelist(address): def isAddressInMyAddressBookSubscriptionsListOrWhitelist(address):
"""
Am I subscribed to this address, is it in my addressbook or whitelist?
"""
if isAddressInMyAddressBook(address): if isAddressInMyAddressBook(address):
return True return True
@ -91,6 +103,11 @@ def isAddressInMyAddressBookSubscriptionsListOrWhitelist(address):
def decodeWalletImportFormat(WIFstring): def decodeWalletImportFormat(WIFstring):
# pylint: disable=inconsistent-return-statements
"""
Convert private key from base58 that's used in the config file to
8-bit binary string
"""
fullString = arithmetic.changebase(WIFstring, 58, 256) fullString = arithmetic.changebase(WIFstring, 58, 256)
privkey = fullString[:-4] privkey = fullString[:-4]
if fullString[-4:] != \ if fullString[-4:] != \
@ -101,7 +118,7 @@ def decodeWalletImportFormat(WIFstring):
' 6 characters of the PRIVATE key: %s', ' 6 characters of the PRIVATE key: %s',
str(WIFstring)[:6] str(WIFstring)[:6]
) )
os._exit(0) os._exit(0) # pylint: disable=protected-access
# return "" # return ""
elif privkey[0] == '\x80': # checksum passed elif privkey[0] == '\x80': # checksum passed
return privkey[1:] return privkey[1:]
@ -111,10 +128,11 @@ def decodeWalletImportFormat(WIFstring):
' the checksum passed but the key doesn\'t begin with hex 80.' ' the checksum passed but the key doesn\'t begin with hex 80.'
' Here is the PRIVATE key: %s', WIFstring ' Here is the PRIVATE key: %s', WIFstring
) )
os._exit(0) os._exit(0) # pylint: disable=protected-access
def reloadMyAddressHashes(): def reloadMyAddressHashes():
"""Reload keys for user's addresses from the config file"""
logger.debug('reloading keys from keys.dat file') logger.debug('reloading keys from keys.dat file')
myECCryptorObjects.clear() myECCryptorObjects.clear()
@ -122,33 +140,29 @@ def reloadMyAddressHashes():
myAddressesByTag.clear() myAddressesByTag.clear()
# myPrivateKeys.clear() # myPrivateKeys.clear()
keyfileSecure = checkSensitiveFilePermissions(state.appdata + 'keys.dat') keyfileSecure = checkSensitiveFilePermissions(os.path.join(
state.appdata, 'keys.dat'))
hasEnabledKeys = False hasEnabledKeys = False
for addressInKeysFile in BMConfigParser().addresses(): for addressInKeysFile in BMConfigParser().addresses():
isEnabled = BMConfigParser().getboolean(addressInKeysFile, 'enabled') isEnabled = BMConfigParser().getboolean(addressInKeysFile, 'enabled')
if isEnabled: if isEnabled:
hasEnabledKeys = True hasEnabledKeys = True
# status # status
_, addressVersionNumber, streamNumber, hash = \ addressVersionNumber, streamNumber, hashobj = decodeAddress(addressInKeysFile)[1:]
decodeAddress(addressInKeysFile)
if addressVersionNumber in (2, 3, 4): if addressVersionNumber in (2, 3, 4):
# Returns a simple 32 bytes of information encoded # Returns a simple 32 bytes of information encoded
# in 64 Hex characters, or null if there was an error. # in 64 Hex characters, or null if there was an error.
privEncryptionKey = hexlify(decodeWalletImportFormat( privEncryptionKey = hexlify(decodeWalletImportFormat(
BMConfigParser().get(addressInKeysFile, 'privencryptionkey')) BMConfigParser().get(addressInKeysFile, 'privencryptionkey')))
)
# It is 32 bytes encoded as 64 hex characters # It is 32 bytes encoded as 64 hex characters
if len(privEncryptionKey) == 64: if len(privEncryptionKey) == 64:
myECCryptorObjects[hash] = \ myECCryptorObjects[hashobj] = \
highlevelcrypto.makeCryptor(privEncryptionKey) highlevelcrypto.makeCryptor(privEncryptionKey)
myAddressesByHash[hash] = addressInKeysFile myAddressesByHash[hashobj] = addressInKeysFile
tag = hashlib.sha512(hashlib.sha512( tag = hashlib.sha512(hashlib.sha512(
encodeVarint(addressVersionNumber) + encodeVarint(addressVersionNumber) +
encodeVarint(streamNumber) + hash).digest() encodeVarint(streamNumber) + hashobj).digest()).digest()[32:]
).digest()[32:]
myAddressesByTag[tag] = addressInKeysFile myAddressesByTag[tag] = addressInKeysFile
else: else:
logger.error( logger.error(
'Error in reloadMyAddressHashes: Can\'t handle' 'Error in reloadMyAddressHashes: Can\'t handle'
@ -161,6 +175,10 @@ def reloadMyAddressHashes():
def reloadBroadcastSendersForWhichImWatching(): def reloadBroadcastSendersForWhichImWatching():
"""
Reinitialize runtime data for the broadcasts I'm subscribed to
from the config file
"""
broadcastSendersForWhichImWatching.clear() broadcastSendersForWhichImWatching.clear()
MyECSubscriptionCryptorObjects.clear() MyECSubscriptionCryptorObjects.clear()
queryreturn = sqlQuery('SELECT address FROM subscriptions where enabled=1') queryreturn = sqlQuery('SELECT address FROM subscriptions where enabled=1')
@ -168,9 +186,9 @@ def reloadBroadcastSendersForWhichImWatching():
for row in queryreturn: for row in queryreturn:
address, = row address, = row
# status # status
_, addressVersionNumber, streamNumber, hash = decodeAddress(address) addressVersionNumber, streamNumber, hashobj = decodeAddress(address)[1:]
if addressVersionNumber == 2: if addressVersionNumber == 2:
broadcastSendersForWhichImWatching[hash] = 0 broadcastSendersForWhichImWatching[hashobj] = 0
# Now, for all addresses, even version 2 addresses, # Now, for all addresses, even version 2 addresses,
# we should create Cryptor objects in a dictionary which we will # we should create Cryptor objects in a dictionary which we will
# use to attempt to decrypt encrypted broadcast messages. # use to attempt to decrypt encrypted broadcast messages.
@ -178,14 +196,14 @@ def reloadBroadcastSendersForWhichImWatching():
if addressVersionNumber <= 3: if addressVersionNumber <= 3:
privEncryptionKey = hashlib.sha512( privEncryptionKey = hashlib.sha512(
encodeVarint(addressVersionNumber) + encodeVarint(addressVersionNumber) +
encodeVarint(streamNumber) + hash encodeVarint(streamNumber) + hashobj
).digest()[:32] ).digest()[:32]
MyECSubscriptionCryptorObjects[hash] = \ MyECSubscriptionCryptorObjects[hashobj] = \
highlevelcrypto.makeCryptor(hexlify(privEncryptionKey)) highlevelcrypto.makeCryptor(hexlify(privEncryptionKey))
else: else:
doubleHashOfAddressData = hashlib.sha512(hashlib.sha512( doubleHashOfAddressData = hashlib.sha512(hashlib.sha512(
encodeVarint(addressVersionNumber) + encodeVarint(addressVersionNumber) +
encodeVarint(streamNumber) + hash encodeVarint(streamNumber) + hashobj
).digest()).digest() ).digest()).digest()
tag = doubleHashOfAddressData[32:] tag = doubleHashOfAddressData[32:]
privEncryptionKey = doubleHashOfAddressData[:32] privEncryptionKey = doubleHashOfAddressData[:32]
@ -194,21 +212,22 @@ def reloadBroadcastSendersForWhichImWatching():
def fixPotentiallyInvalidUTF8Data(text): def fixPotentiallyInvalidUTF8Data(text):
"""Sanitise invalid UTF-8 strings"""
try: try:
unicode(text, 'utf-8') unicode(text, 'utf-8')
return text return text
except: except:
return 'Part of the message is corrupt. The message cannot be' \ return 'Part of the message is corrupt. The message cannot be' \
' displayed the normal way.\n\n' + repr(text) ' displayed the normal way.\n\n' + repr(text)
# Checks sensitive file permissions for inappropriate umask
# during keys.dat creation. (Or unwise subsequent chmod.)
#
# Returns true iff file appears to have appropriate permissions.
def checkSensitiveFilePermissions(filename): def checkSensitiveFilePermissions(filename):
"""
:param str filename: path to the file
:return: True if file appears to have appropriate permissions.
"""
if sys.platform == 'win32': if sys.platform == 'win32':
# TODO: This might deserve extra checks by someone familiar with # .. todo:: This might deserve extra checks by someone familiar with
# Windows systems. # Windows systems.
return True return True
elif sys.platform[:7] == 'freebsd': elif sys.platform[:7] == 'freebsd':
@ -216,30 +235,30 @@ def checkSensitiveFilePermissions(filename):
present_permissions = os.stat(filename)[0] present_permissions = os.stat(filename)[0]
disallowed_permissions = stat.S_IRWXG | stat.S_IRWXO disallowed_permissions = stat.S_IRWXG | stat.S_IRWXO
return present_permissions & disallowed_permissions == 0 return present_permissions & disallowed_permissions == 0
else: try:
try: # Skip known problems for non-Win32 filesystems
# Skip known problems for non-Win32 filesystems # without POSIX permissions.
# without POSIX permissions. fstype = subprocess.check_output(
fstype = subprocess.check_output( 'stat -f -c "%%T" %s' % (filename),
'stat -f -c "%%T" %s' % (filename), shell=True,
shell=True, stderr=subprocess.STDOUT
stderr=subprocess.STDOUT )
) if 'fuseblk' in fstype:
if 'fuseblk' in fstype: logger.info(
logger.info( 'Skipping file permissions check for %s.'
'Skipping file permissions check for %s.' ' Filesystem fuseblk detected.', filename)
' Filesystem fuseblk detected.', filename) return True
return True except:
except: # Swallow exception here, but we might run into trouble later!
# Swallow exception here, but we might run into trouble later! logger.error('Could not determine filesystem type. %s', filename)
logger.error('Could not determine filesystem type. %s', filename) present_permissions = os.stat(filename)[0]
present_permissions = os.stat(filename)[0] disallowed_permissions = stat.S_IRWXG | stat.S_IRWXO
disallowed_permissions = stat.S_IRWXG | stat.S_IRWXO return present_permissions & disallowed_permissions == 0
return present_permissions & disallowed_permissions == 0
# Fixes permissions on a sensitive file. # Fixes permissions on a sensitive file.
def fixSensitiveFilePermissions(filename, hasEnabledKeys): def fixSensitiveFilePermissions(filename, hasEnabledKeys):
"""Try to change file permissions to be more restrictive"""
if hasEnabledKeys: if hasEnabledKeys:
logger.warning( logger.warning(
'Keyfile had insecure permissions, and there were enabled' 'Keyfile had insecure permissions, and there were enabled'
@ -264,6 +283,7 @@ def fixSensitiveFilePermissions(filename, hasEnabledKeys):
def openKeysFile(): def openKeysFile():
"""Open keys file with an external editor"""
if 'linux' in sys.platform: if 'linux' in sys.platform:
subprocess.call(["xdg-open", state.appdata + 'keys.dat']) subprocess.call(["xdg-open", state.appdata + 'keys.dat'])
else: else:

32
src/shutdown.py
View File

@ -1,22 +1,24 @@
"""shutdown function"""
import os import os
import Queue import Queue
import threading import threading
import time import time
from debug import logger
from helper_sql import sqlQuery, sqlStoredProcedure
from helper_threading import StoppableThread
from knownnodes import saveKnownNodes
from inventory import Inventory
from queues import (
addressGeneratorQueue, objectProcessorQueue, UISignalQueue, workerQueue)
import shared import shared
import state import state
from debug import logger
from helper_sql import sqlQuery, sqlStoredProcedure
from inventory import Inventory
from knownnodes import saveKnownNodes
from network import StoppableThread
from queues import (
addressGeneratorQueue, objectProcessorQueue, UISignalQueue, workerQueue)
def doCleanShutdown(): def doCleanShutdown():
# Used to tell proof of work worker threads """
# and the objectProcessorThread to exit. Used to tell all the treads to finish work and exit.
"""
state.shutdown = 1 state.shutdown = 1
objectProcessorQueue.put(('checkShutdownVariable', 'no data')) objectProcessorQueue.put(('checkShutdownVariable', 'no data'))
@ -52,9 +54,11 @@ def doCleanShutdown():
time.sleep(.25) time.sleep(.25)
for thread in threading.enumerate(): for thread in threading.enumerate():
if (thread is not threading.currentThread() and if (
isinstance(thread, StoppableThread) and thread is not threading.currentThread()
thread.name != 'SQL'): and isinstance(thread, StoppableThread)
and thread.name != 'SQL'
):
logger.debug("Waiting for thread %s", thread.name) logger.debug("Waiting for thread %s", thread.name)
thread.join() thread.join()
@ -76,10 +80,10 @@ def doCleanShutdown():
except Queue.Empty: except Queue.Empty:
break break
if shared.thisapp.daemon or not state.enableGUI: # FIXME redundant? if shared.thisapp.daemon or not state.enableGUI: # ..fixme:: redundant?
logger.info('Clean shutdown complete.') logger.info('Clean shutdown complete.')
shared.thisapp.cleanup() shared.thisapp.cleanup()
os._exit(0) os._exit(0) # pylint: disable=protected-access
else: else:
logger.info('Core shutdown complete.') logger.info('Core shutdown complete.')
for thread in threading.enumerate(): for thread in threading.enumerate():

21
src/singleinstance.py
View File

@ -1,8 +1,13 @@
#! /usr/bin/env python """
This is based upon the singleton class from
`tendo <https://github.com/pycontribs/tendo>`_
which is under the Python Software Foundation License version 2
"""
import atexit import atexit
import os import os
import sys import sys
import state import state
try: try:
@ -11,15 +16,10 @@ except ImportError:
pass pass
class singleinstance(object):
class singleinstance:
""" """
Implements a single instance application by creating a lock file Implements a single instance application by creating a lock file
at appdata. at appdata.
This is based upon the singleton class from tendo
https://github.com/pycontribs/tendo
which is under the Python Software Foundation License version 2
""" """
def __init__(self, flavor_id="", daemon=False): def __init__(self, flavor_id="", daemon=False):
self.initialized = False self.initialized = False
@ -40,6 +40,7 @@ class singleinstance:
atexit.register(self.cleanup) atexit.register(self.cleanup)
def lock(self): def lock(self):
"""Obtain single instance lock"""
if self.lockPid is None: if self.lockPid is None:
self.lockPid = os.getpid() self.lockPid = os.getpid()
if sys.platform == 'win32': if sys.platform == 'win32':
@ -52,8 +53,7 @@ class singleinstance:
self.lockfile, self.lockfile,
os.O_CREAT | os.O_EXCL | os.O_RDWR | os.O_TRUNC os.O_CREAT | os.O_EXCL | os.O_RDWR | os.O_TRUNC
) )
except OSError: except OSError as e:
type, e, tb = sys.exc_info()
if e.errno == 13: if e.errno == 13:
print( print(
'Another instance of this application' 'Another instance of this application'
@ -84,6 +84,7 @@ class singleinstance:
self.fp.flush() self.fp.flush()
def cleanup(self): def cleanup(self):
"""Release single instance lock"""
if not self.initialized: if not self.initialized:
return return
if self.daemon and self.lockPid == os.getpid(): if self.daemon and self.lockPid == os.getpid():
@ -94,7 +95,7 @@ class singleinstance:
os.close(self.fd) os.close(self.fd)
else: else:
fcntl.lockf(self.fp, fcntl.LOCK_UN) fcntl.lockf(self.fp, fcntl.LOCK_UN)
except Exception, e: except Exception:
pass pass
return return

15
src/singleton.py
View File

@ -1,6 +1,21 @@
"""
Singleton decorator definition
"""
from functools import wraps
def Singleton(cls): def Singleton(cls):
"""
Decorator implementing the singleton pattern:
it restricts the instantiation of a class to one "single" instance.
"""
instances = {} instances = {}
# https://github.com/sphinx-doc/sphinx/issues/3783
@wraps(cls)
def getinstance(): def getinstance():
"""Find an instance or save newly created one"""
if cls not in instances: if cls not in instances:
instances[cls] = cls() instances[cls] = cls()
return instances[cls] return instances[cls]

45
src/state.py
View File

@ -1,30 +1,43 @@
""" """
src/state.py Global runtime variables.
=================================
""" """
import collections import collections
neededPubkeys = {} neededPubkeys = {}
streamsInWhichIAmParticipating = [] streamsInWhichIAmParticipating = []
# For UPnP
extPort = None extPort = None
# for Tor hidden service """For UPnP"""
socksIP = None socksIP = None
# Network protocols availability, initialised below """for Tor hidden service"""
networkProtocolAvailability = None
appdata = '' # holds the location of the application data storage directory appdata = ''
# Set to 1 by the doCleanShutdown function. """holds the location of the application data storage directory"""
# Used to tell the proof of work worker threads to exit.
shutdown = 0 shutdown = 0
"""
Set to 1 by the `.shutdown.doCleanShutdown` function.
Used to tell the threads to exit.
"""
# Component control flags - set on startup, do not change during runtime # Component control flags - set on startup, do not change during runtime
# The defaults are for standalone GUI (default operating mode) # The defaults are for standalone GUI (default operating mode)
enableNetwork = True # enable network threads enableNetwork = True
enableObjProc = True # enable object processing threads """enable network threads"""
enableAPI = True # enable API (if configured) enableObjProc = True
enableGUI = True # enable GUI (QT or ncurses) """enable object processing thread"""
enableSTDIO = False # enable STDIO threads enableAPI = True
"""enable API (if configured)"""
enableGUI = True
"""enable GUI (QT or ncurses)"""
enableSTDIO = False
"""enable STDIO threads"""
curses = False curses = False
sqlReady = False # set to true by sqlTread when ready for processing
sqlReady = False
"""set to true by `.threads.sqlThread` when ready for processing"""
maximumNumberOfHalfOpenConnections = 0 maximumNumberOfHalfOpenConnections = 0
invThread = None invThread = None
addrThread = None addrThread = None
@ -55,6 +68,8 @@ def resetNetworkProtocolAvailability():
resetNetworkProtocolAvailability() resetNetworkProtocolAvailability()
discoveredPeers = {}
dandelion = 0 dandelion = 0
testmode = False testmode = False

23
src/tests/core.py
View File

@ -17,6 +17,7 @@ from bmconfigparser import BMConfigParser
from helper_msgcoding import MsgEncode, MsgDecode from helper_msgcoding import MsgEncode, MsgDecode
from network import asyncore_pollchoose as asyncore from network import asyncore_pollchoose as asyncore
from network.connectionpool import BMConnectionPool from network.connectionpool import BMConnectionPool
from network.node import Peer
from network.tcp import Socks4aBMConnection, Socks5BMConnection, TCPConnection from network.tcp import Socks4aBMConnection, Socks5BMConnection, TCPConnection
from queues import excQueue from queues import excQueue
@ -30,7 +31,7 @@ def pickle_knownnodes():
with open(knownnodes_file, 'wb') as dst: with open(knownnodes_file, 'wb') as dst:
pickle.dump({ pickle.dump({
stream: { stream: {
state.Peer( Peer(
'%i.%i.%i.%i' % tuple([ '%i.%i.%i.%i' % tuple([
random.randint(1, 255) for i in range(4)]), random.randint(1, 255) for i in range(4)]),
8444): {'lastseen': now, 'rating': 0.1} 8444): {'lastseen': now, 'rating': 0.1}
@ -90,7 +91,7 @@ class TestCore(unittest.TestCase):
"""initial fill script from network.tcp""" """initial fill script from network.tcp"""
BMConfigParser().set('bitmessagesettings', 'dontconnect', 'true') BMConfigParser().set('bitmessagesettings', 'dontconnect', 'true')
try: try:
for peer in (state.Peer("127.0.0.1", 8448),): for peer in (Peer("127.0.0.1", 8448),):
direct = TCPConnection(peer) direct = TCPConnection(peer)
while asyncore.socket_map: while asyncore.socket_map:
print("loop, state = %s" % direct.state) print("loop, state = %s" % direct.state)
@ -147,7 +148,7 @@ class TestCore(unittest.TestCase):
def _initiate_bootstrap(self): def _initiate_bootstrap(self):
BMConfigParser().set('bitmessagesettings', 'dontconnect', 'true') BMConfigParser().set('bitmessagesettings', 'dontconnect', 'true')
self._outdate_knownnodes() self._outdate_knownnodes()
knownnodes.addKnownNode(1, state.Peer('127.0.0.1', 8444), is_self=True) knownnodes.addKnownNode(1, Peer('127.0.0.1', 8444), is_self=True)
knownnodes.cleanupKnownNodes() knownnodes.cleanupKnownNodes()
time.sleep(2) time.sleep(2)
@ -173,6 +174,22 @@ class TestCore(unittest.TestCase):
self.fail( self.fail(
'Failed to connect during %s sec' % (time.time() - _started)) 'Failed to connect during %s sec' % (time.time() - _started))
def test_onionservicesonly(self):
"""test onionservicesonly networking mode"""
BMConfigParser().set('bitmessagesettings', 'onionservicesonly', 'true')
self._initiate_bootstrap()
BMConfigParser().remove_option('bitmessagesettings', 'dontconnect')
for _ in range(360):
time.sleep(1)
for n, peer in enumerate(BMConnectionPool().outboundConnections):
if n > 2:
return
if not peer.host.endswith('.onion'):
self.fail(
'Found non onion hostname %s in outbound connections!'
% peer.host)
self.fail('Failed to connect to at least 3 nodes within 360 sec')
def test_bootstrap(self): def test_bootstrap(self):
"""test bootstrapping""" """test bootstrapping"""
self._initiate_bootstrap() self._initiate_bootstrap()

69
src/tests/test_logger.py Normal file
View File

@ -0,0 +1,69 @@
"""
Testing the logger configuration
"""
import logging
import os
import tempfile
import unittest
class TestLogger(unittest.TestCase):
"""A test case for bmconfigparser"""
conf_template = '''
[loggers]
keys=root
[handlers]
keys=default
[formatters]
keys=default
[formatter_default]
format=%(asctime)s {1} %(message)s
[handler_default]
class=FileHandler
level=NOTSET
formatter=default
args=('{0}', 'w')
[logger_root]
level=DEBUG
handlers=default
'''
def test_fileConfig(self):
"""Put logging.dat with special pattern and check it was used"""
tmp = os.environ['BITMESSAGE_HOME'] = tempfile.gettempdir()
log_config = os.path.join(tmp, 'logging.dat')
log_file = os.path.join(tmp, 'debug.log')
def gen_log_config(pattern):
"""A small closure to generate logging.dat with custom pattern"""
with open(log_config, 'wb') as dst:
dst.write(self.conf_template.format(log_file, pattern))
pattern = r' o_0 '
gen_log_config(pattern)
try:
from pybitmessage.debug import logger, resetLogging
if not os.path.isfile(log_file): # second pass
pattern = r' <===> '
gen_log_config(pattern)
resetLogging()
except ImportError:
self.fail('There is no package pybitmessage. Things gone wrong.')
finally:
os.remove(log_config)
logger_ = logging.getLogger('default')
self.assertEqual(logger, logger_)
logger_.info('Testing the logger...')
self.assertRegexpMatches(open(log_file).read(), pattern)

39
src/tests/test_networkgroup.py Normal file
View File

@ -0,0 +1,39 @@
"""
Test for network group
"""
import unittest
class TestNetworkGroup(unittest.TestCase):
"""
Test case for network group
"""
def test_network_group(self):
"""Test various types of network groups"""
from pybitmessage.protocol import network_group
test_ip = '1.2.3.4'
self.assertEqual('\x01\x02', network_group(test_ip))
test_ip = '127.0.0.1'
self.assertEqual('IPv4', network_group(test_ip))
test_ip = '0102:0304:0506:0708:090A:0B0C:0D0E:0F10'
self.assertEqual(
'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0A\x0B\x0C',
network_group(test_ip))
test_ip = 'bootstrap8444.bitmessage.org'
self.assertEqual(
'bootstrap8444.bitmessage.org',
network_group(test_ip))
test_ip = 'quzwelsuziwqgpt2.onion'
self.assertEqual(
test_ip,
network_group(test_ip))
test_ip = None
self.assertEqual(
None,
network_group(test_ip))

46
src/threads.py Normal file
View File

@ -0,0 +1,46 @@
"""
PyBitmessage does various tasks in separate threads. Most of them inherit
from `.network.StoppableThread`. There are `addressGenerator` for
addresses generation, `objectProcessor` for processing the network objects
passed minimal validation, `singleCleaner` to periodically clean various
internal storages (like inventory and knownnodes) and do forced garbage
collection, `singleWorker` for doing PoW, `sqlThread` for querying sqlite
database.
There are also other threads in the `.network` package.
:func:`set_thread_name` is defined here for the threads that don't inherit from
:class:`.network.StoppableThread`
"""
import threading
try:
import prctl
except ImportError:
def set_thread_name(name):
"""Set a name for the thread for python internal use."""
threading.current_thread().name = name
else:
def set_thread_name(name):
"""Set the thread name for external use (visible from the OS)."""
prctl.set_name(name)
def _thread_name_hack(self):
set_thread_name(self.name)
threading.Thread.__bootstrap_original__(self)
# pylint: disable=protected-access
threading.Thread.__bootstrap_original__ = threading.Thread._Thread__bootstrap
threading.Thread._Thread__bootstrap = _thread_name_hack
from class_addressGenerator import addressGenerator
from class_objectProcessor import objectProcessor
from class_singleCleaner import singleCleaner
from class_singleWorker import singleWorker
from class_sqlThread import sqlThread
__all__ = [
"addressGenerator", "objectProcessor", "singleCleaner", "singleWorker",
"sqlThread"
]

30
src/tr.py
View File

@ -1,21 +1,17 @@
""" """
src/tr.py Translating text
=================================
""" """
# pylint: disable=relative-import
import os import os
import state import state
"""This is used so that the translateText function can be used """
"""when we are in daemon mode and not using any QT functions."""
class translateClass:
class translateClass: # pylint: disable=old-style-class, too-few-public-methods
""" """
This is used so that the translateText function can be used when we are This is used so that the translateText function can be used
in daemon mode and not using any QT functions. when we are in daemon mode and not using any QT functions.
""" """
# pylint: disable=old-style-class,too-few-public-methods
def __init__(self, context, text): def __init__(self, context, text):
self.context = context self.context = context
self.text = text self.text = text
@ -29,12 +25,9 @@ class translateClass: # pylint: disable=old-style-class, too-few-public-m
return self.text return self.text
def _translate(context, text, disambiguation=None, encoding=None, n=None): # pylint: disable=unused-argument def _translate(context, text, disambiguation=None, encoding=None, n=None): # pylint: disable=unused-argument
return translateText(context, text, n) return translateText(context, text, n)
# def _translate(context, text, disambiguation = None, encoding = None, n = None):
# return translateClass(context, text.replace('%','',1))
def translateText(context, text, n=None): def translateText(context, text, n=None):
"""Translate text in context""" """Translate text in context"""
@ -46,12 +39,13 @@ def translateText(context, text, n=None):
try: try:
from PyQt4 import QtCore, QtGui from PyQt4 import QtCore, QtGui
except Exception as err: except Exception as err:
print 'PyBitmessage requires PyQt unless you want to run it as a daemon and interact with it using the API\ print 'PyBitmessage requires PyQt unless you want to run it as a daemon'\
.You can download PyQt from http://www.riverbankcomputing.com/software/pyqt/download\ ' and interact with it using the API.'\
or by searching Google for \'PyQt Download\'.\ ' You can download PyQt from http://www.riverbankcomputing.com/software/pyqt/download'\
If you want to run in daemon mode, see https://bitmessage.org/wiki/Daemon' ' or by searching Google for \'PyQt Download\'.'\
' If you want to run in daemon mode, see https://bitmessage.org/wiki/Daemon'
print 'Error message:', err print 'Error message:', err
os._exit(0) # pylint: disable=protected-access os._exit(0) # pylint: disable=protected-access
if n is None: if n is None:
return QtGui.QApplication.translate(context, text) return QtGui.QApplication.translate(context, text)
return QtGui.QApplication.translate(context, text, None, QtCore.QCoreApplication.CodecForTr, n) return QtGui.QApplication.translate(context, text, None, QtCore.QCoreApplication.CodecForTr, n)

11
src/upnp.py
View File

@ -1,9 +1,6 @@
# pylint: disable=too-many-statements,too-many-branches,protected-access,no-self-use # pylint: disable=too-many-statements,too-many-branches,protected-access,no-self-use
""" """
src/upnp.py Complete UPnP port forwarding implementation in separate thread.
===========
A simple upnp module to forward port for BitMessage
Reference: http://mattscodecave.com/posts/using-python-and-upnp-to-forward-a-port Reference: http://mattscodecave.com/posts/using-python-and-upnp-to-forward-a-port
""" """
@ -21,8 +18,8 @@ import state
import tr import tr
from bmconfigparser import BMConfigParser from bmconfigparser import BMConfigParser
from debug import logger from debug import logger
from helper_threading import StoppableThread from network import BMConnectionPool, StoppableThread
from network.connectionpool import BMConnectionPool from network.node import Peer
def createRequestXML(service, action, arguments=None): def createRequestXML(service, action, arguments=None):
@ -263,7 +260,7 @@ class uPnPThread(StoppableThread):
self.routers.append(newRouter) self.routers.append(newRouter)
self.createPortMapping(newRouter) self.createPortMapping(newRouter)
try: try:
self_peer = state.Peer( self_peer = Peer(
newRouter.GetExternalIPAddress(), newRouter.GetExternalIPAddress(),
self.extPort self.extPort
) )