citizenaspirant/PyBitmessage
1
0
Fork 0
Code Issues 1 Pull Requests Releases Wiki Activity

Network fixes

Browse Source
This commit is contained in:
lakshyacis 2020-01-06 17:14:47 +05:30
parent e37d52d950
commit f0bc74e658
No known key found for this signature in database
GPG Key ID: D2C539C8EC63E9EB
13 changed files with 205 additions and 214 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/network/__init__.py
View File

@ -1,3 +1,6 @@
"""
Network subsystem packages
"""
from addrthread import AddrThread from addrthread import AddrThread
from announcethread import AnnounceThread from announcethread import AnnounceThread
from connectionpool import BMConnectionPool from connectionpool import BMConnectionPool

1
src/network/addrthread.py
View File

@ -12,6 +12,7 @@ from threads import StoppableThread
class AddrThread(StoppableThread): class AddrThread(StoppableThread):
"""(Node) address broadcasting thread"""
name = "AddrBroadcaster" name = "AddrBroadcaster"
def run(self): def run(self):

35
src/network/advanceddispatcher.py
View File

@ -1,9 +1,7 @@
""" """
src/network/advanceddispatcher.py Improved version of asyncore dispatcher
=================================
""" """
# pylint: disable=attribute-defined-outside-init # pylint: disable=attribute-defined-outside-init
import socket import socket
import threading import threading
import time import time
@ -14,7 +12,8 @@ from threads import BusyError, nonBlocking
class ProcessingError(Exception): class ProcessingError(Exception):
"""General class for protocol parser exception, use as a base for others.""" """General class for protocol parser exception,
use as a base for others."""
pass pass
@ -24,7 +23,8 @@ class UnknownStateError(ProcessingError):
class AdvancedDispatcher(asyncore.dispatcher): class AdvancedDispatcher(asyncore.dispatcher):
"""Improved version of asyncore dispatcher, with buffers and protocol state.""" """Improved version of asyncore dispatcher,
with buffers and protocol state."""
# pylint: disable=too-many-instance-attributes # pylint: disable=too-many-instance-attributes
_buf_len = 131072 # 128kB _buf_len = 131072 # 128kB
@ -72,7 +72,8 @@ class AdvancedDispatcher(asyncore.dispatcher):
del self.read_buf[0:length] del self.read_buf[0:length]
def process(self): def process(self):
"""Process (parse) data that's in the buffer, as long as there is enough data and the connection is open.""" """Process (parse) data that's in the buffer,
as long as there is enough data and the connection is open."""
while self.connected and not state.shutdown: while self.connected and not state.shutdown:
try: try:
with nonBlocking(self.processingLock): with nonBlocking(self.processingLock):
@ -104,8 +105,9 @@ class AdvancedDispatcher(asyncore.dispatcher):
if asyncore.maxUploadRate > 0: if asyncore.maxUploadRate > 0:
self.uploadChunk = int(asyncore.uploadBucket) self.uploadChunk = int(asyncore.uploadBucket)
self.uploadChunk = min(self.uploadChunk, len(self.write_buf)) self.uploadChunk = min(self.uploadChunk, len(self.write_buf))
return asyncore.dispatcher.writable(self) and \ return asyncore.dispatcher.writable(self) and (
(self.connecting or (self.connected and self.uploadChunk > 0)) self.connecting or (
self.connected and self.uploadChunk > 0))
def readable(self): def readable(self):
"""Is the read buffer ready to accept data from the network?""" """Is the read buffer ready to accept data from the network?"""
@ -114,13 +116,15 @@ class AdvancedDispatcher(asyncore.dispatcher):
self.downloadChunk = int(asyncore.downloadBucket) self.downloadChunk = int(asyncore.downloadBucket)
try: try:
if self.expectBytes > 0 and not self.fullyEstablished: if self.expectBytes > 0 and not self.fullyEstablished:
self.downloadChunk = min(self.downloadChunk, self.expectBytes - len(self.read_buf)) self.downloadChunk = min(
self.downloadChunk, self.expectBytes - len(self.read_buf))
if self.downloadChunk < 0: if self.downloadChunk < 0:
self.downloadChunk = 0 self.downloadChunk = 0
except AttributeError: except AttributeError:
pass pass
return asyncore.dispatcher.readable(self) and \ return asyncore.dispatcher.readable(self) and (
(self.connecting or self.accepting or (self.connected and self.downloadChunk > 0)) self.connecting or self.accepting or (
self.connected and self.downloadChunk > 0))
def handle_read(self): def handle_read(self):
"""Append incoming data to the read buffer.""" """Append incoming data to the read buffer."""
@ -144,20 +148,21 @@ class AdvancedDispatcher(asyncore.dispatcher):
try: try:
asyncore.dispatcher.handle_connect_event(self) asyncore.dispatcher.handle_connect_event(self)
except socket.error as e: except socket.error as e:
if e.args[0] not in asyncore._DISCONNECTED: # pylint: disable=protected-access # pylint: disable=protected-access
if e.args[0] not in asyncore._DISCONNECTED:
raise raise
def handle_connect(self): def handle_connect(self):
"""Method for handling connection established implementations.""" """Method for handling connection established implementations."""
self.lastTx = time.time() self.lastTx = time.time()
def state_close(self): def state_close(self): # pylint: disable=no-self-use
"""Signal to the processing loop to end.""" """Signal to the processing loop to end."""
# pylint: disable=no-self-use
return False return False
def handle_close(self): def handle_close(self):
"""Callback for connection being closed, but can also be called directly when you want connection to close.""" """Callback for connection being closed,
but can also be called directly when you want connection to close."""
with self.readLock: with self.readLock:
self.read_buf = bytearray() self.read_buf = bytearray()
with self.writeLock: with self.writeLock:

7
src/network/announcethread.py
View File

@ -1,8 +1,6 @@
""" """
src/network/announcethread.py Announce myself (node address)
=================================
""" """
import time import time
import state import state
@ -40,6 +38,7 @@ class AnnounceThread(StoppableThread):
stream, stream,
Peer( Peer(
'127.0.0.1', '127.0.0.1',
BMConfigParser().safeGetInt('bitmessagesettings', 'port')), BMConfigParser().safeGetInt(
'bitmessagesettings', 'port')),
time.time()) time.time())
connection.append_write_buf(assemble_addr([addr])) connection.append_write_buf(assemble_addr([addr]))

17
src/network/assemble.py
View File

@ -1,7 +1,6 @@
""" """
Create bitmessage protocol command packets Create bitmessage protocol command packets
""" """
import struct import struct
import addresses import addresses
@ -13,20 +12,20 @@ from protocol import CreatePacket, encodeHost
def assemble_addr(peerList): def assemble_addr(peerList):
"""Create address command""" """Create address command"""
if isinstance(peerList, Peer): if isinstance(peerList, Peer):
peerList = (peerList) peerList = [peerList]
if not peerList: if not peerList:
return b'' return b''
retval = b'' retval = b''
for i in range(0, len(peerList), MAX_ADDR_COUNT): for i in range(0, len(peerList), MAX_ADDR_COUNT):
payload = addresses.encodeVarint( payload = addresses.encodeVarint(len(peerList[i:i + MAX_ADDR_COUNT]))
len(peerList[i:i + MAX_ADDR_COUNT]))
for stream, peer, timestamp in peerList[i:i + MAX_ADDR_COUNT]: for stream, peer, timestamp in peerList[i:i + MAX_ADDR_COUNT]:
payload += struct.pack( # 64-bit time
'>Q', timestamp) # 64-bit time payload += struct.pack('>Q', timestamp)
payload += struct.pack('>I', stream) payload += struct.pack('>I', stream)
payload += struct.pack( # service bit flags offered by this node
'>q', 1) # service bit flags offered by this node payload += struct.pack('>q', 1)
payload += encodeHost(peer.host) payload += encodeHost(peer.host)
payload += struct.pack('>H', peer.port) # remote port # remote port
payload += struct.pack('>H', peer.port)
retval += CreatePacket('addr', payload) retval += CreatePacket('addr', payload)
return retval return retval

154
src/network/asyncore_pollchoose.py
View File

@ -1,56 +1,11 @@
"""
Basic infrastructure for asynchronous socket service clients and servers.
"""
# -*- Mode: Python -*- # -*- Mode: Python -*-
# Id: asyncore.py,v 2.51 2000/09/07 22:29:26 rushing Exp # Id: asyncore.py,v 2.51 2000/09/07 22:29:26 rushing Exp
# Author: Sam Rushing <rushing@nightmare.com> # Author: Sam Rushing <rushing@nightmare.com>
# pylint: disable=too-many-statements,too-many-branches,no-self-use,too-many-lines,attribute-defined-outside-init # pylint: disable=too-many-branches,too-many-lines,global-statement
# pylint: disable=global-statement # pylint: disable=redefined-builtin,no-self-use
"""
src/network/asyncore_pollchoose.py
==================================
# ======================================================================
# Copyright 1996 by Sam Rushing
#
# All Rights Reserved
#
# Permission to use, copy, modify, and distribute this software and
# its documentation for any purpose and without fee is hereby
# granted, provided that the above copyright notice appear in all
# copies and that both that copyright notice and this permission
# notice appear in supporting documentation, and that the name of Sam
# Rushing not be used in advertising or publicity pertaining to
# distribution of the software without specific, written prior
# permission.
#
# SAM RUSHING DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN
# NO EVENT SHALL SAM RUSHING BE LIABLE FOR ANY SPECIAL, INDIRECT OR
# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
# ======================================================================
Basic infrastructure for asynchronous socket service clients and servers.
There are only two ways to have a program on a single processor do "more
than one thing at a time". Multi-threaded programming is the simplest and
most popular way to do it, but there is another very different technique,
that lets you have nearly all the advantages of multi-threading, without
actually using multiple threads. it's really only practical if your program
is largely I/O bound. If your program is CPU bound, then pre-emptive
scheduled threads are probably what you really need. Network servers are
rarely CPU-bound, however.
If your operating system supports the select() system call in its I/O
library (and nearly all do), then you can use it to juggle multiple
communication channels at once; doing other work while your I/O is taking
place in the "background." Although this strategy can seem strange and
complex, especially at first, it is in many ways easier to understand and
control than multi-threaded programming. The module documented here solves
many of the difficult problems for you, making the task of building
sophisticated high-performance network servers and clients a snap.
"""
import os import os
import select import select
import socket import socket
@ -58,8 +13,9 @@ import sys
import time import time
import warnings import warnings
from errno import ( from errno import (
EADDRINUSE, EAGAIN, EALREADY, EBADF, ECONNABORTED, ECONNREFUSED, ECONNRESET, EHOSTUNREACH, EINPROGRESS, EINTR, EADDRINUSE, EAGAIN, EALREADY, EBADF, ECONNABORTED, ECONNREFUSED,
EINVAL, EISCONN, ENETUNREACH, ENOTCONN, ENOTSOCK, EPIPE, ESHUTDOWN, ETIMEDOUT, EWOULDBLOCK, errorcode ECONNRESET, EHOSTUNREACH, EINPROGRESS, EINTR, EINVAL, EISCONN, ENETUNREACH,
ENOTCONN, ENOTSOCK, EPIPE, ESHUTDOWN, ETIMEDOUT, EWOULDBLOCK, errorcode
) )
from threading import current_thread from threading import current_thread
@ -107,7 +63,8 @@ def _strerror(err):
class ExitNow(Exception): class ExitNow(Exception):
"""We don't use directly but may be necessary as we replace asyncore due to some library raising or expecting it""" """We don't use directly but may be necessary as we replace
asyncore due to some library raising or expecting it"""
pass pass
@ -152,7 +109,8 @@ def write(obj):
def set_rates(download, upload): def set_rates(download, upload):
"""Set throttling rates""" """Set throttling rates"""
global maxDownloadRate, maxUploadRate, downloadBucket, uploadBucket, downloadTimestamp, uploadTimestamp global maxDownloadRate, maxUploadRate, downloadBucket
global uploadBucket, downloadTimestamp, uploadTimestamp
maxDownloadRate = float(download) * 1024 maxDownloadRate = float(download) * 1024
maxUploadRate = float(upload) * 1024 maxUploadRate = float(upload) * 1024
@ -182,7 +140,8 @@ def update_received(download=0):
currentTimestamp = time.time() currentTimestamp = time.time()
receivedBytes += download receivedBytes += download
if maxDownloadRate > 0: if maxDownloadRate > 0:
bucketIncrease = maxDownloadRate * (currentTimestamp - downloadTimestamp) bucketIncrease = \
maxDownloadRate * (currentTimestamp - downloadTimestamp)
downloadBucket += bucketIncrease downloadBucket += bucketIncrease
if downloadBucket > maxDownloadRate: if downloadBucket > maxDownloadRate:
downloadBucket = int(maxDownloadRate) downloadBucket = int(maxDownloadRate)
@ -242,7 +201,6 @@ def readwrite(obj, flags):
def select_poller(timeout=0.0, map=None): def select_poller(timeout=0.0, map=None):
"""A poller which uses select(), available on most platforms.""" """A poller which uses select(), available on most platforms."""
# pylint: disable=redefined-builtin
if map is None: if map is None:
map = socket_map map = socket_map
@ -298,7 +256,6 @@ def select_poller(timeout=0.0, map=None):
def poll_poller(timeout=0.0, map=None): def poll_poller(timeout=0.0, map=None):
"""A poller which uses poll(), available on most UNIXen.""" """A poller which uses poll(), available on most UNIXen."""
# pylint: disable=redefined-builtin
if map is None: if map is None:
map = socket_map map = socket_map
@ -356,7 +313,6 @@ poll2 = poll3 = poll_poller
def epoll_poller(timeout=0.0, map=None): def epoll_poller(timeout=0.0, map=None):
"""A poller which uses epoll(), supported on Linux 2.5.44 and newer.""" """A poller which uses epoll(), supported on Linux 2.5.44 and newer."""
# pylint: disable=redefined-builtin
if map is None: if map is None:
map = socket_map map = socket_map
@ -412,7 +368,7 @@ def epoll_poller(timeout=0.0, map=None):
def kqueue_poller(timeout=0.0, map=None): def kqueue_poller(timeout=0.0, map=None):
"""A poller which uses kqueue(), BSD specific.""" """A poller which uses kqueue(), BSD specific."""
# pylint: disable=redefined-builtin,no-member # pylint: disable=no-member,too-many-statements
if map is None: if map is None:
map = socket_map map = socket_map
@ -440,14 +396,20 @@ def kqueue_poller(timeout=0.0, map=None):
poller_flags |= select.KQ_EV_ENABLE poller_flags |= select.KQ_EV_ENABLE
else: else:
poller_flags |= select.KQ_EV_DISABLE poller_flags |= select.KQ_EV_DISABLE
updates.append(select.kevent(fd, filter=select.KQ_FILTER_READ, flags=poller_flags)) updates.append(
select.kevent(
fd, filter=select.KQ_FILTER_READ,
flags=poller_flags))
if kq_filter & 2 != obj.poller_filter & 2: if kq_filter & 2 != obj.poller_filter & 2:
poller_flags = select.KQ_EV_ADD poller_flags = select.KQ_EV_ADD
if kq_filter & 2: if kq_filter & 2:
poller_flags |= select.KQ_EV_ENABLE poller_flags |= select.KQ_EV_ENABLE
else: else:
poller_flags |= select.KQ_EV_DISABLE poller_flags |= select.KQ_EV_DISABLE
updates.append(select.kevent(fd, filter=select.KQ_FILTER_WRITE, flags=poller_flags)) updates.append(
select.kevent(
fd, filter=select.KQ_FILTER_WRITE,
flags=poller_flags))
obj.poller_filter = kq_filter obj.poller_filter = kq_filter
if not selectables: if not selectables:
@ -481,7 +443,6 @@ def kqueue_poller(timeout=0.0, map=None):
def loop(timeout=30.0, use_poll=False, map=None, count=None, poller=None): def loop(timeout=30.0, use_poll=False, map=None, count=None, poller=None):
"""Poll in a loop, until count or timeout is reached""" """Poll in a loop, until count or timeout is reached"""
# pylint: disable=redefined-builtin
if map is None: if map is None:
map = socket_map map = socket_map
@ -520,9 +481,9 @@ def loop(timeout=30.0, use_poll=False, map=None, count=None, poller=None):
count = count - 1 count = count - 1
class dispatcher: class dispatcher(object):
"""Dispatcher for socket objects""" """Dispatcher for socket objects"""
# pylint: disable=too-many-public-methods,too-many-instance-attributes,old-style-class # pylint: disable=too-many-public-methods,too-many-instance-attributes
debug = False debug = False
connected = False connected = False
@ -537,7 +498,6 @@ class dispatcher:
minTx = 1500 minTx = 1500
def __init__(self, sock=None, map=None): def __init__(self, sock=None, map=None):
# pylint: disable=redefined-builtin
if map is None: if map is None:
self._map = socket_map self._map = socket_map
else: else:
@ -586,8 +546,7 @@ class dispatcher:
def add_channel(self, map=None): def add_channel(self, map=None):
"""Add a channel""" """Add a channel"""
# pylint: disable=redefined-builtin # pylint: disable=attribute-defined-outside-init
if map is None: if map is None:
map = self._map map = self._map
map[self._fileno] = self map[self._fileno] = self
@ -596,8 +555,6 @@ class dispatcher:
def del_channel(self, map=None): def del_channel(self, map=None):
"""Delete a channel""" """Delete a channel"""
# pylint: disable=redefined-builtin
fd = self._fileno fd = self._fileno
if map is None: if map is None:
map = self._map map = self._map
@ -605,11 +562,13 @@ class dispatcher:
del map[fd] del map[fd]
if self._fileno: if self._fileno:
try: try:
kqueue_poller.pollster.control([select.kevent(fd, select.KQ_FILTER_READ, select.KQ_EV_DELETE)], 0) kqueue_poller.pollster.control([select.kevent(
fd, select.KQ_FILTER_READ, select.KQ_EV_DELETE)], 0)
except(AttributeError, KeyError, TypeError, IOError, OSError): except(AttributeError, KeyError, TypeError, IOError, OSError):
pass pass
try: try:
kqueue_poller.pollster.control([select.kevent(fd, select.KQ_FILTER_WRITE, select.KQ_EV_DELETE)], 0) kqueue_poller.pollster.control([select.kevent(
fd, select.KQ_FILTER_WRITE, select.KQ_EV_DELETE)], 0)
except(AttributeError, KeyError, TypeError, IOError, OSError): except(AttributeError, KeyError, TypeError, IOError, OSError):
pass pass
try: try:
@ -627,8 +586,10 @@ class dispatcher:
self.poller_filter = 0 self.poller_filter = 0
self.poller_registered = False self.poller_registered = False
def create_socket(self, family=socket.AF_INET, socket_type=socket.SOCK_STREAM): def create_socket(
self, family=socket.AF_INET, socket_type=socket.SOCK_STREAM):
"""Create a socket""" """Create a socket"""
# pylint: disable=attribute-defined-outside-init
self.family_and_type = family, socket_type self.family_and_type = family, socket_type
sock = socket.socket(family, socket_type) sock = socket.socket(family, socket_type)
sock.setblocking(0) sock.setblocking(0)
@ -636,20 +597,16 @@ class dispatcher:
def set_socket(self, sock, map=None): def set_socket(self, sock, map=None):
"""Set socket""" """Set socket"""
# pylint: disable=redefined-builtin
self.socket = sock self.socket = sock
self._fileno = sock.fileno() self._fileno = sock.fileno()
self.add_channel(map) self.add_channel(map)
def set_reuse_addr(self): def set_reuse_addr(self):
"""try to re-use a server port if possible""" """try to re-use a server port if possible"""
try: try:
self.socket.setsockopt( self.socket.setsockopt(
socket.SOL_SOCKET, socket.SO_REUSEADDR, socket.SOL_SOCKET, socket.SO_REUSEADDR, self.socket.getsockopt(
self.socket.getsockopt(socket.SOL_SOCKET, socket.SOL_SOCKET, socket.SO_REUSEADDR) | 1
socket.SO_REUSEADDR) | 1
) )
except socket.error: except socket.error:
pass pass
@ -704,13 +661,16 @@ class dispatcher:
raise socket.error(err, errorcode[err]) raise socket.error(err, errorcode[err])
def accept(self): def accept(self):
"""Accept incoming connections. Returns either an address pair or None.""" """Accept incoming connections.
Returns either an address pair or None."""
try: try:
conn, addr = self.socket.accept() conn, addr = self.socket.accept()
except TypeError: except TypeError:
return None return None
except socket.error as why: except socket.error as why:
if why.args[0] in (EWOULDBLOCK, WSAEWOULDBLOCK, ECONNABORTED, EAGAIN, ENOTCONN): if why.args[0] in (
EWOULDBLOCK, WSAEWOULDBLOCK, ECONNABORTED,
EAGAIN, ENOTCONN):
return None return None
else: else:
raise raise
@ -769,7 +729,8 @@ class dispatcher:
try: try:
retattr = getattr(self.socket, attr) retattr = getattr(self.socket, attr)
except AttributeError: except AttributeError:
raise AttributeError("%s instance has no attribute '%s'" raise AttributeError(
"%s instance has no attribute '%s'"
% (self.__class__.__name__, attr)) % (self.__class__.__name__, attr))
else: else:
msg = "%(me)s.%(attr)s is deprecated; use %(me)s.socket.%(attr)s"\ msg = "%(me)s.%(attr)s is deprecated; use %(me)s.socket.%(attr)s"\
@ -855,13 +816,8 @@ class dispatcher:
self.log_info( self.log_info(
'uncaptured python exception, closing channel %s (%s:%s %s)' % ( 'uncaptured python exception, closing channel %s (%s:%s %s)' % (
self_repr, self_repr, t, v, tbinfo),
t, 'error')
v,
tbinfo
),
'error'
)
self.handle_close() self.handle_close()
def handle_accept(self): def handle_accept(self):
@ -902,11 +858,8 @@ class dispatcher_with_send(dispatcher):
adds simple buffered output capability, useful for simple clients. adds simple buffered output capability, useful for simple clients.
[for more sophisticated usage use asynchat.async_chat] [for more sophisticated usage use asynchat.async_chat]
""" """
# pylint: disable=redefined-builtin
def __init__(self, sock=None, map=None): def __init__(self, sock=None, map=None):
# pylint: disable=redefined-builtin
dispatcher.__init__(self, sock, map) dispatcher.__init__(self, sock, map)
self.out_buffer = b'' self.out_buffer = b''
@ -941,7 +894,8 @@ def compact_traceback():
"""Return a compact traceback""" """Return a compact traceback"""
t, v, tb = sys.exc_info() t, v, tb = sys.exc_info()
tbinfo = [] tbinfo = []
if not tb: # Must have a traceback # Must have a traceback
if not tb:
raise AssertionError("traceback does not exist") raise AssertionError("traceback does not exist")
while tb: while tb:
tbinfo.append(( tbinfo.append((
@ -961,7 +915,6 @@ def compact_traceback():
def close_all(map=None, ignore_all=False): def close_all(map=None, ignore_all=False):
"""Close all connections""" """Close all connections"""
# pylint: disable=redefined-builtin
if map is None: if map is None:
map = socket_map map = socket_map
@ -998,13 +951,13 @@ def close_all(map=None, ignore_all=False):
if os.name == 'posix': if os.name == 'posix':
import fcntl import fcntl
class file_wrapper: class file_wrapper: # pylint: disable=old-style-class
""" """
Here we override just enough to make a file look like a socket for the purposes of asyncore. Here we override just enough to make a file look
like a socket for the purposes of asyncore.
The passed fd is automatically os.dup()'d The passed fd is automatically os.dup()'d
""" """
# pylint: disable=old-style-class
def __init__(self, fd): def __init__(self, fd):
self.fd = os.dup(fd) self.fd = os.dup(fd)
@ -1019,12 +972,11 @@ if os.name == 'posix':
def getsockopt(self, level, optname, buflen=None): def getsockopt(self, level, optname, buflen=None):
"""Fake getsockopt()""" """Fake getsockopt()"""
if (level == socket.SOL_SOCKET and if (level == socket.SOL_SOCKET and optname == socket.SO_ERROR and
optname == socket.SO_ERROR and
not buflen): not buflen):
return 0 return 0
raise NotImplementedError("Only asyncore specific behaviour " raise NotImplementedError(
"implemented.") "Only asyncore specific behaviour implemented.")
read = recv read = recv
write = send write = send
@ -1041,8 +993,6 @@ if os.name == 'posix':
"""A dispatcher for file_wrapper objects""" """A dispatcher for file_wrapper objects"""
def __init__(self, fd, map=None): def __init__(self, fd, map=None):
# pylint: disable=redefined-builtin
dispatcher.__init__(self, None, map) dispatcher.__init__(self, None, map)
self.connected = True self.connected = True
try: try:

49
src/network/bmobject.py
View File

@ -1,7 +1,6 @@
""" """
BMObject and it's exceptions. BMObject and it's exceptions.
""" """
import logging import logging
import time import time
@ -15,12 +14,14 @@ logger = logging.getLogger('default')
class BMObjectInsufficientPOWError(Exception): class BMObjectInsufficientPOWError(Exception):
"""Exception indicating the object doesn't have sufficient proof of work.""" """Exception indicating the object
doesn't have sufficient proof of work."""
errorCodes = ("Insufficient proof of work") errorCodes = ("Insufficient proof of work")
class BMObjectInvalidDataError(Exception): class BMObjectInvalidDataError(Exception):
"""Exception indicating the data being parsed does not match the specification.""" """Exception indicating the data being parsed
does not match the specification."""
errorCodes = ("Data invalid") errorCodes = ("Data invalid")
@ -30,7 +31,8 @@ class BMObjectExpiredError(Exception):
class BMObjectUnwantedStreamError(Exception): class BMObjectUnwantedStreamError(Exception):
"""Exception indicating the object is in a stream we didn't advertise as being interested in.""" """Exception indicating the object is in a stream
we didn't advertise as being interested in."""
errorCodes = ("Object in unwanted stream") errorCodes = ("Object in unwanted stream")
@ -44,9 +46,8 @@ class BMObjectAlreadyHaveError(Exception):
errorCodes = ("Already have this object") errorCodes = ("Already have this object")
class BMObject(object): class BMObject(object): # pylint: disable=too-many-instance-attributes
"""Bitmessage Object as a class.""" """Bitmessage Object as a class."""
# pylint: disable=too-many-instance-attributes
# max TTL, 28 days and 3 hours # max TTL, 28 days and 3 hours
maxTTL = 28 * 24 * 60 * 60 + 10800 maxTTL = 28 * 24 * 60 * 60 + 10800
@ -81,31 +82,36 @@ class BMObject(object):
raise BMObjectInsufficientPOWError() raise BMObjectInsufficientPOWError()
def checkEOLSanity(self): def checkEOLSanity(self):
"""Check if object's lifetime isn't ridiculously far in the past or future.""" """Check if object's lifetime
isn't ridiculously far in the past or future."""
# EOL sanity check # EOL sanity check
if self.expiresTime - int(time.time()) > BMObject.maxTTL: if self.expiresTime - int(time.time()) > BMObject.maxTTL:
logger.info( logger.info(
'This object\'s End of Life time is too far in the future. Ignoring it. Time is %i', 'This object\'s End of Life time is too far in the future.'
self.expiresTime) ' Ignoring it. Time is %i', self.expiresTime)
# .. todo:: remove from download queue # .. todo:: remove from download queue
raise BMObjectExpiredError() raise BMObjectExpiredError()
if self.expiresTime - int(time.time()) < BMObject.minTTL: if self.expiresTime - int(time.time()) < BMObject.minTTL:
logger.info( logger.info(
'This object\'s End of Life time was too long ago. Ignoring the object. Time is %i', 'This object\'s End of Life time was too long ago.'
self.expiresTime) ' Ignoring the object. Time is %i', self.expiresTime)
# .. todo:: remove from download queue # .. todo:: remove from download queue
raise BMObjectExpiredError() raise BMObjectExpiredError()
def checkStream(self): def checkStream(self):
"""Check if object's stream matches streams we are interested in""" """Check if object's stream matches streams we are interested in"""
if self.streamNumber not in state.streamsInWhichIAmParticipating: if self.streamNumber not in state.streamsInWhichIAmParticipating:
logger.debug('The streamNumber %i isn\'t one we are interested in.', self.streamNumber) logger.debug(
'The streamNumber %i isn\'t one we are interested in.',
self.streamNumber)
raise BMObjectUnwantedStreamError() raise BMObjectUnwantedStreamError()
def checkAlreadyHave(self): def checkAlreadyHave(self):
""" """
Check if we already have the object (so that we don't duplicate it in inventory or advertise it unnecessarily) Check if we already have the object
(so that we don't duplicate it in inventory
or advertise it unnecessarily)
""" """
# if it's a stem duplicate, pretend we don't have it # if it's a stem duplicate, pretend we don't have it
if Dandelion().hasHash(self.inventoryHash): if Dandelion().hasHash(self.inventoryHash):
@ -114,7 +120,8 @@ class BMObject(object):
raise BMObjectAlreadyHaveError() raise BMObjectAlreadyHaveError()
def checkObjectByType(self): def checkObjectByType(self):
"""Call a object type specific check (objects can have additional checks based on their types)""" """Call a object type specific check
(objects can have additional checks based on their types)"""
if self.objectType == protocol.OBJECT_GETPUBKEY: if self.objectType == protocol.OBJECT_GETPUBKEY:
self.checkGetpubkey() self.checkGetpubkey()
elif self.objectType == protocol.OBJECT_PUBKEY: elif self.objectType == protocol.OBJECT_PUBKEY:
@ -125,20 +132,21 @@ class BMObject(object):
self.checkBroadcast() self.checkBroadcast()
# other objects don't require other types of tests # other objects don't require other types of tests
def checkMessage(self): def checkMessage(self): # pylint: disable=no-self-use
""""Message" object type checks.""" """"Message" object type checks."""
# pylint: disable=no-self-use
return return
def checkGetpubkey(self): def checkGetpubkey(self):
""""Getpubkey" object type checks.""" """"Getpubkey" object type checks."""
if len(self.data) < 42: if len(self.data) < 42:
logger.info('getpubkey message doesn\'t contain enough data. Ignoring.') logger.info(
'getpubkey message doesn\'t contain enough data. Ignoring.')
raise BMObjectInvalidError() raise BMObjectInvalidError()
def checkPubkey(self): def checkPubkey(self):
""""Pubkey" object type checks.""" """"Pubkey" object type checks."""
if len(self.data) < 146 or len(self.data) > 440: # sanity check # sanity check
if len(self.data) < 146 or len(self.data) > 440:
logger.info('pubkey object too short or too long. Ignoring.') logger.info('pubkey object too short or too long. Ignoring.')
raise BMObjectInvalidError() raise BMObjectInvalidError()
@ -146,8 +154,9 @@ class BMObject(object):
""""Broadcast" object type checks.""" """"Broadcast" object type checks."""
if len(self.data) < 180: if len(self.data) < 180:
logger.debug( logger.debug(
'The payload length of this broadcast packet is unreasonably low.' 'The payload length of this broadcast'
' Someone is probably trying funny business. Ignoring message.') ' packet is unreasonably low. Someone is probably'
' trying funny business. Ignoring message.')
raise BMObjectInvalidError() raise BMObjectInvalidError()
# this isn't supported anymore # this isn't supported anymore

76
src/network/bmproto.py
View File

@ -1,8 +1,7 @@
""" """
src/network/bmproto.py Bitmessage Protocol
==================================
""" """
# pylint: disable=attribute-defined-outside-init # pylint: disable=attribute-defined-outside-init, too-few-public-methods
import base64 import base64
import hashlib import hashlib
import logging import logging
@ -19,17 +18,16 @@ import state
from bmconfigparser import BMConfigParser from bmconfigparser import BMConfigParser
from inventory import Inventory from inventory import Inventory
from network.advanceddispatcher import AdvancedDispatcher from network.advanceddispatcher import AdvancedDispatcher
from network.constants import (
ADDRESS_ALIVE,
MAX_MESSAGE_SIZE,
MAX_OBJECT_COUNT,
MAX_OBJECT_PAYLOAD_SIZE,
MAX_TIME_OFFSET)
from network.dandelion import Dandelion
from network.bmobject import ( from network.bmobject import (
BMObject, BMObjectInsufficientPOWError, BMObjectInvalidDataError, BMObject, BMObjectInsufficientPOWError, BMObjectInvalidDataError,
BMObjectExpiredError, BMObjectUnwantedStreamError, BMObjectExpiredError, BMObjectUnwantedStreamError,
BMObjectInvalidError, BMObjectAlreadyHaveError) BMObjectInvalidError, BMObjectAlreadyHaveError
)
from network.constants import (
ADDRESS_ALIVE, MAX_MESSAGE_SIZE, MAX_OBJECT_COUNT,
MAX_OBJECT_PAYLOAD_SIZE, MAX_TIME_OFFSET
)
from network.dandelion import Dandelion
from network.proxy import ProxyError from network.proxy import ProxyError
from node import Node, Peer from node import Node, Peer
from objectracker import missingObjects, ObjectTracker from objectracker import missingObjects, ObjectTracker
@ -59,7 +57,8 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
# pylint: disable=too-many-instance-attributes, too-many-public-methods # pylint: disable=too-many-instance-attributes, too-many-public-methods
timeOffsetWrongCount = 0 timeOffsetWrongCount = 0
def __init__(self, address=None, sock=None): # pylint: disable=unused-argument, super-init-not-called def __init__(self, address=None, sock=None):
# pylint: disable=unused-argument, super-init-not-called
AdvancedDispatcher.__init__(self, sock) AdvancedDispatcher.__init__(self, sock)
self.isOutbound = False self.isOutbound = False
# packet/connection from a local IP # packet/connection from a local IP
@ -163,7 +162,8 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
def decode_payload_varint(self): def decode_payload_varint(self):
"""Decode a varint from the payload""" """Decode a varint from the payload"""
value, offset = addresses.decodeVarint(self.payload[self.payloadOffset:]) value, offset = addresses.decodeVarint(
self.payload[self.payloadOffset:])
self.payloadOffset += offset self.payloadOffset += offset
return value return value
@ -185,8 +185,8 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
return Node(services, host, port) return Node(services, host, port)
def decode_payload_content(self, pattern="v"): # pylint: disable=too-many-branches, too-many-statements # pylint: disable=too-many-branches, too-many-statements
def decode_payload_content(self, pattern="v"):
""" """
Decode the payload depending on pattern: Decode the payload depending on pattern:
@ -202,7 +202,8 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
, = end of array , = end of array
""" """
def decode_simple(self, char="v"): # pylint: disable=inconsistent-return-statements # pylint: disable=inconsistent-return-statements
def decode_simple(self, char="v"):
"""Decode the payload using one char pattern""" """Decode the payload using one char pattern"""
if char == "v": if char == "v":
return self.decode_payload_varint() return self.decode_payload_varint()
@ -312,8 +313,11 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
def bm_command_error(self): def bm_command_error(self):
"""Decode an error message and log it""" """Decode an error message and log it"""
fatalStatus, banTime, inventoryVector, errorText = \ err_values = self.decode_payload_content("vvlsls")
self.decode_payload_content("vvlsls") fatalStatus = err_values[0]
# banTime = err_values[1]
# inventoryVector = err_values[2]
errorText = err_values[3]
logger.error( logger.error(
'%s:%i error: %i, %s', self.destination.host, '%s:%i error: %i, %s', self.destination.host,
self.destination.port, fatalStatus, errorText) self.destination.port, fatalStatus, errorText)
@ -408,8 +412,10 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
except KeyError: except KeyError:
pass pass
if self.object.inventoryHash in Inventory() and Dandelion().hasHash(self.object.inventoryHash): if self.object.inventoryHash in Inventory() and Dandelion().hasHash(
Dandelion().removeHash(self.object.inventoryHash, "cycle detection") self.object.inventoryHash):
Dandelion().removeHash(
self.object.inventoryHash, "cycle detection")
Inventory()[self.object.inventoryHash] = ( Inventory()[self.object.inventoryHash] = (
self.object.objectType, self.object.streamNumber, self.object.objectType, self.object.streamNumber,
@ -428,9 +434,9 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
def bm_command_addr(self): def bm_command_addr(self):
"""Incoming addresses, process them""" """Incoming addresses, process them"""
addresses = self._decode_addr() # pylint: disable=redefined-outer-name # pylint: disable=redefined-outer-name
for i in addresses: addresses = self._decode_addr()
seenTime, stream, services, ip, port = i for seenTime, stream, _, ip, port in addresses:
decodedIP = protocol.checkIPAddress(str(ip)) decodedIP = protocol.checkIPAddress(str(ip))
if stream not in state.streamsInWhichIAmParticipating: if stream not in state.streamsInWhichIAmParticipating:
continue continue
@ -441,14 +447,17 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
): ):
peer = Peer(decodedIP, port) peer = Peer(decodedIP, port)
try: try:
if knownnodes.knownNodes[stream][peer]["lastseen"] > seenTime: if knownnodes.knownNodes[stream][peer]["lastseen"] > \
seenTime:
continue continue
except KeyError: except KeyError:
pass pass
if len(knownnodes.knownNodes[stream]) < BMConfigParser().safeGetInt("knownnodes", "maxnodes"): if len(knownnodes.knownNodes[stream]) < \
BMConfigParser().safeGetInt("knownnodes", "maxnodes"):
with knownnodes.knownNodesLock: with knownnodes.knownNodesLock:
try: try:
knownnodes.knownNodes[stream][peer]["lastseen"] = seenTime knownnodes.knownNodes[stream][peer]["lastseen"] = \
seenTime
except (TypeError, KeyError): except (TypeError, KeyError):
knownnodes.knownNodes[stream][peer] = { knownnodes.knownNodes[stream][peer] = {
"lastseen": seenTime, "lastseen": seenTime,
@ -539,7 +548,8 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
length=self.payloadLength, expectBytes=0) length=self.payloadLength, expectBytes=0)
return False return False
def peerValidityChecks(self): # pylint: disable=too-many-return-statements # pylint: disable=too-many-return-statements
def peerValidityChecks(self):
"""Check the validity of the peer""" """Check the validity of the peer"""
if self.remoteProtocolVersion < 3: if self.remoteProtocolVersion < 3:
self.append_write_buf(protocol.assembleErrorMessage( self.append_write_buf(protocol.assembleErrorMessage(
@ -551,8 +561,8 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
return False return False
if self.timeOffset > MAX_TIME_OFFSET: if self.timeOffset > MAX_TIME_OFFSET:
self.append_write_buf(protocol.assembleErrorMessage( self.append_write_buf(protocol.assembleErrorMessage(
errorText="Your time is too far in the future compared to mine." errorText="Your time is too far in the future"
" Closing connection.", fatal=2)) " compared to mine. Closing connection.", fatal=2))
logger.info( logger.info(
"%s's time is too far in the future (%s seconds)." "%s's time is too far in the future (%s seconds)."
" Closing connection to it.", self.destination, self.timeOffset) " Closing connection to it.", self.destination, self.timeOffset)
@ -574,8 +584,8 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
errorText="We don't have shared stream interests." errorText="We don't have shared stream interests."
" Closing connection.", fatal=2)) " Closing connection.", fatal=2))
logger.debug( logger.debug(
'Closed connection to %s because there is no overlapping interest' 'Closed connection to %s because there is no overlapping'
' in streams.', self.destination) ' interest in streams.', self.destination)
return False return False
if self.destination in connectionpool.BMConnectionPool().inboundConnections: if self.destination in connectionpool.BMConnectionPool().inboundConnections:
try: try:
@ -584,8 +594,8 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
errorText="Too many connections from your IP." errorText="Too many connections from your IP."
" Closing connection.", fatal=2)) " Closing connection.", fatal=2))
logger.debug( logger.debug(
'Closed connection to %s because we are already connected' 'Closed connection to %s because we are already'
' to that IP.', self.destination) ' connected to that IP.', self.destination)
return False return False
except: except:
pass pass

8
src/network/connectionchooser.py
View File

@ -1,3 +1,6 @@
"""
Select which node to connect to
"""
# pylint: disable=too-many-branches # pylint: disable=too-many-branches
import logging import logging
import random # nosec import random # nosec
@ -12,6 +15,7 @@ logger = logging.getLogger('default')
def getDiscoveredPeer(): def getDiscoveredPeer():
"""Get a peer from the local peer discovery list"""
try: try:
peer = random.choice(state.discoveredPeers.keys()) peer = random.choice(state.discoveredPeers.keys())
except (IndexError, KeyError): except (IndexError, KeyError):
@ -24,6 +28,7 @@ def getDiscoveredPeer():
def chooseConnection(stream): def chooseConnection(stream):
"""Returns an appropriate connection"""
haveOnion = BMConfigParser().safeGet( haveOnion = BMConfigParser().safeGet(
"bitmessagesettings", "socksproxytype")[0:5] == 'SOCKS' "bitmessagesettings", "socksproxytype")[0:5] == 'SOCKS'
onionOnly = BMConfigParser().safeGetBoolean( onionOnly = BMConfigParser().safeGetBoolean(
@ -49,7 +54,8 @@ def chooseConnection(stream):
logger.warning('Error in %s', peer) logger.warning('Error in %s', peer)
rating = 0 rating = 0
if haveOnion: if haveOnion:
# do not connect to raw IP addresses--keep all traffic within Tor overlay # do not connect to raw IP addresses
# --keep all traffic within Tor overlay
if onionOnly and not peer.host.endswith('.onion'): if onionOnly and not peer.host.endswith('.onion'):
continue continue
# onion addresses have a higher priority when SOCKS # onion addresses have a higher priority when SOCKS

18
src/network/constants.py
View File

@ -3,9 +3,15 @@ Network protocol constants
""" """
ADDRESS_ALIVE = 10800 #: address is online if online less than this many seconds ago #: address is online if online less than this many seconds ago
MAX_ADDR_COUNT = 1000 #: protocol specification says max 1000 addresses in one addr command ADDRESS_ALIVE = 10800
MAX_MESSAGE_SIZE = 1600100 #: ~1.6 MB which is the maximum possible size of an inv message. #: protocol specification says max 1000 addresses in one addr command
MAX_OBJECT_PAYLOAD_SIZE = 2**18 #: 2**18 = 256kB is the maximum size of an object payload MAX_ADDR_COUNT = 1000
MAX_OBJECT_COUNT = 50000 #: protocol specification says max 50000 objects in one inv command #: ~1.6 MB which is the maximum possible size of an inv message.
MAX_TIME_OFFSET = 3600 #: maximum time offset MAX_MESSAGE_SIZE = 1600100
#: 2**18 = 256kB is the maximum size of an object payload
MAX_OBJECT_PAYLOAD_SIZE = 2**18
#: protocol specification says max 50000 objects in one inv command
MAX_OBJECT_COUNT = 50000
#: maximum time offset
MAX_TIME_OFFSET = 3600

10
src/network/dandelion.py
View File

@ -1,10 +1,9 @@
""" """
src/network/dandelion.py Dandelion class definition, tracks stages
========================
""" """
import logging import logging
from collections import namedtuple from collections import namedtuple
from random import choice, sample, expovariate from random import choice, expovariate, sample
from threading import RLock from threading import RLock
from time import time from time import time
@ -28,7 +27,7 @@ logger = logging.getLogger('default')
@Singleton @Singleton
class Dandelion(): # pylint: disable=old-style-class class Dandelion: # pylint: disable=old-style-class
"""Dandelion class for tracking stem/fluff stages.""" """Dandelion class for tracking stem/fluff stages."""
def __init__(self): def __init__(self):
# currently assignable child stems # currently assignable child stems
@ -123,7 +122,8 @@ class Dandelion(): # pylint: disable=old-style-class
self.stem.remove(connection) self.stem.remove(connection)
# active mappings to pointing to the removed node # active mappings to pointing to the removed node
for k in ( for k in (
k for k, v in self.nodeMap.iteritems() if v == connection k for k, v in self.nodeMap.iteritems()
if v == connection
): ):
self.nodeMap[k] = None self.nodeMap[k] = None
for k, v in { for k, v in {

5
src/network/downloadthread.py
View File

@ -1,7 +1,6 @@
""" """
`DownloadThread` class definition `DownloadThread` class definition
""" """
import time import time
import addresses import addresses
@ -30,7 +29,9 @@ class DownloadThread(StoppableThread):
"""Expire pending downloads eventually""" """Expire pending downloads eventually"""
deadline = time.time() - self.requestExpires deadline = time.time() - self.requestExpires
try: try:
toDelete = [k for k, v in missingObjects.iteritems() if v < deadline] toDelete = [
k for k, v in missingObjects.iteritems()
if v < deadline]
except RuntimeError: except RuntimeError:
pass pass
else: else:

16
src/network/invthread.py
View File

@ -1,6 +1,5 @@
""" """
src/network/invthread.py Thread to send inv annoucements
========================
""" """
import Queue import Queue
import random import random
@ -34,7 +33,7 @@ def handleExpiredDandelion(expired):
class InvThread(StoppableThread): class InvThread(StoppableThread):
"""A thread to send inv annoucements.""" """Main thread that sends inv annoucements"""
name = "InvBroadcaster" name = "InvBroadcaster"
@ -43,7 +42,8 @@ class InvThread(StoppableThread):
"""Locally generated inventory items require special handling""" """Locally generated inventory items require special handling"""
Dandelion().addHash(hashId, stream=stream) Dandelion().addHash(hashId, stream=stream)
for connection in BMConnectionPool().connections(): for connection in BMConnectionPool().connections():
if state.dandelion and connection != Dandelion().objectChildStem(hashId): if state.dandelion and connection != \
Dandelion().objectChildStem(hashId):
continue continue
connection.objectsNewToThem[hashId] = time() connection.objectsNewToThem[hashId] = time()
@ -92,15 +92,17 @@ class InvThread(StoppableThread):
random.shuffle(fluffs) random.shuffle(fluffs)
connection.append_write_buf(protocol.CreatePacket( connection.append_write_buf(protocol.CreatePacket(
'inv', 'inv',
addresses.encodeVarint(len(fluffs)) + ''.join(fluffs))) addresses.encodeVarint(
len(fluffs)) + ''.join(fluffs)))
if stems: if stems:
random.shuffle(stems) random.shuffle(stems)
connection.append_write_buf(protocol.CreatePacket( connection.append_write_buf(protocol.CreatePacket(
'dinv', 'dinv',
addresses.encodeVarint(len(stems)) + ''.join(stems))) addresses.encodeVarint(
len(stems)) + ''.join(stems)))
invQueue.iterate() invQueue.iterate()
for i in range(len(chunk)): for _ in range(len(chunk)):
invQueue.task_done() invQueue.task_done()
if Dandelion().refresh < time(): if Dandelion().refresh < time():