575 lines
20 KiB
Python
575 lines
20 KiB
Python
"""
|
|
Low-level protocol-related functions.
|
|
"""
|
|
# pylint: disable=too-many-boolean-expressions,too-many-return-statements
|
|
# pylint: disable=too-many-locals,too-many-statements
|
|
|
|
import base64
|
|
import hashlib
|
|
import random
|
|
import socket
|
|
import sys
|
|
import time
|
|
from binascii import hexlify
|
|
from struct import Struct, pack, unpack
|
|
|
|
import defaults
|
|
import highlevelcrypto
|
|
import state
|
|
from addresses import (
|
|
encodeVarint, decodeVarint, decodeAddress, varintDecodeError)
|
|
from bmconfigparser import config
|
|
from debug import logger
|
|
from helper_sql import sqlExecute
|
|
from network.node import Peer
|
|
from version import softwareVersion
|
|
|
|
# Network constants
|
|
magic = 0xE9BEB4D9
|
|
#: protocol specification says max 1000 addresses in one addr command
|
|
MAX_ADDR_COUNT = 1000
|
|
#: address is online if online less than this many seconds ago
|
|
ADDRESS_ALIVE = 10800
|
|
#: ~1.6 MB which is the maximum possible size of an inv message.
|
|
MAX_MESSAGE_SIZE = 1600100
|
|
#: 2**18 = 256kB is the maximum size of an object payload
|
|
MAX_OBJECT_PAYLOAD_SIZE = 2**18
|
|
#: protocol specification says max 50000 objects in one inv command
|
|
MAX_OBJECT_COUNT = 50000
|
|
#: maximum time offset
|
|
MAX_TIME_OFFSET = 3600
|
|
|
|
# Service flags
|
|
#: This is a normal network node
|
|
NODE_NETWORK = 1
|
|
#: This node supports SSL/TLS in the current connect (python < 2.7.9
|
|
#: only supports an SSL client, so in that case it would only have this
|
|
#: on when the connection is a client).
|
|
NODE_SSL = 2
|
|
# (Proposal) This node may do PoW on behalf of some its peers
|
|
# (PoW offloading/delegating), but it doesn't have to. Clients may have
|
|
# to meet additional requirements (e.g. TLS authentication)
|
|
# NODE_POW = 4
|
|
#: Node supports dandelion
|
|
NODE_DANDELION = 8
|
|
|
|
# Bitfield flags
|
|
BITFIELD_DOESACK = 1
|
|
|
|
# Error types
|
|
STATUS_WARNING = 0
|
|
STATUS_ERROR = 1
|
|
STATUS_FATAL = 2
|
|
|
|
# Object types
|
|
OBJECT_GETPUBKEY = 0
|
|
OBJECT_PUBKEY = 1
|
|
OBJECT_MSG = 2
|
|
OBJECT_BROADCAST = 3
|
|
OBJECT_ONIONPEER = 0x746f72
|
|
OBJECT_I2P = 0x493250
|
|
OBJECT_ADDR = 0x61646472
|
|
|
|
eightBytesOfRandomDataUsedToDetectConnectionsToSelf = pack(
|
|
'>Q', random.randrange(1, 18446744073709551615)) # nosec B311
|
|
|
|
# Compiled struct for packing/unpacking headers
|
|
# New code should use CreatePacket instead of Header.pack
|
|
Header = Struct('!L12sL4s')
|
|
|
|
VersionPacket = Struct('>LqQ20s4s36sH')
|
|
|
|
# Bitfield
|
|
|
|
|
|
def getBitfield(address):
|
|
"""Get a bitfield from an address"""
|
|
# bitfield of features supported by me (see the wiki).
|
|
bitfield = 0
|
|
# send ack
|
|
if not config.safeGetBoolean(address, 'dontsendack'):
|
|
bitfield |= BITFIELD_DOESACK
|
|
return pack('>I', bitfield)
|
|
|
|
|
|
def checkBitfield(bitfieldBinary, flags):
|
|
"""Check if a bitfield matches the given flags"""
|
|
bitfield, = unpack('>I', bitfieldBinary)
|
|
return (bitfield & flags) == flags
|
|
|
|
|
|
def isBitSetWithinBitfield(fourByteString, n):
|
|
"""Check if a particular bit is set in a bitfeld"""
|
|
# Uses MSB 0 bit numbering across 4 bytes of data
|
|
n = 31 - n
|
|
x, = unpack('>L', fourByteString)
|
|
return x & 2**n != 0
|
|
|
|
# Streams
|
|
|
|
|
|
MIN_VALID_STREAM = 1
|
|
MAX_VALID_STREAM = 2**63 - 1
|
|
|
|
# IP addresses
|
|
|
|
|
|
def encodeHost(host):
|
|
"""Encode a given host to be used in low-level socket operations"""
|
|
if host.endswith('.onion'):
|
|
return b'\xfd\x87\xd8\x7e\xeb\x43' + base64.b32decode(
|
|
host.split(".")[0], True)
|
|
elif host.find(':') == -1:
|
|
return b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xFF\xFF' + \
|
|
socket.inet_aton(host)
|
|
return socket.inet_pton(socket.AF_INET6, host)
|
|
|
|
|
|
def networkType(host):
|
|
"""Determine if a host is IPv4, IPv6 or an onion address"""
|
|
if host.endswith('.onion'):
|
|
return 'onion'
|
|
elif host.find(':') == -1:
|
|
return 'IPv4'
|
|
return 'IPv6'
|
|
|
|
|
|
def network_group(host):
|
|
"""Canonical identifier of network group
|
|
simplified, borrowed from
|
|
GetGroup() in src/netaddresses.cpp in bitcoin core"""
|
|
if not isinstance(host, str):
|
|
return None
|
|
network_type = networkType(host)
|
|
try:
|
|
raw_host = encodeHost(host)
|
|
except socket.error:
|
|
return host
|
|
if network_type == 'IPv4':
|
|
decoded_host = checkIPv4Address(raw_host[12:], True)
|
|
if decoded_host:
|
|
# /16 subnet
|
|
return raw_host[12:14]
|
|
elif network_type == 'IPv6':
|
|
decoded_host = checkIPv6Address(raw_host, True)
|
|
if decoded_host:
|
|
# /32 subnet
|
|
return raw_host[0:12]
|
|
else:
|
|
# just host, e.g. for tor
|
|
return host
|
|
# global network type group for local, private, unroutable
|
|
return network_type
|
|
|
|
|
|
def checkIPAddress(host, private=False):
|
|
"""
|
|
Returns hostStandardFormat if it is a valid IP address,
|
|
otherwise returns False
|
|
"""
|
|
if host[0:12] == b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xFF\xFF':
|
|
hostStandardFormat = socket.inet_ntop(socket.AF_INET, host[12:])
|
|
return checkIPv4Address(host[12:], hostStandardFormat, private)
|
|
elif host[0:6] == b'\xfd\x87\xd8\x7e\xeb\x43':
|
|
# Onion, based on BMD/bitcoind
|
|
hostStandardFormat = base64.b32encode(host[6:]).lower() + ".onion"
|
|
if private:
|
|
return False
|
|
return hostStandardFormat
|
|
else:
|
|
try:
|
|
hostStandardFormat = socket.inet_ntop(socket.AF_INET6, host)
|
|
except ValueError:
|
|
return False
|
|
if len(hostStandardFormat) == 0:
|
|
# This can happen on Windows systems which are
|
|
# not 64-bit compatible so let us drop the IPv6 address.
|
|
return False
|
|
return checkIPv6Address(host, hostStandardFormat, private)
|
|
|
|
|
|
def checkIPv4Address(host, hostStandardFormat, private=False):
|
|
"""
|
|
Returns hostStandardFormat if it is an IPv4 address,
|
|
otherwise returns False
|
|
"""
|
|
if host[0:1] == b'\x7F': # 127/8
|
|
if not private:
|
|
logger.debug(
|
|
'Ignoring IP address in loopback range: %s',
|
|
hostStandardFormat)
|
|
return hostStandardFormat if private else False
|
|
if host[0:1] == b'\x0A': # 10/8
|
|
if not private:
|
|
logger.debug(
|
|
'Ignoring IP address in private range: %s', hostStandardFormat)
|
|
return hostStandardFormat if private else False
|
|
if host[0:2] == b'\xC0\xA8': # 192.168/16
|
|
if not private:
|
|
logger.debug(
|
|
'Ignoring IP address in private range: %s', hostStandardFormat)
|
|
return hostStandardFormat if private else False
|
|
if host[0:2] >= b'\xAC\x10' and host[0:2] < b'\xAC\x20': # 172.16/12
|
|
if not private:
|
|
logger.debug(
|
|
'Ignoring IP address in private range: %s', hostStandardFormat)
|
|
return hostStandardFormat if private else False
|
|
return False if private else hostStandardFormat
|
|
|
|
|
|
def checkIPv6Address(host, hostStandardFormat, private=False):
|
|
"""
|
|
Returns hostStandardFormat if it is an IPv6 address,
|
|
otherwise returns False
|
|
"""
|
|
if host == b'\x00' * 15 + b'\x01':
|
|
if not private:
|
|
logger.debug('Ignoring loopback address: %s', hostStandardFormat)
|
|
return False
|
|
try:
|
|
host = [ord(c) for c in host[:2]]
|
|
except TypeError: # python3 has ints already
|
|
pass
|
|
if host[0] == 0xfe and host[1] & 0xc0 == 0x80:
|
|
if not private:
|
|
logger.debug('Ignoring local address: %s', hostStandardFormat)
|
|
return hostStandardFormat if private else False
|
|
if host[0] & 0xfe == 0xfc:
|
|
if not private:
|
|
logger.debug(
|
|
'Ignoring unique local address: %s', hostStandardFormat)
|
|
return hostStandardFormat if private else False
|
|
return False if private else hostStandardFormat
|
|
|
|
|
|
def haveSSL(server=False):
|
|
"""
|
|
Predicate to check if ECDSA server support is required and available
|
|
|
|
python < 2.7.9's ssl library does not support ECDSA server due to
|
|
missing initialisation of available curves, but client works ok
|
|
"""
|
|
if not server:
|
|
return True
|
|
elif sys.version_info >= (2, 7, 9):
|
|
return True
|
|
return False
|
|
|
|
|
|
def checkSocksIP(host):
|
|
"""Predicate to check if we're using a SOCKS proxy"""
|
|
sockshostname = config.safeGet(
|
|
'bitmessagesettings', 'sockshostname')
|
|
try:
|
|
if not state.socksIP:
|
|
state.socksIP = socket.gethostbyname(sockshostname)
|
|
except NameError: # uninitialised
|
|
state.socksIP = socket.gethostbyname(sockshostname)
|
|
except (TypeError, socket.gaierror): # None, resolving failure
|
|
state.socksIP = sockshostname
|
|
return state.socksIP == host
|
|
|
|
|
|
def isProofOfWorkSufficient(
|
|
data, nonceTrialsPerByte=0, payloadLengthExtraBytes=0, recvTime=0):
|
|
"""
|
|
Validate an object's Proof of Work using method described
|
|
:doc:`here </pow>`
|
|
|
|
Arguments:
|
|
int nonceTrialsPerByte (default: from `.defaults`)
|
|
int payloadLengthExtraBytes (default: from `.defaults`)
|
|
float recvTime (optional) UNIX epoch time when object was
|
|
received from the network (default: current system time)
|
|
Returns:
|
|
True if PoW valid and sufficient, False in all other cases
|
|
"""
|
|
if nonceTrialsPerByte < defaults.networkDefaultProofOfWorkNonceTrialsPerByte:
|
|
nonceTrialsPerByte = defaults.networkDefaultProofOfWorkNonceTrialsPerByte
|
|
if payloadLengthExtraBytes < defaults.networkDefaultPayloadLengthExtraBytes:
|
|
payloadLengthExtraBytes = defaults.networkDefaultPayloadLengthExtraBytes
|
|
endOfLifeTime, = unpack('>Q', data[8:16])
|
|
TTL = endOfLifeTime - int(recvTime if recvTime else time.time())
|
|
if TTL < 300:
|
|
TTL = 300
|
|
POW, = unpack('>Q', highlevelcrypto.double_sha512(
|
|
data[:8] + hashlib.sha512(data[8:]).digest())[0:8])
|
|
return POW <= 2 ** 64 / (
|
|
nonceTrialsPerByte * (
|
|
len(data) + payloadLengthExtraBytes
|
|
+ ((TTL * (len(data) + payloadLengthExtraBytes)) / (2 ** 16))))
|
|
|
|
|
|
# Packet creation
|
|
|
|
|
|
def CreatePacket(command, payload=b''):
|
|
"""Construct and return a packet"""
|
|
payload_length = len(payload)
|
|
checksum = hashlib.sha512(payload).digest()[0:4]
|
|
|
|
b = bytearray(Header.size + payload_length)
|
|
Header.pack_into(b, 0, magic, command, payload_length, checksum)
|
|
b[Header.size:] = payload
|
|
return bytes(b)
|
|
|
|
|
|
def assembleAddrMessage(peerList):
|
|
"""Create address command"""
|
|
if isinstance(peerList, Peer):
|
|
peerList = [peerList]
|
|
if not peerList:
|
|
return b''
|
|
retval = b''
|
|
for i in range(0, len(peerList), MAX_ADDR_COUNT):
|
|
payload = encodeVarint(len(peerList[i:i + MAX_ADDR_COUNT]))
|
|
for stream, peer, timestamp in peerList[i:i + MAX_ADDR_COUNT]:
|
|
# 64-bit time
|
|
payload += pack('>Q', timestamp)
|
|
payload += pack('>I', stream)
|
|
# service bit flags offered by this node
|
|
payload += pack('>q', 1)
|
|
payload += encodeHost(peer.host)
|
|
# remote port
|
|
payload += pack('>H', peer.port)
|
|
retval += CreatePacket(b'addr', payload)
|
|
return retval
|
|
|
|
|
|
def assembleVersionMessage( # pylint: disable=too-many-arguments
|
|
remoteHost, remotePort, participatingStreams, dandelion_enabled=True, server=False, nodeid=None,
|
|
):
|
|
"""
|
|
Construct the payload of a version message,
|
|
return the resulting bytes of running `CreatePacket` on it
|
|
"""
|
|
payload = b''
|
|
payload += pack('>L', 3) # protocol version.
|
|
# bitflags of the services I offer.
|
|
payload += pack(
|
|
'>q',
|
|
NODE_NETWORK
|
|
| (NODE_SSL if haveSSL(server) else 0)
|
|
| (NODE_DANDELION if dandelion_enabled else 0)
|
|
)
|
|
payload += pack('>q', int(time.time()))
|
|
|
|
# boolservices of remote connection; ignored by the remote host.
|
|
payload += pack('>q', 1)
|
|
if checkSocksIP(remoteHost) and server:
|
|
# prevent leaking of tor outbound IP
|
|
payload += encodeHost('127.0.0.1')
|
|
payload += pack('>H', 8444)
|
|
else:
|
|
# use first 16 bytes if host data is longer
|
|
# for example in case of onion v3 service
|
|
try:
|
|
payload += encodeHost(remoteHost)[:16]
|
|
except socket.error:
|
|
payload += encodeHost('127.0.0.1')
|
|
payload += pack('>H', remotePort) # remote IPv6 and port
|
|
|
|
# bitflags of the services I offer.
|
|
payload += pack(
|
|
'>q',
|
|
NODE_NETWORK
|
|
| (NODE_SSL if haveSSL(server) else 0)
|
|
| (NODE_DANDELION if dandelion_enabled else 0)
|
|
)
|
|
# = 127.0.0.1. This will be ignored by the remote host.
|
|
# The actual remote connected IP will be used.
|
|
payload += b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xFF\xFF' + pack(
|
|
'>L', 2130706433)
|
|
# we have a separate extPort and incoming over clearnet
|
|
# or outgoing through clearnet
|
|
extport = config.safeGetInt('bitmessagesettings', 'extport')
|
|
if (
|
|
extport and ((server and not checkSocksIP(remoteHost)) or (
|
|
config.get('bitmessagesettings', 'socksproxytype')
|
|
== 'none' and not server))
|
|
):
|
|
payload += pack('>H', extport)
|
|
elif checkSocksIP(remoteHost) and server: # incoming connection over Tor
|
|
payload += pack(
|
|
'>H', config.getint('bitmessagesettings', 'onionport'))
|
|
else: # no extport and not incoming over Tor
|
|
payload += pack(
|
|
'>H', config.getint('bitmessagesettings', 'port'))
|
|
|
|
if nodeid is not None:
|
|
payload += nodeid[0:8]
|
|
else:
|
|
payload += eightBytesOfRandomDataUsedToDetectConnectionsToSelf
|
|
userAgent = ('/PyBitmessage:%s/' % softwareVersion).encode('utf-8')
|
|
payload += encodeVarint(len(userAgent))
|
|
payload += userAgent
|
|
|
|
# Streams
|
|
payload += encodeVarint(len(participatingStreams))
|
|
count = 0
|
|
for stream in sorted(participatingStreams):
|
|
payload += encodeVarint(stream)
|
|
count += 1
|
|
# protocol limit, see specification
|
|
if count >= 160000:
|
|
break
|
|
|
|
return CreatePacket(b'version', payload)
|
|
|
|
|
|
def assembleErrorMessage(fatal=0, banTime=0, inventoryVector='', errorText=''):
|
|
"""
|
|
Construct the payload of an error message,
|
|
return the resulting bytes of running `CreatePacket` on it
|
|
"""
|
|
payload = encodeVarint(fatal)
|
|
payload += encodeVarint(banTime)
|
|
payload += encodeVarint(len(inventoryVector))
|
|
payload += inventoryVector
|
|
payload += encodeVarint(len(errorText))
|
|
payload += errorText
|
|
return CreatePacket(b'error', payload)
|
|
|
|
|
|
# Packet decoding
|
|
|
|
|
|
def decodeObjectParameters(data):
|
|
"""Decode the parameters of a raw object needed to put it in inventory"""
|
|
# BMProto.decode_payload_content("QQIvv")
|
|
expiresTime = unpack('>Q', data[8:16])[0]
|
|
objectType = unpack('>I', data[16:20])[0]
|
|
parserPos = 20 + decodeVarint(data[20:30])[1]
|
|
toStreamNumber = decodeVarint(data[parserPos:parserPos + 10])[0]
|
|
|
|
return objectType, toStreamNumber, expiresTime
|
|
|
|
|
|
def decryptAndCheckPubkeyPayload(data, address):
|
|
"""
|
|
Version 4 pubkeys are encrypted. This function is run when we
|
|
already have the address to which we want to try to send a message.
|
|
The 'data' may come either off of the wire or we might have had it
|
|
already in our inventory when we tried to send a msg to this
|
|
particular address.
|
|
"""
|
|
try:
|
|
addressVersion, streamNumber, ripe = decodeAddress(address)[1:]
|
|
|
|
readPosition = 20 # bypass the nonce, time, and object type
|
|
embeddedAddressVersion, varintLength = decodeVarint(
|
|
data[readPosition:readPosition + 10])
|
|
readPosition += varintLength
|
|
embeddedStreamNumber, varintLength = decodeVarint(
|
|
data[readPosition:readPosition + 10])
|
|
readPosition += varintLength
|
|
# We'll store the address version and stream number
|
|
# (and some more) in the pubkeys table.
|
|
storedData = data[20:readPosition]
|
|
|
|
if addressVersion != embeddedAddressVersion:
|
|
logger.info(
|
|
'Pubkey decryption was UNsuccessful'
|
|
' due to address version mismatch.')
|
|
return 'failed'
|
|
if streamNumber != embeddedStreamNumber:
|
|
logger.info(
|
|
'Pubkey decryption was UNsuccessful'
|
|
' due to stream number mismatch.')
|
|
return 'failed'
|
|
|
|
tag = data[readPosition:readPosition + 32]
|
|
readPosition += 32
|
|
# the time through the tag. More data is appended onto
|
|
# signedData below after the decryption.
|
|
signedData = data[8:readPosition]
|
|
encryptedData = data[readPosition:]
|
|
|
|
# Let us try to decrypt the pubkey
|
|
toAddress, cryptorObject = state.neededPubkeys[tag]
|
|
if toAddress != address:
|
|
logger.critical(
|
|
'decryptAndCheckPubkeyPayload failed due to toAddress'
|
|
' mismatch. This is very peculiar.'
|
|
' toAddress: %s, address %s',
|
|
toAddress, address
|
|
)
|
|
# the only way I can think that this could happen
|
|
# is if someone encodes their address data two different ways.
|
|
# That sort of address-malleability should have been caught
|
|
# by the UI or API and an error given to the user.
|
|
return 'failed'
|
|
try:
|
|
decryptedData = cryptorObject.decrypt(encryptedData)
|
|
except: # noqa:E722
|
|
# FIXME: use a proper exception after `pyelliptic.ecc` is refactored.
|
|
# Someone must have encrypted some data with a different key
|
|
# but tagged it with a tag for which we are watching.
|
|
logger.info('Pubkey decryption was unsuccessful.')
|
|
return 'failed'
|
|
|
|
readPosition = 0
|
|
# bitfieldBehaviors = decryptedData[readPosition:readPosition + 4]
|
|
readPosition += 4
|
|
pubSigningKey = '\x04' + decryptedData[readPosition:readPosition + 64]
|
|
readPosition += 64
|
|
pubEncryptionKey = '\x04' + decryptedData[readPosition:readPosition + 64]
|
|
readPosition += 64
|
|
specifiedNonceTrialsPerByteLength = decodeVarint(
|
|
decryptedData[readPosition:readPosition + 10])[1]
|
|
readPosition += specifiedNonceTrialsPerByteLength
|
|
specifiedPayloadLengthExtraBytesLength = decodeVarint(
|
|
decryptedData[readPosition:readPosition + 10])[1]
|
|
readPosition += specifiedPayloadLengthExtraBytesLength
|
|
storedData += decryptedData[:readPosition]
|
|
signedData += decryptedData[:readPosition]
|
|
signatureLength, signatureLengthLength = decodeVarint(
|
|
decryptedData[readPosition:readPosition + 10])
|
|
readPosition += signatureLengthLength
|
|
signature = decryptedData[readPosition:readPosition + signatureLength]
|
|
|
|
if not highlevelcrypto.verify(
|
|
signedData, signature, hexlify(pubSigningKey)):
|
|
logger.info(
|
|
'ECDSA verify failed (within decryptAndCheckPubkeyPayload)')
|
|
return 'failed'
|
|
|
|
logger.info(
|
|
'ECDSA verify passed (within decryptAndCheckPubkeyPayload)')
|
|
|
|
embeddedRipe = highlevelcrypto.to_ripe(pubSigningKey, pubEncryptionKey)
|
|
|
|
if embeddedRipe != ripe:
|
|
# Although this pubkey object had the tag were were looking for
|
|
# and was encrypted with the correct encryption key,
|
|
# it doesn't contain the correct pubkeys. Someone is
|
|
# either being malicious or using buggy software.
|
|
logger.info(
|
|
'Pubkey decryption was UNsuccessful due to RIPE mismatch.')
|
|
return 'failed'
|
|
|
|
# Everything checked out. Insert it into the pubkeys table.
|
|
|
|
logger.info(
|
|
'within decryptAndCheckPubkeyPayload, '
|
|
'addressVersion: %s, streamNumber: %s\nripe %s\n'
|
|
'publicSigningKey in hex: %s\npublicEncryptionKey in hex: %s',
|
|
addressVersion, streamNumber, hexlify(ripe),
|
|
hexlify(pubSigningKey), hexlify(pubEncryptionKey)
|
|
)
|
|
|
|
t = (address, addressVersion, storedData, int(time.time()), 'yes')
|
|
sqlExecute('''INSERT INTO pubkeys VALUES (?,?,?,?,?)''', *t)
|
|
return 'successful'
|
|
except varintDecodeError:
|
|
logger.info(
|
|
'Pubkey decryption was UNsuccessful due to a malformed varint.')
|
|
return 'failed'
|
|
except Exception:
|
|
logger.critical(
|
|
'Pubkey decryption was UNsuccessful because of'
|
|
' an unhandled exception! This is definitely a bug!',
|
|
exc_info=True
|
|
)
|
|
return 'failed'
|