PyBitmessage-2021-04-27/src/protocol.py

520 lines
18 KiB
Python
Raw Normal View History

2018-05-22 12:34:01 +02:00
"""
Low-level protocol-related functions.
"""
2019-10-28 16:49:57 +01:00
# pylint: disable=too-many-boolean-expressions,too-many-return-statements
# pylint: disable=too-many-locals,too-many-statements
2018-05-22 12:34:01 +02:00
import base64
import hashlib
import random
import socket
import sys
import time
from binascii import hexlify
from struct import Struct, pack, unpack
2018-05-22 12:34:01 +02:00
import defaults
import highlevelcrypto
import state
from addresses import (
encodeVarint, decodeVarint, decodeAddress, varintDecodeError)
from bmconfigparser import BMConfigParser
from debug import logger
from fallback import RIPEMD160Hash
from helper_sql import sqlExecute
from version import softwareVersion
2018-05-22 12:34:01 +02:00
# Service flags
2019-10-28 16:49:57 +01:00
#: This is a normal network node
NODE_NETWORK = 1
2019-10-28 16:49:57 +01:00
#: This node supports SSL/TLS in the current connect (python < 2.7.9
#: only supports an SSL client, so in that case it would only have this
#: on when the connection is a client).
NODE_SSL = 2
2019-10-28 16:49:57 +01:00
# (Proposal) This node may do PoW on behalf of some its peers
# (PoW offloading/delegating), but it doesn't have to. Clients may have
# to meet additional requirements (e.g. TLS authentication)
# NODE_POW = 4
#: Node supports dandelion
NODE_DANDELION = 8
2018-05-22 12:34:01 +02:00
# Bitfield flags
BITFIELD_DOESACK = 1
2018-05-22 12:34:01 +02:00
# Error types
2017-04-16 18:27:15 +02:00
STATUS_WARNING = 0
STATUS_ERROR = 1
STATUS_FATAL = 2
2018-05-22 12:34:01 +02:00
# Object types
OBJECT_GETPUBKEY = 0
OBJECT_PUBKEY = 1
OBJECT_MSG = 2
OBJECT_BROADCAST = 3
OBJECT_ONIONPEER = 0x746f72
OBJECT_I2P = 0x493250
OBJECT_ADDR = 0x61646472
eightBytesOfRandomDataUsedToDetectConnectionsToSelf = pack(
'>Q', random.randrange(1, 18446744073709551615))
2018-05-22 12:34:01 +02:00
# Compiled struct for packing/unpacking headers
# New code should use CreatePacket instead of Header.pack
Header = Struct('!L12sL4s')
VersionPacket = Struct('>LqQ20s4s36sH')
# Bitfield
2018-05-22 12:34:01 +02:00
def getBitfield(address):
2018-05-22 12:34:01 +02:00
"""Get a bitfield from an address"""
# bitfield of features supported by me (see the wiki).
bitfield = 0
# send ack
if not BMConfigParser().safeGetBoolean(address, 'dontsendack'):
bitfield |= BITFIELD_DOESACK
return pack('>I', bitfield)
2018-05-22 12:34:01 +02:00
def checkBitfield(bitfieldBinary, flags):
2018-05-22 12:34:01 +02:00
"""Check if a bitfield matches the given flags"""
bitfield, = unpack('>I', bitfieldBinary)
return (bitfield & flags) == flags
2018-05-22 12:34:01 +02:00
def isBitSetWithinBitfield(fourByteString, n):
2018-05-22 12:34:01 +02:00
"""Check if a particular bit is set in a bitfeld"""
# Uses MSB 0 bit numbering across 4 bytes of data
n = 31 - n
x, = unpack('>L', fourByteString)
return x & 2**n != 0
# ip addresses
2018-05-24 17:59:40 +02:00
def encodeHost(host):
2018-05-22 12:34:01 +02:00
"""Encode a given host to be used in low-level socket operations"""
if host.find('.onion') > -1:
2019-10-28 16:49:57 +01:00
return '\xfd\x87\xd8\x7e\xeb\x43' + base64.b32decode(
host.split(".")[0], True)
elif host.find(':') == -1:
return '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xFF\xFF' + \
socket.inet_aton(host)
2018-05-22 12:34:01 +02:00
return socket.inet_pton(socket.AF_INET6, host)
2017-01-12 19:18:56 +01:00
def networkType(host):
2018-05-22 12:34:01 +02:00
"""Determine if a host is IPv4, IPv6 or an onion address"""
2017-01-12 19:18:56 +01:00
if host.find('.onion') > -1:
return 'onion'
elif host.find(':') == -1:
return 'IPv4'
2018-05-22 12:34:01 +02:00
return 'IPv6'
2017-01-12 19:18:56 +01:00
def network_group(host):
"""Canonical identifier of network group
simplified, borrowed from
GetGroup() in src/netaddresses.cpp in bitcoin core"""
if not isinstance(host, str):
return None
network_type = networkType(host)
try:
raw_host = encodeHost(host)
except socket.error:
return host
if network_type == 'IPv4':
decoded_host = checkIPv4Address(raw_host[12:], True)
if decoded_host:
# /16 subnet
return raw_host[12:14]
elif network_type == 'IPv6':
decoded_host = checkIPv6Address(raw_host, True)
if decoded_host:
# /32 subnet
return raw_host[0:12]
else:
# just host, e.g. for tor
return host
# global network type group for local, private, unroutable
return network_type
def checkIPAddress(host, private=False):
2019-10-28 16:49:57 +01:00
"""
Returns hostStandardFormat if it is a valid IP address,
otherwise returns False
"""
if host[0:12] == '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xFF\xFF':
hostStandardFormat = socket.inet_ntop(socket.AF_INET, host[12:])
return checkIPv4Address(host[12:], hostStandardFormat, private)
elif host[0:6] == '\xfd\x87\xd8\x7e\xeb\x43':
# Onion, based on BMD/bitcoind
hostStandardFormat = base64.b32encode(host[6:]).lower() + ".onion"
if private:
return False
return hostStandardFormat
else:
try:
hostStandardFormat = socket.inet_ntop(socket.AF_INET6, host)
except ValueError:
return False
if hostStandardFormat == "":
2019-10-28 16:49:57 +01:00
# This can happen on Windows systems which are
# not 64-bit compatible so let us drop the IPv6 address.
return False
return checkIPv6Address(host, hostStandardFormat, private)
2018-05-22 12:34:01 +02:00
def checkIPv4Address(host, hostStandardFormat, private=False):
2019-10-28 16:49:57 +01:00
"""
Returns hostStandardFormat if it is an IPv4 address,
otherwise returns False
"""
2018-05-22 12:34:01 +02:00
if host[0] == '\x7F': # 127/8
if not private:
2019-10-28 16:49:57 +01:00
logger.debug(
'Ignoring IP address in loopback range: %s',
hostStandardFormat)
return hostStandardFormat if private else False
2018-05-22 12:34:01 +02:00
if host[0] == '\x0A': # 10/8
if not private:
2019-10-28 16:49:57 +01:00
logger.debug(
'Ignoring IP address in private range: %s', hostStandardFormat)
return hostStandardFormat if private else False
2018-05-22 12:34:01 +02:00
if host[0:2] == '\xC0\xA8': # 192.168/16
if not private:
2019-10-28 16:49:57 +01:00
logger.debug(
'Ignoring IP address in private range: %s', hostStandardFormat)
return hostStandardFormat if private else False
2018-05-22 12:34:01 +02:00
if host[0:2] >= '\xAC\x10' and host[0:2] < '\xAC\x20': # 172.16/12
if not private:
2019-10-28 16:49:57 +01:00
logger.debug(
'Ignoring IP address in private range: %s', hostStandardFormat)
return hostStandardFormat if private else False
return False if private else hostStandardFormat
2018-05-22 12:34:01 +02:00
def checkIPv6Address(host, hostStandardFormat, private=False):
2019-10-28 16:49:57 +01:00
"""
Returns hostStandardFormat if it is an IPv6 address,
otherwise returns False
"""
if host == ('\x00' * 15) + '\x01':
if not private:
2018-05-22 12:34:01 +02:00
logger.debug('Ignoring loopback address: %s', hostStandardFormat)
return False
if host[0] == '\xFE' and (ord(host[1]) & 0xc0) == 0x80:
if not private:
2018-05-22 12:34:01 +02:00
logger.debug('Ignoring local address: %s', hostStandardFormat)
return hostStandardFormat if private else False
if (ord(host[0]) & 0xfe) == 0xfc:
if not private:
2019-10-28 16:49:57 +01:00
logger.debug(
'Ignoring unique local address: %s', hostStandardFormat)
return hostStandardFormat if private else False
return False if private else hostStandardFormat
2018-05-22 12:34:01 +02:00
def haveSSL(server=False):
"""
Predicate to check if ECDSA server support is required and available
python < 2.7.9's ssl library does not support ECDSA server due to
missing initialisation of available curves, but client works ok
"""
if not server:
return True
2018-05-22 12:34:01 +02:00
elif sys.version_info >= (2, 7, 9):
return True
return False
2018-05-22 12:34:01 +02:00
def checkSocksIP(host):
2018-05-22 12:34:01 +02:00
"""Predicate to check if we're using a SOCKS proxy"""
2019-10-28 16:49:57 +01:00
sockshostname = BMConfigParser().safeGet(
'bitmessagesettings', 'sockshostname')
try:
2019-10-28 16:49:57 +01:00
if not state.socksIP:
state.socksIP = socket.gethostbyname(sockshostname)
except NameError: # uninitialised
state.socksIP = socket.gethostbyname(sockshostname)
except (TypeError, socket.gaierror): # None, resolving failure
state.socksIP = sockshostname
return state.socksIP == host
2018-05-22 12:34:01 +02:00
2019-10-28 16:49:57 +01:00
def isProofOfWorkSufficient(
data, nonceTrialsPerByte=0, payloadLengthExtraBytes=0, recvTime=0):
2018-05-25 08:13:01 +02:00
"""
2019-10-28 16:49:57 +01:00
Validate an object's Proof of Work using method described
`here <https://bitmessage.org/wiki/Proof_of_work>`_
2018-05-25 08:13:01 +02:00
Arguments:
2019-10-28 16:49:57 +01:00
int nonceTrialsPerByte (default: from `.defaults`)
int payloadLengthExtraBytes (default: from `.defaults`)
2018-05-25 08:13:01 +02:00
float recvTime (optional) UNIX epoch time when object was
2019-10-28 16:49:57 +01:00
received from the network (default: current system time)
2018-05-25 08:13:01 +02:00
Returns:
True if PoW valid and sufficient, False in all other cases
"""
if nonceTrialsPerByte < defaults.networkDefaultProofOfWorkNonceTrialsPerByte:
nonceTrialsPerByte = defaults.networkDefaultProofOfWorkNonceTrialsPerByte
if payloadLengthExtraBytes < defaults.networkDefaultPayloadLengthExtraBytes:
payloadLengthExtraBytes = defaults.networkDefaultPayloadLengthExtraBytes
endOfLifeTime, = unpack('>Q', data[8:16])
2018-05-25 08:13:01 +02:00
TTL = endOfLifeTime - (int(recvTime) if recvTime else int(time.time()))
if TTL < 300:
TTL = 300
2019-10-28 16:49:57 +01:00
POW, = unpack('>Q', hashlib.sha512(hashlib.sha512(
data[:8] + hashlib.sha512(data[8:]).digest()
).digest()).digest()[0:8])
return POW <= 2 ** 64 / (
nonceTrialsPerByte * (
len(data) + payloadLengthExtraBytes +
((TTL * (len(data) + payloadLengthExtraBytes)) / (2 ** 16))))
# Packet creation
2018-05-24 17:59:40 +02:00
def CreatePacket(command, payload=''):
2019-10-28 16:49:57 +01:00
"""Construct and return a packet"""
payload_length = len(payload)
checksum = hashlib.sha512(payload).digest()[0:4]
2018-05-22 12:34:01 +02:00
b = bytearray(Header.size + payload_length)
Header.pack_into(b, 0, 0xE9BEB4D9, command, payload_length, checksum)
b[Header.size:] = payload
return bytes(b)
2018-05-22 12:34:01 +02:00
2019-10-28 16:49:57 +01:00
def assembleVersionMessage(
remoteHost, remotePort, participatingStreams, server=False, nodeid=None
):
"""
Construct the payload of a version message,
return the resulting bytes of running `CreatePacket` on it
"""
payload = ''
payload += pack('>L', 3) # protocol version.
# bitflags of the services I offer.
2018-05-22 12:34:01 +02:00
payload += pack(
'>q',
NODE_NETWORK |
(NODE_SSL if haveSSL(server) else 0) |
(NODE_DANDELION if state.dandelion else 0)
)
payload += pack('>q', int(time.time()))
2019-10-28 16:49:57 +01:00
# boolservices of remote connection; ignored by the remote host.
payload += pack('>q', 1)
if checkSocksIP(remoteHost) and server:
# prevent leaking of tor outbound IP
payload += encodeHost('127.0.0.1')
payload += pack('>H', 8444)
else:
# use first 16 bytes if host data is longer
# for example in case of onion v3 service
2019-07-12 17:19:06 +02:00
try:
payload += encodeHost(remoteHost)[:16]
except socket.error:
payload += encodeHost('127.0.0.1')
payload += pack('>H', remotePort) # remote IPv6 and port
# bitflags of the services I offer.
2018-05-22 12:34:01 +02:00
payload += pack(
'>q',
NODE_NETWORK |
(NODE_SSL if haveSSL(server) else 0) |
(NODE_DANDELION if state.dandelion else 0)
)
2019-10-28 16:49:57 +01:00
# = 127.0.0.1. This will be ignored by the remote host.
# The actual remote connected IP will be used.
payload += '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xFF\xFF' + pack(
'>L', 2130706433)
# we have a separate extPort and incoming over clearnet
# or outgoing through clearnet
extport = BMConfigParser().safeGetInt('bitmessagesettings', 'extport')
if (
extport and ((server and not checkSocksIP(remoteHost)) or (
2019-10-28 16:49:57 +01:00
BMConfigParser().get('bitmessagesettings', 'socksproxytype')
== 'none' and not server))
):
payload += pack('>H', extport)
2018-05-22 12:34:01 +02:00
elif checkSocksIP(remoteHost) and server: # incoming connection over Tor
2019-10-28 16:49:57 +01:00
payload += pack(
'>H', BMConfigParser().getint('bitmessagesettings', 'onionport'))
else: # no extport and not incoming over Tor
2019-10-28 16:49:57 +01:00
payload += pack(
'>H', BMConfigParser().getint('bitmessagesettings', 'port'))
if nodeid is not None:
payload += nodeid[0:8]
else:
payload += eightBytesOfRandomDataUsedToDetectConnectionsToSelf
userAgent = '/PyBitmessage:' + softwareVersion + '/'
payload += encodeVarint(len(userAgent))
payload += userAgent
# Streams
payload += encodeVarint(len(participatingStreams))
count = 0
for stream in sorted(participatingStreams):
payload += encodeVarint(stream)
count += 1
# protocol limit, see specification
if count >= 160000:
break
return CreatePacket('version', payload)
2018-05-22 12:34:01 +02:00
def assembleErrorMessage(fatal=0, banTime=0, inventoryVector='', errorText=''):
2019-10-28 16:49:57 +01:00
"""
Construct the payload of an error message,
return the resulting bytes of running `CreatePacket` on it
"""
payload = encodeVarint(fatal)
payload += encodeVarint(banTime)
payload += encodeVarint(len(inventoryVector))
payload += inventoryVector
payload += encodeVarint(len(errorText))
payload += errorText
return CreatePacket('error', payload)
# Packet decoding
2018-05-24 17:59:40 +02:00
def decryptAndCheckPubkeyPayload(data, address):
"""
Version 4 pubkeys are encrypted. This function is run when we
already have the address to which we want to try to send a message.
The 'data' may come either off of the wire or we might have had it
already in our inventory when we tried to send a msg to this
particular address.
"""
try:
addressVersion, streamNumber, ripe = decodeAddress(address)[1:]
2018-05-22 12:34:01 +02:00
readPosition = 20 # bypass the nonce, time, and object type
embeddedAddressVersion, varintLength = decodeVarint(
data[readPosition:readPosition + 10])
readPosition += varintLength
embeddedStreamNumber, varintLength = decodeVarint(
data[readPosition:readPosition + 10])
readPosition += varintLength
# We'll store the address version and stream number
# (and some more) in the pubkeys table.
2018-05-22 12:34:01 +02:00
storedData = data[20:readPosition]
if addressVersion != embeddedAddressVersion:
logger.info(
'Pubkey decryption was UNsuccessful'
' due to address version mismatch.')
return 'failed'
if streamNumber != embeddedStreamNumber:
logger.info(
'Pubkey decryption was UNsuccessful'
' due to stream number mismatch.')
return 'failed'
2018-05-22 12:34:01 +02:00
tag = data[readPosition:readPosition + 32]
readPosition += 32
# the time through the tag. More data is appended onto
# signedData below after the decryption.
2018-05-22 12:34:01 +02:00
signedData = data[8:readPosition]
encryptedData = data[readPosition:]
2018-05-22 12:34:01 +02:00
# Let us try to decrypt the pubkey
toAddress, cryptorObject = state.neededPubkeys[tag]
if toAddress != address:
2018-05-22 12:34:01 +02:00
logger.critical(
'decryptAndCheckPubkeyPayload failed due to toAddress'
' mismatch. This is very peculiar.'
' toAddress: %s, address %s',
toAddress, address
)
# the only way I can think that this could happen
# is if someone encodes their address data two different ways.
# That sort of address-malleability should have been caught
# by the UI or API and an error given to the user.
return 'failed'
try:
decryptedData = cryptorObject.decrypt(encryptedData)
except:
# Someone must have encrypted some data with a different key
# but tagged it with a tag for which we are watching.
logger.info('Pubkey decryption was unsuccessful.')
return 'failed'
2018-05-22 12:34:01 +02:00
readPosition = 0
# bitfieldBehaviors = decryptedData[readPosition:readPosition + 4]
readPosition += 4
publicSigningKey = '\x04' + decryptedData[readPosition:readPosition + 64]
readPosition += 64
publicEncryptionKey = '\x04' + decryptedData[readPosition:readPosition + 64]
readPosition += 64
specifiedNonceTrialsPerByteLength = decodeVarint(
decryptedData[readPosition:readPosition + 10])[1]
readPosition += specifiedNonceTrialsPerByteLength
specifiedPayloadLengthExtraBytesLength = decodeVarint(
decryptedData[readPosition:readPosition + 10])[1]
readPosition += specifiedPayloadLengthExtraBytesLength
storedData += decryptedData[:readPosition]
signedData += decryptedData[:readPosition]
signatureLength, signatureLengthLength = decodeVarint(
decryptedData[readPosition:readPosition + 10])
readPosition += signatureLengthLength
signature = decryptedData[readPosition:readPosition + signatureLength]
2018-05-22 12:34:01 +02:00
if not highlevelcrypto.verify(
signedData, signature, hexlify(publicSigningKey)):
logger.info(
'ECDSA verify failed (within decryptAndCheckPubkeyPayload)')
return 'failed'
2018-05-22 12:34:01 +02:00
logger.info(
'ECDSA verify passed (within decryptAndCheckPubkeyPayload)')
sha = hashlib.new('sha512')
sha.update(publicSigningKey + publicEncryptionKey)
embeddedRipe = RIPEMD160Hash(sha.digest()).digest()
2018-05-22 12:34:01 +02:00
if embeddedRipe != ripe:
# Although this pubkey object had the tag were were looking for
# and was encrypted with the correct encryption key,
# it doesn't contain the correct pubkeys. Someone is
# either being malicious or using buggy software.
logger.info(
'Pubkey decryption was UNsuccessful due to RIPE mismatch.')
return 'failed'
2018-05-22 12:34:01 +02:00
# Everything checked out. Insert it into the pubkeys table.
2018-05-22 12:34:01 +02:00
logger.info(
'within decryptAndCheckPubkeyPayload, '
'addressVersion: %s, streamNumber: %s\nripe %s\n'
'publicSigningKey in hex: %s\npublicEncryptionKey in hex: %s',
addressVersion, streamNumber, hexlify(ripe),
hexlify(publicSigningKey), hexlify(publicEncryptionKey)
2018-05-22 12:34:01 +02:00
)
t = (address, addressVersion, storedData, int(time.time()), 'yes')
sqlExecute('''INSERT INTO pubkeys VALUES (?,?,?,?,?)''', *t)
return 'successful'
2018-05-22 12:34:01 +02:00
except varintDecodeError:
logger.info(
'Pubkey decryption was UNsuccessful due to a malformed varint.')
return 'failed'
2018-05-22 12:34:01 +02:00
except Exception:
logger.critical(
'Pubkey decryption was UNsuccessful because of'
2019-10-28 16:49:57 +01:00
' an unhandled exception! This is definitely a bug!',
exc_info=True
)
return 'failed'