@ -6,14 +6,13 @@ import shutil # used for moving the messages.dat file
import sys
import os
from debug import logger
import defaults
import helper_sql
from namecoin import ensureNamecoinOptions
import helper_startup
import paths
import queues
import state
import tr #anslate
import helper_random
import tr
# This thread exists because SQLITE3 is so un-threadsafe that we must
# submit queries to it and it puts results back in a different queue. They
# won't let us just use locks.
@ -68,35 +67,26 @@ class sqlThread(threading.Thread):
' ERROR trying to create database file (message.dat). Error message: %s \n ' % str ( err ) )
os . _exit ( 0 )
if BMConfigParser ( ) . getint ( ' bitmessagesettings ' , ' settingsversion ' ) == 1 :
BMConfigParser ( ) . set ( ' bitmessagesettings ' , ' settingsversion ' , ' 2 ' )
# If the settings version is equal to 2 or 3 then the
# sqlThread will modify the pubkeys table and change
# the settings version to 4.
BMConfigParser ( ) . set ( ' bitmessagesettings ' , ' socksproxytype ' , ' none ' )
BMConfigParser ( ) . set ( ' bitmessagesettings ' , ' sockshostname ' , ' localhost ' )
BMConfigParser ( ) . set ( ' bitmessagesettings ' , ' socksport ' , ' 9050 ' )
BMConfigParser ( ) . set ( ' bitmessagesettings ' , ' socksauthentication ' , ' false ' )
BMConfigParser ( ) . set ( ' bitmessagesettings ' , ' socksusername ' , ' ' )
BMConfigParser ( ) . set ( ' bitmessagesettings ' , ' sockspassword ' , ' ' )
BMConfigParser ( ) . set ( ' bitmessagesettings ' , ' sockslisten ' , ' false ' )
BMConfigParser ( ) . set ( ' bitmessagesettings ' , ' keysencrypted ' , ' false ' )
BMConfigParser ( ) . set ( ' bitmessagesettings ' , ' messagesencrypted ' , ' false ' )
# If the settings version is equal to 2 or 3 then the
# sqlThread will modify the pubkeys table and change
# the settings version to 4.
settingsversion = BMConfigParser ( ) . getint (
' bitmessagesettings ' , ' settingsversion ' )
# People running earlier versions of PyBitmessage do not have the
# usedpersonally field in their pubkeys table. Let's add it.
if BMConfigParser( ) . getint ( ' bitmessagesettings ' , ' settingsversion' ) == 2 :
if settingsversion == 2 :
item = ''' ALTER TABLE pubkeys ADD usedpersonally text DEFAULT ' no ' '''
parameters = ' '
self . cur . execute ( item , parameters )
self . conn . commit ( )
BMConfigParser( ) . set ( ' bitmessagesettings ' , ' settingsversion ' , ' 3 ' )
settingsversion = 3
# People running earlier versions of PyBitmessage do not have the
# encodingtype field in their inbox and sent tables or the read field
# in the inbox table. Let's add them.
if BMConfigParser( ) . getint ( ' bitmessagesettings ' , ' settingsversion' ) == 3 :
if settingsversion == 3 :
item = ''' ALTER TABLE inbox ADD encodingtype int DEFAULT ' 2 ' '''
parameters = ' '
self . cur . execute ( item , parameters )
@ -110,21 +100,13 @@ class sqlThread(threading.Thread):
self . cur . execute ( item , parameters )
self . conn . commit ( )
BMConfigParser( ) . set ( ' bitmessagesettings ' , ' settingsversion ' , ' 4 ' )
settingsversion = 4
if BMConfigParser ( ) . getint ( ' bitmessagesettings ' , ' settingsversion ' ) == 4 :
BMConfigParser ( ) . set ( ' bitmessagesettings ' , ' defaultnoncetrialsperbyte ' , str (
defaults . networkDefaultProofOfWorkNonceTrialsPerByte ) )
BMConfigParser ( ) . set ( ' bitmessagesettings ' , ' defaultpayloadlengthextrabytes ' , str (
defaults . networkDefaultPayloadLengthExtraBytes ) )
BMConfigParser ( ) . set ( ' bitmessagesettings ' , ' settingsversion ' , ' 5 ' )
BMConfigParser ( ) . set (
' bitmessagesettings ' , ' settingsversion ' , str ( settingsversion ) )
BMConfigParser ( ) . save ( )
if BMConfigParser ( ) . getint ( ' bitmessagesettings ' , ' settingsversion ' ) == 5 :
BMConfigParser ( ) . set (
' bitmessagesettings ' , ' maxacceptablenoncetrialsperbyte ' , ' 0 ' )
BMConfigParser ( ) . set (
' bitmessagesettings ' , ' maxacceptablepayloadlengthextrabytes ' , ' 0 ' )
BMConfigParser ( ) . set ( ' bitmessagesettings ' , ' settingsversion ' , ' 6 ' )
helper_startup . updateConfig ( )
# From now on, let us keep a 'version' embedded in the messages.dat
# file so that when we make changes to the database, the database
@ -135,7 +117,8 @@ class sqlThread(threading.Thread):
self . cur . execute ( item , parameters )
if self . cur . fetchall ( ) == [ ] :
# The settings table doesn't exist. We need to make it.
logger . debug ( ' In messages.dat database, creating new \' settings \' table. ' )
logger . debug (
" In messages.dat database, creating new ' settings ' table. " )
self . cur . execute (
''' CREATE TABLE settings (key text, value blob, UNIQUE(key) ON CONFLICT REPLACE) ''' )
self . cur . execute ( ''' INSERT INTO settings VALUES( ' version ' , ' 1 ' ) ''' )
@ -177,17 +160,15 @@ class sqlThread(threading.Thread):
''' update sent set status= ' broadcastqueued ' where status= ' broadcastpending ' ''' )
self . conn . commit ( )
if not BMConfigParser ( ) . has_option ( ' bitmessagesettings ' , ' sockslisten ' ) :
BMConfigParser ( ) . set ( ' bitmessagesettings ' , ' sockslisten ' , ' false ' )
ensureNamecoinOptions ( )
# Let's get rid of the first20bytesofencryptedmessage field in the inventory table.
# Let's get rid of the first20bytesofencryptedmessage field in
# the inventory table.
item = ''' SELECT value FROM settings WHERE key= ' version ' ; '''
parameters = ' '
self . cur . execute ( item , parameters )
if int ( self . cur . fetchall ( ) [ 0 ] [ 0 ] ) == 2 :
logger . debug ( ' In messages.dat database, removing an obsolete field from the inventory table. ' )
logger . debug (
' In messages.dat database, removing an obsolete field from '
' the inventory table. ' )
self . cur . execute (
''' CREATE TEMPORARY TABLE inventory_backup(hash blob, objecttype text, streamnumber int, payload blob, receivedtime integer, UNIQUE(hash) ON CONFLICT REPLACE); ''' )
self . cur . execute (
@ -208,7 +189,9 @@ class sqlThread(threading.Thread):
self . cur . execute ( item , parameters )
currentVersion = int ( self . cur . fetchall ( ) [ 0 ] [ 0 ] )
if currentVersion == 1 or currentVersion == 3 :
logger . debug ( ' In messages.dat database, adding tag field to the inventory table. ' )
logger . debug (
' In messages.dat database, adding tag field to '
' the inventory table. ' )
item = ''' ALTER TABLE inventory ADD tag blob DEFAULT ' ' '''
parameters = ' '
self . cur . execute ( item , parameters )
@ -216,16 +199,6 @@ class sqlThread(threading.Thread):
parameters = ( 4 , )
self . cur . execute ( item , parameters )
if not BMConfigParser ( ) . has_option ( ' bitmessagesettings ' , ' userlocale ' ) :
BMConfigParser ( ) . set ( ' bitmessagesettings ' , ' userlocale ' , ' system ' )
if not BMConfigParser ( ) . has_option ( ' bitmessagesettings ' , ' sendoutgoingconnections ' ) :
BMConfigParser ( ) . set ( ' bitmessagesettings ' , ' sendoutgoingconnections ' , ' True ' )
# Raise the default required difficulty from 1 to 2
# With the change to protocol v3, this is obsolete.
if BMConfigParser ( ) . getint ( ' bitmessagesettings ' , ' settingsversion ' ) == 6 :
BMConfigParser ( ) . set ( ' bitmessagesettings ' , ' settingsversion ' , ' 7 ' )
# Add a new column to the pubkeys table to store the address version.
# We're going to trash all of our pubkeys and let them be redownloaded.
item = ''' SELECT value FROM settings WHERE key= ' version ' ; '''
@ -233,28 +206,15 @@ class sqlThread(threading.Thread):
self . cur . execute ( item , parameters )
currentVersion = int ( self . cur . fetchall ( ) [ 0 ] [ 0 ] )
if currentVersion == 4 :
self . cur . execute ( ''' DROP TABLE pubkeys ''' )
self . cur . execute ( ''' DROP TABLE pubkeys ''' )
self . cur . execute (
''' CREATE TABLE pubkeys (hash blob, addressversion int, transmitdata blob, time int, usedpersonally text, UNIQUE(hash, addressversion) ON CONFLICT REPLACE) ''' )
''' CREATE TABLE pubkeys (hash blob, addressversion int, transmitdata blob, time int, usedpersonally text, UNIQUE(hash, addressversion) ON CONFLICT REPLACE) ''' )
self . cur . execute (
''' delete from inventory where objecttype = ' pubkey ' ; ''' )
item = ''' update settings set value=? WHERE key= ' version ' ; '''
parameters = ( 5 , )
self . cur . execute ( item , parameters )
if not BMConfigParser ( ) . has_option ( ' bitmessagesettings ' , ' useidenticons ' ) :
BMConfigParser ( ) . set ( ' bitmessagesettings ' , ' useidenticons ' , ' True ' )
if not BMConfigParser ( ) . has_option ( ' bitmessagesettings ' , ' identiconsuffix ' ) : # acts as a salt
BMConfigParser ( ) . set ( ' bitmessagesettings ' , ' identiconsuffix ' , ' ' . join ( helper_random . randomchoice ( " 123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz " ) for x in range ( 12 ) ) ) # a twelve character pseudo-password to salt the identicons
#Add settings to support no longer resending messages after a certain period of time even if we never get an ack
if BMConfigParser ( ) . getint ( ' bitmessagesettings ' , ' settingsversion ' ) == 7 :
BMConfigParser ( ) . set (
' bitmessagesettings ' , ' stopresendingafterxdays ' , ' ' )
BMConfigParser ( ) . set (
' bitmessagesettings ' , ' stopresendingafterxmonths ' , ' ' )
BMConfigParser ( ) . set ( ' bitmessagesettings ' , ' settingsversion ' , ' 8 ' )
# Add a new table: objectprocessorqueue with which to hold objects
# that have yet to be processed if the user shuts down Bitmessage.
item = ''' SELECT value FROM settings WHERE key= ' version ' ; '''
@ -262,21 +222,24 @@ class sqlThread(threading.Thread):
self . cur . execute ( item , parameters )
currentVersion = int ( self . cur . fetchall ( ) [ 0 ] [ 0 ] )
if currentVersion == 5 :
self . cur . execute ( ''' DROP TABLE knownnodes ''' )
self . cur . execute ( ''' DROP TABLE knownnodes ''' )
self . cur . execute (
''' CREATE TABLE objectprocessorqueue (objecttype text, data blob, UNIQUE(objecttype, data) ON CONFLICT REPLACE) ''' )
''' CREATE TABLE objectprocessorqueue (objecttype text, data blob, UNIQUE(objecttype, data) ON CONFLICT REPLACE) ''' )
item = ''' update settings set value=? WHERE key= ' version ' ; '''
parameters = ( 6 , )
self . cur . execute ( item , parameters )
# changes related to protocol v3
# In table inventory and objectprocessorqueue, objecttype is now an integer (it was a human-friendly string previously)
# In table inventory and objectprocessorqueue, objecttype is now
# an integer (it was a human-friendly string previously)
item = ''' SELECT value FROM settings WHERE key= ' version ' ; '''
parameters = ' '
self . cur . execute ( item , parameters )
currentVersion = int ( self . cur . fetchall ( ) [ 0 ] [ 0 ] )
if currentVersion == 6 :
logger . debug ( ' In messages.dat database, dropping and recreating the inventory table. ' )
logger . debug (
' In messages.dat database, dropping and recreating '
' the inventory table. ' )
self . cur . execute ( ''' DROP TABLE inventory ''' )
self . cur . execute ( ''' CREATE TABLE inventory (hash blob, objecttype int, streamnumber int, payload blob, expirestime integer, tag blob, UNIQUE(hash) ON CONFLICT REPLACE) ''' )
self . cur . execute ( ''' DROP TABLE objectprocessorqueue ''' )
@ -284,57 +247,26 @@ class sqlThread(threading.Thread):
item = ''' update settings set value=? WHERE key= ' version ' ; '''
parameters = ( 7 , )
self . cur . execute ( item , parameters )
logger . debug ( ' Finished dropping and recreating the inventory table. ' )
# With the change to protocol version 3, reset the user-settable difficulties to 1
if BMConfigParser ( ) . getint ( ' bitmessagesettings ' , ' settingsversion ' ) == 8 :
BMConfigParser ( ) . set ( ' bitmessagesettings ' , ' defaultnoncetrialsperbyte ' , str ( defaults . networkDefaultProofOfWorkNonceTrialsPerByte ) )
BMConfigParser ( ) . set ( ' bitmessagesettings ' , ' defaultpayloadlengthextrabytes ' , str ( defaults . networkDefaultPayloadLengthExtraBytes ) )
previousTotalDifficulty = int ( BMConfigParser ( ) . getint ( ' bitmessagesettings ' , ' maxacceptablenoncetrialsperbyte ' ) ) / 320
previousSmallMessageDifficulty = int ( BMConfigParser ( ) . getint ( ' bitmessagesettings ' , ' maxacceptablepayloadlengthextrabytes ' ) ) / 14000
BMConfigParser ( ) . set ( ' bitmessagesettings ' , ' maxacceptablenoncetrialsperbyte ' , str ( previousTotalDifficulty * 1000 ) )
BMConfigParser ( ) . set ( ' bitmessagesettings ' , ' maxacceptablepayloadlengthextrabytes ' , str ( previousSmallMessageDifficulty * 1000 ) )
BMConfigParser ( ) . set ( ' bitmessagesettings ' , ' settingsversion ' , ' 9 ' )
# Adjust the required POW values for each of this user's addresses to conform to protocol v3 norms.
if BMConfigParser ( ) . getint ( ' bitmessagesettings ' , ' settingsversion ' ) == 9 :
for addressInKeysFile in BMConfigParser ( ) . addressses ( ) :
try :
previousTotalDifficulty = float ( BMConfigParser ( ) . getint ( addressInKeysFile , ' noncetrialsperbyte ' ) ) / 320
previousSmallMessageDifficulty = float ( BMConfigParser ( ) . getint ( addressInKeysFile , ' payloadlengthextrabytes ' ) ) / 14000
if previousTotalDifficulty < = 2 :
previousTotalDifficulty = 1
if previousSmallMessageDifficulty < 1 :
previousSmallMessageDifficulty = 1
BMConfigParser ( ) . set ( addressInKeysFile , ' noncetrialsperbyte ' , str ( int ( previousTotalDifficulty * 1000 ) ) )
BMConfigParser ( ) . set ( addressInKeysFile , ' payloadlengthextrabytes ' , str ( int ( previousSmallMessageDifficulty * 1000 ) ) )
except :
continue
BMConfigParser ( ) . set ( ' bitmessagesettings ' , ' maxdownloadrate ' , ' 0 ' )
BMConfigParser ( ) . set ( ' bitmessagesettings ' , ' maxuploadrate ' , ' 0 ' )
BMConfigParser ( ) . set ( ' bitmessagesettings ' , ' settingsversion ' , ' 10 ' )
BMConfigParser ( ) . save ( )
# sanity check
if BMConfigParser ( ) . getint ( ' bitmessagesettings ' , ' maxacceptablenoncetrialsperbyte ' ) == 0 :
BMConfigParser ( ) . set ( ' bitmessagesettings ' , ' maxacceptablenoncetrialsperbyte ' , str ( defaults . ridiculousDifficulty * defaults . networkDefaultProofOfWorkNonceTrialsPerByte ) )
if BMConfigParser ( ) . getint ( ' bitmessagesettings ' , ' maxacceptablepayloadlengthextrabytes ' ) == 0 :
BMConfigParser ( ) . set ( ' bitmessagesettings ' , ' maxacceptablepayloadlengthextrabytes ' , str ( defaults . ridiculousDifficulty * defaults . networkDefaultPayloadLengthExtraBytes ) )
logger . debug (
' Finished dropping and recreating the inventory table. ' )
# The format of data stored in the pubkeys table has changed. Let's
# clear it, and the pubkeys from inventory, so that they'll be re-downloaded.
# clear it, and the pubkeys from inventory, so that they'll
# be re-downloaded.
item = ''' SELECT value FROM settings WHERE key= ' version ' ; '''
parameters = ' '
self . cur . execute ( item , parameters )
currentVersion = int ( self . cur . fetchall ( ) [ 0 ] [ 0 ] )
if currentVersion == 7 :
logger . debug ( ' In messages.dat database, clearing pubkeys table because the data format has been updated. ' )
logger . debug (
' In messages.dat database, clearing pubkeys table '
' because the data format has been updated. ' )
self . cur . execute (
''' delete from inventory where objecttype = 1; ''' )
self . cur . execute (
''' delete from pubkeys; ''' )
# Any sending messages for which we *thought* that we had the pubkey must
# be rechecked.
# Any sending messages for which we *thought* that we had
# the pubkey must be rechecked.
self . cur . execute (
''' UPDATE sent SET status= ' msgqueued ' WHERE status= ' doingmsgpow ' or status= ' badkey ' ; ''' )
query = ''' update settings set value=? WHERE key= ' version ' ; '''
@ -342,14 +274,17 @@ class sqlThread(threading.Thread):
self . cur . execute ( query , parameters )
logger . debug ( ' Finished clearing currently held pubkeys. ' )
# Add a new column to the inbox table to store the hash of the message signature.
# We'll use this as temporary message UUID in order to detect duplicates.
# Add a new column to the inbox table to store the hash of
# the message signature. We'll use this as temporary message UUID
# in order to detect duplicates.
item = ''' SELECT value FROM settings WHERE key= ' version ' ; '''
parameters = ' '
self . cur . execute ( item , parameters )
currentVersion = int ( self . cur . fetchall ( ) [ 0 ] [ 0 ] )
if currentVersion == 8 :
logger . debug ( ' In messages.dat database, adding sighash field to the inbox table. ' )
logger . debug (
' In messages.dat database, adding sighash field to '
' the inbox table. ' )
item = ''' ALTER TABLE inbox ADD sighash blob DEFAULT ' ' '''
parameters = ' '
self . cur . execute ( item , parameters )
@ -357,17 +292,18 @@ class sqlThread(threading.Thread):
parameters = ( 9 , )
self . cur . execute ( item , parameters )
# TTL is now user-specifiable. Let's add an option to save whatever the user selects.
if not BMConfigParser ( ) . has_option ( ' bitmessagesettings ' , ' ttl ' ) :
BMConfigParser ( ) . set ( ' bitmessagesettings ' , ' ttl ' , ' 367200 ' )
# We'll also need a `sleeptill` field and a `ttl` field. Also we can combine
# the pubkeyretrynumber and msgretrynumber into one.
# We'll also need a `sleeptill` field and a `ttl` field. Also we
# can combine the pubkeyretrynumber and msgretrynumber into one.
item = ''' SELECT value FROM settings WHERE key= ' version ' ; '''
parameters = ' '
self . cur . execute ( item , parameters )
currentVersion = int ( self . cur . fetchall ( ) [ 0 ] [ 0 ] )
if currentVersion == 9 :
logger . info ( ' In messages.dat database, making TTL-related changes: combining the pubkeyretrynumber and msgretrynumber fields into the retrynumber field and adding the sleeptill and ttl fields... ' )
logger . info (
' In messages.dat database, making TTL-related changes: '
' combining the pubkeyretrynumber and msgretrynumber '
' fields into the retrynumber field and adding the '
' sleeptill and ttl fields... ' )
self . cur . execute (
''' CREATE TEMPORARY TABLE sent_backup (msgid blob, toaddress text, toripe blob, fromaddress text, subject text, message text, ackdata blob, lastactiontime integer, status text, retrynumber integer, folder text, encodingtype int) ''' )
self . cur . execute (
@ -406,30 +342,9 @@ class sqlThread(threading.Thread):
logger . debug ( ' In messages.dat database, done adding address field to the pubkeys table and removing the hash field. ' )
self . cur . execute ( ''' update settings set value=10 WHERE key= ' version ' ; ''' )
if not BMConfigParser ( ) . has_option ( ' bitmessagesettings ' , ' onionhostname ' ) :
BMConfigParser ( ) . set ( ' bitmessagesettings ' , ' onionhostname ' , ' ' )
if not BMConfigParser ( ) . has_option ( ' bitmessagesettings ' , ' onionport ' ) :
BMConfigParser ( ) . set ( ' bitmessagesettings ' , ' onionport ' , ' 8444 ' )
if not BMConfigParser ( ) . has_option ( ' bitmessagesettings ' , ' onionbindip ' ) :
BMConfigParser ( ) . set ( ' bitmessagesettings ' , ' onionbindip ' , ' 127.0.0.1 ' )
if not BMConfigParser ( ) . has_option ( ' bitmessagesettings ' , ' smtpdeliver ' ) :
BMConfigParser ( ) . set ( ' bitmessagesettings ' , ' smtpdeliver ' , ' ' )
if not BMConfigParser ( ) . has_option ( ' bitmessagesettings ' , ' hidetrayconnectionnotifications ' ) :
BMConfigParser ( ) . set ( ' bitmessagesettings ' , ' hidetrayconnectionnotifications ' , ' false ' )
if BMConfigParser ( ) . has_option ( ' bitmessagesettings ' , ' maxoutboundconnections ' ) :
try :
if BMConfigParser ( ) . getint ( ' bitmessagesettings ' , ' maxoutboundconnections ' ) < 1 : raise ValueError
except ValueError as err :
BMConfigParser ( ) . remove_option ( ' bitmessagesettings ' , ' maxoutboundconnections ' )
logger . error ( ' Your maximum outbound connections must be a number. ' )
if not BMConfigParser ( ) . has_option ( ' bitmessagesettings ' , ' maxoutboundconnections ' ) :
logger . info ( ' Setting maximum outbound connections to 8. ' )
BMConfigParser ( ) . set ( ' bitmessagesettings ' , ' maxoutboundconnections ' , ' 8 ' )
BMConfigParser ( ) . save ( )
# Are you hoping to add a new option to the keys.dat file of existing
# Bitmessage users or modify the SQLite database? Add it right above this line!
# Bitmessage users or modify the SQLite database? Add it right
# above this line!
try :
testpayload = ' \x00 \x00 '