From c99c3fc78230d298ad78516add14c3ca3bc57da8 Mon Sep 17 00:00:00 2001 From: coffeedogs Date: Wed, 2 May 2018 16:29:55 +0100 Subject: [PATCH 1/5] Fixed: Codacy issues in src/class_*.py --- PULL_REQUEST_TEMPLATE.md | 3 +- src/class_addressGenerator.py | 2 +- src/class_objectProcessor.py | 150 +++++++++++++----------------- src/class_objectProcessorQueue.py | 2 +- src/class_singleCleaner.py | 40 ++++---- src/class_singleWorker.py | 43 ++++----- src/class_smtpDeliver.py | 2 +- src/class_smtpServer.py | 19 +--- src/class_sqlThread.py | 60 +++++------- 9 files changed, 137 insertions(+), 184 deletions(-) diff --git a/PULL_REQUEST_TEMPLATE.md b/PULL_REQUEST_TEMPLATE.md index 91c3bc96..cde23f78 100644 --- a/PULL_REQUEST_TEMPLATE.md +++ b/PULL_REQUEST_TEMPLATE.md @@ -5,10 +5,11 @@ - make the pull request against the ["v0.6" branch](https://github.com/Bitmessage/PyBitmessage/tree/v0.6) - it should be possible to do a fast-forward merge of the pull requests - PGP-sign the commits included in the pull request +- try to use a good editor that removes trailing whitespace, highlights potential python issues and uses unix line endings - You can get paid for merged commits if you register at [Tip4Commit](https://tip4commit.com/github/Bitmessage/PyBitmessage) If for some reason you don't want to use github, you can submit the patch using Bitmessage to the "bitmessage" chan, or to one of the developers. ## Translations For helping with translations, please use [Transifex](https://www.transifex.com/bitmessage-project/pybitmessage/). There is no need to submit pull requests for translations. -For translating technical terms it is recommended to consult the [Microsoft Language Portal](https://www.microsoft.com/Language/en-US/Default.aspx). \ No newline at end of file +For translating technical terms it is recommended to consult the [Microsoft Language Portal](https://www.microsoft.com/Language/en-US/Default.aspx). diff --git a/src/class_addressGenerator.py b/src/class_addressGenerator.py index 2b99e39d..a5750813 100644 --- a/src/class_addressGenerator.py +++ b/src/class_addressGenerator.py @@ -230,7 +230,7 @@ class addressGenerator(threading.Thread, StoppableThread): # need it if we end up passing the info to the API. listOfNewAddressesToSendOutThroughTheAPI = [] - for i in range(numberOfAddressesToMake): + for _ in range(numberOfAddressesToMake): # This next section is a little bit strange. We're # going to generate keys over and over until we find # one that has a RIPEMD hash that starts with either diff --git a/src/class_objectProcessor.py b/src/class_objectProcessor.py index 181ce30e..dd88183b 100644 --- a/src/class_objectProcessor.py +++ b/src/class_objectProcessor.py @@ -3,19 +3,14 @@ import threading import shared import hashlib import random -from struct import unpack, pack -import sys import string -from subprocess import call # used when the API must execute an outside program +from subprocess import call # nosec import traceback from binascii import hexlify -from pyelliptic.openssl import OpenSSL import highlevelcrypto from addresses import * from bmconfigparser import BMConfigParser -import helper_generic -from helper_generic import addDataPadding import helper_bitcoin import helper_inbox import helper_msgcoding @@ -29,20 +24,21 @@ import tr from debug import logger import l10n + class objectProcessor(threading.Thread): """ The objectProcessor thread, of which there is only one, receives network objects (msg, broadcast, pubkey, getpubkey) from the receiveDataThreads. """ def __init__(self): - threading.Thread.__init__(self, name="objectProcessor") """ It may be the case that the last time Bitmessage was running, the user closed it before it finished processing everything in the objectProcessorQueue. Assuming that Bitmessage wasn't closed forcefully, - it should have saved the data in the queue into the objectprocessorqueue + it should have saved the data in the queue into the objectprocessorqueue table. Let's pull it out. """ + threading.Thread.__init__(self, name="objectProcessor") queryreturn = sqlQuery( '''SELECT objecttype, data FROM objectprocessorqueue''') for row in queryreturn: @@ -107,13 +103,12 @@ class objectProcessor(threading.Thread): del shared.ackdataForWhichImWatching[data[readPosition:]] sqlExecute('UPDATE sent SET status=?, lastactiontime=? WHERE ackdata=?', 'ackreceived', - int(time.time()), + int(time.time()), data[readPosition:]) queues.UISignalQueue.put(('updateSentItemStatusByAckdata', (data[readPosition:], tr._translate("MainWindow",'Acknowledgement of the message received %1').arg(l10n.formatTimestamp())))) else: logger.debug('This object is not an acknowledgement bound for me.') - def processgetpubkey(self, data): if len(data) > 200: logger.info('getpubkey is abnormally long. Sanity check failed. Ignoring object.') @@ -173,9 +168,9 @@ class objectProcessor(threading.Thread): except: lastPubkeySendTime = 0 if lastPubkeySendTime > time.time() - 2419200: # If the last time we sent our pubkey was more recent than 28 days ago... - logger.info('Found getpubkey-requested-item in my list of EC hashes BUT we already sent it recently. Ignoring request. The lastPubkeySendTime is: %s' % lastPubkeySendTime) + logger.info('Found getpubkey-requested-item in my list of EC hashes BUT we already sent it recently. Ignoring request. The lastPubkeySendTime is: %s' % lastPubkeySendTime) return - logger.info('Found getpubkey-requested-hash in my list of EC hashes. Telling Worker thread to do the POW for a pubkey message and send it out.') + logger.info('Found getpubkey-requested-hash in my list of EC hashes. Telling Worker thread to do the POW for a pubkey message and send it out.') if requestedAddressVersionNumber == 2: queues.workerQueue.put(( 'doPOWForMyV2Pubkey', requestedHash)) @@ -191,7 +186,6 @@ class objectProcessor(threading.Thread): shared.numberOfPubkeysProcessed += 1 queues.UISignalQueue.put(( 'updateNumberOfPubkeysProcessed', 'no data')) - embeddedTime, = unpack('>Q', data[8:16]) readPosition = 20 # bypass the nonce, time, and object type addressVersion, varintLength = decodeVarint( data[readPosition:readPosition + 10]) @@ -209,7 +203,6 @@ class objectProcessor(threading.Thread): if len(data) < 146: # sanity check. This is the minimum possible length. logger.debug('(within processpubkey) payloadLength less than 146. Sanity check failed.') return - bitfieldBehaviors = data[readPosition:readPosition + 4] readPosition += 4 publicSigningKey = data[readPosition:readPosition + 64] # Is it possible for a public key to be invalid such that trying to @@ -229,21 +222,19 @@ class objectProcessor(threading.Thread): ripeHasher.update(sha.digest()) ripe = ripeHasher.digest() - logger.debug('within recpubkey, addressVersion: %s, streamNumber: %s \n\ ripe %s\n\ publicSigningKey in hex: %s\n\ - publicEncryptionKey in hex: %s' % (addressVersion, - streamNumber, + publicEncryptionKey in hex: %s' % (addressVersion, + streamNumber, hexlify(ripe), hexlify(publicSigningKey), hexlify(publicEncryptionKey) ) ) - address = encodeAddress(addressVersion, streamNumber, ripe) - + queryreturn = sqlQuery( '''SELECT usedpersonally FROM pubkeys WHERE address=? AND usedpersonally='yes' ''', address) if queryreturn != []: # if this pubkey is already in our database and if we have used it personally: @@ -258,16 +249,15 @@ class objectProcessor(threading.Thread): if len(data) < 170: # sanity check. logger.warning('(within processpubkey) payloadLength less than 170. Sanity check failed.') return - bitfieldBehaviors = data[readPosition:readPosition + 4] readPosition += 4 publicSigningKey = '\x04' + data[readPosition:readPosition + 64] readPosition += 64 publicEncryptionKey = '\x04' + data[readPosition:readPosition + 64] readPosition += 64 - specifiedNonceTrialsPerByte, specifiedNonceTrialsPerByteLength = decodeVarint( + _, specifiedNonceTrialsPerByteLength = decodeVarint( data[readPosition:readPosition + 10]) readPosition += specifiedNonceTrialsPerByteLength - specifiedPayloadLengthExtraBytes, specifiedPayloadLengthExtraBytesLength = decodeVarint( + _, specifiedPayloadLengthExtraBytesLength = decodeVarint( data[readPosition:readPosition + 10]) readPosition += specifiedPayloadLengthExtraBytesLength endOfSignedDataPosition = readPosition @@ -287,13 +277,12 @@ class objectProcessor(threading.Thread): ripeHasher = hashlib.new('ripemd160') ripeHasher.update(sha.digest()) ripe = ripeHasher.digest() - logger.debug('within recpubkey, addressVersion: %s, streamNumber: %s \n\ ripe %s\n\ publicSigningKey in hex: %s\n\ - publicEncryptionKey in hex: %s' % (addressVersion, - streamNumber, + publicEncryptionKey in hex: %s' % (addressVersion, + streamNumber, hexlify(ripe), hexlify(publicSigningKey), hexlify(publicEncryptionKey) @@ -320,9 +309,9 @@ class objectProcessor(threading.Thread): if tag not in state.neededPubkeys: logger.info('We don\'t need this v4 pubkey. We didn\'t ask for it.') return - + # Let us try to decrypt the pubkey - toAddress, cryptorObject = state.neededPubkeys[tag] + toAddress, _ = state.neededPubkeys[tag] if shared.decryptAndCheckPubkeyPayload(data, toAddress) == 'successful': # At this point we know that we have been waiting on this pubkey. # This function will command the workerThread to start work on @@ -343,10 +332,10 @@ class objectProcessor(threading.Thread): readPosition = 20 # bypass the nonce, time, and object type msgVersion, msgVersionLength = decodeVarint(data[readPosition:readPosition + 9]) if msgVersion != 1: - logger.info('Cannot understand message versions other than one. Ignoring message.') + logger.info('Cannot understand message versions other than one. Ignoring message.') return readPosition += msgVersionLength - + streamNumberAsClaimedByMsg, streamNumberAsClaimedByMsgLength = decodeVarint( data[readPosition:readPosition + 9]) readPosition += streamNumberAsClaimedByMsgLength @@ -355,7 +344,7 @@ class objectProcessor(threading.Thread): # This is not an acknowledgement bound for me. See if it is a message # bound for me by trying to decrypt it with my private keys. - + for key, cryptorObject in sorted(shared.myECCryptorObjects.items(), key=lambda x: random.random()): try: if initialDecryptionSuccessful: # continue decryption attempts to avoid timing attacks @@ -365,11 +354,11 @@ class objectProcessor(threading.Thread): toRipe = key # This is the RIPE hash of my pubkeys. We need this below to compare to the destination_ripe included in the encrypted data. initialDecryptionSuccessful = True logger.info('EC decryption successful using key associated with ripe hash: %s.' % hexlify(key)) - except Exception as err: + except Exception: pass if not initialDecryptionSuccessful: # This is not a message bound for me. - logger.info('Length of time program spent failing to decrypt this message: %s seconds.' % (time.time() - messageProcessingStartTime,)) + logger.info('Length of time program spent failing to decrypt this message: %s seconds.' % (time.time() - messageProcessingStartTime,)) return # This is a message bound for me. @@ -380,10 +369,10 @@ class objectProcessor(threading.Thread): decryptedData[readPosition:readPosition + 10]) readPosition += sendersAddressVersionNumberLength if sendersAddressVersionNumber == 0: - logger.info('Cannot understand sendersAddressVersionNumber = 0. Ignoring message.') + logger.info('Cannot understand sendersAddressVersionNumber = 0. Ignoring message.') return if sendersAddressVersionNumber > 4: - logger.info('Sender\'s address version number %s not yet supported. Ignoring message.' % sendersAddressVersionNumber) + logger.info('Sender\'s address version number %s not yet supported. Ignoring message.' % sendersAddressVersionNumber) return if len(decryptedData) < 170: logger.info('Length of the unencrypted data is unreasonably short. Sanity check failed. Ignoring message.') @@ -394,7 +383,6 @@ class objectProcessor(threading.Thread): logger.info('sender\'s stream number is 0. Ignoring message.') return readPosition += sendersStreamNumberLength - behaviorBitfield = decryptedData[readPosition:readPosition + 4] readPosition += 4 pubSigningKey = '\x04' + decryptedData[ readPosition:readPosition + 64] @@ -441,7 +429,7 @@ class objectProcessor(threading.Thread): signature = decryptedData[ readPosition:readPosition + signatureLength] signedData = data[8:20] + encodeVarint(1) + encodeVarint(streamNumberAsClaimedByMsg) + decryptedData[:positionOfBottomOfAckData] - + if not highlevelcrypto.verify(signedData, signature, hexlify(pubSigningKey)): logger.debug('ECDSA verify failed') return @@ -458,7 +446,7 @@ class objectProcessor(threading.Thread): ripe.update(sha.digest()) fromAddress = encodeAddress( sendersAddressVersionNumber, sendersStreamNumber, ripe.digest()) - + # Let's store the public key in case we want to reply to this # person. sqlExecute( @@ -468,12 +456,12 @@ class objectProcessor(threading.Thread): decryptedData[:endOfThePublicKeyPosition], int(time.time()), 'yes') - + # Check to see whether we happen to be awaiting this # pubkey in order to send a message. If we are, it will do the POW # and send it. self.possibleNewPubkey(fromAddress) - + # If this message is bound for one of my version 3 addresses (or # higher), then we must check to make sure it meets our demanded # proof of work requirement. If this is bound for one of my chan @@ -539,7 +527,7 @@ class objectProcessor(threading.Thread): except: apiNotifyPath = '' if apiNotifyPath != '': - call([apiNotifyPath, "newMessage"]) + call([apiNotifyPath, "newMessage"]) # nosec # Let us now check and see whether our receiving address is # behaving as a mailing list @@ -565,22 +553,22 @@ class objectProcessor(threading.Thread): # We really should have a discussion about how to # set the TTL for mailing list broadcasts. This is obviously - # hard-coded. - TTL = 2*7*24*60*60 # 2 weeks - t = ('', - toAddress, - ripe, - fromAddress, - subject, - message, - ackdata, + # hard-coded. + TTL = 2*7*24*60*60 # 2 weeks + t = ('', + toAddress, + ripe, + fromAddress, + subject, + message, + ackdata, int(time.time()), # sentTime (this doesn't change) int(time.time()), # lastActionTime - 0, - 'broadcastqueued', - 0, - 'sent', - messageEncodingType, + 0, + 'broadcastqueued', + 0, + 'sent', + messageEncodingType, TTL) helper_sent.insert(t) @@ -601,15 +589,14 @@ class objectProcessor(threading.Thread): ) - messageProcessingStartTime shared.successfullyDecryptMessageTimings.append( timeRequiredToAttemptToDecryptMessage) - sum = 0 + timing_sum = 0 for item in shared.successfullyDecryptMessageTimings: - sum += item + timing_sum += item logger.debug('Time to decrypt this message successfully: %s\n\ Average time for all message decryption successes since startup: %s.' % - (timeRequiredToAttemptToDecryptMessage, sum / len(shared.successfullyDecryptMessageTimings)) + (timeRequiredToAttemptToDecryptMessage, timing_sum / len(shared.successfullyDecryptMessageTimings)) ) - def processbroadcast(self, data): messageProcessingStartTime = time.time() shared.numberOfBroadcastsProcessed += 1 @@ -621,17 +608,15 @@ class objectProcessor(threading.Thread): data[readPosition:readPosition + 9]) readPosition += broadcastVersionLength if broadcastVersion < 4 or broadcastVersion > 5: - logger.info('Cannot decode incoming broadcast versions less than 4 or higher than 5. Assuming the sender isn\'t being silly, you should upgrade Bitmessage because this message shall be ignored.') + logger.info('Cannot decode incoming broadcast versions less than 4 or higher than 5. Assuming the sender isn\'t being silly, you should upgrade Bitmessage because this message shall be ignored.') return cleartextStreamNumber, cleartextStreamNumberLength = decodeVarint( data[readPosition:readPosition + 10]) readPosition += cleartextStreamNumberLength if broadcastVersion == 4: - """ - v4 broadcasts are encrypted the same way the msgs are encrypted. To see if we are interested in a - v4 broadcast, we try to decrypt it. This was replaced with v5 broadcasts which include a tag which - we check instead, just like we do with v4 pubkeys. - """ + # v4 broadcasts are encrypted the same way the msgs are encrypted. To see if we are interested in a + # v4 broadcast, we try to decrypt it. This was replaced with v5 broadcasts which include a tag which + # we check instead, just like we do with v4 pubkeys. signedData = data[8:readPosition] initialDecryptionSuccessful = False for key, cryptorObject in sorted(shared.MyECSubscriptionCryptorObjects.items(), key=lambda x: random.random()): @@ -643,7 +628,7 @@ class objectProcessor(threading.Thread): toRipe = key # This is the RIPE hash of the sender's pubkey. We need this below to compare to the RIPE hash of the sender's address to verify that it was encrypted by with their key rather than some other key. initialDecryptionSuccessful = True logger.info('EC decryption successful using key associated with ripe hash: %s' % hexlify(key)) - except Exception as err: + except Exception: pass # print 'cryptorObject.decrypt Exception:', err if not initialDecryptionSuccessful: @@ -654,7 +639,7 @@ class objectProcessor(threading.Thread): embeddedTag = data[readPosition:readPosition+32] readPosition += 32 if embeddedTag not in shared.MyECSubscriptionCryptorObjects: - logger.debug('We\'re not interested in this broadcast.') + logger.debug('We\'re not interested in this broadcast.') return # We are interested in this broadcast because of its tag. signedData = data[8:readPosition] # We're going to add some more data which is signed further down. @@ -662,8 +647,8 @@ class objectProcessor(threading.Thread): try: decryptedData = cryptorObject.decrypt(data[readPosition:]) logger.debug('EC decryption successful') - except Exception as err: - logger.debug('Broadcast version %s decryption Unsuccessful.' % broadcastVersion) + except Exception: + logger.debug('Broadcast version %s decryption Unsuccessful.' % broadcastVersion) return # At this point this is a broadcast I have decrypted and am # interested in. @@ -676,16 +661,15 @@ class objectProcessor(threading.Thread): return elif broadcastVersion == 5: if sendersAddressVersion < 4: - logger.info('Cannot decode senderAddressVersion less than 4 for broadcast version number 5. Assuming the sender isn\'t being silly, you should upgrade Bitmessage because this message shall be ignored.') + logger.info('Cannot decode senderAddressVersion less than 4 for broadcast version number 5. Assuming the sender isn\'t being silly, you should upgrade Bitmessage because this message shall be ignored.') return readPosition += sendersAddressVersionLength sendersStream, sendersStreamLength = decodeVarint( decryptedData[readPosition:readPosition + 9]) if sendersStream != cleartextStreamNumber: - logger.info('The stream number outside of the encryption on which the POW was completed doesn\'t match the stream number inside the encryption. Ignoring broadcast.') + logger.info('The stream number outside of the encryption on which the POW was completed doesn\'t match the stream number inside the encryption. Ignoring broadcast.') return readPosition += sendersStreamLength - behaviorBitfield = decryptedData[readPosition:readPosition + 4] readPosition += 4 sendersPubSigningKey = '\x04' + \ decryptedData[readPosition:readPosition + 64] @@ -712,13 +696,13 @@ class objectProcessor(threading.Thread): if broadcastVersion == 4: if toRipe != calculatedRipe: - logger.info('The encryption key used to encrypt this message doesn\'t match the keys inbedded in the message itself. Ignoring message.') + logger.info('The encryption key used to encrypt this message doesn\'t match the keys inbedded in the message itself. Ignoring message.') return elif broadcastVersion == 5: calculatedTag = hashlib.sha512(hashlib.sha512(encodeVarint( sendersAddressVersion) + encodeVarint(sendersStream) + calculatedRipe).digest()).digest()[32:] if calculatedTag != embeddedTag: - logger.debug('The tag and encryption key used to encrypt this message doesn\'t match the keys inbedded in the message itself. Ignoring message.') + logger.debug('The tag and encryption key used to encrypt this message doesn\'t match the keys inbedded in the message itself. Ignoring message.') return messageEncodingType, messageEncodingTypeLength = decodeVarint( decryptedData[readPosition:readPosition + 9]) @@ -792,7 +776,7 @@ class objectProcessor(threading.Thread): except: apiNotifyPath = '' if apiNotifyPath != '': - call([apiNotifyPath, "newBroadcast"]) + call([apiNotifyPath, "newBroadcast"]) # nosec # Display timing data logger.info('Time spent processing this interesting broadcast: %s' % (time.time() - messageProcessingStartTime,)) @@ -804,10 +788,10 @@ class objectProcessor(threading.Thread): pubkey, msg, or broadcast message. It might be one that we have been waiting for. Let's check. """ - + # For address versions <= 3, we wait on a key with the correct address version, # stream number, and RIPE hash. - status, addressVersion, streamNumber, ripe = decodeAddress(address) + _, addressVersion, streamNumber, ripe = decodeAddress(address) if addressVersion <=3: if address in state.neededPubkeys: del state.neededPubkeys[address] @@ -840,7 +824,7 @@ class objectProcessor(threading.Thread): if len(ackData) < protocol.Header.size: logger.info('The length of ackData is unreasonably short. Not sending ackData.') return False - + magic,command,payloadLength,checksum = protocol.Header.unpack(ackData[:protocol.Header.size]) if magic != 0xE9BEB4D9: logger.info('Ackdata magic bytes were wrong. Not sending ackData.') @@ -849,13 +833,11 @@ class objectProcessor(threading.Thread): if len(payload) != payloadLength: logger.info('ackData payload length doesn\'t match the payload length specified in the header. Not sending ackdata.') return False - if payloadLength > 1600100: # ~1.6 MB which is the maximum possible size of an inv message. - """ - The largest message should be either an inv or a getdata message at 1.6 MB in size. - That doesn't mean that the object may be that big. The - shared.checkAndShareObjectWithPeers function will verify that it is no larger than - 2^18 bytes. - """ + if payloadLength > 1600100: # ~1.6 MB which is the maximum possible size of an inv message. + # The largest message should be either an inv or a getdata message at 1.6 MB in size. + # That doesn't mean that the object may be that big. The + # shared.checkAndShareObjectWithPeers function will verify that it is no larger than + # 2^18 bytes. return False if checksum != hashlib.sha512(payload).digest()[0:4]: # test the checksum in the message. logger.info('ackdata checksum wrong. Not sending ackdata.') diff --git a/src/class_objectProcessorQueue.py b/src/class_objectProcessorQueue.py index 6309e994..01854a74 100644 --- a/src/class_objectProcessorQueue.py +++ b/src/class_objectProcessorQueue.py @@ -20,7 +20,7 @@ class ObjectProcessorQueue(Queue.Queue): def get(self, block = True, timeout = None): try: item = Queue.Queue.get(self, block, timeout) - except Queue.Empty as e: + except Queue.Empty: raise Queue.Empty() with self.sizeLock: self.curSize -= len(item[1]) diff --git a/src/class_singleCleaner.py b/src/class_singleCleaner.py index b9f35620..5e8e809a 100644 --- a/src/class_singleCleaner.py +++ b/src/class_singleCleaner.py @@ -1,3 +1,22 @@ +""" +The singleCleaner class is a timer-driven thread that cleans data structures +to free memory, resends messages when a remote node doesn't respond, and +sends pong messages to keep connections alive if the network isn't busy. +It cleans these data structures in memory: +inventory (moves data to the on-disk sql database) +inventorySets (clears then reloads data out of sql database) + +It cleans these tables on the disk: +inventory (clears expired objects) +pubkeys (clears pubkeys older than 4 weeks old which we have not used personally) +knownNodes (clears addresses which have not been online for over 3 days) + +It resends messages when there has been no response: +resends getpubkey messages in 5 days (then 10 days, then 20 days, etc...) +resends msg messages in 5 days (then 10 days, then 20 days, etc...) + +""" + import gc import threading import shared @@ -15,25 +34,6 @@ import knownnodes import queues import state -""" -The singleCleaner class is a timer-driven thread that cleans data structures -to free memory, resends messages when a remote node doesn't respond, and -sends pong messages to keep connections alive if the network isn't busy. -It cleans these data structures in memory: -inventory (moves data to the on-disk sql database) -inventorySets (clears then reloads data out of sql database) - -It cleans these tables on the disk: -inventory (clears expired objects) -pubkeys (clears pubkeys older than 4 weeks old which we have not used personally) -knownNodes (clears addresses which have not been online for over 3 days) - -It resends messages when there has been no response: -resends getpubkey messages in 5 days (then 10 days, then 20 days, etc...) -resends msg messages in 5 days (then 10 days, then 20 days, etc...) - -""" - class singleCleaner(threading.Thread, StoppableThread): cycleLength = 300 @@ -61,7 +61,7 @@ class singleCleaner(threading.Thread, StoppableThread): 'updateStatusBar', 'Doing housekeeping (Flushing inventory in memory to disk...)')) Inventory().flush() queues.UISignalQueue.put(('updateStatusBar', '')) - + # If we are running as a daemon then we are going to fill up the UI # queue which will never be handled by a UI. We should clear it to # save memory. diff --git a/src/class_singleWorker.py b/src/class_singleWorker.py index c95d484a..e87cb95d 100644 --- a/src/class_singleWorker.py +++ b/src/class_singleWorker.py @@ -5,7 +5,7 @@ import threading import hashlib from struct import pack # used when the API must execute an outside program -from subprocess import call +from subprocess import call # nosec from binascii import hexlify, unhexlify import tr @@ -228,7 +228,7 @@ class singleWorker(threading.Thread, StoppableThread): # This function also broadcasts out the pubkey message # once it is done with the POW - def doPOWForMyV2Pubkey(self, hash): + def doPOWForMyV2Pubkey(self, adressHash): # Look up my stream number based on my address hash """configSections = shared.config.addresses() for addressInKeysFile in configSections: @@ -239,9 +239,9 @@ class singleWorker(threading.Thread, StoppableThread): if hash == hashFromThisParticularAddress: myAddress = addressInKeysFile break""" - myAddress = shared.myAddressesByHash[hash] + myAddress = shared.myAddressesByHash[adressHash] # status - _, addressVersionNumber, streamNumber, hash = decodeAddress(myAddress) + _, addressVersionNumber, streamNumber, adressHash = decodeAddress(myAddress) # 28 days from now plus or minus five minutes TTL = int(28 * 24 * 60 * 60 + helper_random.randomrandrange(-300, 300)) @@ -293,31 +293,31 @@ class singleWorker(threading.Thread, StoppableThread): # does the necessary POW and sends it out. If it *is* a chan then it # assembles the pubkey and stores is in the pubkey table so that we can # send messages to "ourselves". - def sendOutOrStoreMyV3Pubkey(self, hash): + def sendOutOrStoreMyV3Pubkey(self, adressHash): try: - myAddress = shared.myAddressesByHash[hash] + myAddress = shared.myAddressesByHash[adressHash] except: # The address has been deleted. return if BMConfigParser().safeGetBoolean(myAddress, 'chan'): logger.info('This is a chan address. Not sending pubkey.') return - status, addressVersionNumber, streamNumber, hash = decodeAddress( + _, addressVersionNumber, streamNumber, adressHash = decodeAddress( myAddress) # 28 days from now plus or minus five minutes TTL = int(28 * 24 * 60 * 60 + helper_random.randomrandrange(-300, 300)) embeddedTime = int(time.time() + TTL) + # signedTimeForProtocolV2 = embeddedTime - TTL - """ - According to the protocol specification, the expiresTime - along with the pubkey information is signed. But to be - backwards compatible during the upgrade period, we shall sign - not the expiresTime but rather the current time. There must be - precisely a 28 day difference between the two. After the upgrade - period we'll switch to signing the whole payload with the - expiresTime time. - """ + # According to the protocol specification, the expiresTime + # along with the pubkey information is signed. But to be + # backwards compatible during the upgrade period, we shall sign + # not the expiresTime but rather the current time. There must be + # precisely a 28 day difference between the two. After the upgrade + # period we'll switch to signing the whole payload with the + # expiresTime time. + payload = pack('>Q', (embeddedTime)) payload += '\x00\x00\x00\x01' # object type: pubkey payload += encodeVarint(addressVersionNumber) # Address version number @@ -379,7 +379,7 @@ class singleWorker(threading.Thread, StoppableThread): if shared.BMConfigParser().safeGetBoolean(myAddress, 'chan'): logger.info('This is a chan address. Not sending pubkey.') return - status, addressVersionNumber, streamNumber, hash = decodeAddress( + _, addressVersionNumber, streamNumber, addressHash = decodeAddress( myAddress) # 28 days from now plus or minus five minutes @@ -419,7 +419,7 @@ class singleWorker(threading.Thread, StoppableThread): # when they want to send a message. doubleHashOfAddressData = hashlib.sha512(hashlib.sha512( encodeVarint(addressVersionNumber) + - encodeVarint(streamNumber) + hash + encodeVarint(streamNumber) + addressHash ).digest()).digest() payload += doubleHashOfAddressData[32:] # the tag signature = highlevelcrypto.sign( @@ -462,6 +462,7 @@ class singleWorker(threading.Thread, StoppableThread): # Reset just in case sqlExecute( '''UPDATE sent SET status='broadcastqueued' ''' + '''WHERE status = 'doingbroadcastpow' ''') queryreturn = sqlQuery( '''SELECT fromaddress, subject, message, ''' @@ -848,7 +849,7 @@ class singleWorker(threading.Thread, StoppableThread): # to bypass the address version whose length is definitely 1 readPosition = 1 - streamNumber, streamNumberLength = decodeVarint( + _, streamNumberLength = decodeVarint( pubkeyPayload[readPosition:readPosition + 10]) readPosition += streamNumberLength behaviorBitfield = pubkeyPayload[readPosition:readPosition + 4] @@ -925,7 +926,7 @@ class singleWorker(threading.Thread, StoppableThread): # regardless of what they say is allowed in order # to get our message to propagate through the network. if requiredAverageProofOfWorkNonceTrialsPerByte < \ - defaults.networkDefaultProofOfWorkNonceTrialsPerByte: + defaults.networkDefaultProofOfWorkNonceTrialsPerByte: requiredAverageProofOfWorkNonceTrialsPerByte = \ defaults.networkDefaultProofOfWorkNonceTrialsPerByte if requiredPayloadLengthExtraBytes < \ @@ -1266,7 +1267,7 @@ class singleWorker(threading.Thread, StoppableThread): except: apiNotifyPath = '' if apiNotifyPath != '': - call([apiNotifyPath, "newMessage"]) + call([apiNotifyPath, "newMessage"]) # nosec def requestPubKey(self, toAddress): toStatus, addressVersionNumber, streamNumber, ripe = decodeAddress( diff --git a/src/class_smtpDeliver.py b/src/class_smtpDeliver.py index bb659ebe..f46a61b3 100644 --- a/src/class_smtpDeliver.py +++ b/src/class_smtpDeliver.py @@ -19,7 +19,7 @@ class smtpDeliver(threading.Thread, StoppableThread): def __init__(self, parent=None): threading.Thread.__init__(self, name="smtpDeliver") self.initStop() - + def stopThread(self): try: queues.UISignallerQueue.put(("stopThread", "data")) diff --git a/src/class_smtpServer.py b/src/class_smtpServer.py index b62a7130..216d35be 100644 --- a/src/class_smtpServer.py +++ b/src/class_smtpServer.py @@ -6,7 +6,6 @@ from email.header import decode_header import re import signal import smtpd -import socket import threading import time @@ -16,7 +15,6 @@ from debug import logger from helper_sql import sqlExecute from helper_ackPayload import genAckPayload from helper_threading import StoppableThread -from pyelliptic.openssl import OpenSSL import queues from version import softwareVersion @@ -68,7 +66,6 @@ class smtpServerPyBitmessage(smtpd.SMTPServer): status, addressVersionNumber, streamNumber, ripe = decodeAddress(toAddress) stealthLevel = BMConfigParser().safeGetInt('bitmessagesettings', 'ackstealthlevel') ackdata = genAckPayload(streamNumber, stealthLevel) - t = () sqlExecute( '''INSERT INTO sent VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)''', '', @@ -97,7 +94,7 @@ class smtpServerPyBitmessage(smtpd.SMTPServer): ret.append(unicode(h[0], h[1])) else: ret.append(h[0].decode("utf-8", errors='replace')) - + return ret @@ -151,7 +148,7 @@ class smtpServerPyBitmessage(smtpd.SMTPServer): raise Exception("Bad domain %s", domain) logger.debug("Sending %s to %s about %s", sender, rcpt, msg_subject) self.send(sender, rcpt, msg_subject, body) - logger.info("Relayed %s to %s", sender, rcpt) + logger.info("Relayed %s to %s", sender, rcpt) except Exception as err: logger.error( "Bad to %s: %s", to, repr(err)) continue @@ -162,21 +159,11 @@ class smtpServer(threading.Thread, StoppableThread): threading.Thread.__init__(self, name="smtpServerThread") self.initStop() self.server = smtpServerPyBitmessage(('127.0.0.1', LISTENPORT), None) - + def stopThread(self): super(smtpServer, self).stopThread() self.server.close() return - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) -# for ip in ('127.0.0.1', BMConfigParser().get('bitmessagesettings', 'onionbindip')): - for ip in ('127.0.0.1'): - try: - s.connect((ip, LISTENPORT)) - s.shutdown(socket.SHUT_RDWR) - s.close() - break - except: - pass def run(self): asyncore.loop(1) diff --git a/src/class_sqlThread.py b/src/class_sqlThread.py index a774a24b..45c4d8c0 100644 --- a/src/class_sqlThread.py +++ b/src/class_sqlThread.py @@ -11,9 +11,7 @@ import helper_sql from namecoin import ensureNamecoinOptions import paths import queues -import random import state -import string import tr#anslate import helper_random # This thread exists because SQLITE3 is so un-threadsafe that we must @@ -26,11 +24,11 @@ class sqlThread(threading.Thread): def __init__(self): threading.Thread.__init__(self, name="SQL") - def run(self): + def run(self): self.conn = sqlite3.connect(state.appdata + 'messages.dat') self.conn.text_factory = str self.cur = self.conn.cursor() - + self.cur.execute('PRAGMA secure_delete = true') try: @@ -178,24 +176,11 @@ class sqlThread(threading.Thread): self.cur.execute( '''update sent set status='broadcastqueued' where status='broadcastpending' ''') self.conn.commit() - + if not BMConfigParser().has_option('bitmessagesettings', 'sockslisten'): BMConfigParser().set('bitmessagesettings', 'sockslisten', 'false') - + ensureNamecoinOptions() - - """# Add a new column to the inventory table to store the first 20 bytes of encrypted messages to support Android app - item = '''SELECT value FROM settings WHERE key='version';''' - parameters = '' - self.cur.execute(item, parameters) - if int(self.cur.fetchall()[0][0]) == 1: - print 'upgrading database' - item = '''ALTER TABLE inventory ADD first20bytesofencryptedmessage blob DEFAULT '' ''' - parameters = '' - self.cur.execute(item, parameters) - item = '''update settings set value=? WHERE key='version';''' - parameters = (2,) - self.cur.execute(item, parameters)""" # Let's get rid of the first20bytesofencryptedmessage field in the inventory table. item = '''SELECT value FROM settings WHERE key='version';''' @@ -239,9 +224,6 @@ class sqlThread(threading.Thread): # Raise the default required difficulty from 1 to 2 # With the change to protocol v3, this is obsolete. if BMConfigParser().getint('bitmessagesettings', 'settingsversion') == 6: - """if int(shared.config.get('bitmessagesettings','defaultnoncetrialsperbyte')) == defaults.networkDefaultProofOfWorkNonceTrialsPerByte: - shared.config.set('bitmessagesettings','defaultnoncetrialsperbyte', str(defaults.networkDefaultProofOfWorkNonceTrialsPerByte * 2)) - """ BMConfigParser().set('bitmessagesettings', 'settingsversion', '7') # Add a new column to the pubkeys table to store the address version. @@ -259,7 +241,7 @@ class sqlThread(threading.Thread): item = '''update settings set value=? WHERE key='version';''' parameters = (5,) self.cur.execute(item, parameters) - + if not BMConfigParser().has_option('bitmessagesettings', 'useidenticons'): BMConfigParser().set('bitmessagesettings', 'useidenticons', 'True') if not BMConfigParser().has_option('bitmessagesettings', 'identiconsuffix'): # acts as a salt @@ -271,7 +253,7 @@ class sqlThread(threading.Thread): 'bitmessagesettings', 'stopresendingafterxdays', '') BMConfigParser().set( 'bitmessagesettings', 'stopresendingafterxmonths', '') - BMConfigParser().set('bitmessagesettings', 'settingsversion', '8') + BMConfigParser().set('bitmessagesettings', 'settingsversion', '8') # Add a new table: objectprocessorqueue with which to hold objects # that have yet to be processed if the user shuts down Bitmessage. @@ -286,7 +268,7 @@ class sqlThread(threading.Thread): item = '''update settings set value=? WHERE key='version';''' parameters = (6,) self.cur.execute(item, parameters) - + # changes related to protocol v3 # In table inventory and objectprocessorqueue, objecttype is now an integer (it was a human-friendly string previously) item = '''SELECT value FROM settings WHERE key='version';''' @@ -303,8 +285,8 @@ class sqlThread(threading.Thread): parameters = (7,) self.cur.execute(item, parameters) logger.debug('Finished dropping and recreating the inventory table.') - - # With the change to protocol version 3, reset the user-settable difficulties to 1 + + # With the change to protocol version 3, reset the user-settable difficulties to 1 if BMConfigParser().getint('bitmessagesettings', 'settingsversion') == 8: BMConfigParser().set('bitmessagesettings','defaultnoncetrialsperbyte', str(defaults.networkDefaultProofOfWorkNonceTrialsPerByte)) BMConfigParser().set('bitmessagesettings','defaultpayloadlengthextrabytes', str(defaults.networkDefaultPayloadLengthExtraBytes)) @@ -313,7 +295,7 @@ class sqlThread(threading.Thread): BMConfigParser().set('bitmessagesettings','maxacceptablenoncetrialsperbyte', str(previousTotalDifficulty * 1000)) BMConfigParser().set('bitmessagesettings','maxacceptablepayloadlengthextrabytes', str(previousSmallMessageDifficulty * 1000)) BMConfigParser().set('bitmessagesettings', 'settingsversion', '9') - + # Adjust the required POW values for each of this user's addresses to conform to protocol v3 norms. if BMConfigParser().getint('bitmessagesettings', 'settingsversion') == 9: for addressInKeysFile in BMConfigParser().addressses(): @@ -332,7 +314,7 @@ class sqlThread(threading.Thread): BMConfigParser().set('bitmessagesettings', 'maxuploadrate', '0') BMConfigParser().set('bitmessagesettings', 'settingsversion', '10') BMConfigParser().save() - + # sanity check if BMConfigParser().getint('bitmessagesettings', 'maxacceptablenoncetrialsperbyte') == 0: BMConfigParser().set('bitmessagesettings','maxacceptablenoncetrialsperbyte', str(defaults.ridiculousDifficulty * defaults.networkDefaultProofOfWorkNonceTrialsPerByte)) @@ -361,7 +343,7 @@ class sqlThread(threading.Thread): logger.debug('Finished clearing currently held pubkeys.') # Add a new column to the inbox table to store the hash of the message signature. - # We'll use this as temporary message UUID in order to detect duplicates. + # We'll use this as temporary message UUID in order to detect duplicates. item = '''SELECT value FROM settings WHERE key='version';''' parameters = '' self.cur.execute(item, parameters) @@ -374,11 +356,11 @@ class sqlThread(threading.Thread): item = '''update settings set value=? WHERE key='version';''' parameters = (9,) self.cur.execute(item, parameters) - - # TTL is now user-specifiable. Let's add an option to save whatever the user selects. + + # TTL is now user-specifiable. Let's add an option to save whatever the user selects. if not BMConfigParser().has_option('bitmessagesettings', 'ttl'): BMConfigParser().set('bitmessagesettings', 'ttl', '367200') - # We'll also need a `sleeptill` field and a `ttl` field. Also we can combine + # We'll also need a `sleeptill` field and a `ttl` field. Also we can combine # the pubkeyretrynumber and msgretrynumber into one. item = '''SELECT value FROM settings WHERE key='version';''' parameters = '' @@ -399,16 +381,16 @@ class sqlThread(threading.Thread): logger.info('In messages.dat database, finished making TTL-related changes.') logger.debug('In messages.dat database, adding address field to the pubkeys table.') # We're going to have to calculate the address for each row in the pubkeys - # table. Then we can take out the hash field. + # table. Then we can take out the hash field. self.cur.execute('''ALTER TABLE pubkeys ADD address text DEFAULT '' ''') self.cur.execute('''SELECT hash, addressversion FROM pubkeys''') queryResult = self.cur.fetchall() from addresses import encodeAddress for row in queryResult: - hash, addressVersion = row + addressHash, addressVersion = row address = encodeAddress(addressVersion, 1, hash) item = '''UPDATE pubkeys SET address=? WHERE hash=?;''' - parameters = (address, hash) + parameters = (address, addressHash) self.cur.execute(item, parameters) # Now we can remove the hash field from the pubkeys table. self.cur.execute( @@ -423,7 +405,7 @@ class sqlThread(threading.Thread): self.cur.execute( '''DROP TABLE pubkeys_backup''') logger.debug('In messages.dat database, done adding address field to the pubkeys table and removing the hash field.') self.cur.execute('''update settings set value=10 WHERE key='version';''') - + if not BMConfigParser().has_option('bitmessagesettings', 'onionhostname'): BMConfigParser().set('bitmessagesettings', 'onionhostname', '') if not BMConfigParser().has_option('bitmessagesettings', 'onionport'): @@ -445,10 +427,10 @@ class sqlThread(threading.Thread): BMConfigParser().set('bitmessagesettings', 'maxoutboundconnections', '8') BMConfigParser().save() - + # Are you hoping to add a new option to the keys.dat file of existing # Bitmessage users or modify the SQLite database? Add it right above this line! - + try: testpayload = '\x00\x00' t = ('1234', 1, testpayload, '12345678', 'no') From fe3a86d361546cceeb8c57e9516c49b5d48500a4 Mon Sep 17 00:00:00 2001 From: coffeedogs Date: Mon, 7 May 2018 11:50:51 +0100 Subject: [PATCH 2/5] Fixed: Addressed PR comments --- PULL_REQUEST_TEMPLATE.md | 3 +-- README.md | 1 + src/class_objectProcessorQueue.py | 5 +---- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/PULL_REQUEST_TEMPLATE.md b/PULL_REQUEST_TEMPLATE.md index cde23f78..055943f1 100644 --- a/PULL_REQUEST_TEMPLATE.md +++ b/PULL_REQUEST_TEMPLATE.md @@ -2,8 +2,6 @@ - try to explain what the code is about - try to follow [PEP0008](https://www.python.org/dev/peps/pep-0008/) -- make the pull request against the ["v0.6" branch](https://github.com/Bitmessage/PyBitmessage/tree/v0.6) -- it should be possible to do a fast-forward merge of the pull requests - PGP-sign the commits included in the pull request - try to use a good editor that removes trailing whitespace, highlights potential python issues and uses unix line endings - You can get paid for merged commits if you register at [Tip4Commit](https://tip4commit.com/github/Bitmessage/PyBitmessage) @@ -13,3 +11,4 @@ If for some reason you don't want to use github, you can submit the patch using For helping with translations, please use [Transifex](https://www.transifex.com/bitmessage-project/pybitmessage/). There is no need to submit pull requests for translations. For translating technical terms it is recommended to consult the [Microsoft Language Portal](https://www.microsoft.com/Language/en-US/Default.aspx). + diff --git a/README.md b/README.md index 06cba098..5dfe24df 100644 --- a/README.md +++ b/README.md @@ -29,3 +29,4 @@ References * [Installation](https://bitmessage.org/wiki/Compiling_instructions) * [Discuss on Reddit](https://www.reddit.com/r/bitmessage) * [Chat on Gitter](https://gitter.im/Bitmessage/PyBitmessage) + diff --git a/src/class_objectProcessorQueue.py b/src/class_objectProcessorQueue.py index 01854a74..b6628816 100644 --- a/src/class_objectProcessorQueue.py +++ b/src/class_objectProcessorQueue.py @@ -18,10 +18,7 @@ class ObjectProcessorQueue(Queue.Queue): Queue.Queue.put(self, item, block, timeout) def get(self, block = True, timeout = None): - try: - item = Queue.Queue.get(self, block, timeout) - except Queue.Empty: - raise Queue.Empty() + item = Queue.Queue.get(self, block, timeout) with self.sizeLock: self.curSize -= len(item[1]) return item From ed4a092c51fd4ae93d93336bb3df418a1208001a Mon Sep 17 00:00:00 2001 From: coffeedogs Date: Tue, 8 May 2018 12:13:49 +0100 Subject: [PATCH 3/5] Removed: # nosec directives until further security review. --- src/class_objectProcessor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/class_objectProcessor.py b/src/class_objectProcessor.py index dd88183b..f5d47b2b 100644 --- a/src/class_objectProcessor.py +++ b/src/class_objectProcessor.py @@ -527,7 +527,7 @@ class objectProcessor(threading.Thread): except: apiNotifyPath = '' if apiNotifyPath != '': - call([apiNotifyPath, "newMessage"]) # nosec + call([apiNotifyPath, "newMessage"]) # Let us now check and see whether our receiving address is # behaving as a mailing list @@ -776,7 +776,7 @@ class objectProcessor(threading.Thread): except: apiNotifyPath = '' if apiNotifyPath != '': - call([apiNotifyPath, "newBroadcast"]) # nosec + call([apiNotifyPath, "newBroadcast"]) # Display timing data logger.info('Time spent processing this interesting broadcast: %s' % (time.time() - messageProcessingStartTime,)) From bbf55a0571ddb03724e763b5714441b98be3053d Mon Sep 17 00:00:00 2001 From: coffeedogs Date: Tue, 8 May 2018 12:37:54 +0100 Subject: [PATCH 4/5] Fixed: Addressed PR comments --- PULL_REQUEST_TEMPLATE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/PULL_REQUEST_TEMPLATE.md b/PULL_REQUEST_TEMPLATE.md index 055943f1..a894f907 100644 --- a/PULL_REQUEST_TEMPLATE.md +++ b/PULL_REQUEST_TEMPLATE.md @@ -2,6 +2,7 @@ - try to explain what the code is about - try to follow [PEP0008](https://www.python.org/dev/peps/pep-0008/) +- make the pull request against the ["v0.6" branch](https://github.com/Bitmessage/PyBitmessage/tree/v0.6) - PGP-sign the commits included in the pull request - try to use a good editor that removes trailing whitespace, highlights potential python issues and uses unix line endings - You can get paid for merged commits if you register at [Tip4Commit](https://tip4commit.com/github/Bitmessage/PyBitmessage) From 8979a1bef5ecd9a7f4cd0ee2de95dcb728984436 Mon Sep 17 00:00:00 2001 From: coffeedogs Date: Tue, 8 May 2018 12:53:58 +0100 Subject: [PATCH 5/5] Fixed: Addressed PR comments --- src/class_singleWorker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/class_singleWorker.py b/src/class_singleWorker.py index e87cb95d..d51e124a 100644 --- a/src/class_singleWorker.py +++ b/src/class_singleWorker.py @@ -1267,7 +1267,7 @@ class singleWorker(threading.Thread, StoppableThread): except: apiNotifyPath = '' if apiNotifyPath != '': - call([apiNotifyPath, "newMessage"]) # nosec + call([apiNotifyPath, "newMessage"]) def requestPubKey(self, toAddress): toStatus, addressVersionNumber, streamNumber, ripe = decodeAddress(