Merge branch '1174' into upstream-v0.6

This commit is contained in:
Peter Šurda 2018-03-22 08:31:25 +01:00
commit 5721ca53fd
Signed by: PeterSurda
GPG Key ID: 0C5F50C0B5F37D87
11 changed files with 48 additions and 22 deletions

View File

@ -15,7 +15,7 @@ import random
import state import state
import string import string
import tr#anslate import tr#anslate
import helper_random
# This thread exists because SQLITE3 is so un-threadsafe that we must # This thread exists because SQLITE3 is so un-threadsafe that we must
# submit queries to it and it puts results back in a different queue. They # submit queries to it and it puts results back in a different queue. They
# won't let us just use locks. # won't let us just use locks.
@ -263,7 +263,7 @@ class sqlThread(threading.Thread):
if not BMConfigParser().has_option('bitmessagesettings', 'useidenticons'): if not BMConfigParser().has_option('bitmessagesettings', 'useidenticons'):
BMConfigParser().set('bitmessagesettings', 'useidenticons', 'True') BMConfigParser().set('bitmessagesettings', 'useidenticons', 'True')
if not BMConfigParser().has_option('bitmessagesettings', 'identiconsuffix'): # acts as a salt if not BMConfigParser().has_option('bitmessagesettings', 'identiconsuffix'): # acts as a salt
BMConfigParser().set('bitmessagesettings', 'identiconsuffix', ''.join(random.choice("123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz") for x in range(12))) # a twelve character pseudo-password to salt the identicons BMConfigParser().set('bitmessagesettings', 'identiconsuffix', ''.join(helper_random.randomchoice("123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz") for x in range(12)))# a twelve character pseudo-password to salt the identicons
#Add settings to support no longer resending messages after a certain period of time even if we never get an ack #Add settings to support no longer resending messages after a certain period of time even if we never get an ack
if BMConfigParser().getint('bitmessagesettings', 'settingsversion') == 7: if BMConfigParser().getint('bitmessagesettings', 'settingsversion') == 7:

View File

@ -14,6 +14,7 @@ from bmconfigparser import BMConfigParser
from debug import logger from debug import logger
import messagetypes import messagetypes
from tr import _translate from tr import _translate
import helper_random
BITMESSAGE_ENCODING_IGNORE = 0 BITMESSAGE_ENCODING_IGNORE = 0
BITMESSAGE_ENCODING_TRIVIAL = 1 BITMESSAGE_ENCODING_TRIVIAL = 1
@ -141,8 +142,8 @@ class MsgDecode(object):
if __name__ == '__main__': if __name__ == '__main__':
import random import random
messageData = { messageData = {
"subject": ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(40)), "subject": ''.join(helper_random.randomchoice(string.ascii_lowercase + string.digits) for _ in range(40)),
"body": ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(10000)) "body": ''.join(helper_random.randomchoice(string.ascii_lowercase + string.digits) for _ in range(10000))
} }
obj1 = MsgEncode(messageData, 1) obj1 = MsgEncode(messageData, 1)
obj2 = MsgEncode(messageData, 2) obj2 = MsgEncode(messageData, 2)

View File

@ -11,14 +11,20 @@ def randomBytes(n):
except NotImplementedError: except NotImplementedError:
return OpenSSL.rand(n) return OpenSSL.rand(n)
def randomshuffle(population): def randomshuffle(population):
"""Method randomShuffle. """Method randomShuffle.
shuffle the sequence x in place. shuffle the sequence x in place.
shuffles the elements in list in place, shuffles the elements in list in place,
so they are in a random order. so they are in a random order.
As Shuffle will alter data in-place,
so its input must be a mutable sequence.
In contrast, sample produces a new list
and its input can be much more varied
(tuple, string, xrange, bytearray, set, etc)
""" """
return random.shuffle(population) random.shuffle(population)
def randomsample(population, k): def randomsample(population, k):
@ -27,7 +33,8 @@ def randomsample(population, k):
return a k length list of unique elements return a k length list of unique elements
chosen from the population sequence. chosen from the population sequence.
Used for random sampling Used for random sampling
without replacement without replacement, its called
partial shuffle.
""" """
return random.sample(population, k) return random.sample(population, k)
@ -44,3 +51,13 @@ def randomrandrange(x, y=None):
return random.randrange(x) return random.randrange(x)
else: else:
return random.randrange(x, y) return random.randrange(x, y)
def randomchoice(population):
"""Method randomchoice.
Return a random element from the non-empty
sequence seq. If seq is empty, raises
IndexError.
"""
return random.choice(population)

View File

@ -12,6 +12,7 @@ from distutils.version import StrictVersion
from namecoin import ensureNamecoinOptions from namecoin import ensureNamecoinOptions
import paths import paths
import state import state
import helper_random
storeConfigFilesInSameDirectoryAsProgramByDefault = False # The user may de-select Portable Mode in the settings if they want the config files to stay in the application data folder. storeConfigFilesInSameDirectoryAsProgramByDefault = False # The user may de-select Portable Mode in the settings if they want the config files to stay in the application data folder.
@ -103,7 +104,7 @@ def loadConfig():
BMConfigParser().set('bitmessagesettings', 'dontconnect', 'true') BMConfigParser().set('bitmessagesettings', 'dontconnect', 'true')
BMConfigParser().set('bitmessagesettings', 'userlocale', 'system') BMConfigParser().set('bitmessagesettings', 'userlocale', 'system')
BMConfigParser().set('bitmessagesettings', 'useidenticons', 'True') BMConfigParser().set('bitmessagesettings', 'useidenticons', 'True')
BMConfigParser().set('bitmessagesettings', 'identiconsuffix', ''.join(random.choice("123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz") for x in range(12))) # a twelve character pseudo-password to salt the identicons BMConfigParser().set('bitmessagesettings', 'identiconsuffix', ''.join(helper_random.randomchoice("123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz") for x in range(12)))# a twelve character pseudo-password to salt the identicons
BMConfigParser().set('bitmessagesettings', 'replybelow', 'False') BMConfigParser().set('bitmessagesettings', 'replybelow', 'False')
BMConfigParser().set('bitmessagesettings', 'maxdownloadrate', '0') BMConfigParser().set('bitmessagesettings', 'maxdownloadrate', '0')
BMConfigParser().set('bitmessagesettings', 'maxuploadrate', '0') BMConfigParser().set('bitmessagesettings', 'maxuploadrate', '0')

View File

@ -56,6 +56,7 @@ from threading import current_thread
import warnings import warnings
import os import os
import helper_random
from errno import EALREADY, EINPROGRESS, EWOULDBLOCK, ECONNRESET, EINVAL, \ from errno import EALREADY, EINPROGRESS, EWOULDBLOCK, ECONNRESET, EINVAL, \
ENOTCONN, ESHUTDOWN, EISCONN, EBADF, ECONNABORTED, EPIPE, EAGAIN, \ ENOTCONN, ESHUTDOWN, EISCONN, EBADF, ECONNABORTED, EPIPE, EAGAIN, \
ECONNREFUSED, EHOSTUNREACH, ENETUNREACH, ENOTSOCK, EINTR, ETIMEDOUT, \ ECONNREFUSED, EHOSTUNREACH, ENETUNREACH, ENOTSOCK, EINTR, ETIMEDOUT, \
@ -230,13 +231,13 @@ def select_poller(timeout=0.0, map=None):
if err.args[0] in (WSAENOTSOCK, ): if err.args[0] in (WSAENOTSOCK, ):
return return
for fd in random.sample(r, len(r)): for fd in helper_random.randomsample(r, len(r)):
obj = map.get(fd) obj = map.get(fd)
if obj is None: if obj is None:
continue continue
read(obj) read(obj)
for fd in random.sample(w, len(w)): for fd in helper_random.randomsample(w, len(w)):
obj = map.get(fd) obj = map.get(fd)
if obj is None: if obj is None:
continue continue
@ -292,7 +293,7 @@ def poll_poller(timeout=0.0, map=None):
except socket.error as err: except socket.error as err:
if err.args[0] in (EBADF, WSAENOTSOCK, EINTR): if err.args[0] in (EBADF, WSAENOTSOCK, EINTR):
return return
for fd, flags in random.sample(r, len(r)): for fd, flags in helper_random.randomsample(r, len(r)):
obj = map.get(fd) obj = map.get(fd)
if obj is None: if obj is None:
continue continue
@ -349,7 +350,7 @@ def epoll_poller(timeout=0.0, map=None):
if err.args[0] != EINTR: if err.args[0] != EINTR:
raise raise
r = [] r = []
for fd, flags in random.sample(r, len(r)): for fd, flags in helper_random.randomsample(r, len(r)):
obj = map.get(fd) obj = map.get(fd)
if obj is None: if obj is None:
continue continue
@ -403,7 +404,7 @@ def kqueue_poller(timeout=0.0, map=None):
events = kqueue_poller.pollster.control(updates, selectables, timeout) events = kqueue_poller.pollster.control(updates, selectables, timeout)
if len(events) > 1: if len(events) > 1:
events = random.sample(events, len(events)) events = helper_random.randomsample(events, len(events))
for event in events: for event in events:
fd = event.ident fd = event.ident

View File

@ -23,6 +23,7 @@ from queues import objectProcessorQueue, portCheckerQueue, invQueue, addrQueue
import shared import shared
import state import state
import protocol import protocol
import helper_random
class BMProtoError(ProxyError): class BMProtoError(ProxyError):
errorCodes = ("Protocol error") errorCodes = ("Protocol error")
@ -278,7 +279,7 @@ class BMProto(AdvancedDispatcher, ObjectTracker):
if time.time() < self.skipUntil: if time.time() < self.skipUntil:
return True return True
#TODO make this more asynchronous #TODO make this more asynchronous
random.shuffle(items) helper_random.randomshuffle(items)
for i in map(str, items): for i in map(str, items):
if Dandelion().hasHash(i) and \ if Dandelion().hasHash(i) and \
self != Dandelion().objectChildStem(i): self != Dandelion().objectChildStem(i):

View File

@ -6,10 +6,11 @@ import knownnodes
import protocol import protocol
from queues import portCheckerQueue from queues import portCheckerQueue
import state import state
import helper_random
def getDiscoveredPeer(): def getDiscoveredPeer():
try: try:
peer = random.choice(state.discoveredPeers.keys()) peer = helper_random.randomchoice(state.discoveredPeers.keys())
except (IndexError, KeyError): except (IndexError, KeyError):
raise ValueError raise ValueError
try: try:
@ -29,11 +30,11 @@ def chooseConnection(stream):
except Queue.Empty: except Queue.Empty:
pass pass
# with a probability of 0.5, connect to a discovered peer # with a probability of 0.5, connect to a discovered peer
if random.choice((False, True)) and not haveOnion: if helper_random.randomchoice((False, True)) and not haveOnion:
# discovered peers are already filtered by allowed streams # discovered peers are already filtered by allowed streams
return getDiscoveredPeer() return getDiscoveredPeer()
for _ in range(50): for _ in range(50):
peer = random.choice(knownnodes.knownNodes[stream].keys()) peer = helper_random.randomchoice(knownnodes.knownNodes[stream].keys())
try: try:
rating = knownnodes.knownNodes[stream][peer]["rating"] rating = knownnodes.knownNodes[stream][peer]["rating"]
except TypeError: except TypeError:

View File

@ -16,6 +16,7 @@ import network.asyncore_pollchoose as asyncore
import protocol import protocol
from singleton import Singleton from singleton import Singleton
import state import state
import helper_random
@Singleton @Singleton
class BMConnectionPool(object): class BMConnectionPool(object):
@ -156,7 +157,7 @@ class BMConnectionPool(object):
if established < BMConfigParser().safeGetInt("bitmessagesettings", "maxoutboundconnections"): if established < BMConfigParser().safeGetInt("bitmessagesettings", "maxoutboundconnections"):
for i in range(state.maximumNumberOfHalfOpenConnections - pending): for i in range(state.maximumNumberOfHalfOpenConnections - pending):
try: try:
chosen = chooseConnection(random.choice(self.streams)) chosen = chooseConnection(helper_random.randomchoice(self.streams))
except ValueError: except ValueError:
continue continue
if chosen in self.outboundConnections: if chosen in self.outboundConnections:

View File

@ -10,6 +10,7 @@ from inventory import Inventory
from network.connectionpool import BMConnectionPool from network.connectionpool import BMConnectionPool
import protocol import protocol
from state import missingObjects from state import missingObjects
import helper_random
class DownloadThread(threading.Thread, StoppableThread): class DownloadThread(threading.Thread, StoppableThread):
minPending = 200 minPending = 200
@ -41,7 +42,7 @@ class DownloadThread(threading.Thread, StoppableThread):
requested = 0 requested = 0
# Choose downloading peers randomly # Choose downloading peers randomly
connections = [x for x in BMConnectionPool().inboundConnections.values() + BMConnectionPool().outboundConnections.values() if x.fullyEstablished] connections = [x for x in BMConnectionPool().inboundConnections.values() + BMConnectionPool().outboundConnections.values() if x.fullyEstablished]
random.shuffle(connections) helper_random.randomshuffle(connections)
try: try:
requestChunk = max(int(min(DownloadThread.maxRequestChunk, len(missingObjects)) / len(connections)), 1) requestChunk = max(int(min(DownloadThread.maxRequestChunk, len(missingObjects)) / len(connections)), 1)
except ZeroDivisionError: except ZeroDivisionError:

View File

@ -12,6 +12,7 @@ import traceback
from addresses import calculateInventoryHash from addresses import calculateInventoryHash
from debug import logger from debug import logger
from helper_random import randomBytes from helper_random import randomBytes
import helper_random
from inventory import Inventory from inventory import Inventory
import knownnodes import knownnodes
from network.advanceddispatcher import AdvancedDispatcher from network.advanceddispatcher import AdvancedDispatcher
@ -132,7 +133,7 @@ class TCPConnection(BMProto, TLSDispatcher):
if elemCount > maxAddrCount: if elemCount > maxAddrCount:
elemCount = maxAddrCount elemCount = maxAddrCount
# only if more recent than 3 hours # only if more recent than 3 hours
addrs[stream] = random.sample(filtered.items(), elemCount) addrs[stream] = helper_random.randomsample(filtered.items(), elemCount)
# sent 250 only if the remote isn't interested in it # sent 250 only if the remote isn't interested in it
if len(knownnodes.knownNodes[stream * 2]) > 0 and stream not in self.streams: if len(knownnodes.knownNodes[stream * 2]) > 0 and stream not in self.streams:
filtered = {k: v for k, v in knownnodes.knownNodes[stream*2].items() filtered = {k: v for k, v in knownnodes.knownNodes[stream*2].items()
@ -140,14 +141,14 @@ class TCPConnection(BMProto, TLSDispatcher):
elemCount = len(filtered) elemCount = len(filtered)
if elemCount > maxAddrCount / 2: if elemCount > maxAddrCount / 2:
elemCount = int(maxAddrCount / 2) elemCount = int(maxAddrCount / 2)
addrs[stream * 2] = random.sample(filtered.items(), elemCount) addrs[stream * 2] = helper_random.randomsample(filtered.items(), elemCount)
if len(knownnodes.knownNodes[(stream * 2) + 1]) > 0 and stream not in self.streams: if len(knownnodes.knownNodes[(stream * 2) + 1]) > 0 and stream not in self.streams:
filtered = {k: v for k, v in knownnodes.knownNodes[stream*2+1].items() filtered = {k: v for k, v in knownnodes.knownNodes[stream*2+1].items()
if v["lastseen"] > (int(time.time()) - shared.maximumAgeOfNodesThatIAdvertiseToOthers)} if v["lastseen"] > (int(time.time()) - shared.maximumAgeOfNodesThatIAdvertiseToOthers)}
elemCount = len(filtered) elemCount = len(filtered)
if elemCount > maxAddrCount / 2: if elemCount > maxAddrCount / 2:
elemCount = int(maxAddrCount / 2) elemCount = int(maxAddrCount / 2)
addrs[stream * 2 + 1] = random.sample(filtered.items(), elemCount) addrs[stream * 2 + 1] = helper_random.randomsample(filtered.items(), elemCount)
for substream in addrs.keys(): for substream in addrs.keys():
for peer, params in addrs[substream]: for peer, params in addrs[substream]:
templist.append((substream, peer, params["lastseen"])) templist.append((substream, peer, params["lastseen"]))

View File

@ -1,6 +1,7 @@
import random import random
from threading import RLock from threading import RLock
from time import time from time import time
import helper_random
class RandomTrackingDict(object): class RandomTrackingDict(object):
maxPending = 10 maxPending = 10
@ -82,7 +83,7 @@ class RandomTrackingDict(object):
available = self.len - self.pendingLen available = self.len - self.pendingLen
if count > available: if count > available:
count = available count = available
randomIndex = random.sample(range(self.len - self.pendingLen), count) randomIndex = helper_random.randomsample(range(self.len - self.pendingLen), count)
retval = [self.indexDict[i] for i in randomIndex] retval = [self.indexDict[i] for i in randomIndex]
for i in sorted(randomIndex, reverse=True): for i in sorted(randomIndex, reverse=True):