From f67ffad69d7d4e97b7877904db50fa5163f8a1a0 Mon Sep 17 00:00:00 2001 From: navjot Date: Thu, 12 Nov 2020 22:39:14 +0530 Subject: [PATCH 1/2] written test case for randomtrackingdict module -removed logger from singleinstance module -written test case for randomtrackingdict module -Fixed CQ issues -removed logger from test_randomtrackingdict module -remove main method from randomtrackingdict.py and fixed travis-ci issue -replace print with test failure -fixed flake8 CQ for network.randomtrackingdict module --- src/network/randomtrackingdict.py | 39 ----------------------- src/storage/filesystem.py | 3 +- src/tests/test_randomtrackingdict.py | 47 ++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 40 deletions(-) create mode 100644 src/tests/test_randomtrackingdict.py diff --git a/src/network/randomtrackingdict.py b/src/network/randomtrackingdict.py index e87bf156..cb4e310d 100644 --- a/src/network/randomtrackingdict.py +++ b/src/network/randomtrackingdict.py @@ -1,7 +1,6 @@ """ Track randomize ordered dict """ -import random from threading import RLock from time import time @@ -128,41 +127,3 @@ class RandomTrackingDict(object): self.pendingLen += 1 self.lastPoll = time() return retval - - -if __name__ == '__main__': - - # pylint: disable=redefined-outer-name - def randString(): - """helper function for tests, generates a random string""" - retval = b'' - for _ in range(32): - retval += chr(random.randint(0, 255)) - return retval - - a = [] - k = RandomTrackingDict() - d = {} - - print "populating random tracking dict" - a.append(time()) - for i in range(50000): - k[randString()] = True - a.append(time()) - print "done" - - while k: - retval = k.randomKeys(1000) - if not retval: - print "error getting random keys" - try: - k.randomKeys(100) - print "bad" - except KeyError: - pass - for i in retval: - del k[i] - a.append(time()) - - for x in range(len(a) - 1): - print "%i: %.3f" % (x, a[x + 1] - a[x]) diff --git a/src/storage/filesystem.py b/src/storage/filesystem.py index b19d9272..971d05b9 100644 --- a/src/storage/filesystem.py +++ b/src/storage/filesystem.py @@ -4,6 +4,7 @@ Module for using filesystem (directory with files) for inventory storage import string import time from binascii import hexlify, unhexlify +from debug import logger from os import listdir, makedirs, path, remove, rmdir from threading import RLock @@ -162,7 +163,7 @@ class FilesystemInventory(InventoryStorage): newInventory[streamNumber][hashId] = InventoryItem( objectType, streamNumber, None, expiresTime, tag) except KeyError: - print "error loading %s" % (hexlify(hashId)) + logger.warning('error loading %s', hexlify(hashId)) self._inventory = newInventory # for i, v in self._inventory.items(): # print "loaded stream: %s, %i items" % (i, len(v)) diff --git a/src/tests/test_randomtrackingdict.py b/src/tests/test_randomtrackingdict.py new file mode 100644 index 00000000..5e6b20bb --- /dev/null +++ b/src/tests/test_randomtrackingdict.py @@ -0,0 +1,47 @@ +""" +Tests for RandomTrackingDict class +""" +import random +import unittest + +from network import randomtrackingdict +from time import time + + +class TestRandomTrackingDict(unittest.TestCase): + """Main protocol test case""" + + @staticmethod + def randString(): + """helper function for tests, generates a random string""" + retval = b'' + for _ in range(32): + retval += chr(random.randint(0, 255)) + return retval + + def test_check_randomtrackingdict(self): + """Check the logic of RandomTrackingDict class""" + a = [] + k = randomtrackingdict.RandomTrackingDict() + + a.append(time()) + for i in range(50000): + k[self.randString()] = True + a.append(time()) + + while k: + retval = k.randomKeys(1000) + if not retval: + self.fail("error getting random keys") + + try: + k.randomKeys(100) + self.fail("bad") + except KeyError: + pass + for i in retval: + del k[i] + a.append(time()) + + for x in range(len(a) - 1): + self.assertLess(a[x + 1] - a[x], 10) -- 2.45.1 From b99c6fcf8807dfd1232c15cdf6fb11869445ae61 Mon Sep 17 00:00:00 2001 From: navjot Date: Tue, 22 Dec 2020 15:46:29 +0530 Subject: [PATCH 2/2] fixed travis-ci checks failing issue --- src/network/addrthread.py | 10 ++--- src/network/advanceddispatcher.py | 4 +- src/network/announcethread.py | 10 ++--- src/network/assemble.py | 8 ++-- src/network/asyncore_pollchoose.py | 4 +- src/network/bmobject.py | 10 ++--- src/network/bmproto.py | 22 +++++------ src/network/connectionchooser.py | 8 ++-- src/network/connectionpool.py | 10 ++--- src/network/dandelion.py | 6 +-- src/network/downloadthread.py | 10 ++--- src/network/invthread.py | 12 +++--- src/network/knownnodes.py | 6 +-- src/network/networkthread.py | 8 ++-- src/network/objectracker.py | 6 +-- src/network/proxy.py | 2 +- src/network/randomtrackingdict.py | 2 +- src/network/receivequeuethread.py | 8 ++-- src/network/tcp.py | 38 +++++++++---------- src/network/tls.py | 8 ++-- src/network/udp.py | 6 +-- src/network/uploadthread.py | 10 ++--- src/storage/filesystem.py | 4 +- src/storage/sqlite.py | 2 +- ...rackingdict.py => test_randomtrackdict.py} | 11 ++++-- 25 files changed, 115 insertions(+), 110 deletions(-) rename src/tests/{test_randomtrackingdict.py => test_randomtrackdict.py} (75%) diff --git a/src/network/addrthread.py b/src/network/addrthread.py index 3bf448d8..af2ca3f8 100644 --- a/src/network/addrthread.py +++ b/src/network/addrthread.py @@ -3,11 +3,11 @@ Announce addresses as they are received from other hosts """ import Queue -import state -from helper_random import randomshuffle -from network.assemble import assemble_addr -from network.connectionpool import BMConnectionPool -from queues import addrQueue +from assemble import assemble_addr +from connectionpool import BMConnectionPool +from pybitmessage import state +from pybitmessage.helper_random import randomshuffle +from pybitmessage.queues import addrQueue from threads import StoppableThread diff --git a/src/network/advanceddispatcher.py b/src/network/advanceddispatcher.py index 982be819..da0bff2f 100644 --- a/src/network/advanceddispatcher.py +++ b/src/network/advanceddispatcher.py @@ -6,8 +6,8 @@ import socket import threading import time -import network.asyncore_pollchoose as asyncore -import state +import asyncore_pollchoose as asyncore +from pybitmessage import state from threads import BusyError, nonBlocking diff --git a/src/network/announcethread.py b/src/network/announcethread.py index 19038ab6..1648d0c2 100644 --- a/src/network/announcethread.py +++ b/src/network/announcethread.py @@ -3,13 +3,13 @@ Announce myself (node address) """ import time -import state -from bmconfigparser import BMConfigParser -from network.assemble import assemble_addr -from network.connectionpool import BMConnectionPool -from network.udp import UDPSocket +from assemble import assemble_addr +from connectionpool import BMConnectionPool from node import Peer +from pybitmessage import state +from pybitmessage.bmconfigparser import BMConfigParser from threads import StoppableThread +from udp import UDPSocket class AnnounceThread(StoppableThread): diff --git a/src/network/assemble.py b/src/network/assemble.py index 32fad3e4..f80c7be7 100644 --- a/src/network/assemble.py +++ b/src/network/assemble.py @@ -3,10 +3,10 @@ Create bitmessage protocol command packets """ import struct -import addresses -from network.constants import MAX_ADDR_COUNT -from network.node import Peer -from protocol import CreatePacket, encodeHost +from pybitmessage import addresses +from constants import MAX_ADDR_COUNT +from node import Peer +from pybitmessage.protocol import CreatePacket, encodeHost def assemble_addr(peerList): diff --git a/src/network/asyncore_pollchoose.py b/src/network/asyncore_pollchoose.py index 41757f37..414fe828 100644 --- a/src/network/asyncore_pollchoose.py +++ b/src/network/asyncore_pollchoose.py @@ -17,9 +17,9 @@ from errno import ( ECONNRESET, EHOSTUNREACH, EINPROGRESS, EINTR, EINVAL, EISCONN, ENETUNREACH, ENOTCONN, ENOTSOCK, EPIPE, ESHUTDOWN, ETIMEDOUT, EWOULDBLOCK, errorcode ) -from threading import current_thread +from pybitmessage import helper_random -import helper_random +from threading import current_thread try: from errno import WSAEWOULDBLOCK diff --git a/src/network/bmobject.py b/src/network/bmobject.py index 12b997d7..1bc375ab 100644 --- a/src/network/bmobject.py +++ b/src/network/bmobject.py @@ -4,11 +4,11 @@ BMObject and it's exceptions. import logging import time -import protocol -import state -from addresses import calculateInventoryHash -from inventory import Inventory -from network.dandelion import Dandelion +from dandelion import Dandelion +from pybitmessage import protocol +from pybitmessage import state +from pybitmessage.addresses import calculateInventoryHash +from pybitmessage.inventory import Inventory logger = logging.getLogger('default') diff --git a/src/network/bmproto.py b/src/network/bmproto.py index ef64de94..96ea40bf 100644 --- a/src/network/bmproto.py +++ b/src/network/bmproto.py @@ -10,28 +10,28 @@ import struct import time from binascii import hexlify -import addresses import connectionpool import knownnodes -import protocol -import state -from bmconfigparser import BMConfigParser -from inventory import Inventory -from network.advanceddispatcher import AdvancedDispatcher -from network.bmobject import ( +from advanceddispatcher import AdvancedDispatcher +from bmobject import ( BMObject, BMObjectAlreadyHaveError, BMObjectExpiredError, BMObjectInsufficientPOWError, BMObjectInvalidDataError, BMObjectInvalidError, BMObjectUnwantedStreamError ) -from network.constants import ( +from constants import ( ADDRESS_ALIVE, MAX_MESSAGE_SIZE, MAX_OBJECT_COUNT, MAX_OBJECT_PAYLOAD_SIZE, MAX_TIME_OFFSET ) -from network.dandelion import Dandelion -from network.proxy import ProxyError +from dandelion import Dandelion +from proxy import ProxyError from node import Node, Peer from objectracker import ObjectTracker, missingObjects -from queues import invQueue, objectProcessorQueue, portCheckerQueue +from pybitmessage import addresses +from pybitmessage import protocol +from pybitmessage import state +from pybitmessage.bmconfigparser import BMConfigParser +from pybitmessage.inventory import Inventory +from pybitmessage.queues import invQueue, objectProcessorQueue, portCheckerQueue from randomtrackingdict import RandomTrackingDict logger = logging.getLogger('default') diff --git a/src/network/connectionchooser.py b/src/network/connectionchooser.py index badd98b7..70256a05 100644 --- a/src/network/connectionchooser.py +++ b/src/network/connectionchooser.py @@ -6,10 +6,10 @@ import logging import random # nosec import knownnodes -import protocol -import state -from bmconfigparser import BMConfigParser -from queues import Queue, portCheckerQueue +from pybitmessage import protocol +from pybitmessage import state +from pybitmessage.bmconfigparser import BMConfigParser +from pybitmessage.queues import Queue, portCheckerQueue logger = logging.getLogger('default') diff --git a/src/network/connectionpool.py b/src/network/connectionpool.py index fffc0bc3..f6210633 100644 --- a/src/network/connectionpool.py +++ b/src/network/connectionpool.py @@ -9,15 +9,15 @@ import sys import time import asyncore_pollchoose as asyncore -import helper_random import knownnodes -import protocol -import state -from bmconfigparser import BMConfigParser from connectionchooser import chooseConnection from node import Peer from proxy import Proxy -from singleton import Singleton +from pybitmessage import helper_random +from pybitmessage import protocol +from pybitmessage import state +from pybitmessage.bmconfigparser import BMConfigParser +from pybitmessage.singleton import Singleton from tcp import ( bootstrap, Socks4aBMConnection, Socks5BMConnection, TCPConnection, TCPServer) diff --git a/src/network/dandelion.py b/src/network/dandelion.py index 03f45bd7..3b760ae3 100644 --- a/src/network/dandelion.py +++ b/src/network/dandelion.py @@ -8,9 +8,9 @@ from threading import RLock from time import time import connectionpool -import state -from queues import invQueue -from singleton import Singleton +from pybitmessage import state +from pybitmessage.queues import invQueue +from pybitmessage.singleton import Singleton # randomise routes after 600 seconds REASSIGN_INTERVAL = 600 diff --git a/src/network/downloadthread.py b/src/network/downloadthread.py index 0ae83b5b..399c271d 100644 --- a/src/network/downloadthread.py +++ b/src/network/downloadthread.py @@ -3,13 +3,13 @@ """ import time -import addresses -import helper_random -import protocol from dandelion import Dandelion -from inventory import Inventory -from network.connectionpool import BMConnectionPool +from connectionpool import BMConnectionPool from objectracker import missingObjects +from pybitmessage import addresses +from pybitmessage import helper_random +from pybitmessage import protocol +from pybitmessage.inventory import Inventory from threads import StoppableThread diff --git a/src/network/invthread.py b/src/network/invthread.py index e68b7692..870ecd22 100644 --- a/src/network/invthread.py +++ b/src/network/invthread.py @@ -5,12 +5,12 @@ import Queue import random from time import time -import addresses -import protocol -import state -from network.connectionpool import BMConnectionPool -from network.dandelion import Dandelion -from queues import invQueue +from connectionpool import BMConnectionPool +from dandelion import Dandelion +from pybitmessage import addresses +from pybitmessage import protocol +from pybitmessage import state +from pybitmessage.queues import invQueue from threads import StoppableThread diff --git a/src/network/knownnodes.py b/src/network/knownnodes.py index c92f8e9a..f953a3e3 100644 --- a/src/network/knownnodes.py +++ b/src/network/knownnodes.py @@ -13,9 +13,9 @@ try: except ImportError: from collections import Iterable -import state -from bmconfigparser import BMConfigParser -from network.node import Peer +from node import Peer +from pybitmessage import state +from pybitmessage.bmconfigparser import BMConfigParser state.Peer = Peer diff --git a/src/network/networkthread.py b/src/network/networkthread.py index 61ff6c09..25a40212 100644 --- a/src/network/networkthread.py +++ b/src/network/networkthread.py @@ -1,10 +1,10 @@ """ A thread to handle network concerns """ -import network.asyncore_pollchoose as asyncore -import state -from network.connectionpool import BMConnectionPool -from queues import excQueue +import asyncore_pollchoose as asyncore +from connectionpool import BMConnectionPool +from pybitmessage import state +from pybitmessage.queues import excQueue from threads import StoppableThread diff --git a/src/network/objectracker.py b/src/network/objectracker.py index ca29c023..af41dac6 100644 --- a/src/network/objectracker.py +++ b/src/network/objectracker.py @@ -4,8 +4,8 @@ Module for tracking objects import time from threading import RLock -import network.connectionpool -from network.dandelion import Dandelion +import connectionpool +from dandelion import Dandelion from randomtrackingdict import RandomTrackingDict haveBloom = False @@ -100,7 +100,7 @@ class ObjectTracker(object): def handleReceivedObject(self, streamNumber, hashid): """Handling received object""" - for i in network.connectionpool.BMConnectionPool().connections(): + for i in connectionpool.BMConnectionPool().connections(): if not i.fullyEstablished: continue try: diff --git a/src/network/proxy.py b/src/network/proxy.py index 38676d66..4e90ea2a 100644 --- a/src/network/proxy.py +++ b/src/network/proxy.py @@ -8,7 +8,7 @@ import time import asyncore_pollchoose as asyncore from advanceddispatcher import AdvancedDispatcher -from bmconfigparser import BMConfigParser +from pybitmessage.bmconfigparser import BMConfigParser from node import Peer logger = logging.getLogger('default') diff --git a/src/network/randomtrackingdict.py b/src/network/randomtrackingdict.py index cb4e310d..4b265f82 100644 --- a/src/network/randomtrackingdict.py +++ b/src/network/randomtrackingdict.py @@ -4,7 +4,7 @@ Track randomize ordered dict from threading import RLock from time import time -import helper_random +from pybitmessage import helper_random class RandomTrackingDict(object): diff --git a/src/network/receivequeuethread.py b/src/network/receivequeuethread.py index bf1d8300..0b05b27b 100644 --- a/src/network/receivequeuethread.py +++ b/src/network/receivequeuethread.py @@ -5,10 +5,10 @@ import errno import Queue import socket -import state -from network.advanceddispatcher import UnknownStateError -from network.connectionpool import BMConnectionPool -from queues import receiveDataQueue +from advanceddispatcher import UnknownStateError +from connectionpool import BMConnectionPool +from pybitmessage import state +from pybitmessage.queues import receiveDataQueue from threads import StoppableThread diff --git a/src/network/tcp.py b/src/network/tcp.py index 64153d9d..43feac1c 100644 --- a/src/network/tcp.py +++ b/src/network/tcp.py @@ -2,35 +2,35 @@ TCP protocol handler """ # pylint: disable=too-many-ancestors -import l10n import logging import math import random import socket import time -import addresses import asyncore_pollchoose as asyncore import connectionpool -import helper_random import knownnodes -import protocol -import state -from bmconfigparser import BMConfigParser -from helper_random import randomBytes -from inventory import Inventory -from network.advanceddispatcher import AdvancedDispatcher -from network.assemble import assemble_addr -from network.bmproto import BMProto -from network.constants import MAX_OBJECT_COUNT -from network.dandelion import Dandelion -from network.objectracker import ObjectTracker -from network.socks4a import Socks4aConnection -from network.socks5 import Socks5Connection -from network.tls import TLSDispatcher +from advanceddispatcher import AdvancedDispatcher +from assemble import assemble_addr +from bmproto import BMProto +from constants import MAX_OBJECT_COUNT +from dandelion import Dandelion +from objectracker import ObjectTracker +from socks4a import Socks4aConnection +from socks5 import Socks5Connection +from tls import TLSDispatcher +from pybitmessage import addresses +from pybitmessage import helper_random +from pybitmessage import l10n +from pybitmessage import protocol +from pybitmessage import state +from pybitmessage.bmconfigparser import BMConfigParser +from pybitmessage.helper_random import randomBytes +from pybitmessage.inventory import Inventory +from pybitmessage.queues import invQueue, receiveDataQueue, UISignalQueue +from pybitmessage.tr import _translate from node import Peer -from queues import invQueue, receiveDataQueue, UISignalQueue -from tr import _translate logger = logging.getLogger('default') diff --git a/src/network/tls.py b/src/network/tls.py index 1b325696..87873d5b 100644 --- a/src/network/tls.py +++ b/src/network/tls.py @@ -7,10 +7,10 @@ import socket import ssl import sys -import network.asyncore_pollchoose as asyncore -import paths -from network.advanceddispatcher import AdvancedDispatcher -from queues import receiveDataQueue +import asyncore_pollchoose as asyncore +from advanceddispatcher import AdvancedDispatcher +from pybitmessage import paths +from pybitmessage.queues import receiveDataQueue logger = logging.getLogger('default') diff --git a/src/network/udp.py b/src/network/udp.py index ccc7ade8..cf0fb241 100644 --- a/src/network/udp.py +++ b/src/network/udp.py @@ -5,12 +5,12 @@ import logging import socket import time -import protocol -import state from bmproto import BMProto from node import Peer from objectracker import ObjectTracker -from queues import receiveDataQueue +from pybitmessage import protocol +from pybitmessage import state +from pybitmessage.queues import receiveDataQueue logger = logging.getLogger('default') diff --git a/src/network/uploadthread.py b/src/network/uploadthread.py index 7d80d789..792e4be1 100644 --- a/src/network/uploadthread.py +++ b/src/network/uploadthread.py @@ -3,11 +3,11 @@ """ import time -import helper_random -import protocol -from inventory import Inventory -from network.connectionpool import BMConnectionPool -from network.dandelion import Dandelion +from connectionpool import BMConnectionPool +from dandelion import Dandelion +from pybitmessage import helper_random +from pybitmessage import protocol +from pybitmessage.inventory import Inventory from randomtrackingdict import RandomTrackingDict from threads import StoppableThread diff --git a/src/storage/filesystem.py b/src/storage/filesystem.py index 971d05b9..469627af 100644 --- a/src/storage/filesystem.py +++ b/src/storage/filesystem.py @@ -4,11 +4,11 @@ Module for using filesystem (directory with files) for inventory storage import string import time from binascii import hexlify, unhexlify -from debug import logger from os import listdir, makedirs, path, remove, rmdir from threading import RLock -from paths import lookupAppdataFolder +from pybitmessage.debug import logger +from pybitmessage.paths import lookupAppdataFolder from storage import InventoryItem, InventoryStorage diff --git a/src/storage/sqlite.py b/src/storage/sqlite.py index 50a2034e..4048087b 100644 --- a/src/storage/sqlite.py +++ b/src/storage/sqlite.py @@ -5,7 +5,7 @@ import sqlite3 import time from threading import RLock -from helper_sql import SqlBulkExecute, sqlExecute, sqlQuery +from pybitmessage.helper_sql import SqlBulkExecute, sqlExecute, sqlQuery from storage import InventoryItem, InventoryStorage diff --git a/src/tests/test_randomtrackingdict.py b/src/tests/test_randomtrackdict.py similarity index 75% rename from src/tests/test_randomtrackingdict.py rename to src/tests/test_randomtrackdict.py index 5e6b20bb..a164c61a 100644 --- a/src/tests/test_randomtrackingdict.py +++ b/src/tests/test_randomtrackdict.py @@ -4,12 +4,13 @@ Tests for RandomTrackingDict class import random import unittest -from network import randomtrackingdict from time import time class TestRandomTrackingDict(unittest.TestCase): - """Main protocol test case""" + """ + Main protocol test case + """ @staticmethod def randString(): @@ -21,8 +22,12 @@ class TestRandomTrackingDict(unittest.TestCase): def test_check_randomtrackingdict(self): """Check the logic of RandomTrackingDict class""" + # from network import RandomTrackingDict + # from pybitmessage.network.randomtrackingdict import RandomTrackingDict + # from network.randomtrackingdict import RandomTrackingDict + from pybitmessage import network a = [] - k = randomtrackingdict.RandomTrackingDict() + k = network.randomtrackingdict.RandomTrackingDict() a.append(time()) for i in range(50000): -- 2.45.1