diff --git a/src/bmconfigparser.py b/src/bmconfigparser.py index 328cf0c7..bb7f67d0 100644 --- a/src/bmconfigparser.py +++ b/src/bmconfigparser.py @@ -7,7 +7,7 @@ import os import shutil from datetime import datetime -import state +from paths import get_active_config_folder from singleton import Singleton BMConfigDefaults = { @@ -124,7 +124,16 @@ class BMConfigParser(ConfigParser.SafeConfigParser): return [ x for x in BMConfigParser().sections() if x.startswith('BM-')] + def _reset(self): + """Reset current config. There doesn't appear to be a built in + method for this""" + sections = self.sections() + for x in sections: + self.remove_section(x) + def read(self, filenames): + """Read config and populate defaults""" + self._reset() ConfigParser.ConfigParser.read(self, filenames) for section in self.sections(): for option in self.options(section): @@ -144,7 +153,7 @@ class BMConfigParser(ConfigParser.SafeConfigParser): def save(self): """Save the runtime config onto the filesystem""" - fileName = os.path.join(state.appdata, 'keys.dat') + fileName = os.path.join(get_active_config_folder(), 'keys.dat') fileNameBak = '.'.join([ fileName, datetime.now().strftime("%Y%j%H%M%S%f"), 'bak']) # create a backup copy to prevent the accidental loss due to diff --git a/src/class_sqlThread.py b/src/class_sqlThread.py index 7e9eb6c5..115de6b6 100644 --- a/src/class_sqlThread.py +++ b/src/class_sqlThread.py @@ -28,7 +28,9 @@ class sqlThread(threading.Thread): def run(self): # pylint: disable=too-many-locals, too-many-branches, too-many-statements """Process SQL queries from `.helper_sql.sqlSubmitQueue`""" - self.conn = sqlite3.connect(state.appdata + 'messages.dat') + self.conn = sqlite3.connect( + os.path.join(paths.get_active_config_folder(), + 'messages.dat')) self.conn.text_factory = str self.cur = self.conn.cursor() diff --git a/src/debug.py b/src/debug.py index cab07275..add197dd 100644 --- a/src/debug.py +++ b/src/debug.py @@ -42,7 +42,8 @@ import os import sys import helper_startup -import state + +from paths import get_active_config_folder helper_startup.loadConfig() @@ -67,7 +68,7 @@ def configureLogging(): sys.excepthook = log_uncaught_exceptions fail_msg = '' try: - logging_config = os.path.join(state.appdata, 'logging.dat') + logging_config = os.path.join(get_active_config_folder(), 'logging.dat') logging.config.fileConfig( logging_config, disable_existing_loggers=False) return ( @@ -103,7 +104,7 @@ def configureLogging(): 'class': 'logging.handlers.RotatingFileHandler', 'formatter': 'default', 'level': log_level, - 'filename': os.path.join(state.appdata, 'debug.log'), + 'filename': os.path.join(get_active_config_folder(), 'debug.log'), 'maxBytes': 2097152, # 2 MiB 'backupCount': 1, 'encoding': 'UTF-8', @@ -140,14 +141,23 @@ def resetLogging(): """Reconfigure logging in runtime when state.appdata dir changed""" # pylint: disable=global-statement, used-before-assignment global logger - for i in logger.handlers: - logger.removeHandler(i) + for i in list(logger.handlers): + try: + logger.removeHandler(i) + except ValueError: + pass i.flush() i.close() configureLogging() logger = logging.getLogger('default') +def flush_logs(): + """Tell handlers to flush""" + for i in list(logger.handlers): + i.flush() + + # ! preconfigured, msg = configureLogging() logger = logging.getLogger('default') diff --git a/src/network/knownnodes.py b/src/network/knownnodes.py index 07871c7c..69a00e1e 100644 --- a/src/network/knownnodes.py +++ b/src/network/knownnodes.py @@ -16,6 +16,7 @@ except ImportError: import state from bmconfigparser import BMConfigParser from network.node import Peer +from paths import get_active_config_folder knownNodesLock = threading.RLock() """Thread lock for knownnodes modification""" @@ -91,7 +92,7 @@ def pickle_deserialize_old_knownnodes(source): def saveKnownNodes(dirName=None): """Save knownnodes to filesystem""" if dirName is None: - dirName = state.appdata + dirName = get_active_config_folder() with knownNodesLock: with open(os.path.join(dirName, 'knownnodes.dat'), 'wb') as output: json_serialize_knownnodes(output) @@ -149,7 +150,9 @@ def createDefaultKnownNodes(): def readKnownNodes(): """Load knownnodes from filesystem""" try: - with open(state.appdata + 'knownnodes.dat', 'rb') as source: + with open(os.path.join(get_active_config_folder(), + 'knownnodes.dat'), + 'rb') as source: with knownNodesLock: try: json_deserialize_knownnodes(source) diff --git a/src/paths.py b/src/paths.py index e2f8c97e..c0c124b6 100644 --- a/src/paths.py +++ b/src/paths.py @@ -8,6 +8,8 @@ import sys from datetime import datetime from shutil import move +import state + logger = logging.getLogger('default') # When using py2exe or py2app, the variable frozen is added to the sys @@ -31,6 +33,28 @@ def lookupExeFolder(): return exeFolder +def set_appdata_folder(path=None): + """Changes appdata location. Does not move any files, just where + where it's looking when loading/saving later""" + if not path: + try: + # default + del os.environ['BITMESSAGE_HOME'] + except KeyError: + pass + else: + os.environ['BITMESSAGE_HOME'] = path + + state.appdata = lookupAppdataFolder() + return state.appdata + + +def get_active_config_folder(): + """Returns current config directory, workaround for inconsistent + imports""" + return state.appdata + + def lookupAppdataFolder(): """Returns path of the folder where application data is stored""" APPNAME = "PyBitmessage" diff --git a/src/shared.py b/src/shared.py index 3a6fbc31..231c262b 100644 --- a/src/shared.py +++ b/src/shared.py @@ -17,11 +17,11 @@ from binascii import hexlify # Project imports. import highlevelcrypto -import state from addresses import decodeAddress, encodeVarint from bmconfigparser import BMConfigParser from debug import logger from helper_sql import sqlQuery +from paths import get_active_config_folder from pyelliptic import arithmetic @@ -114,7 +114,7 @@ def reloadMyAddressHashes(): # myPrivateKeys.clear() keyfileSecure = checkSensitiveFilePermissions(os.path.join( - state.appdata, 'keys.dat')) + get_active_config_folder(), 'keys.dat')) hasEnabledKeys = False for addressInKeysFile in BMConfigParser().addresses(): isEnabled = BMConfigParser().getboolean(addressInKeysFile, 'enabled') @@ -144,7 +144,7 @@ def reloadMyAddressHashes(): if not keyfileSecure: fixSensitiveFilePermissions(os.path.join( - state.appdata, 'keys.dat'), hasEnabledKeys) + get_active_config_folder(), 'keys.dat'), hasEnabledKeys) def reloadBroadcastSendersForWhichImWatching(): diff --git a/src/tests/test_config.py b/src/tests/test_config.py index 35ddd3fa..3046224c 100644 --- a/src/tests/test_config.py +++ b/src/tests/test_config.py @@ -45,6 +45,7 @@ class TestProcessConfig(TestProcessProto): def test_config_defaults(self): """Test settings in the generated config""" self._stop_process() + self._kill_process() config = BMConfigParser() config.read(os.path.join(self.home, 'keys.dat')) diff --git a/src/tests/test_logger.py b/src/tests/test_logger.py index 57448911..6afe0a26 100644 --- a/src/tests/test_logger.py +++ b/src/tests/test_logger.py @@ -7,6 +7,8 @@ import os import tempfile import unittest +from pybitmessage.paths import set_appdata_folder + class TestLogger(unittest.TestCase): """A test case for bmconfigparser""" @@ -35,11 +37,14 @@ level=DEBUG handlers=default ''' - def test_fileConfig(self): + def test_fileConfig(self): # pylint: disable=invalid-name """Put logging.dat with special pattern and check it was used""" - tmp = os.environ['BITMESSAGE_HOME'] = tempfile.gettempdir() + # FIXME: this doesn't always work, probably due to module + # load order. Workaround through .travis.yml + tmp = set_appdata_folder(tempfile.gettempdir()) log_config = os.path.join(tmp, 'logging.dat') log_file = os.path.join(tmp, 'debug.log') + pass_ = 1 def gen_log_config(pattern): """A small closure to generate logging.dat with custom pattern""" @@ -48,11 +53,14 @@ handlers=default pattern = r' o_0 ' gen_log_config(pattern) + pass_ = 1 try: - from pybitmessage.debug import logger, resetLogging + # pylint: disable=import-outside-toplevel + from pybitmessage.debug import logger, resetLogging, flush_logs if not os.path.isfile(log_file): # second pass pattern = r' <===> ' + pass_ = 2 gen_log_config(pattern) resetLogging() except ImportError: @@ -60,10 +68,17 @@ handlers=default finally: os.remove(log_config) + # pylint: disable=invalid-name + self.longMessage = True + logger_ = logging.getLogger('default') - self.assertEqual(logger, logger_) + self.assertEqual(logger, logger_, ", pass {}". + format(pass_)) logger_.info('Testing the logger...') + flush_logs() - self.assertRegexpMatches(open(log_file).read(), pattern) + # pylint: disable=deprecated-method + self.assertRegexpMatches(open(log_file).read(), pattern, + ", pass {}".format(pass_)) diff --git a/src/tests/test_process.py b/src/tests/test_process.py index 73a6e493..571a562f 100644 --- a/src/tests/test_process.py +++ b/src/tests/test_process.py @@ -5,12 +5,15 @@ Common reusable code for tests and tests for pybitmessage process. import os import signal import subprocess # nosec +import sys import tempfile import time import unittest import psutil +from pybitmessage.paths import set_appdata_folder + def put_signal_file(path, filename): """Creates file, presence of which is a signal about some event.""" @@ -23,7 +26,26 @@ class TestProcessProto(unittest.TestCase): it starts pybitmessage in setUpClass() and stops it in tearDownClass() """ _process_cmd = ['pybitmessage', '-d'] - _threads_count = 15 + _threads_count_min = 15 + _threads_count_max = 16 + _threads_names = [ + 'PyBitmessage', + 'addressGenerato', + 'singleWorker', + 'SQL', + 'objectProcessor', + 'singleCleaner', + 'singleAPI', + 'Asyncore', + 'ReceiveQueue_0', + 'ReceiveQueue_1', + 'ReceiveQueue_2', + 'Announcer', + 'InvBroadcaster', + 'AddrBroadcaster', + 'Downloader', + 'Uploader' + ] _files = ( 'keys.dat', 'debug.log', 'messages.dat', 'knownnodes.dat', '.api_started', 'unittest.lock' @@ -32,10 +54,28 @@ class TestProcessProto(unittest.TestCase): @classmethod def setUpClass(cls): """Setup environment and start pybitmessage""" - cls.home = os.environ['BITMESSAGE_HOME'] = tempfile.gettempdir() + cls.home = set_appdata_folder(tempfile.gettempdir()) + cls._cleanup_files() put_signal_file(cls.home, 'unittest.lock') + starttime = int(time.time()) subprocess.call(cls._process_cmd) # nosec - time.sleep(5) + timeout = starttime + 30 + while time.time() <= timeout: + try: + if os.path.exists(os.path.join(cls.home, + 'singleton.lock')): + pstat = os.stat(os.path.join(cls.home, 'singleton.lock')) + if starttime <= pstat.st_mtime and pstat.st_size > 0: + break + except OSError: + break + time.sleep(1) + if time.time() >= timeout: + raise psutil.TimeoutExpired( + "Timeout starting {}".format(cls._process_cmd)) + # wait a bit for the program to fully start + # 10 sec should be enough + time.sleep(10) cls.pid = int(cls._get_readline('singleton.lock')) cls.process = psutil.Process(cls.pid) @@ -56,6 +96,18 @@ class TestProcessProto(unittest.TestCase): return False return True + @classmethod + def _kill_process(cls, timeout=5): + try: + cls.process.send_signal(signal.SIGKILL) + cls.process.wait(timeout) + # Windows or already dead + except (AttributeError, psutil.NoSuchProcess): + return True + # except psutil.TimeoutExpired propagates, it means something is very + # wrong + return True + @classmethod def _cleanup_files(cls): for pfile in cls._files: @@ -69,33 +121,57 @@ class TestProcessProto(unittest.TestCase): """Ensures that pybitmessage stopped and removes files""" try: if not cls._stop_process(): - print(open(os.path.join(cls.home, 'debug.log'), 'rb').read()) cls.process.kill() - except psutil.NoSuchProcess: + cls.process.wait(5) + except (psutil.TimeoutExpired, psutil.NoSuchProcess): pass finally: cls._cleanup_files() def _test_threads(self): - # only count for now - # because of https://github.com/giampaolo/psutil/issues/613 - # PyBitmessage - # - addressGenerator - # - singleWorker - # - SQL - # - objectProcessor - # - singleCleaner - # - singleAPI - # - Asyncore - # - ReceiveQueue_0 - # - ReceiveQueue_1 - # - ReceiveQueue_2 - # - Announcer - # - InvBroadcaster - # - AddrBroadcaster - # - Downloader - self.assertEqual( - len(self.process.threads()), self._threads_count) + """Test number and names of threads""" + + # pylint: disable=invalid-name + self.longMessage = True + + try: + # using ps for posix platforms + # because of https://github.com/giampaolo/psutil/issues/613 + thread_names = subprocess.check_output([ + "ps", "-L", "-o", "comm=", "--pid", + str(self.process.pid) + ]).split() + except: # pylint: disable=bare-except + thread_names = [] + + running_threads = len(thread_names) + if 0 < running_threads < 30: # adequacy check + extra_threads = [] + missing_threads = [] + for thread_name in thread_names: + if thread_name not in self._threads_names: + extra_threads.append(thread_name) + for thread_name in self._threads_names: + if thread_name not in thread_names: + missing_threads.append(thread_name) + + msg = "Missing threads: {}, Extra threads: {}".format( + ",".join(missing_threads), ",".join(extra_threads)) + else: + running_threads = self.process.num_threads() + if sys.platform.startswith('win'): + running_threads -= 1 # one extra thread on Windows! + msg = "Unexpected running thread count" + + self.assertGreaterEqual( + running_threads, + self._threads_count_min, + msg) + + self.assertLessEqual( + running_threads, + self._threads_count_max, + msg) class TestProcessShutdown(TestProcessProto): @@ -110,7 +186,14 @@ class TestProcessShutdown(TestProcessProto): @classmethod def tearDownClass(cls): """Special teardown because pybitmessage is already stopped""" - cls._cleanup_files() + try: + if cls.process.is_running(): + cls.process.kill() + cls.process.wait(5) + except (psutil.TimeoutExpired, psutil.NoSuchProcess): + pass + finally: + cls._cleanup_files() class TestProcess(TestProcessProto):