diff --git a/.travis.yml b/.travis.yml
index 03a89471..a1a314d9 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -15,7 +15,8 @@ addons:
 install:
   - pip install -r requirements.txt
   - python setup.py install
+  - export PYTHONWARNINGS=all
 script:
   - python checkdeps.py
   - xvfb-run src/bitmessagemain.py -t
-  - python setup.py test
+  - python -bm tests
diff --git a/requirements.txt b/requirements.txt
index c55e5cf1..f9972a7e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,4 @@
+coverage
 python_prctl
 psutil
 pycrypto
diff --git a/setup.py b/setup.py
index 05e34e06..56d87308 100644
--- a/setup.py
+++ b/setup.py
@@ -4,7 +4,6 @@ import os
 import platform
 import shutil
 import sys
-import unittest
 
 from setuptools import setup, Extension
 from setuptools.command.install import install
@@ -46,14 +45,6 @@ class InstallCmd(install):
         return install.run(self)
 
 
-def unittest_discover():
-    """Explicit test suite creation"""
-    if sys.hexversion >= 0x3000000:
-        from pybitmessage import pathmagic
-        pathmagic.setup()
-    return unittest.TestLoader().discover('pybitmessage.tests')
-
-
 if __name__ == "__main__":
     here = os.path.abspath(os.path.dirname(__file__))
     with open(os.path.join(here, 'README.md')) as f:
@@ -125,7 +116,7 @@ if __name__ == "__main__":
         #keywords='',
         install_requires=installRequires,
         tests_require=requirements,
-        test_suite='setup.unittest_discover',
+        test_suite='tests.unittest_discover',
         extras_require=EXTRAS_REQUIRE,
         classifiers=[
             "License :: OSI Approved :: MIT License"
diff --git a/src/bitmessagemain.py b/src/bitmessagemain.py
index 29eaf13e..b54805bb 100755
--- a/src/bitmessagemain.py
+++ b/src/bitmessagemain.py
@@ -378,11 +378,7 @@ class Main(object):
             test_core_result = test_core.run()
             self.stop()
             test_core.cleanup()
-            sys.exit(
-                'Core tests failed!'
-                if test_core_result.errors or test_core_result.failures
-                else 0
-            )
+            sys.exit(not test_core_result.wasSuccessful())
 
     @staticmethod
     def daemonize():
diff --git a/tests.py b/tests.py
new file mode 100644
index 00000000..f0c82f77
--- /dev/null
+++ b/tests.py
@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+"""Custom tests runner script for tox and python3"""
+import random  # noseq
+import sys
+import unittest
+
+
+def unittest_discover():
+    """Explicit test suite creation"""
+    if sys.hexversion >= 0x3000000:
+        from pybitmessage import pathmagic
+        pathmagic.setup()
+    loader = unittest.defaultTestLoader
+    # randomize the order of tests in test cases
+    loader.sortTestMethodsUsing = lambda a, b: random.randint(-1, 1)
+    # pybitmessage symlink may disappear on Windows
+    return loader.discover('src.tests')
+
+
+if __name__ == "__main__":
+    result = unittest.TextTestRunner(verbosity=2).run(unittest_discover())
+    sys.exit(not result.wasSuccessful())
diff --git a/tox.ini b/tox.ini
new file mode 100644
index 00000000..211afd47
--- /dev/null
+++ b/tox.ini
@@ -0,0 +1,34 @@
+[tox]
+envlist = reset,py{27,37,38},stats
+skip_missing_interpreters = true
+
+[testenv]
+setenv =
+    BITMESSAGE_HOME = {envtmpdir}
+    PYTHONWARNINGS = all
+deps = -rrequirements.txt
+commands =
+    python checkdeps.py
+    coverage run -a src/bitmessagemain.py -t
+    coverage run -a -m tests
+
+[testenv:reset]
+commands = coverage erase
+
+[testenv:stats]
+commands =
+    coverage report
+    coverage xml
+
+[coverage:run]
+source = src
+omit =
+    */lib*
+    tests.py
+    */tests/*
+    src/version.py
+    */__init__.py
+    src/fallback/umsgpack/*
+
+[coverage:report]
+ignore_errors = true