Added: Code quality tool

fabfile/README.md

@@ -0,0 +1,55 @@
+# Fabric
+
+[Fabric](https://www.fabfile.org) is a Python library for performing devops tasks. You can thing of it a bit like a
+makefile on steroids for Python. Its api abstracts away the clunky way you would run shell commands in Python, check
+return values and manage stdio. Tasks may be targetted at particular hosts or group of hosts.
+
+# Using Fabric
+
+    $ cd PyBitmessage
+    $ fab <task_name>
+
+For a list of available commands:
+
+    $ fab -l
+
+General fabric commandline help
+
+    $ fab -h
+
+Arguments can be given:
+
+    $ fab task1:arg1=arg1value,arg2=arg2value task2:option1
+
+Tasks target hosts. Hosts can be specified with -H, or roles can be defined and you can target groups of hosts with -R.
+Furthermore, you can use -- to run arbitrary shell commands rather than tasks:
+
+    $ fab -H localhost task1
+    $ fab -R webservers -- sudo /etc/httpd restart
+
+# Getting started
+
+ * Install Fabric, fabric-virtualenv, virtualenvwrapper system-wide (log in to a new terminal afterwards if you're installing
+   virtualenvwrappers for the first time)
+    $ sudo apt install Fabric virtualenvwrapper; sudo pip install fabric-virtualenv
+ * Create a virtualenv called pybitmessage and install fabfile/requirements.txt
+    $ mkvirtualenv --system-site-packages pybitmessage-devops
+    $ pip install -r fabfile/requirements.txt
+ * Ensure you can ssh localhost with no intervention, which may include:
+   * settings in ~/.ssh/config
+   * authorized_keys file
+   * load ssh key
+   * check(!) and accept the host key
+ * From the PyBitmessage directory you can now run fab commands!
+
+# Rationale
+
+There are a number of advantages that should benefit us:
+
+ * Common tasks can be writen in Python and executed consistently
+ * Common tasks are now under source control
+ * All developers can run the same commands, if the underlying command sequence for a task changes (after review, obv)
+   the user does not have to care
+ * Tasks can be combined either programmatically or on the commandline and run in series or parallel
+ * Whoee environemnts can be managed very effectively in conjunction with a configuration management system
+

fabfile/__init__.py

@@ -0,0 +1,14 @@
+"""
+Fabric is a Python library for performing devops tasks. If you have Fabric installed (systemwide or via pip) you can
+run commands like this:
+
+    $ fab code_quality
+
+For a list of commands
+
+"""
+
+from fabfile.tasks import code_quality
+
+
+__all__ = ["code_quality"]

fabfile/lib.py

@@ -0,0 +1,133 @@
+"""A library of functions and constrants for tasks to make use of."""
+
+import os
+<<<<<<< HEAD
+=======
+import sys
+>>>>>>> 81c272dbc332d842b7f8ab548ebeef2dd9a73995
+
+from fabric.api import run, hide
+from fabric.context_managers import settings
+from fabvenv import virtualenv
+
+
+FABRIC_ROOT = os.path.dirname(__file__)
+PROJECT_ROOT = os.path.dirname(FABRIC_ROOT)
+<<<<<<< HEAD
+VENV_ROOT = os.path.expanduser(os.path.join('~', '.virtualenvs', 'pybitmessage'))
+=======
+VENV_ROOT = os.path.expanduser(os.path.join('~', '.virtualenvs', 'pybitmessage-devops'))
+
+
+def coerce_bool(value):
+    """Coerce a value into a boolean"""
+    if isinstance(value, bool):
+        return value
+    elif any(
+            [
+                value in [0, '0'],
+                value.lower().startswith('n'),
+            ]
+    ):
+        return False
+    elif any(
+            [
+                value in [1, '1'],
+                value.lower().startswith('y'),
+            ]
+    ):
+        return True
+    else:
+        print "Bad boolean value {}".format(value)
+        sys.exit(1)
+
+
+def flatten(data):
+    """Recursively flatten lists"""
+    result = []
+    for item in data:
+        if isinstance(item, list):
+            result.append(flatten(item))
+        else:
+            result.append(item)
+    return result
+>>>>>>> 81c272dbc332d842b7f8ab548ebeef2dd9a73995
+
+
+def pycodestyle(path_to_file):
+    """Run pycodestyle on a file"""
+    with virtualenv(VENV_ROOT):
+<<<<<<< HEAD
+        with hide('warnings', 'running'):
+=======
+        with hide('warnings', 'running', 'stdout', 'stderr'):  # pylint: disable=not-context-manager
+>>>>>>> 81c272dbc332d842b7f8ab548ebeef2dd9a73995
+            with settings(warn_only=True):
+                return run(
+                    'pycodestyle --config={0} {1}'.format(
+                        os.path.join(
+                            PROJECT_ROOT,
+                            'setup.cfg',
+                        ),
+                        path_to_file,
+                    ),
+                )
+
+
+def flake8(path_to_file):
+    """Run flake8 on a file"""
+    with virtualenv(VENV_ROOT):
+<<<<<<< HEAD
+        with hide('warnings', 'running'):
+=======
+        with hide('warnings', 'running', 'stdout'):  # pylint: disable=not-context-manager
+>>>>>>> 81c272dbc332d842b7f8ab548ebeef2dd9a73995
+            with settings(warn_only=True):
+                return run(
+                    'flake8 --config={0} {1}'.format(
+                        os.path.join(
+                            PROJECT_ROOT,
+                            'setup.cfg',
+                        ),
+                        path_to_file,
+                    ),
+                )
+
+
+def pylint(path_to_file):
+    """Run pylint on a file"""
+    with virtualenv(VENV_ROOT):
+<<<<<<< HEAD
+        with hide('warnings', 'running', 'stdout', 'stderr'):
+=======
+        with hide('warnings', 'running', 'stdout', 'stderr'):  # pylint: disable=not-context-manager
+>>>>>>> 81c272dbc332d842b7f8ab548ebeef2dd9a73995
+            with settings(warn_only=True):
+                return run(
+                    'pylint --rcfile={0} {1}'.format(
+                        os.path.join(
+                            PROJECT_ROOT,
+                            'setup.cfg',
+                        ),
+                        path_to_file,
+                    ),
+                )
+
+
+def autopep8(path_to_file):
+    """Run autopep8 on a file"""
+    with virtualenv(VENV_ROOT):
+<<<<<<< HEAD
+        with hide('running'):
+            with settings(warn_only=True):
+                return run(
+                    "autopep8 -ai --max-line-length=119 --hang-closing {}".format(
+=======
+        with hide('running'):  # pylint: disable=not-context-manager
+            with settings(warn_only=True):
+                return run(
+                    "autopep8 --experimental --aggressive --aggressive -i --max-line-length=119 {}".format(
+>>>>>>> 81c272dbc332d842b7f8ab548ebeef2dd9a73995
+                        path_to_file
+                    ),
+                )

fabfile/requirements.txt

@@ -0,0 +1,9 @@
+# These requirements are for the Fabric commands that support devops tasks for PyBitmessage, not for running
+# PyBitmessage itself.
+# TODO: Consider moving to an extra_requires group in setup.py
+
+pycodestyle==2.3.1  # https://github.com/PyCQA/pycodestyle/issues/741
+flake8
+pylint
+-e git://github.com/hhatto/autopep8.git@ver1.2.2#egg=autopep8  # Needed for fixing E712
+pep8  # autopep8 doesn't seem to like pycodestyle

fabfile/tasks.py

@@ -0,0 +1,269 @@
+"""
+<<<<<<< HEAD
+Fabric is a Python library for performing devops tasks. If you have Fabric installed (systemwide or via pip) you can
+run commands from within the project directory like this:
+
+    $ fab code_quality
+
+For a list of commands:
+
+    $ fab -l
+
+Known bugs:
+
+=======
+Fabric tasks for PyBitmessage devops operations.
+>>>>>>> 81c272dbc332d842b7f8ab548ebeef2dd9a73995
+"""
+
+import os
+import sys
+
+<<<<<<< HEAD
+import fabric
+from fabric.api import run, task, hide, cd
+from fabfile.lib import pycodestyle, flake8, pylint, autopep8, PROJECT_ROOT, VENV_ROOT
+from fabvenv import virtualenv
+
+
+fabric.state.output['running'] = False
+
+
+@task
+def code_quality(top=10, verbose=True, details=False, fix=True, filename=None):
+    """Intended to be temporary until we have improved code quality and have safeguards to maintain it in place."""
+
+    end = int(top) if int(top) else -1
+
+    if verbose == "0":
+        verbose = False
+    if not isinstance(verbose, bool):
+        print "Bad verbosity value, try 0"
+        sys.exit(1)
+
+    if details == "1":
+        details = True
+    if not isinstance(details, bool):
+        print "Bad details value, try 1"
+        sys.exit(1)
+
+    if fix == "0":
+        fix = False
+    if not isinstance(fix, bool):
+        print "Bad fix value, try 0"
+        sys.exit(1)
+
+    with hide('warnings'):
+=======
+from fabric.api import run, task, hide, cd
+from fabvenv import virtualenv
+
+from fabfile.lib import pycodestyle, flake8, pylint, autopep8, PROJECT_ROOT, VENV_ROOT, coerce_bool, flatten
+
+
+def get_results_for_tools_applied_to_file_list(file_list):
+    """Take a list of files and resuln the results of applying the tools"""
+    results = []
+    for path_to_file in file_list:
+        result = {}
+
+        result['pycodestyle_violations'] = [
+            i
+            for i in pycodestyle(path_to_file).split("\n")
+            if i
+        ]
+        result['flake8_violations'] = [
+            i
+            for i in flake8(path_to_file).split("\n")
+            if i
+        ]
+        result['pylint_violations'] = [
+            i
+            for i in pylint(path_to_file).split("\n")
+            if all([
+                i,
+                not i.startswith(' '),
+                not i.startswith('\r'),
+                not i.startswith('-'),
+                not i.startswith('Y'),
+                not i.startswith('*'),
+                not i.startswith('Using config file'),
+            ])
+        ]
+        result['path_to_file'] = path_to_file
+        result['total_violations'] = sum(
+            [
+                len(result['pycodestyle_violations']),
+                len(result['flake8_violations']),
+                len(result['pylint_violations']),
+            ]
+        )
+        results.append(result)
+        return results
+
+
+def print_item(item, verbose, details):
+    """Print an item with the appropriate verbosity / detail"""
+    if verbose:
+        line = "{0} {1} {2} {3} {4}".format(
+            item['total_violations'],
+            len(item['pycodestyle_violations']),
+            len(item['flake8_violations']),
+            len(item['pylint_violations']),
+            item['path_to_file'],
+        )
+    else:
+        line = item['path_to_file']
+    print line
+
+    if details:
+        print "pycodestyle:"
+        for detail in flatten(item['pycodestyle_violations']):
+            print detail
+        print
+
+        print "flake8:"
+        for detail in flatten(item['flake8_violations']):
+            print detail
+        print
+
+        print "pylint:"
+        for detail in flatten(item['pylint_violations']):
+            print detail
+        print
+
+
+@task
+def code_quality(top=10, verbose=True, details=False, fix=False, filename=None):
+    """
+    Check code quality.
+
+    By default this command will analyse each Python file in the project with a variety of tools and display the count
+    or details of the violations discovered, sorted by most iolations first.
+
+    Default usage:
+
+    $ fab -H localhost code_quality
+
+    Options:
+        top: Display / fix only the top violating files, a value of 0 will display / fix all files
+        verbose: Display a header and the counts. Without this you just get the filenames in order
+        details: Display the violations one per line after the count/file summary
+        fix: Run autopep8 on the displayed file(s)
+        filename: Rather than analysing all files and displaying / fixing the top N, just analyse / diaply / fix the
+        specified file
+
+    Intended to be temporary until we have improved code quality and have safeguards to maintain it in place.
+
+    """
+
+    verbose = coerce_bool(verbose)
+    details = coerce_bool(details)
+    fix = coerce_bool(fix)
+    end = int(top) if int(top) else -1
+
+    with hide('warnings', 'running'):  # pylint: disable=not-context-manager
+>>>>>>> 81c272dbc332d842b7f8ab548ebeef2dd9a73995
+        with virtualenv(VENV_ROOT):
+            if filename:
+                filename = os.path.abspath(filename)
+                if not os.path.exists(filename):
+                    print "Bad filename, specify a Python file"
+                    sys.exit(1)
+                else:
+                    file_list = [filename]
+            else:
+<<<<<<< HEAD
+                with cd(PROJECT_ROOT):
+                    file_list = run('find . -name "*.py"', capture=True).split("\n")
+
+    results = []
+    for path_to_file in file_list:
+
+        results.append(
+            {
+                'pycodestyle_violations': [
+                    i
+                    for i in pycodestyle(path_to_file).split("\n")
+                    if i
+                ],
+                'flake8_violations': [
+                    i
+                    for i in flake8(path_to_file).split("\n")
+                    if i
+                ],
+                'pylint_violations': [
+                    i
+                    for i in pylint(path_to_file).split("\n")
+                    if all([
+                        i,
+                        not i.startswith(' '),
+                        not i.startswith('-'),
+                        not i.startswith('Y'),
+                        not i.startswith('*'),
+                    ])
+                ],
+                'path_to_file': path_to_file,
+            }
+        )
+=======
+                with cd(PROJECT_ROOT):  # pylint: disable=not-context-manager
+                    file_list = run('find . -name "*.py"').split("\n")
+
+    if fix:
+        for path_to_file in file_list:
+            autopep8(path_to_file)
+
+    results = get_results_for_tools_applied_to_file_list(file_list)
+>>>>>>> 81c272dbc332d842b7f8ab548ebeef2dd9a73995
+
+    if verbose:
+        print "\ntotal pycodestyle flake8 pylint path_to_file\n"
+
+<<<<<<< HEAD
+    for item in sorted(
+            results,
+            reverse=True,
+            key=lambda x: len(x['pycodestyle_violations']) + len(x['flake8_violations']) + len(x['pylint_violations']),
+    )[:end]:
+
+        violation_detail = item['pycodestyle_violations'] + item['flake8_violations'] + item['pylint_violations'],
+
+        if verbose:
+            line = "{0} {1} {2} {3} {4}".format(
+                len(violation_detail[0]),
+                len(item['pycodestyle_violations']),
+                len(item['flake8_violations']),
+                len(item['pylint_violations']),
+                item['path_to_file'],
+            )
+        else:
+            line = item['path_to_file']
+
+        print line
+        if details:
+            for detail_line in violation_detail:
+                if isinstance(detail_line, list):
+                    for subline in detail_line:
+                        if isinstance(subline, list):
+                            for subsubline in subline:
+                                print subsubline
+                        else:
+                            print subline
+                else:
+                    print detail_line
+            print
+
+        if fix:
+            autopep8_output = [i for i in autopep8(item['path_to_file']) if i]
+            print autopep8_output
+=======
+    # Sort and slice
+    for item in sorted(
+            results,
+            reverse=True,
+            key=lambda x: x['total_violations']
+    )[:end]:
+
+        print_item(item, verbose, details)
+>>>>>>> 81c272dbc332d842b7f8ab548ebeef2dd9a73995