From 81777e29c9dd1abccfd763df40a88c84f3ac73a4 Mon Sep 17 00:00:00 2001 From: coffeedogs Date: Tue, 15 May 2018 15:57:56 +0100 Subject: [PATCH] Added: Fabric taskrunner to check python files and report violations --- fabfile/README.md | 87 +++++++++++++++++++++ fabfile/__init__.py | 32 ++++++++ fabfile/lib.py | 150 +++++++++++++++++++++++++++++++++++ fabfile/requirements.txt | 9 +++ fabfile/tasks.py | 165 +++++++++++++++++++++++++++++++++++++++ setup.cfg | 10 +++ 6 files changed, 453 insertions(+) create mode 100644 fabfile/README.md create mode 100644 fabfile/__init__.py create mode 100644 fabfile/lib.py create mode 100644 fabfile/requirements.txt create mode 100644 fabfile/tasks.py create mode 100644 setup.cfg diff --git a/fabfile/README.md b/fabfile/README.md new file mode 100644 index 00000000..3d4d41be --- /dev/null +++ b/fabfile/README.md @@ -0,0 +1,87 @@ +# Fabric + +[Fabric](https://www.fabfile.org) is a Python library for performing devops tasks. You can thing of it a bit like a +makefile on steroids for Python. Its api abstracts away the clunky way you would run shell commands in Python, check +return values and manage stdio. Tasks may be targetted at particular hosts or group of hosts. + +# Using Fabric + + $ cd PyBitmessage + $ fab + +For a list of available commands: + + $ fab -l + +General fabric commandline help + + $ fab -h + +Arguments can be given: + + $ fab task1:arg1=arg1value,arg2=arg2value task2:option1 + +Tasks target hosts. Hosts can be specified with -H, or roles can be defined and you can target groups of hosts with -R. +Furthermore, you can use -- to run arbitrary shell commands rather than tasks: + + $ fab -H localhost task1 + $ fab -R webservers -- sudo /etc/httpd restart + +# Getting started + + * Install [Fabric](http://docs.fabfile.org/en/1.14/usage/fab.html), + [fabric-virtualenv](https://pypi.org/project/fabric-virtualenv/) and + [virtualenvwrapper](https://virtualenvwrapper.readthedocs.io/en/latest/) + system-wide using your preferred method. + * Create a virtualenv called pybitmessage and install fabfile/requirements.txt + $ mkvirtualenv -r fabfile/requirements.txt --system-site-packages pybitmessage-devops + * Ensure you can ssh localhost with no intervention, which may include: + * ssh [sshd_config server] and [ssh_config client] configuration + * authorized_keys file + * load ssh key + * check(!) and accept the host key + * From the PyBitmessage directory you can now run fab commands! + +# Rationale + +There are a number of advantages that should benefit us: + + * Common tasks can be writen in Python and executed consistently + * Common tasks are now under source control + * All developers can run the same commands, if the underlying command sequence for a task changes (after review, obv) + the user does not have to care + * Tasks can be combined either programmatically or on the commandline and run in series or parallel + * Whole environments can be managed very effectively in conjunction with a configuration management system + + +# /etc/ssh/sshd_config + +If you're going to be using ssh to connect to localhost you want to avoid weakening your security. The best way of +doing this is blocking port 22 with a firewall. As a belt and braces approach you can also edit the +/etc/ssh/sshd_config file to restrict login further: + +``` +PubkeyAuthentication no + +... + +Match ::1 + PubkeyAuthentication yes +``` +Adapted from [stackexchange](https://unix.stackexchange.com/questions/406245/limit-ssh-access-to-specific-clients-by-ip-address) + + +# ~/.ssh/config + +Fabric will honour your ~/.ssh/config file for your convenience. Since you will spend more time with this key unlocked +than others you should use a different key: + +``` +Host localhost + HostName localhost + IdentityFile ~/.ssh/id_rsa_localhost + +Host github + HostName github.com + IdentityFile ~/.ssh/id_rsa_github +``` diff --git a/fabfile/__init__.py b/fabfile/__init__.py new file mode 100644 index 00000000..67a68967 --- /dev/null +++ b/fabfile/__init__.py @@ -0,0 +1,32 @@ +""" +Fabric is a Python library for performing devops tasks. If you have Fabric installed (systemwide or via pip) you can +run commands like this: + + $ fab code_quality + +For a list of commands: + + $ fab -l + +For help on fabric itself: + + $ fab -h + +For more help on a particular command +""" + +from fabric.api import env + +from fabfile.tasks import code_quality + + +# Without this, `fab -l` would display the whole docstring as preamble +__doc__ = "" + +# This list defines which tasks are made available to the user +__all__ = [ + "code_quality", +] + +# Honour the user's ssh client configuration +env.use_ssh_config = True diff --git a/fabfile/lib.py b/fabfile/lib.py new file mode 100644 index 00000000..e22af53a --- /dev/null +++ b/fabfile/lib.py @@ -0,0 +1,150 @@ +# pylint: disable=not-context-manager +""" +A library of functions and constants for tasks to make use of. + +""" + +import os +import sys + +from fabric.api import run, hide +from fabric.context_managers import settings, shell_env +from fabvenv import virtualenv + + +FABRIC_ROOT = os.path.dirname(__file__) +PROJECT_ROOT = os.path.dirname(FABRIC_ROOT) +VENV_ROOT = os.path.expanduser(os.path.join('~', '.virtualenvs', 'pybitmessage-devops')) +PYTHONPATH = os.path.join(PROJECT_ROOT, 'src',) + + +def coerce_bool(value): + """Coerce a value into a boolean""" + if isinstance(value, bool): + return value + elif any( + [ + value in [0, '0'], + value.lower().startswith('n'), + ] + ): + return False + elif any( + [ + value in [1, '1'], + value.lower().startswith('y'), + ] + ): + return True + else: + sys.exit("Bad boolean value {}".format(value)) + + +def flatten(data): + """Recursively flatten lists""" + result = [] + for item in data: + if isinstance(item, list): + result.append(flatten(item)) + else: + result.append(item) + return result + + +def pycodestyle(path_to_file): + """Run pycodestyle on a file""" + with virtualenv(VENV_ROOT): + with hide('warnings', 'running', 'stdout', 'stderr'): + with settings(warn_only=True): + return run( + 'pycodestyle --config={0} {1}'.format( + os.path.join( + PROJECT_ROOT, + 'setup.cfg', + ), + path_to_file, + ), + ) + + +def flake8(path_to_file): + """Run flake8 on a file""" + with virtualenv(VENV_ROOT): + with hide('warnings', 'running', 'stdout'): + with settings(warn_only=True): + return run( + 'flake8 --config={0} {1}'.format( + os.path.join( + PROJECT_ROOT, + 'setup.cfg', + ), + path_to_file, + ), + ) + + +def pylint(path_to_file): + """Run pylint on a file""" + with virtualenv(VENV_ROOT): + with hide('warnings', 'running', 'stdout', 'stderr'): + with settings(warn_only=True): + with shell_env(PYTHONPATH=PYTHONPATH): + return run( + 'pylint --rcfile={0} {1}'.format( + os.path.join( + PROJECT_ROOT, + 'setup.cfg', + ), + path_to_file, + ), + ) + + +def autopep8(path_to_file): + """Run autopep8 on a file""" + with virtualenv(VENV_ROOT): + with hide('running'): + with settings(warn_only=True): + return run( + "autopep8 --experimental --aggressive --aggressive -i --max-line-length=119 {}".format( + path_to_file + ), + ) + + +def get_filtered_pycodestyle_output(path_to_file): + """Clean up the raw results for pycodestyle""" + + return [ + i + for i in pycodestyle(path_to_file).split(os.linesep) + if i + ] + + +def get_filtered_flake8_output(path_to_file): + """Clean up the raw results for flake8""" + + return [ + i + for i in flake8(path_to_file).split(os.linesep) + if i + ] + + +def get_filtered_pylint_output(path_to_file): + """Clean up the raw results for pylint""" + + return [ + i + for i in pylint(path_to_file).split(os.linesep) + if all([ + i, + not i.startswith(' '), + not i.startswith('\r'), + not i.startswith('-'), + not i.startswith('Y'), + not i.startswith('*'), + not i.startswith('Using config file'), + ]) + ] diff --git a/fabfile/requirements.txt b/fabfile/requirements.txt new file mode 100644 index 00000000..0d0a3962 --- /dev/null +++ b/fabfile/requirements.txt @@ -0,0 +1,9 @@ +# These requirements are for the Fabric commands that support devops tasks for PyBitmessage, not for running +# PyBitmessage itself. +# TODO: Consider moving to an extra_requires group in setup.py + +pycodestyle==2.3.1 # https://github.com/PyCQA/pycodestyle/issues/741 +flake8 +pylint +-e git://github.com/hhatto/autopep8.git@ver1.2.2#egg=autopep8 # Needed for fixing E712 +pep8 # autopep8 doesn't seem to like pycodestyle diff --git a/fabfile/tasks.py b/fabfile/tasks.py new file mode 100644 index 00000000..22c5d147 --- /dev/null +++ b/fabfile/tasks.py @@ -0,0 +1,165 @@ +# pylint: disable=not-context-manager +""" +Fabric tasks for PyBitmessage devops operations. + +# pylint: disable=not-context-manager +""" + +import os +import sys + +from fabric.api import run, task, hide, cd +from fabvenv import virtualenv + +from fabfile.lib import ( + autopep8, PROJECT_ROOT, VENV_ROOT, coerce_bool, flatten, + get_filtered_pycodestyle_output, get_filtered_flake8_output, get_filtered_pylint_output, +) + + +def get_tool_results(file_list): + """Take a list of files and resuln the results of applying the tools""" + + results = [] + for path_to_file in file_list: + result = {} + result['pycodestyle_violations'] = get_filtered_pycodestyle_output(path_to_file) + result['flake8_violations'] = get_filtered_flake8_output(path_to_file) + result['pylint_violations'] = get_filtered_pylint_output(path_to_file) + result['path_to_file'] = path_to_file + result['total_violations'] = sum([ + len(result['pycodestyle_violations']), + len(result['flake8_violations']), + len(result['pylint_violations']), + ]) + results.append(result) + return results + + +def print_results(results, top, verbose, details): + """Print an item with the appropriate verbosity / detail""" + + if verbose: + print ''.join( + [ + os.linesep, + 'total pycodestyle flake8 pylint path_to_file', + os.linesep, + ] + ) + + for item in sort_and_slice(results, top): + + if verbose: + line = "{0} {1} {2} {3} {4}".format( + item['total_violations'], + len(item['pycodestyle_violations']), + len(item['flake8_violations']), + len(item['pylint_violations']), + item['path_to_file'], + ) + else: + line = item['path_to_file'] + print line + + if details: + print "pycodestyle:" + for detail in flatten(item['pycodestyle_violations']): + print detail + print + + print "flake8:" + for detail in flatten(item['flake8_violations']): + print detail + print + + print "pylint:" + for detail in flatten(item['pylint_violations']): + print detail + print + + +def sort_and_slice(results, top): + """Sort dictionary items by the `total_violations` key and honour top""" + + returnables = [] + for item in sorted( + results, + reverse=True, + key=lambda x: x['total_violations'] + )[:top]: + returnables.append(item) + return returnables + + +def generate_file_list(filename): + """Return an unfiltered list of absolute paths to the files to act on""" + + with hide('warnings', 'running', 'stdout'): + with virtualenv(VENV_ROOT): + + if filename: + filename = os.path.abspath(filename) + if not os.path.exists(filename): + print "Bad filename, specify a Python file" + sys.exit(1) + else: + file_list = [filename] + + else: + with cd(PROJECT_ROOT): + file_list = [ + os.path.abspath(i.rstrip('\r')) + for i in run('find . -name "*.py"').split(os.linesep) + ] + + return file_list + + +@task +def code_quality(verbose=True, details=False, fix=False, filename=None, top=10): + """ + Check code quality. + + By default this command will analyse each Python file in the project with a variety of tools and display the count + or details of the violations discovered, sorted by most violations first. + + Default usage: + + $ fab -H localhost code_quality + + :param top: Display / fix only the top N violating files, a value of 0 will display / fix all files + :type top: int, default 10 + :param verbose: Display a header and the counts, without this you just get the filenames in order + :type verbose: bool, default True + :param details: Display the violations one per line after the count / file summary + :type details: bool, default False + :param fix: Run autopep8 aggressively on the displayed file(s) + :type fix: bool, default False + :param filename: Rather than analysing all files and displaying / fixing the top N, just analyse / display / fix + the specified file + :type filename: string, valid path to a file, default all files in the project + :return: This fabric task has an exit status equal to the total number of violations and uses stdio but it does + not return anything if you manage to call it successfully from Python + :rtype: None + + Intended to be temporary until we have improved code quality and have safeguards to maintain it in place. + + """ + + verbose = coerce_bool(verbose) + details = coerce_bool(details) + fix = coerce_bool(fix) + top = int(top) or -1 + + file_list = generate_file_list(filename) + results = get_tool_results(file_list) + + if fix: + for item in sort_and_slice(results, top): + autopep8(item['path_to_file']) + # Recalculate results after autopep8 to surprise the user the least + results = get_tool_results(file_list) + + print_results(results, top, verbose, details) + sys.exit(sum([item['total_violations'] for item in results])) diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..32abcdc7 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,10 @@ +[pycodestyle] +max-line-length = 119 + +[flake8] +max-line-length = 119 +ignore = E722 + +# pylint +[MESSAGES CONTROL] +disable=invalid-name,bare-except