diff --git a/fabfile/README.md b/fabfile/README.md new file mode 100644 index 00000000..53430915 --- /dev/null +++ b/fabfile/README.md @@ -0,0 +1,55 @@ +# Fabric + +[Fabric](https://www.fabfile.org) is a Python library for performing devops tasks. You can thing of it a bit like a +makefile on steroids for Python. Its api abstracts away the clunky way you would run shell commands in Python, check +return values and manage stdio. Tasks may be targetted at particular hosts or group of hosts. + +# Using Fabric + + $ cd PyBitmessage + $ fab + +For a list of available commands: + + $ fab -l + +General fabric commandline help + + $ fab -h + +Arguments can be given: + + $ fab task1:arg1=arg1value,arg2=arg2value task2:option1 + +Tasks target hosts. Hosts can be specified with -H, or roles can be defined and you can target groups of hosts with -R. +Furthermore, you can use -- to run arbitrary shell commands rather than tasks: + + $ fab -H localhost task1 + $ fab -R webservers -- sudo /etc/httpd restart + +# Getting started + + * Install Fabric, fabric-virtualenv, virtualenvwrapper system-wide (log in to a new terminal afterwards if you're installing + virtualenvwrappers for the first time) + $ sudo apt install Fabric virtualenvwrapper; sudo pip install fabric-virtualenv + * Create a virtualenv called pybitmessage and install fabfile/requirements.txt + $ mkvirtualenv --system-site-packages pybitmessage-devops + $ pip install -r fabfile/requirements.txt + * Ensure you can ssh localhost with no intervention, which may include: + * settings in ~/.ssh/config + * authorized_keys file + * load ssh key + * check(!) and accept the host key + * From the PyBitmessage directory you can now run fab commands! + +# Rationale + +There are a number of advantages that should benefit us: + + * Common tasks can be writen in Python and executed consistently + * Common tasks are now under source control + * All developers can run the same commands, if the underlying command sequence for a task changes (after review, obv) + the user does not have to care + * Tasks can be combined either programmatically or on the commandline and run in series or parallel + * Whoee environemnts can be managed very effectively in conjunction with a configuration management system + diff --git a/fabfile/__init__.py b/fabfile/__init__.py new file mode 100644 index 00000000..5d252a68 --- /dev/null +++ b/fabfile/__init__.py @@ -0,0 +1,14 @@ +""" +Fabric is a Python library for performing devops tasks. If you have Fabric installed (systemwide or via pip) you can +run commands like this: + + $ fab code_quality + +For a list of commands + +""" + +from fabfile.tasks import code_quality + + +__all__ = ["code_quality"] diff --git a/fabfile/lib.py b/fabfile/lib.py new file mode 100644 index 00000000..8c54ab32 --- /dev/null +++ b/fabfile/lib.py @@ -0,0 +1,133 @@ +"""A library of functions and constrants for tasks to make use of.""" + +import os +<<<<<<< HEAD +======= +import sys +>>>>>>> 81c272dbc332d842b7f8ab548ebeef2dd9a73995 + +from fabric.api import run, hide +from fabric.context_managers import settings +from fabvenv import virtualenv + + +FABRIC_ROOT = os.path.dirname(__file__) +PROJECT_ROOT = os.path.dirname(FABRIC_ROOT) +<<<<<<< HEAD +VENV_ROOT = os.path.expanduser(os.path.join('~', '.virtualenvs', 'pybitmessage')) +======= +VENV_ROOT = os.path.expanduser(os.path.join('~', '.virtualenvs', 'pybitmessage-devops')) + + +def coerce_bool(value): + """Coerce a value into a boolean""" + if isinstance(value, bool): + return value + elif any( + [ + value in [0, '0'], + value.lower().startswith('n'), + ] + ): + return False + elif any( + [ + value in [1, '1'], + value.lower().startswith('y'), + ] + ): + return True + else: + print "Bad boolean value {}".format(value) + sys.exit(1) + + +def flatten(data): + """Recursively flatten lists""" + result = [] + for item in data: + if isinstance(item, list): + result.append(flatten(item)) + else: + result.append(item) + return result +>>>>>>> 81c272dbc332d842b7f8ab548ebeef2dd9a73995 + + +def pycodestyle(path_to_file): + """Run pycodestyle on a file""" + with virtualenv(VENV_ROOT): +<<<<<<< HEAD + with hide('warnings', 'running'): +======= + with hide('warnings', 'running', 'stdout', 'stderr'): # pylint: disable=not-context-manager +>>>>>>> 81c272dbc332d842b7f8ab548ebeef2dd9a73995 + with settings(warn_only=True): + return run( + 'pycodestyle --config={0} {1}'.format( + os.path.join( + PROJECT_ROOT, + 'setup.cfg', + ), + path_to_file, + ), + ) + + +def flake8(path_to_file): + """Run flake8 on a file""" + with virtualenv(VENV_ROOT): +<<<<<<< HEAD + with hide('warnings', 'running'): +======= + with hide('warnings', 'running', 'stdout'): # pylint: disable=not-context-manager +>>>>>>> 81c272dbc332d842b7f8ab548ebeef2dd9a73995 + with settings(warn_only=True): + return run( + 'flake8 --config={0} {1}'.format( + os.path.join( + PROJECT_ROOT, + 'setup.cfg', + ), + path_to_file, + ), + ) + + +def pylint(path_to_file): + """Run pylint on a file""" + with virtualenv(VENV_ROOT): +<<<<<<< HEAD + with hide('warnings', 'running', 'stdout', 'stderr'): +======= + with hide('warnings', 'running', 'stdout', 'stderr'): # pylint: disable=not-context-manager +>>>>>>> 81c272dbc332d842b7f8ab548ebeef2dd9a73995 + with settings(warn_only=True): + return run( + 'pylint --rcfile={0} {1}'.format( + os.path.join( + PROJECT_ROOT, + 'setup.cfg', + ), + path_to_file, + ), + ) + + +def autopep8(path_to_file): + """Run autopep8 on a file""" + with virtualenv(VENV_ROOT): +<<<<<<< HEAD + with hide('running'): + with settings(warn_only=True): + return run( + "autopep8 -ai --max-line-length=119 --hang-closing {}".format( +======= + with hide('running'): # pylint: disable=not-context-manager + with settings(warn_only=True): + return run( + "autopep8 --experimental --aggressive --aggressive -i --max-line-length=119 {}".format( +>>>>>>> 81c272dbc332d842b7f8ab548ebeef2dd9a73995 + path_to_file + ), + ) diff --git a/fabfile/requirements.txt b/fabfile/requirements.txt new file mode 100644 index 00000000..0d0a3962 --- /dev/null +++ b/fabfile/requirements.txt @@ -0,0 +1,9 @@ +# These requirements are for the Fabric commands that support devops tasks for PyBitmessage, not for running +# PyBitmessage itself. +# TODO: Consider moving to an extra_requires group in setup.py + +pycodestyle==2.3.1 # https://github.com/PyCQA/pycodestyle/issues/741 +flake8 +pylint +-e git://github.com/hhatto/autopep8.git@ver1.2.2#egg=autopep8 # Needed for fixing E712 +pep8 # autopep8 doesn't seem to like pycodestyle diff --git a/fabfile/tasks.py b/fabfile/tasks.py new file mode 100644 index 00000000..b155601a --- /dev/null +++ b/fabfile/tasks.py @@ -0,0 +1,269 @@ +""" +<<<<<<< HEAD +Fabric is a Python library for performing devops tasks. If you have Fabric installed (systemwide or via pip) you can +run commands from within the project directory like this: + + $ fab code_quality + +For a list of commands: + + $ fab -l + +Known bugs: + +======= +Fabric tasks for PyBitmessage devops operations. +>>>>>>> 81c272dbc332d842b7f8ab548ebeef2dd9a73995 +""" + +import os +import sys + +<<<<<<< HEAD +import fabric +from fabric.api import run, task, hide, cd +from fabfile.lib import pycodestyle, flake8, pylint, autopep8, PROJECT_ROOT, VENV_ROOT +from fabvenv import virtualenv + + +fabric.state.output['running'] = False + + +@task +def code_quality(top=10, verbose=True, details=False, fix=True, filename=None): + """Intended to be temporary until we have improved code quality and have safeguards to maintain it in place.""" + + end = int(top) if int(top) else -1 + + if verbose == "0": + verbose = False + if not isinstance(verbose, bool): + print "Bad verbosity value, try 0" + sys.exit(1) + + if details == "1": + details = True + if not isinstance(details, bool): + print "Bad details value, try 1" + sys.exit(1) + + if fix == "0": + fix = False + if not isinstance(fix, bool): + print "Bad fix value, try 0" + sys.exit(1) + + with hide('warnings'): +======= +from fabric.api import run, task, hide, cd +from fabvenv import virtualenv + +from fabfile.lib import pycodestyle, flake8, pylint, autopep8, PROJECT_ROOT, VENV_ROOT, coerce_bool, flatten + + +def get_results_for_tools_applied_to_file_list(file_list): + """Take a list of files and resuln the results of applying the tools""" + results = [] + for path_to_file in file_list: + result = {} + + result['pycodestyle_violations'] = [ + i + for i in pycodestyle(path_to_file).split("\n") + if i + ] + result['flake8_violations'] = [ + i + for i in flake8(path_to_file).split("\n") + if i + ] + result['pylint_violations'] = [ + i + for i in pylint(path_to_file).split("\n") + if all([ + i, + not i.startswith(' '), + not i.startswith('\r'), + not i.startswith('-'), + not i.startswith('Y'), + not i.startswith('*'), + not i.startswith('Using config file'), + ]) + ] + result['path_to_file'] = path_to_file + result['total_violations'] = sum( + [ + len(result['pycodestyle_violations']), + len(result['flake8_violations']), + len(result['pylint_violations']), + ] + ) + results.append(result) + return results + + +def print_item(item, verbose, details): + """Print an item with the appropriate verbosity / detail""" + if verbose: + line = "{0} {1} {2} {3} {4}".format( + item['total_violations'], + len(item['pycodestyle_violations']), + len(item['flake8_violations']), + len(item['pylint_violations']), + item['path_to_file'], + ) + else: + line = item['path_to_file'] + print line + + if details: + print "pycodestyle:" + for detail in flatten(item['pycodestyle_violations']): + print detail + print + + print "flake8:" + for detail in flatten(item['flake8_violations']): + print detail + print + + print "pylint:" + for detail in flatten(item['pylint_violations']): + print detail + print + + +@task +def code_quality(top=10, verbose=True, details=False, fix=False, filename=None): + """ + Check code quality. + + By default this command will analyse each Python file in the project with a variety of tools and display the count + or details of the violations discovered, sorted by most iolations first. + + Default usage: + + $ fab -H localhost code_quality + + Options: + top: Display / fix only the top violating files, a value of 0 will display / fix all files + verbose: Display a header and the counts. Without this you just get the filenames in order + details: Display the violations one per line after the count/file summary + fix: Run autopep8 on the displayed file(s) + filename: Rather than analysing all files and displaying / fixing the top N, just analyse / diaply / fix the + specified file + + Intended to be temporary until we have improved code quality and have safeguards to maintain it in place. + + """ + + verbose = coerce_bool(verbose) + details = coerce_bool(details) + fix = coerce_bool(fix) + end = int(top) if int(top) else -1 + + with hide('warnings', 'running'): # pylint: disable=not-context-manager +>>>>>>> 81c272dbc332d842b7f8ab548ebeef2dd9a73995 + with virtualenv(VENV_ROOT): + if filename: + filename = os.path.abspath(filename) + if not os.path.exists(filename): + print "Bad filename, specify a Python file" + sys.exit(1) + else: + file_list = [filename] + else: +<<<<<<< HEAD + with cd(PROJECT_ROOT): + file_list = run('find . -name "*.py"', capture=True).split("\n") + + results = [] + for path_to_file in file_list: + + results.append( + { + 'pycodestyle_violations': [ + i + for i in pycodestyle(path_to_file).split("\n") + if i + ], + 'flake8_violations': [ + i + for i in flake8(path_to_file).split("\n") + if i + ], + 'pylint_violations': [ + i + for i in pylint(path_to_file).split("\n") + if all([ + i, + not i.startswith(' '), + not i.startswith('-'), + not i.startswith('Y'), + not i.startswith('*'), + ]) + ], + 'path_to_file': path_to_file, + } + ) +======= + with cd(PROJECT_ROOT): # pylint: disable=not-context-manager + file_list = run('find . -name "*.py"').split("\n") + + if fix: + for path_to_file in file_list: + autopep8(path_to_file) + + results = get_results_for_tools_applied_to_file_list(file_list) +>>>>>>> 81c272dbc332d842b7f8ab548ebeef2dd9a73995 + + if verbose: + print "\ntotal pycodestyle flake8 pylint path_to_file\n" + +<<<<<<< HEAD + for item in sorted( + results, + reverse=True, + key=lambda x: len(x['pycodestyle_violations']) + len(x['flake8_violations']) + len(x['pylint_violations']), + )[:end]: + + violation_detail = item['pycodestyle_violations'] + item['flake8_violations'] + item['pylint_violations'], + + if verbose: + line = "{0} {1} {2} {3} {4}".format( + len(violation_detail[0]), + len(item['pycodestyle_violations']), + len(item['flake8_violations']), + len(item['pylint_violations']), + item['path_to_file'], + ) + else: + line = item['path_to_file'] + + print line + if details: + for detail_line in violation_detail: + if isinstance(detail_line, list): + for subline in detail_line: + if isinstance(subline, list): + for subsubline in subline: + print subsubline + else: + print subline + else: + print detail_line + print + + if fix: + autopep8_output = [i for i in autopep8(item['path_to_file']) if i] + print autopep8_output +======= + # Sort and slice + for item in sorted( + results, + reverse=True, + key=lambda x: x['total_violations'] + )[:end]: + + print_item(item, verbose, details) +>>>>>>> 81c272dbc332d842b7f8ab548ebeef2dd9a73995