319 lines
11 KiB
Raw Normal View History

# pylint: disable=not-context-manager
Fabric tasks for PyBitmessage devops operations.
Note that where tasks declare params to be bools, they use coerce_bool() and so will accept any commandline (string)
representation of true or false that coerce_bool() understands.
import os
import sys
from fabric.api import run, task, hide, cd, settings, sudo
from fabric.contrib.project import rsync_project
from fabvenv import virtualenv
from fabfile.lib import (
autopep8, PROJECT_ROOT, VENV_ROOT, coerce_bool, flatten, filelist_from_git, default_hosts,
get_filtered_pycodestyle_output, get_filtered_flake8_output, get_filtered_pylint_output,
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(os.path.dirname(__file__)), 'src'))) # noqa:E402
from version import softwareVersion # pylint: disable=wrong-import-position
def get_tool_results(file_list):
"""Take a list of files and resuln the results of applying the tools"""
results = []
for path_to_file in file_list:
result = {}
result['pycodestyle_violations'] = get_filtered_pycodestyle_output(path_to_file)
result['flake8_violations'] = get_filtered_flake8_output(path_to_file)
result['pylint_violations'] = get_filtered_pylint_output(path_to_file)
result['path_to_file'] = path_to_file
result['total_violations'] = sum([
return results
def print_results(results, top, verbose, details):
"""Print an item with the appropriate verbosity / detail"""
if verbose and results:
print ''.join(
'total pycodestyle flake8 pylint path_to_file',
for item in sort_and_slice(results, top):
if verbose:
line = "{0} {1} {2} {3} {4}".format(
line = item['path_to_file']
print line
if details:
print "pycodestyle:"
for detail in flatten(item['pycodestyle_violations']):
print detail
print "flake8:"
for detail in flatten(item['flake8_violations']):
print detail
print "pylint:"
for detail in flatten(item['pylint_violations']):
print detail
def sort_and_slice(results, top):
"""Sort dictionary items by the `total_violations` key and honour top"""
returnables = []
for item in sorted(
key=lambda x: x['total_violations']
return returnables
def generate_file_list(filename):
"""Return an unfiltered list of absolute paths to the files to act on"""
with hide('warnings', 'running', 'stdout'):
with virtualenv(VENV_ROOT):
if filename:
filename = os.path.abspath(filename)
if not os.path.exists(filename):
print "Bad filename, specify a Python file"
file_list = [filename]
with cd(PROJECT_ROOT):
file_list = [
for i in run('find . -name "*.py"').split(os.linesep)
return file_list
def create_dependency_graphs():
To better understand the relationship between methods, dependency graphs can be drawn between functions and
Since the resulting images are large and could be out of date on the next commit, storing them in the repo is
pointless. Instead, it makes sense to build a dependency graph for a particular, documented version of the code.
.. todo:: Consider saving a hash of the intermediate dotty file so unnecessary image generation can be avoided.
with virtualenv(VENV_ROOT):
with hide('running', 'stdout'):
# .. todo:: consider a better place to put this, use a particular commit
with cd(PROJECT_ROOT):
with settings(warn_only=True):
if run('stat pyan').return_code:
run('git clone')
with cd(os.path.join(PROJECT_ROOT, 'pyan')):
run('git checkout pre-python3')
# .. todo:: Use better settings. This is MVP to make a diagram
with cd(PROJECT_ROOT):
file_list = run("find . -type f -name '*.py' ! -path './src/.eggs/*'").split('\r\n')
for cmd in [
'neato -Goverlap=false -Tpng > deps-neato.png',
'sfdp -Goverlap=false -Tpng > deps-sfdp.png',
'dot -Goverlap=false -Tpng > deps-dot.png',
pyan_cmd = './pyan/ {} --dot'.format(' '.join(file_list))
sed_cmd = r"sed s'/http\-old/http_old/'g" # dot doesn't like dashes
run('|'.join([pyan_cmd, sed_cmd, cmd]))
run('mv *.png docs/_build/html/_static/')
def code_quality(verbose=True, details=False, fix=False, filename=None, top=10, rev=None):
Check code quality.
By default this command will analyse each Python file in the project with a variety of tools and display the count
or details of the violations discovered, sorted by most violations first.
Default usage:
$ fab -H localhost code_quality
:param rev: If not None, act on files changed since this commit. 'cached/staged' and 'working' have special meaning
:type rev: str or None, default None
:param top: Display / fix only the top N violating files, a value of 0 will display / fix all files
:type top: int, default 10
:param verbose: Display a header and the counts, without this you just get the filenames in order
:type verbose: bool, default True
:param details: Display the violations one per line after the count / file summary
:type details: bool, default False
:param fix: Run autopep8 aggressively on the displayed file(s)
:type fix: bool, default False
:param filename: Don't test/fix the top N, just the specified file
:type filename: string, valid path to a file, default all files in the project
:return: None, exit status equals total number of violations
:rtype: None
Intended to be temporary until we have improved code quality and have safeguards to maintain it in place.
# pylint: disable=too-many-arguments
verbose = coerce_bool(verbose)
details = coerce_bool(details)
fix = coerce_bool(fix)
top = int(top) or -1
file_list = generate_file_list(filename) if not rev else filelist_from_git(rev)
results = get_tool_results(file_list)
if fix:
for item in sort_and_slice(results, top):
# Recalculate results after autopep8 to surprise the user the least
results = get_tool_results(file_list)
print_results(results, top, verbose, details)
sys.exit(sum([item['total_violations'] for item in results]))
def test():
"""Run tests on the code"""
with cd(PROJECT_ROOT):
with virtualenv(VENV_ROOT):
run('pip uninstall -y pybitmessage')
run('python install')
run('pybitmessage -t')
run('python test')
def build_docs(dep_graph=False, apidoc=True):
Build the documentation locally.
:param dep_graph: Build the dependency graphs
:type dep_graph: Bool, default False
:param apidoc: Build the automatically generated rst files from the source code
:type apidoc: Bool, default True
Default usage:
$ fab -H localhost build_docs
First, a dependency graph is generated and converted into an image that is referenced in the development page.
Next, the sphinx-apidoc command is (usually) run which searches the code. As part of this it loads the modules and
if this has side-effects then they will be evident. Any documentation strings that make use of Python documentation
conventions (like parameter specification) or the Restructured Text (RsT) syntax will be extracted.
Next, the `make html` command is run to generate HTML output. Other formats (epub, pdf) are available.
.. todo:: support other languages
apidoc = coerce_bool(apidoc)
if coerce_bool(dep_graph):
with virtualenv(VENV_ROOT):
with hide('running'):
apidoc_result = 0
if apidoc:
run('mkdir -p {}'.format(os.path.join(PROJECT_ROOT, 'docs', 'autodoc')))
with cd(os.path.join(PROJECT_ROOT, 'docs', 'autodoc')):
with settings(warn_only=True):
run('rm *.rst')
with cd(os.path.join(PROJECT_ROOT, 'docs')):
apidoc_result = run('sphinx-apidoc -o autodoc ..').return_code
with cd(os.path.join(PROJECT_ROOT, 'docs')):
make_result = run('make html').return_code
return_code = apidoc_result + make_result
def push_docs(path=None):
Upload the generated docs to a public server.
Default usage:
$ fab -H localhost push_docs
.. todo:: support other languages
.. todo:: integrate with configuration management data to get web root and webserver restart command
# Making assumptions
WEB_ROOT = path if path is not None else os.path.join('var', 'www', 'html', 'pybitmessage', 'en', 'latest')
VERSION_ROOT = os.path.join(os.path.dirname(WEB_ROOT), softwareVersion)
local_dir=os.path.join(PROJECT_ROOT, 'docs', '_build', 'html')
result = run('ln -sf {0} {1}'.format(WEB_ROOT, VERSION_ROOT))
if result.return_code:
print 'Linking the new release failed'
# More assumptions
sudo('systemctl restart apache2')
def clean():
"""Clean up files generated by fabric commands."""
with hide('running', 'stdout'):
with cd(PROJECT_ROOT):
run(r"find . -name '*.pyc' -exec rm '{}' \;")