coffeedogs
4 years ago
6 changed files with 453 additions and 0 deletions
@ -0,0 +1,87 @@ |
|||
# Fabric |
|||
|
|||
[Fabric](https://www.fabfile.org) is a Python library for performing devops tasks. You can thing of it a bit like a |
|||
makefile on steroids for Python. Its api abstracts away the clunky way you would run shell commands in Python, check |
|||
return values and manage stdio. Tasks may be targetted at particular hosts or group of hosts. |
|||
|
|||
# Using Fabric |
|||
|
|||
$ cd PyBitmessage |
|||
$ fab <task_name> |
|||
|
|||
For a list of available commands: |
|||
|
|||
$ fab -l |
|||
|
|||
General fabric commandline help |
|||
|
|||
$ fab -h |
|||
|
|||
Arguments can be given: |
|||
|
|||
$ fab task1:arg1=arg1value,arg2=arg2value task2:option1 |
|||
|
|||
Tasks target hosts. Hosts can be specified with -H, or roles can be defined and you can target groups of hosts with -R. |
|||
Furthermore, you can use -- to run arbitrary shell commands rather than tasks: |
|||
|
|||
$ fab -H localhost task1 |
|||
$ fab -R webservers -- sudo /etc/httpd restart |
|||
|
|||
# Getting started |
|||
|
|||
* Install [Fabric](http://docs.fabfile.org/en/1.14/usage/fab.html), |
|||
[fabric-virtualenv](https://pypi.org/project/fabric-virtualenv/) and |
|||
[virtualenvwrapper](https://virtualenvwrapper.readthedocs.io/en/latest/) |
|||
system-wide using your preferred method. |
|||
* Create a virtualenv called pybitmessage and install fabfile/requirements.txt |
|||
$ mkvirtualenv -r fabfile/requirements.txt --system-site-packages pybitmessage-devops |
|||
* Ensure you can ssh localhost with no intervention, which may include: |
|||
* ssh [sshd_config server] and [ssh_config client] configuration |
|||
* authorized_keys file |
|||
* load ssh key |
|||
* check(!) and accept the host key |
|||
* From the PyBitmessage directory you can now run fab commands! |
|||
|
|||
# Rationale |
|||
|
|||
There are a number of advantages that should benefit us: |
|||
|
|||
* Common tasks can be writen in Python and executed consistently |
|||
* Common tasks are now under source control |
|||
* All developers can run the same commands, if the underlying command sequence for a task changes (after review, obv) |
|||
the user does not have to care |
|||
* Tasks can be combined either programmatically or on the commandline and run in series or parallel |
|||
* Whole environments can be managed very effectively in conjunction with a configuration management system |
|||
|
|||
<a name="sshd_config"></a> |
|||
# /etc/ssh/sshd_config |
|||
|
|||
If you're going to be using ssh to connect to localhost you want to avoid weakening your security. The best way of |
|||
doing this is blocking port 22 with a firewall. As a belt and braces approach you can also edit the |
|||
/etc/ssh/sshd_config file to restrict login further: |
|||
|
|||
``` |
|||
PubkeyAuthentication no |
|||
|
|||
... |
|||
|
|||
Match ::1 |
|||
PubkeyAuthentication yes |
|||
``` |
|||
Adapted from [stackexchange](https://unix.stackexchange.com/questions/406245/limit-ssh-access-to-specific-clients-by-ip-address) |
|||
|
|||
<a name="ssh_config"></a> |
|||
# ~/.ssh/config |
|||
|
|||
Fabric will honour your ~/.ssh/config file for your convenience. Since you will spend more time with this key unlocked |
|||
than others you should use a different key: |
|||
|
|||
``` |
|||
Host localhost |
|||
HostName localhost |
|||
IdentityFile ~/.ssh/id_rsa_localhost |
|||
|
|||
Host github |
|||
HostName github.com |
|||
IdentityFile ~/.ssh/id_rsa_github |
|||
``` |
|||
@ -0,0 +1,32 @@ |
|||
""" |
|||
Fabric is a Python library for performing devops tasks. If you have Fabric installed (systemwide or via pip) you can |
|||
run commands like this: |
|||
|
|||
$ fab code_quality |
|||
|
|||
For a list of commands: |
|||
|
|||
$ fab -l |
|||
|
|||
For help on fabric itself: |
|||
|
|||
$ fab -h |
|||
|
|||
For more help on a particular command |
|||
""" |
|||
|
|||
from fabric.api import env |
|||
|
|||
from fabfile.tasks import code_quality |
|||
|
|||
|
|||
# Without this, `fab -l` would display the whole docstring as preamble |
|||
__doc__ = "" |
|||
|
|||
# This list defines which tasks are made available to the user |
|||
__all__ = [ |
|||
"code_quality", |
|||
] |
|||
|
|||
# Honour the user's ssh client configuration |
|||
env.use_ssh_config = True |
|||
@ -0,0 +1,150 @@ |
|||
# pylint: disable=not-context-manager |
|||
""" |
|||
A library of functions and constants for tasks to make use of. |
|||
|
|||
""" |
|||
|
|||
import os |
|||
import sys |
|||
|
|||
from fabric.api import run, hide |
|||
from fabric.context_managers import settings, shell_env |
|||
from fabvenv import virtualenv |
|||
|
|||
|
|||
FABRIC_ROOT = os.path.dirname(__file__) |
|||
PROJECT_ROOT = os.path.dirname(FABRIC_ROOT) |
|||
VENV_ROOT = os.path.expanduser(os.path.join('~', '.virtualenvs', 'pybitmessage-devops')) |
|||
PYTHONPATH = os.path.join(PROJECT_ROOT, 'src',) |
|||
|
|||
|
|||
def coerce_bool(value): |
|||
"""Coerce a value into a boolean""" |
|||
if isinstance(value, bool): |
|||
return value |
|||
elif any( |
|||
[ |
|||
value in [0, '0'], |
|||
value.lower().startswith('n'), |
|||
] |
|||
): |
|||
return False |
|||
elif any( |
|||
[ |
|||
value in [1, '1'], |
|||
value.lower().startswith('y'), |
|||
] |
|||
): |
|||
return True |
|||
else: |
|||
sys.exit("Bad boolean value {}".format(value)) |
|||
|
|||
|
|||
def flatten(data): |
|||
"""Recursively flatten lists""" |
|||
result = [] |
|||
for item in data: |
|||
if isinstance(item, list): |
|||
result.append(flatten(item)) |
|||
else: |
|||
result.append(item) |
|||
return result |
|||
|
|||
|
|||
def pycodestyle(path_to_file): |
|||
"""Run pycodestyle on a file""" |
|||
with virtualenv(VENV_ROOT): |
|||
with hide('warnings', 'running', 'stdout', 'stderr'): |
|||
with settings(warn_only=True): |
|||
return run( |
|||
'pycodestyle --config={0} {1}'.format( |
|||
os.path.join( |
|||
PROJECT_ROOT, |
|||
'setup.cfg', |
|||
), |
|||
path_to_file, |
|||
), |
|||
) |
|||
|
|||
|
|||
def flake8(path_to_file): |
|||
"""Run flake8 on a file""" |
|||
with virtualenv(VENV_ROOT): |
|||
with hide('warnings', 'running', 'stdout'): |
|||
with settings(warn_only=True): |
|||
return run( |
|||
'flake8 --config={0} {1}'.format( |
|||
os.path.join( |
|||
PROJECT_ROOT, |
|||
'setup.cfg', |
|||
), |
|||
path_to_file, |
|||
), |
|||
) |
|||
|
|||
|
|||
def pylint(path_to_file): |
|||
"""Run pylint on a file""" |
|||
with virtualenv(VENV_ROOT): |
|||
with hide('warnings', 'running', 'stdout', 'stderr'): |
|||
with settings(warn_only=True): |
|||
with shell_env(PYTHONPATH=PYTHONPATH): |
|||
return run( |
|||
'pylint --rcfile={0} {1}'.format( |
|||
os.path.join( |
|||
PROJECT_ROOT, |
|||
'setup.cfg', |
|||
), |
|||
path_to_file, |
|||
), |
|||
) |
|||
|
|||
|
|||
def autopep8(path_to_file): |
|||
"""Run autopep8 on a file""" |
|||
with virtualenv(VENV_ROOT): |
|||
with hide('running'): |
|||
with settings(warn_only=True): |
|||
return run( |
|||
"autopep8 --experimental --aggressive --aggressive -i --max-line-length=119 {}".format( |
|||
path_to_file |
|||
), |
|||
) |
|||
|
|||
|
|||
def get_filtered_pycodestyle_output(path_to_file): |
|||
"""Clean up the raw results for pycodestyle""" |
|||
|
|||
return [ |
|||
i |
|||
for i in pycodestyle(path_to_file).split(os.linesep) |
|||
if i |
|||
] |
|||
|
|||
|
|||
def get_filtered_flake8_output(path_to_file): |
|||
"""Clean up the raw results for flake8""" |
|||
|
|||
return [ |
|||
i |
|||
for i in flake8(path_to_file).split(os.linesep) |
|||
if i |
|||
] |
|||
|
|||
|
|||
def get_filtered_pylint_output(path_to_file): |
|||
"""Clean up the raw results for pylint""" |
|||
|
|||
return [ |
|||
i |
|||
for i in pylint(path_to_file).split(os.linesep) |
|||
if all([ |
|||
i, |
|||
not i.startswith(' '), |
|||
not i.startswith('\r'), |
|||
not i.startswith('-'), |
|||
not i.startswith('Y'), |
|||
not i.startswith('*'), |
|||
not i.startswith('Using config file'), |
|||
]) |
|||
] |
|||
@ -0,0 +1,9 @@ |
|||
# These requirements are for the Fabric commands that support devops tasks for PyBitmessage, not for running |
|||
# PyBitmessage itself. |
|||
# TODO: Consider moving to an extra_requires group in setup.py |
|||
|
|||
pycodestyle==2.3.1 # https://github.com/PyCQA/pycodestyle/issues/741 |
|||
flake8 |
|||
pylint |
|||
-e git://github.com/hhatto/autopep8.git@ver1.2.2#egg=autopep8 # Needed for fixing E712 |
|||
pep8 # autopep8 doesn't seem to like pycodestyle |
|||
@ -0,0 +1,165 @@ |
|||
# pylint: disable=not-context-manager |
|||
""" |
|||
Fabric tasks for PyBitmessage devops operations. |
|||
|
|||
# pylint: disable=not-context-manager |
|||
""" |
|||
|
|||
import os |
|||
import sys |
|||
|
|||
from fabric.api import run, task, hide, cd |
|||
from fabvenv import virtualenv |
|||
|
|||
from fabfile.lib import ( |
|||
autopep8, PROJECT_ROOT, VENV_ROOT, coerce_bool, flatten, |
|||
get_filtered_pycodestyle_output, get_filtered_flake8_output, get_filtered_pylint_output, |
|||
) |
|||
|
|||
|
|||
def get_tool_results(file_list): |
|||
"""Take a list of files and resuln the results of applying the tools""" |
|||
|
|||
results = [] |
|||
for path_to_file in file_list: |
|||
result = {} |
|||
result['pycodestyle_violations'] = get_filtered_pycodestyle_output(path_to_file) |
|||
result['flake8_violations'] = get_filtered_flake8_output(path_to_file) |
|||
result['pylint_violations'] = get_filtered_pylint_output(path_to_file) |
|||
result['path_to_file'] = path_to_file |
|||
result['total_violations'] = sum([ |
|||
len(result['pycodestyle_violations']), |
|||
len(result['flake8_violations']), |
|||
len(result['pylint_violations']), |
|||
]) |
|||
results.append(result) |
|||
return results |
|||
|
|||
|
|||
def print_results(results, top, verbose, details): |
|||
"""Print an item with the appropriate verbosity / detail""" |
|||
|
|||
if verbose: |
|||
print ''.join( |
|||
[ |
|||
os.linesep, |
|||
'total pycodestyle flake8 pylint path_to_file', |
|||
os.linesep, |
|||
] |
|||
) |
|||
|
|||
for item in sort_and_slice(results, top): |
|||
|
|||
if verbose: |
|||
line = "{0} {1} {2} {3} {4}".format( |
|||
item['total_violations'], |
|||
len(item['pycodestyle_violations']), |
|||
len(item['flake8_violations']), |
|||
len(item['pylint_violations']), |
|||
item['path_to_file'], |
|||
) |
|||
else: |
|||
line = item['path_to_file'] |
|||
print line |
|||
|
|||
if details: |
|||
print "pycodestyle:" |
|||
for detail in flatten(item['pycodestyle_violations']): |
|||
print detail |
|||
print |
|||
|
|||
print "flake8:" |
|||
for detail in flatten(item['flake8_violations']): |
|||
print detail |
|||
print |
|||
|
|||
print "pylint:" |
|||
for detail in flatten(item['pylint_violations']): |
|||
print detail |
|||
print |
|||
|
|||
|
|||
def sort_and_slice(results, top): |
|||
"""Sort dictionary items by the `total_violations` key and honour top""" |
|||
|
|||
returnables = [] |
|||
for item in sorted( |
|||
results, |
|||
reverse=True, |
|||
key=lambda x: x['total_violations'] |
|||
)[:top]: |
|||
returnables.append(item) |
|||
return returnables |
|||
|
|||
|
|||
def generate_file_list(filename): |
|||
"""Return an unfiltered list of absolute paths to the files to act on""" |
|||
|
|||
with hide('warnings', 'running', 'stdout'): |
|||
with virtualenv(VENV_ROOT): |
|||
|
|||
if filename: |
|||
filename = os.path.abspath(filename) |
|||
if not os.path.exists(filename): |
|||
print "Bad filename, specify a Python file" |
|||
sys.exit(1) |
|||
else: |
|||
file_list = [filename] |
|||
|
|||
else: |
|||
with cd(PROJECT_ROOT): |
|||
file_list = [ |
|||
os.path.abspath(i.rstrip('\r')) |
|||
for i in run('find . -name "*.py"').split(os.linesep) |
|||
] |
|||
|
|||
return file_list |
|||
|
|||
|
|||
@task |
|||
def code_quality(verbose=True, details=False, fix=False, filename=None, top=10): |
|||
""" |
|||
Check code quality. |
|||
|
|||
By default this command will analyse each Python file in the project with a variety of tools and display the count |
|||
or details of the violations discovered, sorted by most violations first. |
|||
|
|||
Default usage: |
|||
|
|||
$ fab -H localhost code_quality |
|||
|
|||
:param top: Display / fix only the top N violating files, a value of 0 will display / fix all files |
|||
:type top: int, default 10 |
|||
:param verbose: Display a header and the counts, without this you just get the filenames in order |
|||
:type verbose: bool, default True |
|||
:param details: Display the violations one per line after the count / file summary |
|||
:type details: bool, default False |
|||
:param fix: Run autopep8 aggressively on the displayed file(s) |
|||
:type fix: bool, default False |
|||
:param filename: Rather than analysing all files and displaying / fixing the top N, just analyse / display / fix |
|||
the specified file |
|||
:type filename: string, valid path to a file, default all files in the project |
|||
:return: This fabric task has an exit status equal to the total number of violations and uses stdio but it does |
|||
not return anything if you manage to call it successfully from Python |
|||
:rtype: None |
|||
|
|||
Intended to be temporary until we have improved code quality and have safeguards to maintain it in place. |
|||
|
|||
""" |
|||
|
|||
verbose = coerce_bool(verbose) |
|||
details = coerce_bool(details) |
|||
fix = coerce_bool(fix) |
|||
top = int(top) or -1 |
|||
|
|||
file_list = generate_file_list(filename) |
|||
results = get_tool_results(file_list) |
|||
|
|||
if fix: |
|||
for item in sort_and_slice(results, top): |
|||
autopep8(item['path_to_file']) |
|||
# Recalculate results after autopep8 to surprise the user the least |
|||
results = get_tool_results(file_list) |
|||
|
|||
print_results(results, top, verbose, details) |
|||
sys.exit(sum([item['total_violations'] for item in results])) |
|||
@ -0,0 +1,10 @@ |
|||
[pycodestyle] |
|||
max-line-length = 119 |
|||
|
|||
[flake8] |
|||
max-line-length = 119 |
|||
ignore = E722 |
|||
|
|||
# pylint |
|||
[MESSAGES CONTROL] |
|||
disable=invalid-name,bare-except |
|||
Loading…
Reference in new issue