Bitmessage/PyBitmessage-2024-12-19
Archived
2
0
Fork 0
Code Issues 168 Pull Requests 52 Actions Packages Projects Releases 5 Wiki Activity

Added: Code quality tool

Browse Source
This commit is contained in:
coffeedogs 2018-05-11 09:01:49 +01:00
parent 7445665963
commit 41a217cc8e
No known key found for this signature in database
GPG Key ID: 9D818C503D0B7E70
5 changed files with 480 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
fabfile/README.md Normal file
View File

@ -0,0 +1,55 @@
# Fabric
[Fabric](https://www.fabfile.org) is a Python library for performing devops tasks. You can thing of it a bit like a
makefile on steroids for Python. Its api abstracts away the clunky way you would run shell commands in Python, check
return values and manage stdio. Tasks may be targetted at particular hosts or group of hosts.
# Using Fabric
$ cd PyBitmessage
$ fab <task_name>
For a list of available commands:
$ fab -l
General fabric commandline help
$ fab -h
Arguments can be given:
$ fab task1:arg1=arg1value,arg2=arg2value task2:option1
Tasks target hosts. Hosts can be specified with -H, or roles can be defined and you can target groups of hosts with -R.
Furthermore, you can use -- to run arbitrary shell commands rather than tasks:
$ fab -H localhost task1
$ fab -R webservers -- sudo /etc/httpd restart
# Getting started
* Install Fabric, fabric-virtualenv, virtualenvwrapper system-wide (log in to a new terminal afterwards if you're installing
virtualenvwrappers for the first time)
$ sudo apt install Fabric virtualenvwrapper; sudo pip install fabric-virtualenv
* Create a virtualenv called pybitmessage and install fabfile/requirements.txt
$ mkvirtualenv --system-site-packages pybitmessage-devops
$ pip install -r fabfile/requirements.txt
* Ensure you can ssh localhost with no intervention, which may include:
* settings in ~/.ssh/config
* authorized_keys file
* load ssh key
* check(!) and accept the host key
* From the PyBitmessage directory you can now run fab commands!
# Rationale
There are a number of advantages that should benefit us:
* Common tasks can be writen in Python and executed consistently
* Common tasks are now under source control
* All developers can run the same commands, if the underlying command sequence for a task changes (after review, obv)
the user does not have to care
* Tasks can be combined either programmatically or on the commandline and run in series or parallel
* Whoee environemnts can be managed very effectively in conjunction with a configuration management system

14
fabfile/__init__.py Normal file
View File

@ -0,0 +1,14 @@
"""
Fabric is a Python library for performing devops tasks. If you have Fabric installed (systemwide or via pip) you can
run commands like this:
$ fab code_quality
For a list of commands
"""
from fabfile.tasks import code_quality
__all__ = ["code_quality"]

133
fabfile/lib.py Normal file
View File

@ -0,0 +1,133 @@
"""A library of functions and constrants for tasks to make use of."""
import os
<<<<<<< HEAD
=======
import sys
>>>>>>> 81c272dbc332d842b7f8ab548ebeef2dd9a73995
from fabric.api import run, hide
from fabric.context_managers import settings
from fabvenv import virtualenv
FABRIC_ROOT = os.path.dirname(__file__)
PROJECT_ROOT = os.path.dirname(FABRIC_ROOT)
<<<<<<< HEAD
VENV_ROOT = os.path.expanduser(os.path.join('~', '.virtualenvs', 'pybitmessage'))
=======
VENV_ROOT = os.path.expanduser(os.path.join('~', '.virtualenvs', 'pybitmessage-devops'))
def coerce_bool(value):
"""Coerce a value into a boolean"""
if isinstance(value, bool):
return value
elif any(
[
value in [0, '0'],
value.lower().startswith('n'),
]
):
return False
elif any(
[
value in [1, '1'],
value.lower().startswith('y'),
]
):
return True
else:
print "Bad boolean value {}".format(value)
sys.exit(1)
def flatten(data):
"""Recursively flatten lists"""
result = []
for item in data:
if isinstance(item, list):
result.append(flatten(item))
else:
result.append(item)
return result
>>>>>>> 81c272dbc332d842b7f8ab548ebeef2dd9a73995
def pycodestyle(path_to_file):
"""Run pycodestyle on a file"""
with virtualenv(VENV_ROOT):
<<<<<<< HEAD
with hide('warnings', 'running'):
=======
with hide('warnings', 'running', 'stdout', 'stderr'): # pylint: disable=not-context-manager
>>>>>>> 81c272dbc332d842b7f8ab548ebeef2dd9a73995
with settings(warn_only=True):
return run(
'pycodestyle --config={0} {1}'.format(
os.path.join(
PROJECT_ROOT,
'setup.cfg',
),
path_to_file,
),
)
def flake8(path_to_file):
"""Run flake8 on a file"""
with virtualenv(VENV_ROOT):
<<<<<<< HEAD
with hide('warnings', 'running'):
=======
with hide('warnings', 'running', 'stdout'): # pylint: disable=not-context-manager
>>>>>>> 81c272dbc332d842b7f8ab548ebeef2dd9a73995
with settings(warn_only=True):
return run(
'flake8 --config={0} {1}'.format(
os.path.join(
PROJECT_ROOT,
'setup.cfg',
),
path_to_file,
),
)
def pylint(path_to_file):
"""Run pylint on a file"""
with virtualenv(VENV_ROOT):
<<<<<<< HEAD
with hide('warnings', 'running', 'stdout', 'stderr'):
=======
with hide('warnings', 'running', 'stdout', 'stderr'): # pylint: disable=not-context-manager
>>>>>>> 81c272dbc332d842b7f8ab548ebeef2dd9a73995
with settings(warn_only=True):
return run(
'pylint --rcfile={0} {1}'.format(
os.path.join(
PROJECT_ROOT,
'setup.cfg',
),
path_to_file,
),
)
def autopep8(path_to_file):
"""Run autopep8 on a file"""
with virtualenv(VENV_ROOT):
<<<<<<< HEAD
with hide('running'):
with settings(warn_only=True):
return run(
"autopep8 -ai --max-line-length=119 --hang-closing {}".format(
=======
with hide('running'): # pylint: disable=not-context-manager
with settings(warn_only=True):
return run(
"autopep8 --experimental --aggressive --aggressive -i --max-line-length=119 {}".format(
>>>>>>> 81c272dbc332d842b7f8ab548ebeef2dd9a73995
path_to_file
),
)

9
fabfile/requirements.txt Normal file
View File

@ -0,0 +1,9 @@
# These requirements are for the Fabric commands that support devops tasks for PyBitmessage, not for running
# PyBitmessage itself.
# TODO: Consider moving to an extra_requires group in setup.py
pycodestyle==2.3.1 # https://github.com/PyCQA/pycodestyle/issues/741
flake8
pylint
-e git://github.com/hhatto/autopep8.git@ver1.2.2#egg=autopep8 # Needed for fixing E712
pep8 # autopep8 doesn't seem to like pycodestyle

269
fabfile/tasks.py Normal file
View File

@ -0,0 +1,269 @@
"""
<<<<<<< HEAD
Fabric is a Python library for performing devops tasks. If you have Fabric installed (systemwide or via pip) you can
run commands from within the project directory like this:
$ fab code_quality
For a list of commands:
$ fab -l
Known bugs:
=======
Fabric tasks for PyBitmessage devops operations.
>>>>>>> 81c272dbc332d842b7f8ab548ebeef2dd9a73995
"""
import os
import sys
<<<<<<< HEAD
import fabric
from fabric.api import run, task, hide, cd
from fabfile.lib import pycodestyle, flake8, pylint, autopep8, PROJECT_ROOT, VENV_ROOT
from fabvenv import virtualenv
fabric.state.output['running'] = False
@task
def code_quality(top=10, verbose=True, details=False, fix=True, filename=None):
"""Intended to be temporary until we have improved code quality and have safeguards to maintain it in place."""
end = int(top) if int(top) else -1
if verbose == "0":
verbose = False
if not isinstance(verbose, bool):
print "Bad verbosity value, try 0"
sys.exit(1)
if details == "1":
details = True
if not isinstance(details, bool):
print "Bad details value, try 1"
sys.exit(1)
if fix == "0":
fix = False
if not isinstance(fix, bool):
print "Bad fix value, try 0"
sys.exit(1)
with hide('warnings'):
=======
from fabric.api import run, task, hide, cd
from fabvenv import virtualenv
from fabfile.lib import pycodestyle, flake8, pylint, autopep8, PROJECT_ROOT, VENV_ROOT, coerce_bool, flatten
def get_results_for_tools_applied_to_file_list(file_list):
"""Take a list of files and resuln the results of applying the tools"""
results = []
for path_to_file in file_list:
result = {}
result['pycodestyle_violations'] = [
i
for i in pycodestyle(path_to_file).split("\n")
if i
]
result['flake8_violations'] = [
i
for i in flake8(path_to_file).split("\n")
if i
]
result['pylint_violations'] = [
i
for i in pylint(path_to_file).split("\n")
if all([
i,
not i.startswith(' '),
not i.startswith('\r'),
not i.startswith('-'),
not i.startswith('Y'),
not i.startswith('*'),
not i.startswith('Using config file'),
])
]
result['path_to_file'] = path_to_file
result['total_violations'] = sum(
[
len(result['pycodestyle_violations']),
len(result['flake8_violations']),
len(result['pylint_violations']),
]
)
results.append(result)
return results
def print_item(item, verbose, details):
"""Print an item with the appropriate verbosity / detail"""
if verbose:
line = "{0} {1} {2} {3} {4}".format(
item['total_violations'],
len(item['pycodestyle_violations']),
len(item['flake8_violations']),
len(item['pylint_violations']),
item['path_to_file'],
)
else:
line = item['path_to_file']
print line
if details:
print "pycodestyle:"
for detail in flatten(item['pycodestyle_violations']):
print detail
print
print "flake8:"
for detail in flatten(item['flake8_violations']):
print detail
print
print "pylint:"
for detail in flatten(item['pylint_violations']):
print detail
print
@task
def code_quality(top=10, verbose=True, details=False, fix=False, filename=None):
"""
Check code quality.
By default this command will analyse each Python file in the project with a variety of tools and display the count
or details of the violations discovered, sorted by most iolations first.
Default usage:
$ fab -H localhost code_quality
Options:
top: Display / fix only the top violating files, a value of 0 will display / fix all files
verbose: Display a header and the counts. Without this you just get the filenames in order
details: Display the violations one per line after the count/file summary
fix: Run autopep8 on the displayed file(s)
filename: Rather than analysing all files and displaying / fixing the top N, just analyse / diaply / fix the
specified file
Intended to be temporary until we have improved code quality and have safeguards to maintain it in place.
"""
verbose = coerce_bool(verbose)
details = coerce_bool(details)
fix = coerce_bool(fix)
end = int(top) if int(top) else -1
with hide('warnings', 'running'): # pylint: disable=not-context-manager
>>>>>>> 81c272dbc332d842b7f8ab548ebeef2dd9a73995
with virtualenv(VENV_ROOT):
if filename:
filename = os.path.abspath(filename)
if not os.path.exists(filename):
print "Bad filename, specify a Python file"
sys.exit(1)
else:
file_list = [filename]
else:
<<<<<<< HEAD
with cd(PROJECT_ROOT):
file_list = run('find . -name "*.py"', capture=True).split("\n")
results = []
for path_to_file in file_list:
results.append(
{
'pycodestyle_violations': [
i
for i in pycodestyle(path_to_file).split("\n")
if i
],
'flake8_violations': [
i
for i in flake8(path_to_file).split("\n")
if i
],
'pylint_violations': [
i
for i in pylint(path_to_file).split("\n")
if all([
i,
not i.startswith(' '),
not i.startswith('-'),
not i.startswith('Y'),
not i.startswith('*'),
])
],
'path_to_file': path_to_file,
}
)
=======
with cd(PROJECT_ROOT): # pylint: disable=not-context-manager
file_list = run('find . -name "*.py"').split("\n")
if fix:
for path_to_file in file_list:
autopep8(path_to_file)
results = get_results_for_tools_applied_to_file_list(file_list)
>>>>>>> 81c272dbc332d842b7f8ab548ebeef2dd9a73995
if verbose:
print "\ntotal pycodestyle flake8 pylint path_to_file\n"
<<<<<<< HEAD
for item in sorted(
results,
reverse=True,
key=lambda x: len(x['pycodestyle_violations']) + len(x['flake8_violations']) + len(x['pylint_violations']),
)[:end]:
violation_detail = item['pycodestyle_violations'] + item['flake8_violations'] + item['pylint_violations'],
if verbose:
line = "{0} {1} {2} {3} {4}".format(
len(violation_detail[0]),
len(item['pycodestyle_violations']),
len(item['flake8_violations']),
len(item['pylint_violations']),
item['path_to_file'],
)
else:
line = item['path_to_file']
print line
if details:
for detail_line in violation_detail:
if isinstance(detail_line, list):
for subline in detail_line:
if isinstance(subline, list):
for subsubline in subline:
print subsubline
else:
print subline
else:
print detail_line
print
if fix:
autopep8_output = [i for i in autopep8(item['path_to_file']) if i]
print autopep8_output
=======
# Sort and slice
for item in sorted(
results,
reverse=True,
key=lambda x: x['total_violations']
)[:end]:
print_item(item, verbose, details)
>>>>>>> 81c272dbc332d842b7f8ab548ebeef2dd9a73995