From 4cc616526a70920baea4473ee92b410593e244f9 Mon Sep 17 00:00:00 2001 From: Dmitri Bogomolov <4glitch@gmail.com> Date: Sun, 5 Sep 2021 20:37:31 +0300 Subject: [PATCH] Remove stale unused files (closes: #1337) --- bandit.yml | 4 - configure | 1 - fabfile/README.md | 101 ------------- fabfile/__init__.py | 36 ----- fabfile/lib.py | 200 ------------------------ fabfile/requirements.txt | 9 -- fabfile/tasks.py | 318 --------------------------------------- 7 files changed, 669 deletions(-) delete mode 100644 bandit.yml delete mode 100755 configure delete mode 100644 fabfile/README.md delete mode 100644 fabfile/__init__.py delete mode 100644 fabfile/lib.py delete mode 100644 fabfile/requirements.txt delete mode 100644 fabfile/tasks.py diff --git a/bandit.yml b/bandit.yml deleted file mode 100644 index 4d24be14..00000000 --- a/bandit.yml +++ /dev/null @@ -1,4 +0,0 @@ -# Codacy uses Bandit. - -# Asserts are accepted throughout the project. -skips: ['B101'] diff --git a/configure b/configure deleted file mode 100755 index 0519ecba..00000000 --- a/configure +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/fabfile/README.md b/fabfile/README.md deleted file mode 100644 index 5e90147c..00000000 --- a/fabfile/README.md +++ /dev/null @@ -1,101 +0,0 @@ -# Fabric - -[Fabric](https://www.fabfile.org) is a Python library for performing devops tasks. You can think of it a bit like a -makefile on steroids for Python. Its api abstracts away the clunky way you would run shell commands in Python, check -return values and manage stdio. Tasks may be targetted at particular hosts or group of hosts. - -# Using Fabric - - $ cd PyBitmessage - $ fab - -For a list of available commands: - - $ fab -l - -General fabric commandline help - - $ fab -h - -Arguments can be given: - - $ fab task1:arg1=arg1value,arg2=arg2value task2:option1 - -Tasks target hosts. Hosts can be specified with -H, or roles can be defined and you can target groups of hosts with -R. -Furthermore, you can use -- to run arbitrary shell commands rather than tasks: - - $ fab -H localhost task1 - $ fab -R webservers -- sudo /etc/httpd restart - -# Getting started - - * Install [Fabric](http://docs.fabfile.org/en/1.14/usage/fab.html), - [fabric-virtualenv](https://pypi.org/project/fabric-virtualenv/) and - [virtualenvwrapper](https://virtualenvwrapper.readthedocs.io/en/latest/) - system-wide using your preferred method. - * Create a virtualenv called pybitmessage and install fabfile/requirements.txt - $ mkvirtualenv -r fabfile/requirements.txt --system-site-packages pybitmessage-devops - * Ensure you can ssh localhost with no intervention, which may include: - * ssh [sshd_config server] and [ssh_config client] configuration - * authorized_keys file - * load ssh key - * check(!) and accept the host key - * From the PyBitmessage directory you can now run fab commands! - -# Rationale - -There are a number of advantages that should benefit us: - - * Common tasks can be written in Python and executed consistently - * Common tasks are now under source control - * All developers can run the same commands, if the underlying command sequence for a task changes (after review, obv) - the user does not have to care - * Tasks can be combined either programmatically or on the commandline and run in series or parallel - * Whole environments can be managed very effectively in conjunction with a configuration management system - - -# /etc/ssh/sshd_config - -If you're going to be using ssh to connect to localhost you want to avoid weakening your security. The best way of -doing this is blocking port 22 with a firewall. As a belt and braces approach you can also edit the -/etc/ssh/sshd_config file to restrict login further: - -``` -PubkeyAuthentication no - -... - -Match ::1 - PubkeyAuthentication yes -``` -Adapted from [stackexchange](https://unix.stackexchange.com/questions/406245/limit-ssh-access-to-specific-clients-by-ip-address) - - -# ~/.ssh/config - -Fabric will honour your ~/.ssh/config file for your convenience. Since you will spend more time with this key unlocked -than others you should use a different key: - -``` -Host localhost - HostName localhost - IdentityFile ~/.ssh/id_rsa_localhost - -Host github - HostName github.com - IdentityFile ~/.ssh/id_rsa_github -``` - -# Ideas for further development - -## Smaller - - * Decorators and context managers are useful for accepting common params like verbosity, force or doing command-level help - * if `git status` or `git status --staged` produce results, prefer that to generate the file list - - -## Larger - - * Support documentation translations, aim for current transifex'ed languages - * Fabric 2 is finally out, go @bitprophet! Invoke/Fabric2 is a rewrite of Fabric supporting Python3. Probably makes - sense for us to stick to the battle-hardened 1.x branch, at least until we support Python3. diff --git a/fabfile/__init__.py b/fabfile/__init__.py deleted file mode 100644 index 9aec62bb..00000000 --- a/fabfile/__init__.py +++ /dev/null @@ -1,36 +0,0 @@ -""" -Fabric is a Python library for performing devops tasks. If you have Fabric installed (systemwide or via pip) you can -run commands like this: - - $ fab code_quality - -For a list of commands: - - $ fab -l - -For help on fabric itself: - - $ fab -h - -For more help on a particular command -""" - -from fabric.api import env - -from fabfile.tasks import code_quality, build_docs, push_docs, clean, test - - -# Without this, `fab -l` would display the whole docstring as preamble -__doc__ = "" - -# This list defines which tasks are made available to the user -__all__ = [ - "code_quality", - "test", - "build_docs", - "push_docs", - "clean", -] - -# Honour the user's ssh client configuration -env.use_ssh_config = True diff --git a/fabfile/lib.py b/fabfile/lib.py deleted file mode 100644 index 7af40231..00000000 --- a/fabfile/lib.py +++ /dev/null @@ -1,200 +0,0 @@ -# pylint: disable=not-context-manager -""" -A library of functions and constants for tasks to make use of. - -""" - -import os -import sys -import re -from functools import wraps - -from fabric.api import run, hide, cd, env -from fabric.context_managers import settings, shell_env -from fabvenv import virtualenv - - -FABRIC_ROOT = os.path.dirname(__file__) -PROJECT_ROOT = os.path.dirname(FABRIC_ROOT) -VENV_ROOT = os.path.expanduser(os.path.join('~', '.virtualenvs', 'pybitmessage-devops')) -PYTHONPATH = os.path.join(PROJECT_ROOT, 'src',) - - -def coerce_list(value): - """Coerce a value into a list""" - if isinstance(value, str): - return value.split(',') - else: - sys.exit("Bad string value {}".format(value)) - - -def coerce_bool(value): - """Coerce a value into a boolean""" - if isinstance(value, bool): - return value - elif any( - [ - value in [0, '0'], - value.lower().startswith('n'), - ] - ): - return False - elif any( - [ - value in [1, '1'], - value.lower().startswith('y'), - ] - ): - return True - else: - sys.exit("Bad boolean value {}".format(value)) - - -def flatten(data): - """Recursively flatten lists""" - result = [] - for item in data: - if isinstance(item, list): - result.append(flatten(item)) - else: - result.append(item) - return result - - -def filelist_from_git(rev=None): - """Return a list of files based on git output""" - cmd = 'git diff --name-only' - if rev: - if rev in ['cached', 'staged']: - cmd += ' --{}'.format(rev) - elif rev == 'working': - pass - else: - cmd += ' -r {}'.format(rev) - - with cd(PROJECT_ROOT): - with hide('running', 'stdout'): - results = [] - ansi_escape = re.compile(r'\x1B\[[0-?]*[ -/]*[@-~]') - clean = ansi_escape.sub('', run(cmd)) - clean = re.sub('\n', '', clean) - for line in clean.split('\r'): - if line.endswith(".py"): - results.append(os.path.abspath(line)) - return results - - -def pycodestyle(path_to_file): - """Run pycodestyle on a file""" - with virtualenv(VENV_ROOT): - with hide('warnings', 'running', 'stdout', 'stderr'): - with settings(warn_only=True): - return run( - 'pycodestyle --config={0} {1}'.format( - os.path.join( - PROJECT_ROOT, - 'setup.cfg', - ), - path_to_file, - ), - ) - - -def flake8(path_to_file): - """Run flake8 on a file""" - with virtualenv(VENV_ROOT): - with hide('warnings', 'running', 'stdout'): - with settings(warn_only=True): - return run( - 'flake8 --config={0} {1}'.format( - os.path.join( - PROJECT_ROOT, - 'setup.cfg', - ), - path_to_file, - ), - ) - - -def pylint(path_to_file): - """Run pylint on a file""" - with virtualenv(VENV_ROOT): - with hide('warnings', 'running', 'stdout', 'stderr'): - with settings(warn_only=True): - with shell_env(PYTHONPATH=PYTHONPATH): - return run( - 'pylint --rcfile={0} {1}'.format( - os.path.join( - PROJECT_ROOT, - 'setup.cfg', - ), - path_to_file, - ), - ) - - -def autopep8(path_to_file): - """Run autopep8 on a file""" - with virtualenv(VENV_ROOT): - with hide('running'): - with settings(warn_only=True): - return run( - "autopep8 --experimental --aggressive --aggressive -i --max-line-length=119 {}".format( - path_to_file - ), - ) - - -def get_filtered_pycodestyle_output(path_to_file): - """Clean up the raw results for pycodestyle""" - - return [ - i - for i in pycodestyle(path_to_file).split(os.linesep) - if i - ] - - -def get_filtered_flake8_output(path_to_file): - """Clean up the raw results for flake8""" - - return [ - i - for i in flake8(path_to_file).split(os.linesep) - if i - ] - - -def get_filtered_pylint_output(path_to_file): - """Clean up the raw results for pylint""" - - return [ - i - for i in pylint(path_to_file).split(os.linesep) - if all([ - i, - not i.startswith(' '), - not i.startswith('\r'), - not i.startswith('-'), - not i.startswith('Y'), - not i.startswith('*'), - not i.startswith('Using config file'), - ]) - ] - - -def default_hosts(hosts): - """Decorator to apply default hosts to a task""" - - def real_decorator(func): - """Manipulate env""" - env.hosts = env.hosts or hosts - - @wraps(func) - def wrapper(*args, **kwargs): - """Original function called from here""" - return func(*args, **kwargs) - - return wrapper - - return real_decorator diff --git a/fabfile/requirements.txt b/fabfile/requirements.txt deleted file mode 100644 index 0d0a3962..00000000 --- a/fabfile/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -# These requirements are for the Fabric commands that support devops tasks for PyBitmessage, not for running -# PyBitmessage itself. -# TODO: Consider moving to an extra_requires group in setup.py - -pycodestyle==2.3.1 # https://github.com/PyCQA/pycodestyle/issues/741 -flake8 -pylint --e git://github.com/hhatto/autopep8.git@ver1.2.2#egg=autopep8 # Needed for fixing E712 -pep8 # autopep8 doesn't seem to like pycodestyle diff --git a/fabfile/tasks.py b/fabfile/tasks.py deleted file mode 100644 index fb05937d..00000000 --- a/fabfile/tasks.py +++ /dev/null @@ -1,318 +0,0 @@ -# pylint: disable=not-context-manager -""" -Fabric tasks for PyBitmessage devops operations. - -Note that where tasks declare params to be bools, they use coerce_bool() and so will accept any commandline (string) -representation of true or false that coerce_bool() understands. - -""" - -import os -import sys - -from fabric.api import run, task, hide, cd, settings, sudo -from fabric.contrib.project import rsync_project -from fabvenv import virtualenv - -from fabfile.lib import ( - autopep8, PROJECT_ROOT, VENV_ROOT, coerce_bool, flatten, filelist_from_git, default_hosts, - get_filtered_pycodestyle_output, get_filtered_flake8_output, get_filtered_pylint_output, -) - -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(os.path.dirname(__file__)), 'src'))) # noqa:E402 -from version import softwareVersion # pylint: disable=wrong-import-position - - -def get_tool_results(file_list): - """Take a list of files and resuln the results of applying the tools""" - - results = [] - for path_to_file in file_list: - result = {} - result['pycodestyle_violations'] = get_filtered_pycodestyle_output(path_to_file) - result['flake8_violations'] = get_filtered_flake8_output(path_to_file) - result['pylint_violations'] = get_filtered_pylint_output(path_to_file) - result['path_to_file'] = path_to_file - result['total_violations'] = sum([ - len(result['pycodestyle_violations']), - len(result['flake8_violations']), - len(result['pylint_violations']), - ]) - results.append(result) - return results - - -def print_results(results, top, verbose, details): - """Print an item with the appropriate verbosity / detail""" - - if verbose and results: - print ''.join( - [ - os.linesep, - 'total pycodestyle flake8 pylint path_to_file', - os.linesep, - ] - ) - - for item in sort_and_slice(results, top): - - if verbose: - line = "{0} {1} {2} {3} {4}".format( - item['total_violations'], - len(item['pycodestyle_violations']), - len(item['flake8_violations']), - len(item['pylint_violations']), - item['path_to_file'], - ) - else: - line = item['path_to_file'] - print line - - if details: - print "pycodestyle:" - for detail in flatten(item['pycodestyle_violations']): - print detail - print - - print "flake8:" - for detail in flatten(item['flake8_violations']): - print detail - print - - print "pylint:" - for detail in flatten(item['pylint_violations']): - print detail - print - - -def sort_and_slice(results, top): - """Sort dictionary items by the `total_violations` key and honour top""" - - returnables = [] - for item in sorted( - results, - reverse=True, - key=lambda x: x['total_violations'] - )[:top]: - returnables.append(item) - return returnables - - -def generate_file_list(filename): - """Return an unfiltered list of absolute paths to the files to act on""" - - with hide('warnings', 'running', 'stdout'): - with virtualenv(VENV_ROOT): - - if filename: - filename = os.path.abspath(filename) - if not os.path.exists(filename): - print "Bad filename, specify a Python file" - sys.exit(1) - else: - file_list = [filename] - - else: - with cd(PROJECT_ROOT): - file_list = [ - os.path.abspath(i.rstrip('\r')) - for i in run('find . -name "*.py"').split(os.linesep) - ] - - return file_list - - -def create_dependency_graphs(): - """ - To better understand the relationship between methods, dependency graphs can be drawn between functions and - methods. - - Since the resulting images are large and could be out of date on the next commit, storing them in the repo is - pointless. Instead, it makes sense to build a dependency graph for a particular, documented version of the code. - - .. todo:: Consider saving a hash of the intermediate dotty file so unnecessary image generation can be avoided. - - """ - with virtualenv(VENV_ROOT): - with hide('running', 'stdout'): - - # .. todo:: consider a better place to put this, use a particular commit - with cd(PROJECT_ROOT): - with settings(warn_only=True): - if run('stat pyan').return_code: - run('git clone https://github.com/davidfraser/pyan.git') - with cd(os.path.join(PROJECT_ROOT, 'pyan')): - run('git checkout pre-python3') - - # .. todo:: Use better settings. This is MVP to make a diagram - with cd(PROJECT_ROOT): - file_list = run("find . -type f -name '*.py' ! -path './src/.eggs/*'").split('\r\n') - for cmd in [ - 'neato -Goverlap=false -Tpng > deps-neato.png', - 'sfdp -Goverlap=false -Tpng > deps-sfdp.png', - 'dot -Goverlap=false -Tpng > deps-dot.png', - ]: - pyan_cmd = './pyan/pyan.py {} --dot'.format(' '.join(file_list)) - sed_cmd = r"sed s'/http\-old/http_old/'g" # dot doesn't like dashes - run('|'.join([pyan_cmd, sed_cmd, cmd])) - - run('mv *.png docs/_build/html/_static/') - - -@task -@default_hosts(['localhost']) -def code_quality(verbose=True, details=False, fix=False, filename=None, top=10, rev=None): - """ - Check code quality. - - By default this command will analyse each Python file in the project with a variety of tools and display the count - or details of the violations discovered, sorted by most violations first. - - Default usage: - - $ fab -H localhost code_quality - - :param rev: If not None, act on files changed since this commit. 'cached/staged' and 'working' have special meaning - :type rev: str or None, default None - :param top: Display / fix only the top N violating files, a value of 0 will display / fix all files - :type top: int, default 10 - :param verbose: Display a header and the counts, without this you just get the filenames in order - :type verbose: bool, default True - :param details: Display the violations one per line after the count / file summary - :type details: bool, default False - :param fix: Run autopep8 aggressively on the displayed file(s) - :type fix: bool, default False - :param filename: Don't test/fix the top N, just the specified file - :type filename: string, valid path to a file, default all files in the project - :return: None, exit status equals total number of violations - :rtype: None - - Intended to be temporary until we have improved code quality and have safeguards to maintain it in place. - - """ - # pylint: disable=too-many-arguments - - verbose = coerce_bool(verbose) - details = coerce_bool(details) - fix = coerce_bool(fix) - top = int(top) or -1 - - file_list = generate_file_list(filename) if not rev else filelist_from_git(rev) - results = get_tool_results(file_list) - - if fix: - for item in sort_and_slice(results, top): - autopep8(item['path_to_file']) - # Recalculate results after autopep8 to surprise the user the least - results = get_tool_results(file_list) - - print_results(results, top, verbose, details) - sys.exit(sum([item['total_violations'] for item in results])) - - -@task -@default_hosts(['localhost']) -def test(): - """Run tests on the code""" - - with cd(PROJECT_ROOT): - with virtualenv(VENV_ROOT): - - run('pip uninstall -y pybitmessage') - run('python setup.py install') - - run('pybitmessage -t') - run('python setup.py test') - - -@task -@default_hosts(['localhost']) -def build_docs(dep_graph=False, apidoc=True): - """ - Build the documentation locally. - - :param dep_graph: Build the dependency graphs - :type dep_graph: Bool, default False - :param apidoc: Build the automatically generated rst files from the source code - :type apidoc: Bool, default True - - Default usage: - - $ fab -H localhost build_docs - - Implementation: - - First, a dependency graph is generated and converted into an image that is referenced in the development page. - - Next, the sphinx-apidoc command is (usually) run which searches the code. As part of this it loads the modules and - if this has side-effects then they will be evident. Any documentation strings that make use of Python documentation - conventions (like parameter specification) or the Restructured Text (RsT) syntax will be extracted. - - Next, the `make html` command is run to generate HTML output. Other formats (epub, pdf) are available. - - .. todo:: support other languages - - """ - - apidoc = coerce_bool(apidoc) - - if coerce_bool(dep_graph): - create_dependency_graphs() - - with virtualenv(VENV_ROOT): - with hide('running'): - - apidoc_result = 0 - if apidoc: - run('mkdir -p {}'.format(os.path.join(PROJECT_ROOT, 'docs', 'autodoc'))) - with cd(os.path.join(PROJECT_ROOT, 'docs', 'autodoc')): - with settings(warn_only=True): - run('rm *.rst') - with cd(os.path.join(PROJECT_ROOT, 'docs')): - apidoc_result = run('sphinx-apidoc -o autodoc ..').return_code - - with cd(os.path.join(PROJECT_ROOT, 'docs')): - make_result = run('make html').return_code - return_code = apidoc_result + make_result - - sys.exit(return_code) - - -@task -@default_hosts(['localhost']) -def push_docs(path=None): - """ - Upload the generated docs to a public server. - - Default usage: - - $ fab -H localhost push_docs - - .. todo:: support other languages - .. todo:: integrate with configuration management data to get web root and webserver restart command - - """ - - # Making assumptions - WEB_ROOT = path if path is not None else os.path.join('var', 'www', 'html', 'pybitmessage', 'en', 'latest') - VERSION_ROOT = os.path.join(os.path.dirname(WEB_ROOT), softwareVersion) - - rsync_project( - remote_dir=VERSION_ROOT, - local_dir=os.path.join(PROJECT_ROOT, 'docs', '_build', 'html') - ) - result = run('ln -sf {0} {1}'.format(WEB_ROOT, VERSION_ROOT)) - if result.return_code: - print 'Linking the new release failed' - - # More assumptions - sudo('systemctl restart apache2') - - -@task -@default_hosts(['localhost']) -def clean(): - """Clean up files generated by fabric commands.""" - with hide('running', 'stdout'): - with cd(PROJECT_ROOT): - run(r"find . -name '*.pyc' -exec rm '{}' \;")