From 8b7b85b8f189e5eb4c785f31b5763379412a6058 Mon Sep 17 00:00:00 2001 From: Rafael Bodill Date: Wed, 28 Nov 2018 21:22:58 +0200 Subject: [PATCH] Initial commit --- README.md | 111 ++++++++ library/storage/syncthing/syncthing_device.py | 232 +++++++++++++++ library/storage/syncthing/syncthing_folder.py | 268 ++++++++++++++++++ playbooks/install_syncthing.yml | 28 ++ playbooks/manage.yml | 61 ++++ 5 files changed, 700 insertions(+) create mode 100644 README.md create mode 100644 library/storage/syncthing/syncthing_device.py create mode 100644 library/storage/syncthing/syncthing_folder.py create mode 100644 playbooks/install_syncthing.yml create mode 100644 playbooks/manage.yml diff --git a/README.md b/README.md new file mode 100644 index 0000000..39de1ca --- /dev/null +++ b/README.md @@ -0,0 +1,111 @@ +# Ansible Modules for Syncthing + +Collection of modules for [Syncthing](https://syncthing.net) management. + +## Install + +Copy the `./library` directory to your Ansible project and ensure your +`ansible.cfg` has this line: + +```ini +[defaults] +library = ./library +``` + +Please note this module was test on: + +* Ubuntu 16.04 with Syncthing v0.14.52 + +Please report successful usage on other platforms/versions. + +## Usage + +See [example playbooks](./playbooks) for robust feature usage: + +* [install_syncthing.yml] - Install Syncthing on Ubuntu (with systemd) +* [manage.yml] - Ensure Syncthing devices and folders across devices + +[install_syncthing.yml]: http:// +[manage.yml]: http:// + +## Modules + +### Module: `syncthing_device` + +Manage synced devices. Add, remove or pause devices using ID. + +Examples: + +```yml +# Add a device to share with, use auto-configuration +- name: Add syncthing device + syncthing_device: + id: ABCDEFG-ABCDEFG-ABCDEFG-ABCDEFG-ABCDEFG-ABCDEFG-ABCDEFG-ABCDEFG + name: my-device-name + +# Add a device to share with +- name: Add syncthing device + syncthing_device: + id: ABCDEFG-ABCDEFG-ABCDEFG-ABCDEFG-ABCDEFG-ABCDEFG-ABCDEFG-ABCDEFG + name: my-other-device + host: http://127.0.0.1:8384 + api_key: aBCDeFG1h2IJKlmNopq3rs45uvwxy6Zz + +# Pause an existing device +- name: Pause syncthing device + syncthing_device: + id: ABCDEFG-ABCDEFG-ABCDEFG-ABCDEFG-ABCDEFG-ABCDEFG-ABCDEFG-ABCDEFG + name: my-device-name + state: pause + +# Remove an existing device +- name: Remove syncthing device + syncthing_device: + id: ABCDEFG-ABCDEFG-ABCDEFG-ABCDEFG-ABCDEFG-ABCDEFG-ABCDEFG-ABCDEFG + name: my-device-name + state: absent +``` + +### Module: `syncthing_folder` + +Manage synced devices. Add, remove or pause devices using ID. + +Examples: + +```yml +# Add a folder to synchronize with another device, use auto-configuration +- name: Add syncthing folder + syncthing_folder: + path: ~/Documents + id: documents + devices: + - ABCDEFG-ABCDEFG-ABCDEFG-ABCDEFG-ABCDEFG-ABCDEFG-ABCDEFG-ABCDEFG + +# Add a folder to share with several devices, specify host and api key +- name: Add syncthing folder + syncthing_folder: + path: ~/Downloads + id: downloads + devices: + - ABCDEFG-ABCDEFG-ABCDEFG-ABCDEFG-ABCDEFG-ABCDEFG-ABCDEFG-ABCDEFG + - GFEDCBA-GFEDCBA-GFEDCBA-GFEDCBA-GFEDCBA-GFEDCBA-GFEDCBA-GFEDCBA + host: http://127.0.0.1:8384 + api_key: aBCDeFG1h2IJKlmNopq3rs45uvwxy6Zz + +# Pause an existing folder +- name: Pause syncthing folder + syncthing_folder: + id: downloads + state: pause + +# Remove an existing folder +- name: Remove syncthing folder + syncthing_folder: + id: downloads + state: absent +``` + +## License + +Copyright: (c) 2018, Rafael Bodill `` +GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) diff --git a/library/storage/syncthing/syncthing_device.py b/library/storage/syncthing/syncthing_device.py new file mode 100644 index 0000000..17fffd3 --- /dev/null +++ b/library/storage/syncthing/syncthing_device.py @@ -0,0 +1,232 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# Copyright: (c) 2018, Rafael Bodill +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +ANSIBLE_METADATA = { + 'metadata_version': '1.1', + 'status': ['preview'], + 'supported_by': 'community' +} + +DOCUMENTATION = ''' +--- +module: syncthing_device + +short_description: Manage Syncthing devices + +version_added: "2.7" + +description: + - "This is my longer description explaining my sample module" + +options: + id: + description: + - This is the unique id of this new device + required: true + name: + description: + - The name for this new device + required: false + host: + description: + - Host to connect to, including port + default: http://127.0.0.1:8384 + api_key: + description: + - API key to use for authentication with host. + If not provided, will try to auto-configure from filesystem. + required: false + timeout: + description: + - The socket level timeout in seconds + default: 30 + state: + description: + - Use present/absent to ensure device is added, or not. + default: present + choices: ['absent', 'present', 'paused'] + +author: + - Rafael Bodill (@rafi) +''' + +EXAMPLES = ''' +# Add a device to share with +- name: Add syncthing device + syncthing_device: + id: 1234-1234-1234-1234 + name: my-server-name +''' + +RETURN = ''' +response: + description: The API response, in-case of an error. + type: dict +''' + +import os +import json +import platform +from xml.etree.ElementTree import parse + +from ansible.module_utils.basic import AnsibleModule +from ansible.module_utils.urls import fetch_url, url_argument_spec + +SYNCTHING_API_URI = "/rest/system/config" +if platform.system() == 'Windows': + DEFAULT_ST_CONFIG_LOCATION = '%localappdata%/Syncthing/config.xml' +elif platform.system() == 'Darwin': + DEFAULT_ST_CONFIG_LOCATION = '$HOME/Library/Application Support/Syncthing/config.xml' +else: + DEFAULT_ST_CONFIG_LOCATION = '$HOME/.config/syncthing/config.xml' + + +def make_headers(host, api_key): + url = '{}{}'.format(host, SYNCTHING_API_URI) + headers = {'X-Api-Key': api_key } + return url, headers + +def get_key_from_filesystem(module): + try: + stconfigfile = os.path.expandvars(DEFAULT_ST_CONFIG_LOCATION) + stconfig = parse(stconfigfile) + root = stconfig.getroot() + gui = root.find('gui') + api_key = gui.find('apikey').text + return api_key + except Exception: + module.fail_json(msg="Auto-configuration failed. Please specify" + "the API key manually.") + +# Fetch Syncthing configuration +def get_config(module): + url, headers = make_headers(module.params['host'], module.params['api_key']) + resp, info = fetch_url( + module, url, data=None, headers=headers, + method='GET', timeout=module.params['timeout']) + + if not info or info['status'] != 200: + result['response'] = info + module.fail_json(msg='Error occured while calling host', **result) + + try: + content = resp.read() + except AttributeError: + result['content'] = info.pop('body', '') + result['response'] = str(info) + module.fail_json(msg='Error occured while reading response', **result) + + return json.loads(content) + +# Post the new configuration to Syncthing API +def post_config(module, config, result): + url, headers = make_headers(module.params['host'], module.params['api_key']) + headers['Content-Type'] = 'application/json' + + result['msg'] = config + resp, info = fetch_url( + module, url, data=json.dumps(config), headers=headers, + method='POST', timeout=module.params['timeout']) + + if not info or info['status'] != 200: + result['response'] = str(info) + module.fail_json(msg='Error occured while posting new config', **result) + +# Returns an object of a new device +def create_device(params): + device = { + 'addresses': [ + 'dynamic' + ], + 'allowedNetworks': [], + 'autoAcceptFolders': False, + 'certName': '', + 'compression': 'metadata', + 'deviceID': params['id'], + 'ignoredFolders': [], + 'introducedBy': '', + 'introducer': False, + 'maxRecvKbps': 0, + 'maxSendKbps': 0, + 'name': params['name'], + 'paused': True if params['state'] == 'paused' else False, + 'pendingFolders': [], + 'skipIntroductionRemovals': False + } + return device + +def run_module(): + # module arguments + module_args = url_argument_spec() + module_args.update(dict( + id=dict(type='str', required=True), + name=dict(type='str', required=False), + host=dict(type='str', default='http://127.0.0.1:8384'), + api_key=dict(type='str', required=False, no_log=True), + timeout=dict(type='int', default=30), + state=dict(type='str', default='present', + choices=['absent', 'present', 'pause']), + )) + + # seed the result dict in the object + result = { + "changed": False, + "response": None, + } + + # the AnsibleModule object will be our abstraction working with Ansible + module = AnsibleModule( + argument_spec=module_args, + supports_check_mode=True + ) + + if module.params['state'] != 'absent' and not module.params['name']: + module.fail_json(msg='You must provide a name when creating', **result) + + if module.check_mode: + return result + + # Auto-configuration: Try to fetch API key from filesystem + if not module.params['api_key']: + module.params['api_key'] = get_key_from_filesystem(module) + + config = get_config(module) + if module.params['state'] == 'absent': + # Remove device from list, if found + for idx, device in enumerate(config['devices']): + if device['deviceID'] == module.params['id']: + config['devices'].pop(idx) + result['changed'] = True + break + else: + # Bail-out if device is already added + for device in config['devices']: + if device['deviceID'] == module.params['id']: + want_pause = module.params['state'] == 'pause' + if (want_pause and device['paused']) or \ + (not want_pause and not device['paused']): + module.exit_json(**result) + else: + device['paused'] = want_pause + result['changed'] = True + break + + # Append the new device into configuration + if not result['changed']: + device = create_device(module.params) + config['devices'].append(device) + result['changed'] = True + + if result['changed']: + post_config(module, config, result) + + module.exit_json(**result) + +def main(): + run_module() + +if __name__ == '__main__': + main() diff --git a/library/storage/syncthing/syncthing_folder.py b/library/storage/syncthing/syncthing_folder.py new file mode 100644 index 0000000..027734d --- /dev/null +++ b/library/storage/syncthing/syncthing_folder.py @@ -0,0 +1,268 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# Copyright: (c) 2018, Rafael Bodill +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +ANSIBLE_METADATA = { + 'metadata_version': '1.1', + 'status': ['preview'], + 'supported_by': 'community' +} + +DOCUMENTATION = ''' +--- +module: syncthing_folder + +short_description: Manage Syncthing folders + +version_added: "2.7" + +description: + - "This is my longer description explaining my sample module" + +options: + id: + description: + - This is the unique id of this new folder + required: true + label: + description: + - The label for this new folder + required: false + path: + description: + - This is the path of the folder + required: false + devices: + description: + - List of devices to share folder with + required: false + host: + description: + - Host to connect to, including port + default: http://127.0.0.1:8384 + api_key: + description: + - API key to use for authentication with host. + If not provided, will try to auto-configure from filesystem. + required: false + timeout: + description: + - The socket level timeout in seconds + default: 30 + state: + description: + - Use present/absent to ensure folder is shared, or not. + default: present + choices: ['absent', 'present', 'paused'] + +author: + - Rafael Bodill (@rafi) +''' + +EXAMPLES = ''' +# Add a folder to synchronize with another device +- name: Add syncthing folder + syncthing_folder: + id: box + path: ~/box + devices: + - 1234-1234-1234-1234 +''' + +RETURN = ''' +response: + description: The API response, in-case of an error. + type: dict +''' + +import os +import json +import platform +from xml.etree.ElementTree import parse + +from ansible.module_utils.basic import AnsibleModule +from ansible.module_utils.urls import fetch_url, url_argument_spec + +SYNCTHING_API_URI = "/rest/system/config" +if platform.system() == 'Windows': + DEFAULT_ST_CONFIG_LOCATION = '%localappdata%/Syncthing/config.xml' +elif platform.system() == 'Darwin': + DEFAULT_ST_CONFIG_LOCATION = '$HOME/Library/Application Support/Syncthing/config.xml' +else: + DEFAULT_ST_CONFIG_LOCATION = '$HOME/.config/syncthing/config.xml' + + +def make_headers(host, api_key): + url = '{}{}'.format(host, SYNCTHING_API_URI) + headers = {'X-Api-Key': api_key } + return url, headers + +def get_key_from_filesystem(module): + try: + stconfigfile = os.path.expandvars(DEFAULT_ST_CONFIG_LOCATION) + stconfig = parse(stconfigfile) + root = stconfig.getroot() + gui = root.find('gui') + api_key = gui.find('apikey').text + return api_key + except Exception: + module.fail_json(msg="Auto-configuration failed. Please specify" + "the API key manually.") + +# Fetch Syncthing configuration +def get_config(module): + url, headers = make_headers(module.params['host'], module.params['api_key']) + resp, info = fetch_url( + module, url, data=None, headers=headers, + method='GET', timeout=module.params['timeout']) + + if not info or info['status'] != 200: + result['response'] = info + module.fail_json(msg='Error occured while calling host', **result) + + try: + content = resp.read() + except AttributeError: + result['content'] = info.pop('body', '') + result['response'] = str(info) + module.fail_json(msg='Error occured while reading response', **result) + + return json.loads(content) + +# Post the new configuration to Syncthing API +def post_config(module, config, result): + url, headers = make_headers(module.params['host'], module.params['api_key']) + headers['Content-Type'] = 'application/json' + + result['msg'] = config + resp, info = fetch_url( + module, url, data=json.dumps(config), headers=headers, + method='POST', timeout=module.params['timeout']) + + if not info or info['status'] != 200: + result['response'] = str(info) + module.fail_json(msg='Error occured while posting new config', **result) + +# Returns an object of a new folder +def create_folder(params): + folder = { + 'autoNormalize': True, + 'copiers': 0, + 'devices': [], + 'disableSparseFiles': False, + 'disableTempIndexes': False, + 'filesystemType': 'basic', + 'fsWatcherDelayS': 10, + 'fsWatcherEnabled': True, + 'hashers': 0, + 'id': params['id'], + 'ignoreDelete': False, + 'ignorePerms': False, + 'label': params['label'] if params['label'] else params['id'], + 'markerName': '.stfolder', + 'maxConflicts': -1, + 'minDiskFree': { + 'unit': '%', + 'value': 1 + }, + 'order': 'random', + 'path': params['path'], + 'paused': True if params['state'] == 'paused' else False, + 'pullerMaxPendingKiB': 0, + 'pullerPauseS': 0, + 'rescanIntervalS': 3600, + 'scanProgressIntervalS': 0, + 'type': 'sendreceive', + 'useLargeBlocks': False, + 'versioning': { + 'params': {}, + 'type': '' + }, + 'weakHashThresholdPct': 25 + } + + # Collect wanted devices to share folder with + for device_id in params['devices']: + folder['devices'].append({ + 'deviceID': device_id, + 'introducedBy': '', + }) + + return folder + +def run_module(): + # module arguments + module_args = url_argument_spec() + module_args.update(dict( + id=dict(type='str', required=True), + label=dict(type='str', required=False), + path=dict(type='path', required=False), + devices=dict(type='list', required=False, default=False), + host=dict(type='str', default='http://127.0.0.1:8384'), + api_key=dict(type='str', required=False, no_log=True), + timeout=dict(type='int', default=30), + state=dict(type='str', default='present', + choices=['absent', 'present', 'pause']), + )) + + # seed the result dict in the object + result = { + "changed": False, + "response": None, + } + + # the AnsibleModule object will be our abstraction working with Ansible + module = AnsibleModule( + argument_spec=module_args, + supports_check_mode=True + ) + + if module.params['state'] != 'absent' and not module.params['path']: + module.fail_json(msg='You must provide a path when creating', **result) + + if module.check_mode: + return result + + # Auto-configuration: Try to fetch API key from filesystem + if not module.params['api_key']: + module.params['api_key'] = get_key_from_filesystem(module) + + config = get_config(module) + if module.params['state'] == 'absent': + # Remove folder from list, if found + for idx, folder in enumerate(config['folders']): + if folder['id'] == module.params['id']: + config['folders'].pop(idx) + result['changed'] = True + break + else: + # Bail-out if folder is already added + for folder in config['folders']: + if folder['id'] == module.params['id']: + want_pause = module.params['state'] == 'pause' + if (want_pause and folder['paused']) or \ + (not want_pause and not folder['paused']): + module.exit_json(**result) + else: + folder['paused'] = want_pause + result['changed'] = True + break + + # Append the new folder into configuration + if not result['changed']: + folder = create_folder(module.params) + config['folders'].append(folder) + result['changed'] = True + + if result['changed']: + post_config(module, config, result) + + module.exit_json(**result) + +def main(): + run_module() + +if __name__ == '__main__': + main() diff --git a/playbooks/install_syncthing.yml b/playbooks/install_syncthing.yml new file mode 100644 index 0000000..d084d30 --- /dev/null +++ b/playbooks/install_syncthing.yml @@ -0,0 +1,28 @@ +--- + +- hosts: all + vars: + syncthing_apt_key_id: "D26E6ED000654A3E" + syncthing_apt_key_url: "https://syncthing.net/release-key.txt" + syncthing_apt_repository: "deb http://apt.syncthing.net/ syncthing release" + syncthing_service_user: root + + tasks: + - name: Add Syncthing repository key + apt_key: + id: "{{ syncthing_apt_key_id }}" + url: "{{ syncthing_apt_key_url }}" + keyring: "/etc/apt/trusted.gpg.d/syncthing.gpg" + + - name: Add Syncthing repository + apt_repository: + repo: "{{ syncthing_apt_repository }}" + + - name: Install Syncthing package + apt: name=syncthing + + - name: Enable and ensure Syncthing is started + systemd: + name: syncthing@{{ syncthing_service_user }} + enabled: yes + state: started diff --git a/playbooks/manage.yml b/playbooks/manage.yml new file mode 100644 index 0000000..f4011bb --- /dev/null +++ b/playbooks/manage.yml @@ -0,0 +1,61 @@ +--- + +- hosts: syncthing + vars: + syncthing_api: 127.0.0.1:8384 + syncthing_service_user: deploy + syncthing_folders: + - path: /srv/data/media + id: media + group: staff + mode: "0775" + - id: default + state: absent + + tasks: + - name: Ensure directories state + file: + path: '{{ item.path }}' + state: '{{ "absent" if item.state|d("") == "absent" else "directory" }}' + owner: '{{ syncthing_service_user }}' + group: '{{ item.group | default("root") }}' + mode: '{{ item.mode | default("0755") }}' + with_items: "{{ syncthing_folders }}" + + - name: Get system config to grab unique id of each machine + uri: + url: "http://{{ syncthing_api }}/rest/system/config" + return_content: no + status_code: [ 200, 403 ] + register: syncthing_config_raw + check_mode: no + + - name: Set each machine with a unique id + set_fact: + syncthing_id: "{{ syncthing_config_raw.x_syncthing_id }}" + syncthing_ids: [] + + - name: Prepare a list of all machine ids + set_fact: + syncthing_ids: "{{ syncthing_ids + [ hostvars[item].syncthing_id ] }}" + when: inventory_hostname != item + with_items: "{{ groups['syncthing'] }}" + + - name: Ensure syncthing devices + syncthing_device: + id: "{{ hostvars[item].syncthing_id }}" + name: "{{ item }}" + become: yes + become_user: "{{ syncthing_service_user }}" + when: inventory_hostname != item + with_items: "{{ groups['syncthing'] }}" + + - name: Ensure syncthing folders + syncthing_folder: + path: "{{ item.path | default(omit) }}" + id: "{{ item.id }}" + devices: "{{ syncthing_ids }}" + state: "{{ item.state | default('present') }}" + become: yes + become_user: "{{ syncthing_service_user }}" + with_items: "{{ syncthing_folders }}"