diff --git a/buildbot_transifex/webhook.py b/buildbot_transifex/webhook.py index 9312f10..b8b9c90 100644 --- a/buildbot_transifex/webhook.py +++ b/buildbot_transifex/webhook.py @@ -1,8 +1,19 @@ + +import sys + +import os import base64 + +import time import json import re import hmac +import pprint import hashlib +import requests +from subprocess import call +from base64 import b64encode + from buildbot.process.properties import Properties from buildbot.util import bytes2unicode, unicode2bytes from buildbot.www.hooks.base import BaseHookHandler @@ -12,204 +23,156 @@ from twisted.python import log from dateutil.parser import parse as dateparse -_HEADER_USER_AGENT = 'User-Agent' +HTTP_USER_AGENT = 'User-Agent' _HEADER_SIGNATURE = 'X-TX-Signature' _EVENT_KEY = 'event' +branch = 'v0.6' +transifexSecret = "" +transifexUsername = "" +transifexPassword = "" +transifex_webhook_url = 'https://buildbot.bitmessage.org/change_hook/transifex' +transifex_dict = {} +secret = "" +master = "" + +from transifex.api import transifex_api + +transifex_api.setup(auth='my_token') +org = transifex_api.Organization.get(slug='my_org') +proj = org.fetch('projects').get(slug='my_project') +lang = transifex_api.Language.get(code='') +resource = proj.fetch('resources').get(slug='my_resource') +url = transifex_api.ResourceTranslationsAsyncDownload.download( + resource=resource, language=lang,content_encoding = 'text', + file_type = 'default', pseudo = False +) class TransifexHandler(BaseHookHandler): - # def verifyGitHubSignature (environ, payload_body): - # signature = 'sha1=' + hmac.new(gitHubSecret, payload_body, sha1).hexdigest() - # try: - # if signature != environ.get('X-TX-Signature'): - # return False - # return True - # except: - # return False + def __init__(self, transifex_dict, secret, master,): + self.secret = secret + self.master = master + self.transifex_dict = transifex_dict - # def verifyTransifexSignature (environ, payload_body): - # signature = b64encode(hmac.new(transifexSecret, payload_body, sha1).digest()) - # try: - # debug(signature) - # if signature != environ.get('HTTP_X_TX_SIGNATURE'): - # return False - # return True - # except: - # return False + def verifyTransifexSignature( + self, request, content, rendered_secret, signature, header_signature + ): + http_verb = 'POST' + http_url_path = request.headers('X-TX-Url') + http_gmt_date = request.headers('Date') + content_md5 = hashlib.md5(content).hexdigest() + msg = b'\n'.join([ + http_verb, http_url_path, http_gmt_date, content_md5 + ]) + tx_signature = base64.b64encode( + hmac.new( + key=rendered_secret, + msg=msg, + digestmod=hashlib.sha256 + ).digest() + ) + if tx_signature() != header_signature: + raise ValueError('Invalid secret') + + try: + if signature != os.environ.get('HTTP_X_TX_SIGNATURE'): + return False + return True + except: + return False - # def returnMessage(status = False, message = "Unimplemented"): - # output = json.dumps({"status": "OK" if status else "FAIL", "message": message}) - # return [output, [('Content-type', 'text/plain'), - # ('Content-Length', str(len(output))) - # ]] + def updateLocalTranslationDestination(self, ts, lang, branch): + call(["git", "pull", "--all", "-q"]) + call(["git", "stash", "-q"]) + call(["git", "checkout", "-q", branch]) + call(["git", "checkout", "-q", "-b", "translate_" + lang + "_" + str(ts)]) + call(["git", "branch", "-q", "--set-upstream-to=origin/v0.6"]) - # def application(environ, start_response): - # status = '200 OK' - # output = '' - # lockWait() - # length = int(environ.get('CONTENT_LENGTH', '0')) - # body = environ['wsgi.input'].read(length) - - # if "Transifex" in environ.get("HTTP_USER_AGENT"): - # # debug(environ) - # # debug(body) - # if not verifyTransifexSignature(environ, body): - # debug ("Verify Transifex Signature fail, but fuck them") - # else: - # debug ("Verify Transifex Signature ok") - # # output, responseHeaders = returnMessage(False, "Checksum bad") - # # start_response(status, responseHeaders) - # # unlock() - # # return [output] - # try: - # # debug(body) - # payload = parse_qs(body) - # # debug(payload) - # if 'pybitmessage' in payload['project'] and 'pybitmessage' in payload['resource']: - # if 'translated' in payload and '100' in payload['translated']: - # ts = int(time.time()) - # updateLocalTranslationDestination(ts, payload['language'][0].lower()) - # downloadTranslatedLanguage(ts, payload['language'][0]) - # response = commitTranslatedLanguage(ts, payload['language'][0].lower()) - # if response.ok: - # output, responseHeaders = returnMessage(True, "Processed.") - # else: - # output, responseHeaders = returnMessage(False, "Error: %i." % (response.status_code)) - # else: - # output, responseHeaders = returnMessage(False, "Nothing to do") - # else: - # output, responseHeaders = returnMessage(False, "Nothing to do") - # except: - # output, responseHeaders = returnMessage(True, "Not processing") - # else: - # debug("Unknown command %s" % (environ.get("HTTP_X_GITHUB_EVENT"))) - # output, responseHeaders = returnMessage(True, "Unknown command, ignoring") - def __init__(self, transifex_dict=None): - super(TransifexHandler, self).__init__(*args, **kwargs) - - def process_translation_completed(self, payload, event_type, codebase): - refname = payload["ref"] + def downloadTranslatedLanguage(ts, lang): + headers = {"Authorization": "Basic " + b64encode(transifexUsername + ":" + transifexPassword)} + fname = "bitmessage_" + lang.lower() + ".po" + with open("src/translations/" + fname, "wt") as handle: + response = requests.get("https://www.transifex.com/api/2/project/pybitmessage/resource/pybitmessage/translation/" + lang + "/", + headers=headers) + if response.ok: + content = json.loads(response.content)["content"] + handle.write(content.encode("utf-8")) + return response + def process_translation_completed(self, transifex_dict, event_type, codebase): + # refname = payload["ref"] + payload = transifex_dict changes = [] - # We only care about regular heads or tags - match = re.match(r"^refs/(heads|tags)/(.+)$", refname) - if event_type == 'translation_completed': - pass + # match = re.match(r"^refs/(heads|tags)/(.+)$", refname) + if "Transifex" in os.environ.get("HTTP_USER_AGENT"): + if not self.verifyTransifexSignature( + request, content, rendered_secret, signature, header_signature): + print('Invalid transifex signature!') + else: + payload = self.transifex_dict + if 'pybitmessage-test' in payload['transifex_data']['project'] and 'messagespot' in payload['transifex_data']['translation_completed']['resource']: + if 'translation_completed' in payload['transifex_data'] and '100' in payload['transifex_data']['translated']: + ts = int(time.time()) + lang = payload['transifex_data']['translation_completed']['language'] + branch = payload['transifex_data']['branch'] + self.updateLocalTranslationDestination(ts, lang.lower(), branch) + self.downloadTranslatedLanguage(ts, lang.lower()) - if not match: - log.msg("Ignoring refname '{}': Not a branch or tag".format(refname)) - return changes - - branch = match.group(2) repository = payload['repository'] repo_url = repository['ssh_url'] project = repository['full_name'] commits = payload['commits'] - if isinstance(self.options, dict) and self.options.get('onlyIncludePushCommit', False): - commits = commits[:1] + # if isinstance(self.options, dict) and self.options.get('onlyIncludePushCommit', False): + # commits = commits[:1] - for commit in commits: - files = [] - for kind in ('added', 'modified', 'removed'): - files.extend(commit.get(kind, []) or []) - timestamp = dateparse(commit['timestamp']) - change = { - 'author': '{} <{}>'.format(commit['author']['name'], - commit['author']['email']), - 'files': files, - 'comments': commit['message'], - 'revision': commit['id'], - 'when_timestamp': timestamp, - 'branch': branch, - 'revlink': commit['url'], - 'repository': repo_url, - 'project': project, - 'category': event_type, - 'properties': { - 'event': event_type, - 'repository_name': repository['name'], - 'owner': repository["owner"]["username"] - }, - } - if codebase is not None: - change['codebase'] = codebase - changes.insert(0, change) - return changes + # for commit in commits: + # files = [] + # for kind in ('added', 'modified', 'removed'): + # files.extend(commit.get(kind, []) or []) + # timestamp = dateparse(commit['timestamp']) + # change = { + # 'author': '{} <{}>'.format(commit['author']['name'], + # commit['author']['email']), + # 'files': files, + # 'comments': commit['message'], + # 'revision': commit['id'], + # 'when_timestamp': timestamp, + # 'branch': branch, + # 'revlink': commit['url'], + # 'repository': repo_url, + # 'project': project, + # 'category': event_type, + # 'properties': { + # 'event': event_type, + # 'repository_name': repository['name'], + # 'owner': repository["owner"]["username"] + # }, + # } + # if codebase is not None: + # change['codebase'] = codebase + # changes.insert(0, change) + # return changes - def process_review_completed(self, payload, event_type, codebase): - action = payload['action'] - if event_type == 'review_completed': - pass - # Only handle potential new stuff, ignore close/. - # Merge itself is handled by the regular branch push message - if action not in ['opened', 'synchronized', 'edited', 'reopened']: - log.msg("Transifex Pull Request event '{}' ignored".format(action)) - return [] - # pull_request = payload['pull_request'] - # if not pull_request['mergeable']: - # log.msg("Transifex Pull Request ignored because it is not mergeable.") - # return [] - # if pull_request['merged']: - # log.msg("Transifex Pull Request ignored because it is already merged.") - # return [] - # timestamp = dateparse(pull_request['updated_at']) - # base = pull_request['base'] - # head = pull_request['head'] - repository = payload['repository'] - change = { - 'author': '{} <{}>'.format(pull_request['user']['full_name'], - pull_request['user']['email']), - # 'comments': 'PR#{}: {}\n\n{}'.format( - # pull_request['number'], - # pull_request['title'], - # pull_request['body']), - 'revision': base['sha'], - 'when_timestamp': timestamp, - 'branch': base['ref'], - # 'revlink': pull_request['html_url'], - 'repository': base['repo']['ssh_url'], - 'project': repository['full_name'], - 'category': event_type, - 'properties': { - 'event': event_type, - 'base_branch': base['ref'], - 'base_sha': base['sha'], - 'base_repo_id': base['repo_id'], - 'base_repository': base['repo']['clone_url'], - 'base_git_ssh_url': base['repo']['ssh_url'], - 'head_branch': head['ref'], - 'head_sha': head['sha'], - 'head_repo_id': head['repo_id'], - 'head_repository': head['repo']['clone_url'], - 'head_git_ssh_url': head['repo']['ssh_url'], - 'head_owner': head['repo']['owner']['username'], - 'head_reponame': head['repo']['name'], - # 'pr_id': pull_request['id'], - # 'pr_number': pull_request['number'], - 'repository_name': repository['name'], - 'owner': repository["owner"]["username"], - }, - } - if codebase is not None: - change['codebase'] = codebase - return [change] + def process_review_completed(self, payload, transifex_data): + pass - def _transform_variables(payload): + + def _transform_variables(self, payload=transifex_dict): retval = { - project: payload.get('project'), - repository = [payload.get('resource')], - branch = payload.get('language') + 'project': payload[''].get('project'), + 'repository': [payload.get('resource')], + 'branch': payload.get('language') } return retval @defer.inlineCallbacks def getChanges(self, request): - secret = None + self.secret = None if isinstance(self.options, dict): - secret = self.options.get('secret') + self.secret = self.options.get('secret') try: content = request.content.read() content_text = bytes2unicode(content) @@ -217,35 +180,16 @@ class TransifexHandler(BaseHookHandler): except Exception as exception: raise ValueError('Error loading JSON: ' + str(exception)) - - if secret is not None: + if self.secret is not None: p = Properties() p.master = self.master - rendered_secret = yield p.render(secret) + rendered_secret = yield p.render(self.secret) signature = hmac.new( unicode2bytes(rendered_secret), unicode2bytes(content_text.strip()), digestmod=hashlib.sha256) header_signature = bytes2unicode( request.getHeader(_HEADER_SIGNATURE)) - - http_verb = 'POST' - http_url_path = request.headers('X-TX-Url') - http_gmt_date = request.headers('Date') - content_md5 = hashlib.md5(content).hexdigest() - msg = b'\n'.join([ - http_verb, http_url_path, http_gmt_date, content_md5 - ]) - tx_signature = base64.b64encode( - hmac.new( - key=rendered_secret, - msg=msg, - digestmod=hashlib.sha256 - ).digest() - ) - if tx_signature() != header_signature: - raise ValueError('Invalid secret') - event_type = bytes2unicode(payload.get(_EVENT_KEY), "None") log.msg("Received event '{}' from transifex".format(event_type))