From f2587e071d9c7fcf469163f8a3e3e075bfcd4070 Mon Sep 17 00:00:00 2001 From: shekhar-cis Date: Mon, 2 Jan 2023 20:30:09 +0530 Subject: [PATCH 1/5] Update webhook and setup --- buildbot_transifex/webhook.py | 119 +++++++++++++++++++++++++++------- setup.py | 27 +++----- 2 files changed, 104 insertions(+), 42 deletions(-) diff --git a/buildbot_transifex/webhook.py b/buildbot_transifex/webhook.py index f2c4864..9312f10 100644 --- a/buildbot_transifex/webhook.py +++ b/buildbot_transifex/webhook.py @@ -16,16 +16,87 @@ _HEADER_USER_AGENT = 'User-Agent' _HEADER_SIGNATURE = 'X-TX-Signature' _EVENT_KEY = 'event' - class TransifexHandler(BaseHookHandler): + # def verifyGitHubSignature (environ, payload_body): + # signature = 'sha1=' + hmac.new(gitHubSecret, payload_body, sha1).hexdigest() + # try: + # if signature != environ.get('X-TX-Signature'): + # return False + # return True + # except: + # return False + + # def verifyTransifexSignature (environ, payload_body): + # signature = b64encode(hmac.new(transifexSecret, payload_body, sha1).digest()) + # try: + # debug(signature) + # if signature != environ.get('HTTP_X_TX_SIGNATURE'): + # return False + # return True + # except: + # return False + + # def returnMessage(status = False, message = "Unimplemented"): + # output = json.dumps({"status": "OK" if status else "FAIL", "message": message}) + # return [output, [('Content-type', 'text/plain'), + # ('Content-Length', str(len(output))) + # ]] + + # def application(environ, start_response): + # status = '200 OK' + # output = '' + # lockWait() + # length = int(environ.get('CONTENT_LENGTH', '0')) + # body = environ['wsgi.input'].read(length) + + # if "Transifex" in environ.get("HTTP_USER_AGENT"): + # # debug(environ) + # # debug(body) + # if not verifyTransifexSignature(environ, body): + # debug ("Verify Transifex Signature fail, but fuck them") + # else: + # debug ("Verify Transifex Signature ok") + # # output, responseHeaders = returnMessage(False, "Checksum bad") + # # start_response(status, responseHeaders) + # # unlock() + # # return [output] + # try: + # # debug(body) + # payload = parse_qs(body) + # # debug(payload) + # if 'pybitmessage' in payload['project'] and 'pybitmessage' in payload['resource']: + # if 'translated' in payload and '100' in payload['translated']: + # ts = int(time.time()) + # updateLocalTranslationDestination(ts, payload['language'][0].lower()) + # downloadTranslatedLanguage(ts, payload['language'][0]) + # response = commitTranslatedLanguage(ts, payload['language'][0].lower()) + # if response.ok: + # output, responseHeaders = returnMessage(True, "Processed.") + # else: + # output, responseHeaders = returnMessage(False, "Error: %i." % (response.status_code)) + # else: + # output, responseHeaders = returnMessage(False, "Nothing to do") + # else: + # output, responseHeaders = returnMessage(False, "Nothing to do") + # except: + # output, responseHeaders = returnMessage(True, "Not processing") + # else: + # debug("Unknown command %s" % (environ.get("HTTP_X_GITHUB_EVENT"))) + # output, responseHeaders = returnMessage(True, "Unknown command, ignoring") + def __init__(self, transifex_dict=None): + super(TransifexHandler, self).__init__(*args, **kwargs) + def process_translation_completed(self, payload, event_type, codebase): refname = payload["ref"] changes = [] - + # We only care about regular heads or tags match = re.match(r"^refs/(heads|tags)/(.+)$", refname) + if event_type == 'translation_completed': + pass + if not match: log.msg("Ignoring refname '{}': Not a branch or tag".format(refname)) return changes @@ -68,36 +139,37 @@ class TransifexHandler(BaseHookHandler): changes.insert(0, change) return changes - def process_review_compoleted(self, payload, event_type, codebase): + def process_review_completed(self, payload, event_type, codebase): action = payload['action'] - + if event_type == 'review_completed': + pass # Only handle potential new stuff, ignore close/. # Merge itself is handled by the regular branch push message if action not in ['opened', 'synchronized', 'edited', 'reopened']: - log.msg("Gitea Pull Request event '{}' ignored".format(action)) + log.msg("Transifex Pull Request event '{}' ignored".format(action)) return [] - pull_request = payload['pull_request'] - if not pull_request['mergeable']: - log.msg("Gitea Pull Request ignored because it is not mergeable.") - return [] - if pull_request['merged']: - log.msg("Gitea Pull Request ignored because it is already merged.") - return [] - timestamp = dateparse(pull_request['updated_at']) - base = pull_request['base'] - head = pull_request['head'] + # pull_request = payload['pull_request'] + # if not pull_request['mergeable']: + # log.msg("Transifex Pull Request ignored because it is not mergeable.") + # return [] + # if pull_request['merged']: + # log.msg("Transifex Pull Request ignored because it is already merged.") + # return [] + # timestamp = dateparse(pull_request['updated_at']) + # base = pull_request['base'] + # head = pull_request['head'] repository = payload['repository'] change = { 'author': '{} <{}>'.format(pull_request['user']['full_name'], pull_request['user']['email']), - 'comments': 'PR#{}: {}\n\n{}'.format( - pull_request['number'], - pull_request['title'], - pull_request['body']), + # 'comments': 'PR#{}: {}\n\n{}'.format( + # pull_request['number'], + # pull_request['title'], + # pull_request['body']), 'revision': base['sha'], 'when_timestamp': timestamp, 'branch': base['ref'], - 'revlink': pull_request['html_url'], + # 'revlink': pull_request['html_url'], 'repository': base['repo']['ssh_url'], 'project': repository['full_name'], 'category': event_type, @@ -115,8 +187,8 @@ class TransifexHandler(BaseHookHandler): 'head_git_ssh_url': head['repo']['ssh_url'], 'head_owner': head['repo']['owner']['username'], 'head_reponame': head['repo']['name'], - 'pr_id': pull_request['id'], - 'pr_number': pull_request['number'], + # 'pr_id': pull_request['id'], + # 'pr_number': pull_request['number'], 'repository_name': repository['name'], 'owner': repository["owner"]["username"], }, @@ -189,5 +261,4 @@ class TransifexHandler(BaseHookHandler): return (changes, 'transifex') -# Plugin name -transifex = TransifexHandler +transifex = TransifexHandler(transifex_dict) diff --git a/setup.py b/setup.py index 7f030f6..84a0eb0 100644 --- a/setup.py +++ b/setup.py @@ -6,32 +6,23 @@ from setuptools import setup with open("README.md", "r") as fh: long_description = fh.read() -VERSION = "1.7.2" +VERSION = "0.1" -setup(name='buildbot-gitea', +setup(name='buildbot-transifex', version=VERSION, - description='buildbot plugin for integration with Gitea.', - author='Marvin Pohl', - author_email='hello@lab132.com', - url='https://github.com/lab132/buildbot-gitea', - long_description=long_description, + description='buildbot plugin for integration with transifex.', + author='', + author_email='', + url='', + long_description='Transifex webhook', long_description_content_type="text/markdown", - packages=['buildbot_gitea'], + packages=['buildbot_transifex'], install_requires=[ "buildbot>=3.0.0" ], entry_points={ "buildbot.webhooks": [ - "gitea = buildbot_gitea.webhook:gitea" - ], - "buildbot.steps": [ - "Gitea = buildbot_gitea.step_source:Gitea" - ], - "buildbot.reporters": [ - "GiteaStatusPush = buildbot_gitea.reporter:GiteaStatusPush" - ], - "buildbot.util": [ - "GiteaAuth = buildbot_gitea.auth:GiteaAuth" + "transifex = buildbot_transifex.webhook:transifex" ] }, classifiers=[ -- 2.40.1 From 1b29a5639cca326d849150ec1b611fe00505d674 Mon Sep 17 00:00:00 2001 From: shekhar-cis Date: Wed, 4 Jan 2023 20:51:06 +0530 Subject: [PATCH 2/5] WIP[downloadTranslatedLanguage() and updateLocalTranslationDestination()] --- buildbot_transifex/webhook.py | 341 +++++++++++++++------------------- 1 file changed, 147 insertions(+), 194 deletions(-) diff --git a/buildbot_transifex/webhook.py b/buildbot_transifex/webhook.py index 9312f10..b29e234 100644 --- a/buildbot_transifex/webhook.py +++ b/buildbot_transifex/webhook.py @@ -1,8 +1,19 @@ + +import sys + +import os import base64 + +import time import json import re import hmac +import pprint import hashlib +import requests +from subprocess import call +from base64 import b64encode + from buildbot.process.properties import Properties from buildbot.util import bytes2unicode, unicode2bytes from buildbot.www.hooks.base import BaseHookHandler @@ -15,201 +26,157 @@ from dateutil.parser import parse as dateparse _HEADER_USER_AGENT = 'User-Agent' _HEADER_SIGNATURE = 'X-TX-Signature' _EVENT_KEY = 'event' +transifexSecret = "" +transifexUsername = "" +transifexPassword = "" +transifex_dict = {} +secret = "" +master = "" + +gitHubToken = os.environ('gitHubToken') class TransifexHandler(BaseHookHandler): - # def verifyGitHubSignature (environ, payload_body): - # signature = 'sha1=' + hmac.new(gitHubSecret, payload_body, sha1).hexdigest() - # try: - # if signature != environ.get('X-TX-Signature'): - # return False - # return True - # except: - # return False + def __init__(self, master, secret, transifex_dict): + self.secret = secret + self.master = master + self.transifex_dict = transifex_dict - # def verifyTransifexSignature (environ, payload_body): - # signature = b64encode(hmac.new(transifexSecret, payload_body, sha1).digest()) - # try: - # debug(signature) - # if signature != environ.get('HTTP_X_TX_SIGNATURE'): - # return False - # return True - # except: - # return False - # def returnMessage(status = False, message = "Unimplemented"): - # output = json.dumps({"status": "OK" if status else "FAIL", "message": message}) - # return [output, [('Content-type', 'text/plain'), - # ('Content-Length', str(len(output))) - # ]] + def returnMessage(self, status = False, message = "Unimplemented"): + output = json.dumps({"status": "OK" if status else "FAIL", "message": message}) + return [output, [('Content-type', 'text/plain'), + ('Content-Length', str(len(output))) + ]] - # def application(environ, start_response): - # status = '200 OK' - # output = '' - # lockWait() - # length = int(environ.get('CONTENT_LENGTH', '0')) - # body = environ['wsgi.input'].read(length) + def verifyTransifexSignature( + self, request, content, rendered_secret, signature, header_signature + ): + http_verb = 'POST' + http_url_path = request.headers('X-TX-Url') + http_gmt_date = request.headers('Date') + content_md5 = hashlib.md5(content).hexdigest() + msg = b'\n'.join([ + http_verb, http_url_path, http_gmt_date, content_md5 + ]) + tx_signature = base64.b64encode( + hmac.new( + key=rendered_secret, + msg=msg, + digestmod=hashlib.sha256 + ).digest() + ) + if tx_signature() != header_signature: + raise ValueError('Invalid secret') - # if "Transifex" in environ.get("HTTP_USER_AGENT"): - # # debug(environ) - # # debug(body) - # if not verifyTransifexSignature(environ, body): - # debug ("Verify Transifex Signature fail, but fuck them") - # else: - # debug ("Verify Transifex Signature ok") - # # output, responseHeaders = returnMessage(False, "Checksum bad") - # # start_response(status, responseHeaders) - # # unlock() - # # return [output] - # try: - # # debug(body) - # payload = parse_qs(body) - # # debug(payload) - # if 'pybitmessage' in payload['project'] and 'pybitmessage' in payload['resource']: - # if 'translated' in payload and '100' in payload['translated']: - # ts = int(time.time()) - # updateLocalTranslationDestination(ts, payload['language'][0].lower()) - # downloadTranslatedLanguage(ts, payload['language'][0]) - # response = commitTranslatedLanguage(ts, payload['language'][0].lower()) - # if response.ok: - # output, responseHeaders = returnMessage(True, "Processed.") - # else: - # output, responseHeaders = returnMessage(False, "Error: %i." % (response.status_code)) - # else: - # output, responseHeaders = returnMessage(False, "Nothing to do") - # else: - # output, responseHeaders = returnMessage(False, "Nothing to do") - # except: - # output, responseHeaders = returnMessage(True, "Not processing") - # else: - # debug("Unknown command %s" % (environ.get("HTTP_X_GITHUB_EVENT"))) - # output, responseHeaders = returnMessage(True, "Unknown command, ignoring") - def __init__(self, transifex_dict=None): - super(TransifexHandler, self).__init__(*args, **kwargs) + try: + if signature != os.environ.get('HTTP_X_TX_SIGNATURE'): + return False + return True + except: + return False - def process_translation_completed(self, payload, event_type, codebase): - refname = payload["ref"] + def downloadTranslatedLanguage(self, ts, lang): + headers = {"Authorization": "Basic " + b64encode(transifexUsername + ":" + transifexPassword)} + fname = "bitmessage_" + lang.lower() + ".po" + with open("src/translations/" + fname, "wt") as handle: + response = requests.get("https://www.transifex.com/api/2/project/pybitmessage/resource/pybitmessage/translation/" + lang + "/", + headers=headers) + if response.ok: + content = json.loads(response.content)["content"] + handle.write(content.encode("utf-8")) + return response + def commitTranslatedLanguage(self, ts, lang): + call(["kivy", "src/translations/messages.pro"]) + call(["git", "add", "src/translations/bitmessage_" + lang + ".ts", "src/translations/bitmessage_" + lang + ".qm"]) + call(["git", "commit", "-q", "-S", "-m", "Auto-updated language %s from transifex" % (lang)]) + newbranch = "translate_" + lang + "_" + str(ts) + call(["git", "push", "-q", "translations", newbranch + ":" + newbranch]) + branch = transifex_dict['branch'] + + request = { + "title": "Translation update " + lang, + "body": "Auto-updated from transifex", + "head": "PyBitmessageTranslations:" + newbranch, + "base": branch + } + headers = {"Authorization": "token " + gitHubToken} + response = requests.post("https://api.github.com/repos/Bitmessage/PyBitmessage/pulls", + headers=headers, data=json.dumps(request)) + return response + + + def process_translation_completed(self, payload, transifex_dict, event_type, codebase): changes = [] - - # We only care about regular heads or tags - match = re.match(r"^refs/(heads|tags)/(.+)$", refname) - if event_type == 'translation_completed': - pass + transifex_response = self._transform_variables(payload, transifex_dict) + if 'pybitmessage-test' in transifex_response['project'] and 'messagespot' in transifex_response['resource']: + if 'translation_completed' in transifex_response['event'] and 100 in transifex_response['translated']: + ts = int(time.time()) + lang = transifex_response['language'] + branch = transifex_dict['branch'] + self.downloadTranslatedLanguage(ts, lang.lower()) + response = self.commitTranslatedLanguage(ts, lang.lower()) + if response.ok: + output, responseHeaders = self.returnMessage(True, "Processed.") + else: + output, responseHeaders = self.returnMessage(False, "Error: %i." % (response.status_code)) + else: + output, responseHeaders = self.returnMessage(False, "Nothing to do") + else: + output, responseHeaders = self.returnMessage(False, "Nothing to do") - if not match: - log.msg("Ignoring refname '{}': Not a branch or tag".format(refname)) - return changes + # if isinstance(self.options, dict) and self.options.get('onlyIncludePushCommit', False): + # commits = commits[:1] - branch = match.group(2) - - repository = payload['repository'] - repo_url = repository['ssh_url'] - project = repository['full_name'] - - commits = payload['commits'] - if isinstance(self.options, dict) and self.options.get('onlyIncludePushCommit', False): - commits = commits[:1] - - for commit in commits: - files = [] - for kind in ('added', 'modified', 'removed'): - files.extend(commit.get(kind, []) or []) - timestamp = dateparse(commit['timestamp']) - change = { - 'author': '{} <{}>'.format(commit['author']['name'], - commit['author']['email']), - 'files': files, - 'comments': commit['message'], - 'revision': commit['id'], - 'when_timestamp': timestamp, - 'branch': branch, - 'revlink': commit['url'], - 'repository': repo_url, - 'project': project, - 'category': event_type, - 'properties': { - 'event': event_type, - 'repository_name': repository['name'], - 'owner': repository["owner"]["username"] - }, - } - if codebase is not None: - change['codebase'] = codebase - changes.insert(0, change) + # for commit in commits: + # files = [] + # for kind in ('added', 'modified', 'removed'): + # files.extend(commit.get(kind, []) or []) + # timestamp = dateparse(commit['timestamp']) + # change = { + # 'author': '{} <{}>'.format(commit['author']['name'], + # commit['author']['email']), + # 'files': files, + # 'comments': commit['message'], + # 'revision': commit['id'], + # 'when_timestamp': timestamp, + # 'branch': branch, + # 'revlink': commit['url'], + # 'repository': repo_url, + # 'project': project, + # 'category': event_type, + # 'properties': { + # 'event': event_type, + # 'repository_name': repository['name'], + # 'owner': repository["owner"]["username"] + # }, + # } + # if codebase is not None: + # change['codebase'] = codebase + # changes.insert(0, change) return changes - def process_review_completed(self, payload, event_type, codebase): - action = payload['action'] - if event_type == 'review_completed': - pass - # Only handle potential new stuff, ignore close/. - # Merge itself is handled by the regular branch push message - if action not in ['opened', 'synchronized', 'edited', 'reopened']: - log.msg("Transifex Pull Request event '{}' ignored".format(action)) - return [] - # pull_request = payload['pull_request'] - # if not pull_request['mergeable']: - # log.msg("Transifex Pull Request ignored because it is not mergeable.") - # return [] - # if pull_request['merged']: - # log.msg("Transifex Pull Request ignored because it is already merged.") - # return [] - # timestamp = dateparse(pull_request['updated_at']) - # base = pull_request['base'] - # head = pull_request['head'] - repository = payload['repository'] - change = { - 'author': '{} <{}>'.format(pull_request['user']['full_name'], - pull_request['user']['email']), - # 'comments': 'PR#{}: {}\n\n{}'.format( - # pull_request['number'], - # pull_request['title'], - # pull_request['body']), - 'revision': base['sha'], - 'when_timestamp': timestamp, - 'branch': base['ref'], - # 'revlink': pull_request['html_url'], - 'repository': base['repo']['ssh_url'], - 'project': repository['full_name'], - 'category': event_type, - 'properties': { - 'event': event_type, - 'base_branch': base['ref'], - 'base_sha': base['sha'], - 'base_repo_id': base['repo_id'], - 'base_repository': base['repo']['clone_url'], - 'base_git_ssh_url': base['repo']['ssh_url'], - 'head_branch': head['ref'], - 'head_sha': head['sha'], - 'head_repo_id': head['repo_id'], - 'head_repository': head['repo']['clone_url'], - 'head_git_ssh_url': head['repo']['ssh_url'], - 'head_owner': head['repo']['owner']['username'], - 'head_reponame': head['repo']['name'], - # 'pr_id': pull_request['id'], - # 'pr_number': pull_request['number'], - 'repository_name': repository['name'], - 'owner': repository["owner"]["username"], - }, - } - if codebase is not None: - change['codebase'] = codebase - return [change] + def process_review_completed(self, payload, transifex_data): + pass - def _transform_variables(payload): - retval = { - project: payload.get('project'), - repository = [payload.get('resource')], - branch = payload.get('language') + + def _transform_variables(self, payload, transifex_dict): + transifex_variables = { + 'project': payload['project'], + "translated": payload['translated'], + "resource": payload['resource'], + "event": payload['event'], + "language": payload['language'] } - return retval + + return transifex_variables @defer.inlineCallbacks def getChanges(self, request): - secret = None + self.secret = None if isinstance(self.options, dict): - secret = self.options.get('secret') + self.secret = self.options.get('secret') try: content = request.content.read() content_text = bytes2unicode(content) @@ -217,34 +184,20 @@ class TransifexHandler(BaseHookHandler): except Exception as exception: raise ValueError('Error loading JSON: ' + str(exception)) - - if secret is not None: + if self.secret is not None: p = Properties() p.master = self.master - rendered_secret = yield p.render(secret) + rendered_secret = yield p.render(self.secret) signature = hmac.new( unicode2bytes(rendered_secret), unicode2bytes(content_text.strip()), digestmod=hashlib.sha256) header_signature = bytes2unicode( request.getHeader(_HEADER_SIGNATURE)) - - http_verb = 'POST' - http_url_path = request.headers('X-TX-Url') - http_gmt_date = request.headers('Date') - content_md5 = hashlib.md5(content).hexdigest() - msg = b'\n'.join([ - http_verb, http_url_path, http_gmt_date, content_md5 - ]) - tx_signature = base64.b64encode( - hmac.new( - key=rendered_secret, - msg=msg, - digestmod=hashlib.sha256 - ).digest() + self.verifyTransifexSignature( + request, content, rendered_secret, + signature, header_signature ) - if tx_signature() != header_signature: - raise ValueError('Invalid secret') event_type = bytes2unicode(payload.get(_EVENT_KEY), "None") log.msg("Received event '{}' from transifex".format(event_type)) -- 2.40.1 From 89305ec6026f5a4cabe5f0a9d00700c472290a5f Mon Sep 17 00:00:00 2001 From: shekhar-cis Date: Fri, 6 Jan 2023 20:38:43 +0530 Subject: [PATCH 3/5] Update Webhook response variabel --- buildbot_transifex/webhook.py | 186 +++++++++++++--------------------- 1 file changed, 72 insertions(+), 114 deletions(-) diff --git a/buildbot_transifex/webhook.py b/buildbot_transifex/webhook.py index b29e234..b383cb1 100644 --- a/buildbot_transifex/webhook.py +++ b/buildbot_transifex/webhook.py @@ -1,18 +1,14 @@ - -import sys - +"""Transifex webhook handler """ import os import base64 import time import json -import re import hmac -import pprint import hashlib -import requests from subprocess import call from base64 import b64encode +import requests from buildbot.process.properties import Properties from buildbot.util import bytes2unicode, unicode2bytes @@ -25,152 +21,94 @@ from dateutil.parser import parse as dateparse _HEADER_USER_AGENT = 'User-Agent' _HEADER_SIGNATURE = 'X-TX-Signature' +_HEADER_URL_PATH = 'X-TX-Url' +HTTP_DATE = 'date' _EVENT_KEY = 'event' -transifexSecret = "" -transifexUsername = "" -transifexPassword = "" -transifex_dict = {} -secret = "" -master = "" +transifex_request_data = {} -gitHubToken = os.environ('gitHubToken') class TransifexHandler(BaseHookHandler): - def __init__(self, master, secret, transifex_dict): + def __init__(self, master, secret, options, transifex_dict): + if not options: + options = {} self.secret = secret self.master = master + self.options = options self.transifex_dict = transifex_dict def returnMessage(self, status = False, message = "Unimplemented"): output = json.dumps({"status": "OK" if status else "FAIL", "message": message}) - return [output, [('Content-type', 'text/plain'), - ('Content-Length', str(len(output))) - ]] + return [output, [('Content-type', 'application/json')]] def verifyTransifexSignature( - self, request, content, rendered_secret, signature, header_signature + self, request, content, signature, header_signature ): http_verb = 'POST' - http_url_path = request.headers('X-TX-Url') - http_gmt_date = request.headers('Date') + http_url_path = request.getHeader(_HEADER_URL_PATH) + http_gmt_date = request.getHeader(HTTP_DATE) content_md5 = hashlib.md5(content).hexdigest() msg = b'\n'.join([ http_verb, http_url_path, http_gmt_date, content_md5 ]) tx_signature = base64.b64encode( hmac.new( - key=rendered_secret, + key=self.rendered_secret, msg=msg, digestmod=hashlib.sha256 ).digest() ) - if tx_signature() != header_signature: - raise ValueError('Invalid secret') - - try: - if signature != os.environ.get('HTTP_X_TX_SIGNATURE'): - return False - return True - except: + if tx_signature != header_signature: return False - def downloadTranslatedLanguage(self, ts, lang): - headers = {"Authorization": "Basic " + b64encode(transifexUsername + ":" + transifexPassword)} - fname = "bitmessage_" + lang.lower() + ".po" - with open("src/translations/" + fname, "wt") as handle: - response = requests.get("https://www.transifex.com/api/2/project/pybitmessage/resource/pybitmessage/translation/" + lang + "/", - headers=headers) - if response.ok: - content = json.loads(response.content)["content"] - handle.write(content.encode("utf-8")) - return response - - def commitTranslatedLanguage(self, ts, lang): - call(["kivy", "src/translations/messages.pro"]) - call(["git", "add", "src/translations/bitmessage_" + lang + ".ts", "src/translations/bitmessage_" + lang + ".qm"]) - call(["git", "commit", "-q", "-S", "-m", "Auto-updated language %s from transifex" % (lang)]) - newbranch = "translate_" + lang + "_" + str(ts) - call(["git", "push", "-q", "translations", newbranch + ":" + newbranch]) - branch = transifex_dict['branch'] - - request = { - "title": "Translation update " + lang, - "body": "Auto-updated from transifex", - "head": "PyBitmessageTranslations:" + newbranch, - "base": branch - } - headers = {"Authorization": "token " + gitHubToken} - response = requests.post("https://api.github.com/repos/Bitmessage/PyBitmessage/pulls", - headers=headers, data=json.dumps(request)) - return response - + if signature != request.getHeader(_HEADER_SIGNATURE): + return False def process_translation_completed(self, payload, transifex_dict, event_type, codebase): changes = [] transifex_response = self._transform_variables(payload, transifex_dict) if 'pybitmessage-test' in transifex_response['project'] and 'messagespot' in transifex_response['resource']: - if 'translation_completed' in transifex_response['event'] and 100 in transifex_response['translated']: + if 'translation_completed' in transifex_response['event']: ts = int(time.time()) lang = transifex_response['language'] - branch = transifex_dict['branch'] - self.downloadTranslatedLanguage(ts, lang.lower()) - response = self.commitTranslatedLanguage(ts, lang.lower()) - if response.ok: - output, responseHeaders = self.returnMessage(True, "Processed.") - else: - output, responseHeaders = self.returnMessage(False, "Error: %i." % (response.status_code)) - else: - output, responseHeaders = self.returnMessage(False, "Nothing to do") - else: - output, responseHeaders = self.returnMessage(False, "Nothing to do") + return - # if isinstance(self.options, dict) and self.options.get('onlyIncludePushCommit', False): + # if isinstance(self.options, dict): # commits = commits[:1] - # for commit in commits: - # files = [] - # for kind in ('added', 'modified', 'removed'): - # files.extend(commit.get(kind, []) or []) - # timestamp = dateparse(commit['timestamp']) - # change = { - # 'author': '{} <{}>'.format(commit['author']['name'], - # commit['author']['email']), - # 'files': files, - # 'comments': commit['message'], - # 'revision': commit['id'], - # 'when_timestamp': timestamp, - # 'branch': branch, - # 'revlink': commit['url'], - # 'repository': repo_url, - # 'project': project, - # 'category': event_type, - # 'properties': { - # 'event': event_type, - # 'repository_name': repository['name'], - # 'owner': repository["owner"]["username"] - # }, - # } - # if codebase is not None: - # change['codebase'] = codebase - # changes.insert(0, change) + # # for commit in commits: + # # files = [] + # # for kind in ('added', 'modified', 'removed'): + # # files.extend(commit.get(kind, []) or []) + change = { + 'author': 'buildbot-transifex, + 'resource': transifex_response['resource'], + 'branch': transifex_dict['branch'], + 'project': transifex_response['project'], + 'event': event_type, + 'properties': { + 'branch': branch, + 'revision': revision, + 'language': lang, + 'resource': resource, + 'project': project + } + } + if codebase is not None: + change['codebase'] = codebase + changes.insert(0, change) return changes - def process_review_completed(self, payload, transifex_data): - pass - - def _transform_variables(self, payload, transifex_dict): - transifex_variables = { - 'project': payload['project'], - "translated": payload['translated'], - "resource": payload['resource'], - "event": payload['event'], - "language": payload['language'] + project = payload.get('project', 'None') + transform_values = { + project: { + "repository": "https://github.com/Bitmessage/PyBitmessage", + "branch": "v0.6" + } } - - return transifex_variables + return transform_values @defer.inlineCallbacks def getChanges(self, request): @@ -187,19 +125,39 @@ class TransifexHandler(BaseHookHandler): if self.secret is not None: p = Properties() p.master = self.master - rendered_secret = yield p.render(self.secret) + option = self.options + self.rendered_secret = yield p.render(self.secret) signature = hmac.new( - unicode2bytes(rendered_secret), + unicode2bytes(self.rendered_secret), unicode2bytes(content_text.strip()), digestmod=hashlib.sha256) header_signature = bytes2unicode( request.getHeader(_HEADER_SIGNATURE)) self.verifyTransifexSignature( - request, content, rendered_secret, + request, content, self.rendered_secret, signature, header_signature ) - event_type = bytes2unicode(payload.get(_EVENT_KEY), "None") + event_type = payload.get("event", "None") + language = payload.get("language", 'None') + project = payload.get("project", 'None') + resource = payload.get("resource", 'None') + + transifex_request_data['branch'] = "v0.6" + transifex_request_data['revision'] = "" + transifex_request_data["properties"] = "langugage" + transifex_request_data["properties"] = "resource" + transifex_request_data["properties"] = "project" + + transifex_request_data["properties"] = { + "branch": branch, + "revision": revision + } + transifex_request_data["changes"] = { + "author": "buildbot-transifex", + "repository": project, + } + log.msg("Received event '{}' from transifex".format(event_type)) codebase = "" -- 2.40.1 From ab6d1083ecf9add9e7ce523d5a8933d858b99ece Mon Sep 17 00:00:00 2001 From: shekhar-cis Date: Mon, 9 Jan 2023 21:56:28 +0530 Subject: [PATCH 4/5] Updated method names --- buildbot_transifex/webhook.py | 121 +++++++++++++--------------------- 1 file changed, 47 insertions(+), 74 deletions(-) diff --git a/buildbot_transifex/webhook.py b/buildbot_transifex/webhook.py index b383cb1..2912dbc 100644 --- a/buildbot_transifex/webhook.py +++ b/buildbot_transifex/webhook.py @@ -1,14 +1,12 @@ """Transifex webhook handler """ -import os import base64 - -import time -import json -import hmac import hashlib -from subprocess import call -from base64 import b64encode +import hmac +import json +import os import requests +import re +import time from buildbot.process.properties import Properties from buildbot.util import bytes2unicode, unicode2bytes @@ -17,37 +15,36 @@ from buildbot.www.hooks.base import BaseHookHandler from twisted.internet import defer from twisted.python import log -from dateutil.parser import parse as dateparse _HEADER_USER_AGENT = 'User-Agent' _HEADER_SIGNATURE = 'X-TX-Signature' _HEADER_URL_PATH = 'X-TX-Url' -HTTP_DATE = 'date' +_HTTP_DATE = 'date' _EVENT_KEY = 'event' -transifex_request_data = {} +author = 'buildbot-transifex' class TransifexHandler(BaseHookHandler): - def __init__(self, master, secret, options, transifex_dict): + def __init__(self, master, secret, transifex_to_github_map, options=None): if not options: options = {} self.secret = secret self.master = master self.options = options - self.transifex_dict = transifex_dict + self.transifex_to_github_map = transifex_to_github_map def returnMessage(self, status = False, message = "Unimplemented"): output = json.dumps({"status": "OK" if status else "FAIL", "message": message}) return [output, [('Content-type', 'application/json')]] - def verifyTransifexSignature( + def _verifyTransifexSignature( self, request, content, signature, header_signature ): http_verb = 'POST' http_url_path = request.getHeader(_HEADER_URL_PATH) - http_gmt_date = request.getHeader(HTTP_DATE) + http_gmt_date = request.getHeader(_HTTP_DATE) content_md5 = hashlib.md5(content).hexdigest() msg = b'\n'.join([ http_verb, http_url_path, http_gmt_date, content_md5 @@ -60,39 +57,25 @@ class TransifexHandler(BaseHookHandler): ).digest() ) if tx_signature != header_signature: - return False - + raise ValueError("Tx Signature mismatch") + if signature != request.getHeader(_HEADER_SIGNATURE): - return False + raise ValueError("Signature mismatch") + return True - def process_translation_completed(self, payload, transifex_dict, event_type, codebase): + def process_translation_completed(self, payload, codebase): changes = [] - transifex_response = self._transform_variables(payload, transifex_dict) - if 'pybitmessage-test' in transifex_response['project'] and 'messagespot' in transifex_response['resource']: - if 'translation_completed' in transifex_response['event']: - ts = int(time.time()) - lang = transifex_response['language'] - return - - # if isinstance(self.options, dict): - # commits = commits[:1] - - # # for commit in commits: - # # files = [] - # # for kind in ('added', 'modified', 'removed'): - # # files.extend(commit.get(kind, []) or []) + translated_request = self._transform_variables(payload['project'], payload['resource']) change = { - 'author': 'buildbot-transifex, - 'resource': transifex_response['resource'], - 'branch': transifex_dict['branch'], - 'project': transifex_response['project'], - 'event': event_type, + 'author': "buildbot-transifex", + 'branch': translated_request["branch"], + 'project': translated_request["project"], 'properties': { - 'branch': branch, - 'revision': revision, - 'language': lang, - 'resource': resource, - 'project': project + "transifex_language": payload.get("language", "None"), + "transifex_event": payload.get("event", "None"), + "transifex_project": payload.get("project", "None"), + "transifex_resource": payload.get("resource", "None"), + "transifex_branch": "v0.6" } } if codebase is not None: @@ -100,18 +83,22 @@ class TransifexHandler(BaseHookHandler): changes.insert(0, change) return changes - def _transform_variables(self, payload, transifex_dict): - project = payload.get('project', 'None') - transform_values = { - project: { - "repository": "https://github.com/Bitmessage/PyBitmessage", - "branch": "v0.6" - } + def _transform_variables(self, transifex_project, transifex_resource): + if transifex_project is None: + raise ValueError("Unknown project %s from transifex".format(transifex_project)) + key = "{}/{}".format(transifex_project, transifex_resource) + _map = self.map[key] + repository = _map["repository"] + project = re.sub(r'^.*/(.*?)(\.git)?$', r'\1', repository) + return{ + 'project': project, + 'repository': repository, + 'branch': _map["branch"], } - return transform_values @defer.inlineCallbacks def getChanges(self, request): + change = {} self.secret = None if isinstance(self.options, dict): self.secret = self.options.get('secret') @@ -126,36 +113,23 @@ class TransifexHandler(BaseHookHandler): p = Properties() p.master = self.master option = self.options - self.rendered_secret = yield p.render(self.secret) + rendered_secret = yield p.render(self.secret) signature = hmac.new( - unicode2bytes(self.rendered_secret), + unicode2bytes(rendered_secret), unicode2bytes(content_text.strip()), digestmod=hashlib.sha256) header_signature = bytes2unicode( request.getHeader(_HEADER_SIGNATURE)) - self.verifyTransifexSignature( - request, content, self.rendered_secret, - signature, header_signature - ) - + self._verifyTransifexSignature(request, content, rendered_secret, signature, header_signature) event_type = payload.get("event", "None") - language = payload.get("language", 'None') - project = payload.get("project", 'None') - resource = payload.get("resource", 'None') - transifex_request_data['branch'] = "v0.6" - transifex_request_data['revision'] = "" - transifex_request_data["properties"] = "langugage" - transifex_request_data["properties"] = "resource" - transifex_request_data["properties"] = "project" + mapped_request = self._transform_variables(payload['project'], payload['resource']) - transifex_request_data["properties"] = { - "branch": branch, - "revision": revision - } - transifex_request_data["changes"] = { - "author": "buildbot-transifex", - "repository": project, + change["changes"] = { + "author": author, + "repository": mapped_request["repository"], + "project": mapped_request["project"], + "branch": mapped_request["branch"] } log.msg("Received event '{}' from transifex".format(event_type)) @@ -171,5 +145,4 @@ class TransifexHandler(BaseHookHandler): return (changes, 'transifex') - -transifex = TransifexHandler(transifex_dict) +transifex = TransifexHandler -- 2.40.1 From 8091be197d04c9e802d4b4956ec6bd6362536738 Mon Sep 17 00:00:00 2001 From: shekhar-cis Date: Fri, 13 Jan 2023 12:30:22 +0530 Subject: [PATCH 5/5] Fixed change and property --- buildbot_transifex/webhook.py | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/buildbot_transifex/webhook.py b/buildbot_transifex/webhook.py index 2912dbc..6ed87d8 100644 --- a/buildbot_transifex/webhook.py +++ b/buildbot_transifex/webhook.py @@ -66,16 +66,18 @@ class TransifexHandler(BaseHookHandler): def process_translation_completed(self, payload, codebase): changes = [] translated_request = self._transform_variables(payload['project'], payload['resource']) + ts = int(time.time()) change = { - 'author': "buildbot-transifex", + 'author': author, 'branch': translated_request["branch"], + 'branch': translated_request["repository"], 'project': translated_request["project"], 'properties': { "transifex_language": payload.get("language", "None"), "transifex_event": payload.get("event", "None"), "transifex_project": payload.get("project", "None"), "transifex_resource": payload.get("resource", "None"), - "transifex_branch": "v0.6" + "transifex_branch": "translate_" + payload['language'] + "_" + str(ts) } } if codebase is not None: @@ -83,10 +85,10 @@ class TransifexHandler(BaseHookHandler): changes.insert(0, change) return changes - def _transform_variables(self, transifex_project, transifex_resource): + def _transform_variables(self, transifex_project): if transifex_project is None: raise ValueError("Unknown project %s from transifex".format(transifex_project)) - key = "{}/{}".format(transifex_project, transifex_resource) + key = transifex_project _map = self.map[key] repository = _map["repository"] project = re.sub(r'^.*/(.*?)(\.git)?$', r'\1', repository) @@ -122,16 +124,6 @@ class TransifexHandler(BaseHookHandler): request.getHeader(_HEADER_SIGNATURE)) self._verifyTransifexSignature(request, content, rendered_secret, signature, header_signature) event_type = payload.get("event", "None") - - mapped_request = self._transform_variables(payload['project'], payload['resource']) - - change["changes"] = { - "author": author, - "repository": mapped_request["repository"], - "project": mapped_request["project"], - "branch": mapped_request["branch"] - } - log.msg("Received event '{}' from transifex".format(event_type)) codebase = "" -- 2.40.1