Update webhook and setup #1
|
@ -1,8 +1,19 @@
|
|||
|
||||
import sys
|
||||
|
||||
|
||||
import os
|
||||
import base64
|
||||
|
||||
import time
|
||||
import json
|
||||
import re
|
||||
import hmac
|
||||
PeterSurda
commented
no need no need `logging`
PeterSurda
commented
no need for no need for `logging`
|
||||
import pprint
|
||||
import hashlib
|
||||
import requests
|
||||
PeterSurda
commented
we already imported base64 we already imported base64
|
||||
from subprocess import call
|
||||
from base64 import b64encode
|
||||
|
||||
from buildbot.process.properties import Properties
|
||||
from buildbot.util import bytes2unicode, unicode2bytes
|
||||
from buildbot.www.hooks.base import BaseHookHandler
|
||||
|
@ -15,220 +26,32 @@ from dateutil.parser import parse as dateparse
|
|||
_HEADER_USER_AGENT = 'User-Agent'
|
||||
PeterSurda
commented
why? why?
PeterSurda
commented
`_HEADER_DATE`
|
||||
_HEADER_SIGNATURE = 'X-TX-Signature'
|
||||
_EVENT_KEY = 'event'
|
||||
PeterSurda
commented
This is specific to a request so it shouldn't be a module variable. Wrong scope. At the very least, if there are multiple requests in parallel, this will cause collisions. This is specific to a request so it shouldn't be a module variable. Wrong scope. At the very least, if there are multiple requests in parallel, this will cause collisions.
|
||||
transifexSecret = ""
|
||||
PeterSurda
commented
should be inside the map should be inside the map
PeterSurda
commented
we don't need any of this we don't need any of this
|
||||
transifexUsername = ""
|
||||
PeterSurda
commented
should be arguments passed to constructor should be arguments passed to constructor
|
||||
transifexPassword = ""
|
||||
transifex_dict = {}
|
||||
secret = ""
|
||||
PeterSurda
commented
should be extracted from properties or buildbot runtime variables should be extracted from properties or buildbot runtime variables
|
||||
master = ""
|
||||
|
||||
PeterSurda
commented
`options` should probably be the last and default to `None` or something
|
||||
gitHubToken = os.environ('gitHubToken')
|
||||
PeterSurda marked this conversation as resolved
Outdated
|
||||
|
||||
class TransifexHandler(BaseHookHandler):
|
||||
|
||||
PeterSurda marked this conversation as resolved
Outdated
PeterSurda
commented
something like something like `transifex_to_github_map` is a better name
|
||||
# def verifyGitHubSignature (environ, payload_body):
|
||||
# signature = 'sha1=' + hmac.new(gitHubSecret, payload_body, sha1).hexdigest()
|
||||
# try:
|
||||
# if signature != environ.get('X-TX-Signature'):
|
||||
# return False
|
||||
# return True
|
||||
# except:
|
||||
# return False
|
||||
|
||||
# def verifyTransifexSignature (environ, payload_body):
|
||||
# signature = b64encode(hmac.new(transifexSecret, payload_body, sha1).digest())
|
||||
# try:
|
||||
# debug(signature)
|
||||
# if signature != environ.get('HTTP_X_TX_SIGNATURE'):
|
||||
# return False
|
||||
# return True
|
||||
# except:
|
||||
# return False
|
||||
|
||||
# def returnMessage(status = False, message = "Unimplemented"):
|
||||
# output = json.dumps({"status": "OK" if status else "FAIL", "message": message})
|
||||
# return [output, [('Content-type', 'text/plain'),
|
||||
# ('Content-Length', str(len(output)))
|
||||
# ]]
|
||||
|
||||
# def application(environ, start_response):
|
||||
# status = '200 OK'
|
||||
# output = ''
|
||||
# lockWait()
|
||||
# length = int(environ.get('CONTENT_LENGTH', '0'))
|
||||
# body = environ['wsgi.input'].read(length)
|
||||
|
||||
# if "Transifex" in environ.get("HTTP_USER_AGENT"):
|
||||
# # debug(environ)
|
||||
# # debug(body)
|
||||
# if not verifyTransifexSignature(environ, body):
|
||||
# debug ("Verify Transifex Signature fail, but fuck them")
|
||||
# else:
|
||||
# debug ("Verify Transifex Signature ok")
|
||||
# # output, responseHeaders = returnMessage(False, "Checksum bad")
|
||||
# # start_response(status, responseHeaders)
|
||||
# # unlock()
|
||||
# # return [output]
|
||||
# try:
|
||||
# # debug(body)
|
||||
# payload = parse_qs(body)
|
||||
# # debug(payload)
|
||||
# if 'pybitmessage' in payload['project'] and 'pybitmessage' in payload['resource']:
|
||||
# if 'translated' in payload and '100' in payload['translated']:
|
||||
# ts = int(time.time())
|
||||
# updateLocalTranslationDestination(ts, payload['language'][0].lower())
|
||||
# downloadTranslatedLanguage(ts, payload['language'][0])
|
||||
# response = commitTranslatedLanguage(ts, payload['language'][0].lower())
|
||||
# if response.ok:
|
||||
# output, responseHeaders = returnMessage(True, "Processed.")
|
||||
# else:
|
||||
# output, responseHeaders = returnMessage(False, "Error: %i." % (response.status_code))
|
||||
# else:
|
||||
# output, responseHeaders = returnMessage(False, "Nothing to do")
|
||||
# else:
|
||||
# output, responseHeaders = returnMessage(False, "Nothing to do")
|
||||
# except:
|
||||
# output, responseHeaders = returnMessage(True, "Not processing")
|
||||
# else:
|
||||
# debug("Unknown command %s" % (environ.get("HTTP_X_GITHUB_EVENT")))
|
||||
# output, responseHeaders = returnMessage(True, "Unknown command, ignoring")
|
||||
def __init__(self, transifex_dict=None):
|
||||
super(TransifexHandler, self).__init__(*args, **kwargs)
|
||||
|
||||
def process_translation_completed(self, payload, event_type, codebase):
|
||||
refname = payload["ref"]
|
||||
|
||||
changes = []
|
||||
|
||||
# We only care about regular heads or tags
|
||||
match = re.match(r"^refs/(heads|tags)/(.+)$", refname)
|
||||
if event_type == 'translation_completed':
|
||||
pass
|
||||
|
||||
if not match:
|
||||
log.msg("Ignoring refname '{}': Not a branch or tag".format(refname))
|
||||
return changes
|
||||
|
||||
branch = match.group(2)
|
||||
|
||||
repository = payload['repository']
|
||||
repo_url = repository['ssh_url']
|
||||
project = repository['full_name']
|
||||
|
||||
commits = payload['commits']
|
||||
if isinstance(self.options, dict) and self.options.get('onlyIncludePushCommit', False):
|
||||
commits = commits[:1]
|
||||
|
||||
for commit in commits:
|
||||
files = []
|
||||
for kind in ('added', 'modified', 'removed'):
|
||||
files.extend(commit.get(kind, []) or [])
|
||||
timestamp = dateparse(commit['timestamp'])
|
||||
change = {
|
||||
'author': '{} <{}>'.format(commit['author']['name'],
|
||||
commit['author']['email']),
|
||||
'files': files,
|
||||
'comments': commit['message'],
|
||||
'revision': commit['id'],
|
||||
'when_timestamp': timestamp,
|
||||
'branch': branch,
|
||||
'revlink': commit['url'],
|
||||
'repository': repo_url,
|
||||
'project': project,
|
||||
'category': event_type,
|
||||
'properties': {
|
||||
'event': event_type,
|
||||
'repository_name': repository['name'],
|
||||
'owner': repository["owner"]["username"]
|
||||
},
|
||||
}
|
||||
if codebase is not None:
|
||||
change['codebase'] = codebase
|
||||
changes.insert(0, change)
|
||||
return changes
|
||||
|
||||
def process_review_completed(self, payload, event_type, codebase):
|
||||
action = payload['action']
|
||||
if event_type == 'review_completed':
|
||||
pass
|
||||
# Only handle potential new stuff, ignore close/.
|
||||
# Merge itself is handled by the regular branch push message
|
||||
if action not in ['opened', 'synchronized', 'edited', 'reopened']:
|
||||
log.msg("Transifex Pull Request event '{}' ignored".format(action))
|
||||
return []
|
||||
# pull_request = payload['pull_request']
|
||||
# if not pull_request['mergeable']:
|
||||
# log.msg("Transifex Pull Request ignored because it is not mergeable.")
|
||||
# return []
|
||||
# if pull_request['merged']:
|
||||
# log.msg("Transifex Pull Request ignored because it is already merged.")
|
||||
# return []
|
||||
# timestamp = dateparse(pull_request['updated_at'])
|
||||
# base = pull_request['base']
|
||||
# head = pull_request['head']
|
||||
repository = payload['repository']
|
||||
change = {
|
||||
'author': '{} <{}>'.format(pull_request['user']['full_name'],
|
||||
pull_request['user']['email']),
|
||||
# 'comments': 'PR#{}: {}\n\n{}'.format(
|
||||
# pull_request['number'],
|
||||
# pull_request['title'],
|
||||
# pull_request['body']),
|
||||
'revision': base['sha'],
|
||||
'when_timestamp': timestamp,
|
||||
'branch': base['ref'],
|
||||
# 'revlink': pull_request['html_url'],
|
||||
'repository': base['repo']['ssh_url'],
|
||||
'project': repository['full_name'],
|
||||
'category': event_type,
|
||||
'properties': {
|
||||
'event': event_type,
|
||||
'base_branch': base['ref'],
|
||||
'base_sha': base['sha'],
|
||||
'base_repo_id': base['repo_id'],
|
||||
'base_repository': base['repo']['clone_url'],
|
||||
'base_git_ssh_url': base['repo']['ssh_url'],
|
||||
'head_branch': head['ref'],
|
||||
'head_sha': head['sha'],
|
||||
'head_repo_id': head['repo_id'],
|
||||
'head_repository': head['repo']['clone_url'],
|
||||
'head_git_ssh_url': head['repo']['ssh_url'],
|
||||
'head_owner': head['repo']['owner']['username'],
|
||||
'head_reponame': head['repo']['name'],
|
||||
# 'pr_id': pull_request['id'],
|
||||
# 'pr_number': pull_request['number'],
|
||||
'repository_name': repository['name'],
|
||||
'owner': repository["owner"]["username"],
|
||||
},
|
||||
}
|
||||
if codebase is not None:
|
||||
change['codebase'] = codebase
|
||||
return [change]
|
||||
|
||||
def _transform_variables(payload):
|
||||
retval = {
|
||||
project: payload.get('project'),
|
||||
repository = [payload.get('resource')],
|
||||
branch = payload.get('language')
|
||||
}
|
||||
return retval
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def getChanges(self, request):
|
||||
secret = None
|
||||
if isinstance(self.options, dict):
|
||||
secret = self.options.get('secret')
|
||||
try:
|
||||
content = request.content.read()
|
||||
content_text = bytes2unicode(content)
|
||||
payload = json.loads(content_text)
|
||||
except Exception as exception:
|
||||
raise ValueError('Error loading JSON: ' + str(exception))
|
||||
def __init__(self, master, secret, transifex_dict):
|
||||
self.secret = secret
|
||||
self.master = master
|
||||
self.transifex_dict = transifex_dict
|
||||
|
||||
|
||||
PeterSurda marked this conversation as resolved
Outdated
PeterSurda
commented
`_verifyTransifexSignature`
|
||||
if secret is not None:
|
||||
p = Properties()
|
||||
p.master = self.master
|
||||
rendered_secret = yield p.render(secret)
|
||||
signature = hmac.new(
|
||||
unicode2bytes(rendered_secret),
|
||||
unicode2bytes(content_text.strip()),
|
||||
digestmod=hashlib.sha256)
|
||||
header_signature = bytes2unicode(
|
||||
request.getHeader(_HEADER_SIGNATURE))
|
||||
def returnMessage(self, status = False, message = "Unimplemented"):
|
||||
output = json.dumps({"status": "OK" if status else "FAIL", "message": message})
|
||||
return [output, [('Content-type', 'text/plain'),
|
||||
('Content-Length', str(len(output)))
|
||||
PeterSurda
commented
- content-length is usually set by the web server automatically
- content-type should maybe be `application/json`
|
||||
]]
|
||||
|
||||
def verifyTransifexSignature(
|
||||
self, request, content, rendered_secret, signature, header_signature
|
||||
PeterSurda marked this conversation as resolved
Outdated
PeterSurda
commented
rendered_secret can be extracted directly, no need to pass it rendered_secret can be extracted directly, no need to pass it
|
||||
):
|
||||
http_verb = 'POST'
|
||||
http_url_path = request.headers('X-TX-Url')
|
||||
http_gmt_date = request.headers('Date')
|
||||
|
@ -246,6 +69,136 @@ class TransifexHandler(BaseHookHandler):
|
|||
if tx_signature() != header_signature:
|
||||
PeterSurda
commented
`tx_signature` isn't a function/method
|
||||
raise ValueError('Invalid secret')
|
||||
PeterSurda marked this conversation as resolved
Outdated
|
||||
|
||||
PeterSurda
commented
don't have weird constants here, rather process is through the map. don't have weird constants here, rather process is through the map.
PeterSurda
commented
`repository` missing
|
||||
try:
|
||||
PeterSurda
commented
`transifex_to_github_map` is an object attribute, no need to pass it around
|
||||
if signature != os.environ.get('HTTP_X_TX_SIGNATURE'):
|
||||
PeterSurda marked this conversation as resolved
Outdated
PeterSurda
commented
- use constant
- use a default value, then no need for try/except
|
||||
return False
|
||||
PeterSurda
commented
error handling missing here error handling missing here
|
||||
return True
|
||||
except:
|
||||
PeterSurda
commented
if you use if you use `.get` with a default value, you can avoid `try`/`except`
|
||||
return False
|
||||
PeterSurda
commented
`language = payload.get('language', "None")`
|
||||
|
||||
PeterSurda
commented
this should be the randomly generated one, the branch of the PR this should be the randomly generated one, the branch of the PR
|
||||
def downloadTranslatedLanguage(self, ts, lang):
|
||||
PeterSurda marked this conversation as resolved
Outdated
PeterSurda
commented
This should be in the buildbot job, not here. The webhook only transforms data structures and puts them into a database.
This should be in the buildbot job, not here. The webhook only transforms data structures and puts them into a database.
- `"tx pull -l language {}".format(options["language").` or something like that
PeterSurda
commented
this line ( this line (`'resource': ...`) should be remove
|
||||
headers = {"Authorization": "Basic " + b64encode(transifexUsername + ":" + transifexPassword)}
|
||||
PeterSurda
commented
something like something like `translated_request` or `mapped_request` would be more understandable. There is not response coming from transifex.
|
||||
fname = "bitmessage_" + lang.lower() + ".po"
|
||||
PeterSurda
commented
not translating anything. There is nothing here that translates transifex payload information into github information. not translating anything. There is nothing here that translates transifex payload information into github information.
|
||||
with open("src/translations/" + fname, "wt") as handle:
|
||||
response = requests.get("https://www.transifex.com/api/2/project/pybitmessage/resource/pybitmessage/translation/" + lang + "/",
|
||||
headers=headers)
|
||||
if response.ok:
|
||||
content = json.loads(response.content)["content"]
|
||||
PeterSurda
commented
this is a sub-property this is a sub-property
|
||||
handle.write(content.encode("utf-8"))
|
||||
PeterSurda
commented
from map from map
|
||||
return response
|
||||
PeterSurda
commented
- no need super
- needs `master`
- doesn't save `transifex_dict`
- `secret` also missing
PeterSurda
commented
from map from map
|
||||
|
||||
def commitTranslatedLanguage(self, ts, lang):
|
||||
call(["kivy", "src/translations/messages.pro"])
|
||||
call(["git", "add", "src/translations/bitmessage_" + lang + ".ts", "src/translations/bitmessage_" + lang + ".qm"])
|
||||
call(["git", "commit", "-q", "-S", "-m", "Auto-updated language %s from transifex" % (lang)])
|
||||
newbranch = "translate_" + lang + "_" + str(ts)
|
||||
call(["git", "push", "-q", "translations", newbranch + ":" + newbranch])
|
||||
branch = transifex_dict['branch']
|
||||
PeterSurda
commented
maybe add error handling, e.g. maybe add error handling, e.g.
```raise ValueError("Unknown project %s from transifex".format(transifex_project)"```
|
||||
|
||||
request = {
|
||||
PeterSurda
commented
missing functionality, the project should be extracted from the missing functionality, the project should be extracted from the `repository` variable
|
||||
"title": "Translation update " + lang,
|
||||
"body": "Auto-updated from transifex",
|
||||
"head": "PyBitmessageTranslations:" + newbranch,
|
||||
"base": branch
|
||||
PeterSurda
commented
this shouldn't be here this shouldn't be here
|
||||
}
|
||||
headers = {"Authorization": "token " + gitHubToken}
|
||||
response = requests.post("https://api.github.com/repos/Bitmessage/PyBitmessage/pulls",
|
||||
headers=headers, data=json.dumps(request))
|
||||
return response
|
||||
|
||||
PeterSurda
commented
- either don't verify user agent at all, or verify it inside verifyTransifexSignature
- the verification should be called before the factory pattern
|
||||
|
||||
def process_translation_completed(self, payload, transifex_dict, event_type, codebase):
|
||||
changes = []
|
||||
transifex_response = self._transform_variables(payload, transifex_dict)
|
||||
if 'pybitmessage-test' in transifex_response['project'] and 'messagespot' in transifex_response['resource']:
|
||||
if 'translation_completed' in transifex_response['event'] and 100 in transifex_response['translated']:
|
||||
ts = int(time.time())
|
||||
lang = transifex_response['language']
|
||||
branch = transifex_dict['branch']
|
||||
self.downloadTranslatedLanguage(ts, lang.lower())
|
||||
response = self.commitTranslatedLanguage(ts, lang.lower())
|
||||
if response.ok:
|
||||
output, responseHeaders = self.returnMessage(True, "Processed.")
|
||||
else:
|
||||
PeterSurda
commented
we can just use we can just use `rendered_secret`, not `self.rendered_secret`
|
||||
output, responseHeaders = self.returnMessage(False, "Error: %i." % (response.status_code))
|
||||
else:
|
||||
output, responseHeaders = self.returnMessage(False, "Nothing to do")
|
||||
else:
|
||||
PeterSurda
commented
duplicate duplicate
|
||||
output, responseHeaders = self.returnMessage(False, "Nothing to do")
|
||||
|
||||
# if isinstance(self.options, dict) and self.options.get('onlyIncludePushCommit', False):
|
||||
PeterSurda
commented
since we're raising inside the since we're raising inside the `_verifyTransifexSignature`, no need for `if` here anymore
|
||||
# commits = commits[:1]
|
||||
|
||||
PeterSurda
commented
this is incomplete, we shouldn't proceed if the verification fails this is incomplete, we shouldn't proceed if the verification fails
|
||||
# for commit in commits:
|
||||
PeterSurda
commented
we're not doing anything with the return value of the verification we're not doing anything with the return value of the verification
|
||||
# files = []
|
||||
# for kind in ('added', 'modified', 'removed'):
|
||||
# files.extend(commit.get(kind, []) or [])
|
||||
# timestamp = dateparse(commit['timestamp'])
|
||||
# change = {
|
||||
PeterSurda
commented
duplicate duplicate
|
||||
# 'author': '{} <{}>'.format(commit['author']['name'],
|
||||
# commit['author']['email']),
|
||||
# 'files': files,
|
||||
# 'comments': commit['message'],
|
||||
# 'revision': commit['id'],
|
||||
PeterSurda
commented
we shouldn't be modifying we shouldn't be modifying `request_data`, that's input
|
||||
# 'when_timestamp': timestamp,
|
||||
PeterSurda
commented
maybe maybe `mapped_request`
|
||||
# 'branch': branch,
|
||||
# 'revlink': commit['url'],
|
||||
# 'repository': repo_url,
|
||||
PeterSurda
commented
from the map from the map
PeterSurda
commented
maybe make maybe make `"buildbot-transifex`" a class variable or a module variable?
|
||||
# 'project': project,
|
||||
PeterSurda
commented
not doing anything useful not doing anything useful
|
||||
# 'category': event_type,
|
||||
PeterSurda
commented
this is github syntax, needs to be translated into transifex syntax, maybe even ignored as transifex may not have this kind of variable this is github syntax, needs to be translated into transifex syntax, maybe even ignored as transifex may not have this kind of variable
PeterSurda
commented
```
transifex_request_data["properties"] = {
"transifex_language": language,
"transifex_resource": resource,
"transifex_project": project
}
```
|
||||
# 'properties': {
|
||||
# 'event': event_type,
|
||||
# 'repository_name': repository['name'],
|
||||
# 'owner': repository["owner"]["username"]
|
||||
# },
|
||||
# }
|
||||
# if codebase is not None:
|
||||
# change['codebase'] = codebase
|
||||
# changes.insert(0, change)
|
||||
return changes
|
||||
|
||||
def process_review_completed(self, payload, transifex_data):
|
||||
pass
|
||||
|
||||
PeterSurda
commented
- here you need to translate from transifex syntax using the map to github syntax
- also add additional transifex variables as custom properties
|
||||
|
||||
def _transform_variables(self, payload, transifex_dict):
|
||||
transifex_variables = {
|
||||
PeterSurda
commented
Probably something like:
- we need to translate from transifex structure to github/gitea structure
- the key should be constructed from transifex data, and the value should be a dictionary of key/values referencing a github/gitea project
Probably something like:
```
key = payload.get("project", "")
value = {
"repository": "https://github.com/Bitmessage/PyBitmessage",
"branch": "v0.6"
}
```
|
||||
'project': payload['project'],
|
||||
"translated": payload['translated'],
|
||||
"resource": payload['resource'],
|
||||
"event": payload['event'],
|
||||
"language": payload['language']
|
||||
}
|
||||
|
||||
return transifex_variables
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def getChanges(self, request):
|
||||
self.secret = None
|
||||
if isinstance(self.options, dict):
|
||||
self.secret = self.options.get('secret')
|
||||
try:
|
||||
content = request.content.read()
|
||||
content_text = bytes2unicode(content)
|
||||
payload = json.loads(content_text)
|
||||
except Exception as exception:
|
||||
raise ValueError('Error loading JSON: ' + str(exception))
|
||||
|
||||
PeterSurda
commented
- add a section to verify all mandatory options are present (e.g. "language", "project", "resource")
|
||||
if self.secret is not None:
|
||||
p = Properties()
|
||||
p.master = self.master
|
||||
rendered_secret = yield p.render(self.secret)
|
||||
signature = hmac.new(
|
||||
unicode2bytes(rendered_secret),
|
||||
unicode2bytes(content_text.strip()),
|
||||
digestmod=hashlib.sha256)
|
||||
header_signature = bytes2unicode(
|
||||
request.getHeader(_HEADER_SIGNATURE))
|
||||
self.verifyTransifexSignature(
|
||||
request, content, rendered_secret,
|
||||
signature, header_signature
|
||||
)
|
||||
|
||||
event_type = bytes2unicode(payload.get(_EVENT_KEY), "None")
|
||||
PeterSurda
commented
- ~~maybe we don't need `bytes2unicode`?~~ (leftover from gitea hook)
- we probably need `event_type = payload.get("event", "none")
|
||||
log.msg("Received event '{}' from transifex".format(event_type))
|
||||
PeterSurda
commented
- check out https://git.bitmessage.org/Bitmessage/buildbot_multibuild/src/branch/master/lib/worker_multibuild.py to see the mandatory components
- `project` can be extracted from `repository` (is a substring)
- `revision` let's ignore it for now
- `branch` is in the map, just like `repository`
|
||||
|
||||
|
|
isort