forked from Bitmessage/buildbot-transifex
WIP[downloadTranslatedLanguage() and updateLocalTranslationDestination()]
This commit is contained in:
parent
f2587e071d
commit
4b38bcc252
GPG Key ID:
F4F00AB04E83F9A7
|
|
@ -1,8 +1,19 @@
|
|||
|
||||
import sys
|
||||
|
||||
import os
|
||||
import base64
|
||||
|
||||
import time
|
||||
import json
|
||||
import re
|
||||
import hmac
|
||||
import pprint
|
||||
import hashlib
|
||||
import requests
|
||||
from subprocess import call
|
||||
from base64 import b64encode
|
||||
|
||||
from buildbot.process.properties import Properties
|
||||
from buildbot.util import bytes2unicode, unicode2bytes
|
||||
from buildbot.www.hooks.base import BaseHookHandler
|
||||
|
|
@ -12,204 +23,156 @@ from twisted.python import log
|
|||
|
||||
from dateutil.parser import parse as dateparse
|
||||
|
||||
_HEADER_USER_AGENT = 'User-Agent'
|
||||
HTTP_USER_AGENT = 'User-Agent'
|
||||
_HEADER_SIGNATURE = 'X-TX-Signature'
|
||||
_EVENT_KEY = 'event'
|
||||
branch = 'v0.6'
|
||||
transifexSecret = ""
|
||||
transifexUsername = ""
|
||||
transifexPassword = ""
|
||||
transifex_webhook_url = 'https://buildbot.bitmessage.org/change_hook/transifex'
|
||||
transifex_dict = {}
|
||||
secret = ""
|
||||
master = ""
|
||||
|
||||
from transifex.api import transifex_api
|
||||
|
||||
transifex_api.setup(auth='my_token')
|
||||
org = transifex_api.Organization.get(slug='my_org')
|
||||
proj = org.fetch('projects').get(slug='my_project')
|
||||
lang = transifex_api.Language.get(code='<lang>')
|
||||
resource = proj.fetch('resources').get(slug='my_resource')
|
||||
url = transifex_api.ResourceTranslationsAsyncDownload.download(
|
||||
resource=resource, language=lang,content_encoding = 'text',
|
||||
file_type = 'default', pseudo = False
|
||||
)
|
||||
|
||||
class TransifexHandler(BaseHookHandler):
|
||||
|
||||
# def verifyGitHubSignature (environ, payload_body):
|
||||
# signature = 'sha1=' + hmac.new(gitHubSecret, payload_body, sha1).hexdigest()
|
||||
# try:
|
||||
# if signature != environ.get('X-TX-Signature'):
|
||||
# return False
|
||||
# return True
|
||||
# except:
|
||||
# return False
|
||||
def __init__(self, transifex_dict, secret, master,):
|
||||
self.secret = secret
|
||||
self.master = master
|
||||
self.transifex_dict = transifex_dict
|
||||
|
||||
# def verifyTransifexSignature (environ, payload_body):
|
||||
# signature = b64encode(hmac.new(transifexSecret, payload_body, sha1).digest())
|
||||
# try:
|
||||
# debug(signature)
|
||||
# if signature != environ.get('HTTP_X_TX_SIGNATURE'):
|
||||
# return False
|
||||
# return True
|
||||
# except:
|
||||
# return False
|
||||
def verifyTransifexSignature(
|
||||
self, request, content, rendered_secret, signature, header_signature
|
||||
):
|
||||
http_verb = 'POST'
|
||||
http_url_path = request.headers('X-TX-Url')
|
||||
http_gmt_date = request.headers('Date')
|
||||
content_md5 = hashlib.md5(content).hexdigest()
|
||||
msg = b'\n'.join([
|
||||
http_verb, http_url_path, http_gmt_date, content_md5
|
||||
])
|
||||
tx_signature = base64.b64encode(
|
||||
hmac.new(
|
||||
key=rendered_secret,
|
||||
msg=msg,
|
||||
digestmod=hashlib.sha256
|
||||
).digest()
|
||||
)
|
||||
if tx_signature() != header_signature:
|
||||
raise ValueError('Invalid secret')
|
||||
|
||||
try:
|
||||
if signature != os.environ.get('HTTP_X_TX_SIGNATURE'):
|
||||
return False
|
||||
return True
|
||||
except:
|
||||
return False
|
||||
|
||||
# def returnMessage(status = False, message = "Unimplemented"):
|
||||
# output = json.dumps({"status": "OK" if status else "FAIL", "message": message})
|
||||
# return [output, [('Content-type', 'text/plain'),
|
||||
# ('Content-Length', str(len(output)))
|
||||
# ]]
|
||||
def updateLocalTranslationDestination(self, ts, lang, branch):
|
||||
call(["git", "pull", "--all", "-q"])
|
||||
call(["git", "stash", "-q"])
|
||||
call(["git", "checkout", "-q", branch])
|
||||
call(["git", "checkout", "-q", "-b", "translate_" + lang + "_" + str(ts)])
|
||||
call(["git", "branch", "-q", "--set-upstream-to=origin/v0.6"])
|
||||
|
||||
# def application(environ, start_response):
|
||||
# status = '200 OK'
|
||||
# output = ''
|
||||
# lockWait()
|
||||
# length = int(environ.get('CONTENT_LENGTH', '0'))
|
||||
# body = environ['wsgi.input'].read(length)
|
||||
|
||||
# if "Transifex" in environ.get("HTTP_USER_AGENT"):
|
||||
# # debug(environ)
|
||||
# # debug(body)
|
||||
# if not verifyTransifexSignature(environ, body):
|
||||
# debug ("Verify Transifex Signature fail, but fuck them")
|
||||
# else:
|
||||
# debug ("Verify Transifex Signature ok")
|
||||
# # output, responseHeaders = returnMessage(False, "Checksum bad")
|
||||
# # start_response(status, responseHeaders)
|
||||
# # unlock()
|
||||
# # return [output]
|
||||
# try:
|
||||
# # debug(body)
|
||||
# payload = parse_qs(body)
|
||||
# # debug(payload)
|
||||
# if 'pybitmessage' in payload['project'] and 'pybitmessage' in payload['resource']:
|
||||
# if 'translated' in payload and '100' in payload['translated']:
|
||||
# ts = int(time.time())
|
||||
# updateLocalTranslationDestination(ts, payload['language'][0].lower())
|
||||
# downloadTranslatedLanguage(ts, payload['language'][0])
|
||||
# response = commitTranslatedLanguage(ts, payload['language'][0].lower())
|
||||
# if response.ok:
|
||||
# output, responseHeaders = returnMessage(True, "Processed.")
|
||||
# else:
|
||||
# output, responseHeaders = returnMessage(False, "Error: %i." % (response.status_code))
|
||||
# else:
|
||||
# output, responseHeaders = returnMessage(False, "Nothing to do")
|
||||
# else:
|
||||
# output, responseHeaders = returnMessage(False, "Nothing to do")
|
||||
# except:
|
||||
# output, responseHeaders = returnMessage(True, "Not processing")
|
||||
# else:
|
||||
# debug("Unknown command %s" % (environ.get("HTTP_X_GITHUB_EVENT")))
|
||||
# output, responseHeaders = returnMessage(True, "Unknown command, ignoring")
|
||||
def __init__(self, transifex_dict=None):
|
||||
super(TransifexHandler, self).__init__(*args, **kwargs)
|
||||
|
||||
def process_translation_completed(self, payload, event_type, codebase):
|
||||
refname = payload["ref"]
|
||||
def downloadTranslatedLanguage(ts, lang):
|
||||
headers = {"Authorization": "Basic " + b64encode(transifexUsername + ":" + transifexPassword)}
|
||||
fname = "bitmessage_" + lang.lower() + ".po"
|
||||
with open("src/translations/" + fname, "wt") as handle:
|
||||
response = requests.get("https://www.transifex.com/api/2/project/pybitmessage/resource/pybitmessage/translation/" + lang + "/",
|
||||
headers=headers)
|
||||
if response.ok:
|
||||
content = json.loads(response.content)["content"]
|
||||
handle.write(content.encode("utf-8"))
|
||||
return response
|
||||
|
||||
def process_translation_completed(self, transifex_dict, event_type, codebase):
|
||||
# refname = payload["ref"]
|
||||
payload = transifex_dict
|
||||
changes = []
|
||||
|
||||
# We only care about regular heads or tags
|
||||
match = re.match(r"^refs/(heads|tags)/(.+)$", refname)
|
||||
if event_type == 'translation_completed':
|
||||
pass
|
||||
# match = re.match(r"^refs/(heads|tags)/(.+)$", refname)
|
||||
if "Transifex" in os.environ.get("HTTP_USER_AGENT"):
|
||||
if not self.verifyTransifexSignature(
|
||||
request, content, rendered_secret, signature, header_signature):
|
||||
print('Invalid transifex signature!')
|
||||
else:
|
||||
payload = self.transifex_dict
|
||||
if 'pybitmessage-test' in payload['transifex_data']['project'] and 'messagespot' in payload['transifex_data']['translation_completed']['resource']:
|
||||
if 'translation_completed' in payload['transifex_data'] and '100' in payload['transifex_data']['translated']:
|
||||
ts = int(time.time())
|
||||
lang = payload['transifex_data']['translation_completed']['language']
|
||||
branch = payload['transifex_data']['branch']
|
||||
self.updateLocalTranslationDestination(ts, lang.lower(), branch)
|
||||
self.downloadTranslatedLanguage(ts, lang.lower())
|
||||
|
||||
if not match:
|
||||
log.msg("Ignoring refname '{}': Not a branch or tag".format(refname))
|
||||
return changes
|
||||
|
||||
branch = match.group(2)
|
||||
|
||||
repository = payload['repository']
|
||||
repo_url = repository['ssh_url']
|
||||
project = repository['full_name']
|
||||
|
||||
commits = payload['commits']
|
||||
if isinstance(self.options, dict) and self.options.get('onlyIncludePushCommit', False):
|
||||
commits = commits[:1]
|
||||
# if isinstance(self.options, dict) and self.options.get('onlyIncludePushCommit', False):
|
||||
# commits = commits[:1]
|
||||
|
||||
for commit in commits:
|
||||
files = []
|
||||
for kind in ('added', 'modified', 'removed'):
|
||||
files.extend(commit.get(kind, []) or [])
|
||||
timestamp = dateparse(commit['timestamp'])
|
||||
change = {
|
||||
'author': '{} <{}>'.format(commit['author']['name'],
|
||||
commit['author']['email']),
|
||||
'files': files,
|
||||
'comments': commit['message'],
|
||||
'revision': commit['id'],
|
||||
'when_timestamp': timestamp,
|
||||
'branch': branch,
|
||||
'revlink': commit['url'],
|
||||
'repository': repo_url,
|
||||
'project': project,
|
||||
'category': event_type,
|
||||
'properties': {
|
||||
'event': event_type,
|
||||
'repository_name': repository['name'],
|
||||
'owner': repository["owner"]["username"]
|
||||
},
|
||||
}
|
||||
if codebase is not None:
|
||||
change['codebase'] = codebase
|
||||
changes.insert(0, change)
|
||||
return changes
|
||||
# for commit in commits:
|
||||
# files = []
|
||||
# for kind in ('added', 'modified', 'removed'):
|
||||
# files.extend(commit.get(kind, []) or [])
|
||||
# timestamp = dateparse(commit['timestamp'])
|
||||
# change = {
|
||||
# 'author': '{} <{}>'.format(commit['author']['name'],
|
||||
# commit['author']['email']),
|
||||
# 'files': files,
|
||||
# 'comments': commit['message'],
|
||||
# 'revision': commit['id'],
|
||||
# 'when_timestamp': timestamp,
|
||||
# 'branch': branch,
|
||||
# 'revlink': commit['url'],
|
||||
# 'repository': repo_url,
|
||||
# 'project': project,
|
||||
# 'category': event_type,
|
||||
# 'properties': {
|
||||
# 'event': event_type,
|
||||
# 'repository_name': repository['name'],
|
||||
# 'owner': repository["owner"]["username"]
|
||||
# },
|
||||
# }
|
||||
# if codebase is not None:
|
||||
# change['codebase'] = codebase
|
||||
# changes.insert(0, change)
|
||||
# return changes
|
||||
|
||||
def process_review_completed(self, payload, event_type, codebase):
|
||||
action = payload['action']
|
||||
if event_type == 'review_completed':
|
||||
pass
|
||||
# Only handle potential new stuff, ignore close/.
|
||||
# Merge itself is handled by the regular branch push message
|
||||
if action not in ['opened', 'synchronized', 'edited', 'reopened']:
|
||||
log.msg("Transifex Pull Request event '{}' ignored".format(action))
|
||||
return []
|
||||
# pull_request = payload['pull_request']
|
||||
# if not pull_request['mergeable']:
|
||||
# log.msg("Transifex Pull Request ignored because it is not mergeable.")
|
||||
# return []
|
||||
# if pull_request['merged']:
|
||||
# log.msg("Transifex Pull Request ignored because it is already merged.")
|
||||
# return []
|
||||
# timestamp = dateparse(pull_request['updated_at'])
|
||||
# base = pull_request['base']
|
||||
# head = pull_request['head']
|
||||
repository = payload['repository']
|
||||
change = {
|
||||
'author': '{} <{}>'.format(pull_request['user']['full_name'],
|
||||
pull_request['user']['email']),
|
||||
# 'comments': 'PR#{}: {}\n\n{}'.format(
|
||||
# pull_request['number'],
|
||||
# pull_request['title'],
|
||||
# pull_request['body']),
|
||||
'revision': base['sha'],
|
||||
'when_timestamp': timestamp,
|
||||
'branch': base['ref'],
|
||||
# 'revlink': pull_request['html_url'],
|
||||
'repository': base['repo']['ssh_url'],
|
||||
'project': repository['full_name'],
|
||||
'category': event_type,
|
||||
'properties': {
|
||||
'event': event_type,
|
||||
'base_branch': base['ref'],
|
||||
'base_sha': base['sha'],
|
||||
'base_repo_id': base['repo_id'],
|
||||
'base_repository': base['repo']['clone_url'],
|
||||
'base_git_ssh_url': base['repo']['ssh_url'],
|
||||
'head_branch': head['ref'],
|
||||
'head_sha': head['sha'],
|
||||
'head_repo_id': head['repo_id'],
|
||||
'head_repository': head['repo']['clone_url'],
|
||||
'head_git_ssh_url': head['repo']['ssh_url'],
|
||||
'head_owner': head['repo']['owner']['username'],
|
||||
'head_reponame': head['repo']['name'],
|
||||
# 'pr_id': pull_request['id'],
|
||||
# 'pr_number': pull_request['number'],
|
||||
'repository_name': repository['name'],
|
||||
'owner': repository["owner"]["username"],
|
||||
},
|
||||
}
|
||||
if codebase is not None:
|
||||
change['codebase'] = codebase
|
||||
return [change]
|
||||
def process_review_completed(self, payload, transifex_data):
|
||||
pass
|
||||
|
||||
def _transform_variables(payload):
|
||||
|
||||
def _transform_variables(self, payload=transifex_dict):
|
||||
retval = {
|
||||
project: payload.get('project'),
|
||||
repository = [payload.get('resource')],
|
||||
branch = payload.get('language')
|
||||
'project': payload[''].get('project'),
|
||||
'repository': [payload.get('resource')],
|
||||
'branch': payload.get('language')
|
||||
}
|
||||
return retval
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def getChanges(self, request):
|
||||
secret = None
|
||||
self.secret = None
|
||||
if isinstance(self.options, dict):
|
||||
secret = self.options.get('secret')
|
||||
self.secret = self.options.get('secret')
|
||||
try:
|
||||
content = request.content.read()
|
||||
content_text = bytes2unicode(content)
|
||||
|
|
@ -217,35 +180,16 @@ class TransifexHandler(BaseHookHandler):
|
|||
except Exception as exception:
|
||||
raise ValueError('Error loading JSON: ' + str(exception))
|
||||
|
||||
|
||||
if secret is not None:
|
||||
if self.secret is not None:
|
||||
p = Properties()
|
||||
p.master = self.master
|
||||
rendered_secret = yield p.render(secret)
|
||||
rendered_secret = yield p.render(self.secret)
|
||||
signature = hmac.new(
|
||||
unicode2bytes(rendered_secret),
|
||||
unicode2bytes(content_text.strip()),
|
||||
digestmod=hashlib.sha256)
|
||||
header_signature = bytes2unicode(
|
||||
request.getHeader(_HEADER_SIGNATURE))
|
||||
|
||||
http_verb = 'POST'
|
||||
http_url_path = request.headers('X-TX-Url')
|
||||
http_gmt_date = request.headers('Date')
|
||||
content_md5 = hashlib.md5(content).hexdigest()
|
||||
msg = b'\n'.join([
|
||||
http_verb, http_url_path, http_gmt_date, content_md5
|
||||
])
|
||||
tx_signature = base64.b64encode(
|
||||
hmac.new(
|
||||
key=rendered_secret,
|
||||
msg=msg,
|
||||
digestmod=hashlib.sha256
|
||||
).digest()
|
||||
)
|
||||
if tx_signature() != header_signature:
|
||||
raise ValueError('Invalid secret')
|
||||
|
||||
event_type = bytes2unicode(payload.get(_EVENT_KEY), "None")
|
||||
log.msg("Received event '{}' from transifex".format(event_type))
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue