shekhar-cis
/
buildbot-transifex
forked from Bitmessage/buildbot-transifex
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

WIP[downloadTranslatedLanguage() and updateLocalTranslationDestination()]

This commit is contained in:
shekhar-cis 2023-01-04 20:51:06 +05:30
parent f2587e071d
commit 4b38bcc252
Signed by: shekhar-cis
GPG Key ID: F4F00AB04E83F9A7
1 changed files with 135 additions and 191 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

326
buildbot_transifex/webhook.py
View File

@ -1,8 +1,19 @@
import sys
import os
import base64 import base64
import time
import json import json
import re import re
import hmac import hmac
import pprint
import hashlib import hashlib
import requests
from subprocess import call
from base64 import b64encode
from buildbot.process.properties import Properties from buildbot.process.properties import Properties
from buildbot.util import bytes2unicode, unicode2bytes from buildbot.util import bytes2unicode, unicode2bytes
from buildbot.www.hooks.base import BaseHookHandler from buildbot.www.hooks.base import BaseHookHandler
@ -12,204 +23,156 @@ from twisted.python import log
from dateutil.parser import parse as dateparse from dateutil.parser import parse as dateparse
_HEADER_USER_AGENT = 'User-Agent' HTTP_USER_AGENT = 'User-Agent'
_HEADER_SIGNATURE = 'X-TX-Signature' _HEADER_SIGNATURE = 'X-TX-Signature'
_EVENT_KEY = 'event' _EVENT_KEY = 'event'
branch = 'v0.6'
transifexSecret = ""
transifexUsername = ""
transifexPassword = ""
transifex_webhook_url = 'https://buildbot.bitmessage.org/change_hook/transifex'
transifex_dict = {}
secret = ""
master = ""
from transifex.api import transifex_api
transifex_api.setup(auth='my_token')
org = transifex_api.Organization.get(slug='my_org')
proj = org.fetch('projects').get(slug='my_project')
lang = transifex_api.Language.get(code='<lang>')
resource = proj.fetch('resources').get(slug='my_resource')
url = transifex_api.ResourceTranslationsAsyncDownload.download(
resource=resource, language=lang,content_encoding = 'text',
file_type = 'default', pseudo = False
)
class TransifexHandler(BaseHookHandler): class TransifexHandler(BaseHookHandler):
# def verifyGitHubSignature (environ, payload_body): def __init__(self, transifex_dict, secret, master,):
# signature = 'sha1=' + hmac.new(gitHubSecret, payload_body, sha1).hexdigest() self.secret = secret
# try: self.master = master
# if signature != environ.get('X-TX-Signature'): self.transifex_dict = transifex_dict
# return False
# return True
# except:
# return False
# def verifyTransifexSignature (environ, payload_body): def verifyTransifexSignature(
# signature = b64encode(hmac.new(transifexSecret, payload_body, sha1).digest()) self, request, content, rendered_secret, signature, header_signature
# try: ):
# debug(signature) http_verb = 'POST'
# if signature != environ.get('HTTP_X_TX_SIGNATURE'): http_url_path = request.headers('X-TX-Url')
# return False http_gmt_date = request.headers('Date')
# return True content_md5 = hashlib.md5(content).hexdigest()
# except: msg = b'\n'.join([
# return False http_verb, http_url_path, http_gmt_date, content_md5
])
tx_signature = base64.b64encode(
hmac.new(
key=rendered_secret,
msg=msg,
digestmod=hashlib.sha256
).digest()
)
if tx_signature() != header_signature:
raise ValueError('Invalid secret')
try:
if signature != os.environ.get('HTTP_X_TX_SIGNATURE'):
return False
return True
except:
return False
# def returnMessage(status = False, message = "Unimplemented"): def updateLocalTranslationDestination(self, ts, lang, branch):
# output = json.dumps({"status": "OK" if status else "FAIL", "message": message}) call(["git", "pull", "--all", "-q"])
# return [output, [('Content-type', 'text/plain'), call(["git", "stash", "-q"])
# ('Content-Length', str(len(output))) call(["git", "checkout", "-q", branch])
# ]] call(["git", "checkout", "-q", "-b", "translate_" + lang + "_" + str(ts)])
call(["git", "branch", "-q", "--set-upstream-to=origin/v0.6"])
# def application(environ, start_response): def downloadTranslatedLanguage(ts, lang):
# status = '200 OK' headers = {"Authorization": "Basic " + b64encode(transifexUsername + ":" + transifexPassword)}
# output = '' fname = "bitmessage_" + lang.lower() + ".po"
# lockWait() with open("src/translations/" + fname, "wt") as handle:
# length = int(environ.get('CONTENT_LENGTH', '0')) response = requests.get("https://www.transifex.com/api/2/project/pybitmessage/resource/pybitmessage/translation/" + lang + "/",
# body = environ['wsgi.input'].read(length) headers=headers)
if response.ok:
# if "Transifex" in environ.get("HTTP_USER_AGENT"): content = json.loads(response.content)["content"]
# # debug(environ) handle.write(content.encode("utf-8"))
# # debug(body) return response
# if not verifyTransifexSignature(environ, body):
# debug ("Verify Transifex Signature fail, but fuck them")
# else:
# debug ("Verify Transifex Signature ok")
# # output, responseHeaders = returnMessage(False, "Checksum bad")
# # start_response(status, responseHeaders)
# # unlock()
# # return [output]
# try:
# # debug(body)
# payload = parse_qs(body)
# # debug(payload)
# if 'pybitmessage' in payload['project'] and 'pybitmessage' in payload['resource']:
# if 'translated' in payload and '100' in payload['translated']:
# ts = int(time.time())
# updateLocalTranslationDestination(ts, payload['language'][0].lower())
# downloadTranslatedLanguage(ts, payload['language'][0])
# response = commitTranslatedLanguage(ts, payload['language'][0].lower())
# if response.ok:
# output, responseHeaders = returnMessage(True, "Processed.")
# else:
# output, responseHeaders = returnMessage(False, "Error: %i." % (response.status_code))
# else:
# output, responseHeaders = returnMessage(False, "Nothing to do")
# else:
# output, responseHeaders = returnMessage(False, "Nothing to do")
# except:
# output, responseHeaders = returnMessage(True, "Not processing")
# else:
# debug("Unknown command %s" % (environ.get("HTTP_X_GITHUB_EVENT")))
# output, responseHeaders = returnMessage(True, "Unknown command, ignoring")
def __init__(self, transifex_dict=None):
super(TransifexHandler, self).__init__(*args, **kwargs)
def process_translation_completed(self, payload, event_type, codebase):
refname = payload["ref"]
def process_translation_completed(self, transifex_dict, event_type, codebase):
# refname = payload["ref"]
payload = transifex_dict
changes = [] changes = []
# We only care about regular heads or tags # We only care about regular heads or tags
match = re.match(r"^refs/(heads|tags)/(.+)$", refname) # match = re.match(r"^refs/(heads|tags)/(.+)$", refname)
if event_type == 'translation_completed': if "Transifex" in os.environ.get("HTTP_USER_AGENT"):
pass if not self.verifyTransifexSignature(
request, content, rendered_secret, signature, header_signature):
print('Invalid transifex signature!')
else:
payload = self.transifex_dict
if 'pybitmessage-test' in payload['transifex_data']['project'] and 'messagespot' in payload['transifex_data']['translation_completed']['resource']:
if 'translation_completed' in payload['transifex_data'] and '100' in payload['transifex_data']['translated']:
ts = int(time.time())
lang = payload['transifex_data']['translation_completed']['language']
branch = payload['transifex_data']['branch']
self.updateLocalTranslationDestination(ts, lang.lower(), branch)
self.downloadTranslatedLanguage(ts, lang.lower())
if not match:
log.msg("Ignoring refname '{}': Not a branch or tag".format(refname))
return changes
branch = match.group(2)
repository = payload['repository'] repository = payload['repository']
repo_url = repository['ssh_url'] repo_url = repository['ssh_url']
project = repository['full_name'] project = repository['full_name']
commits = payload['commits'] commits = payload['commits']
if isinstance(self.options, dict) and self.options.get('onlyIncludePushCommit', False): # if isinstance(self.options, dict) and self.options.get('onlyIncludePushCommit', False):
commits = commits[:1] # commits = commits[:1]
for commit in commits: # for commit in commits:
files = [] # files = []
for kind in ('added', 'modified', 'removed'): # for kind in ('added', 'modified', 'removed'):
files.extend(commit.get(kind, []) or []) # files.extend(commit.get(kind, []) or [])
timestamp = dateparse(commit['timestamp']) # timestamp = dateparse(commit['timestamp'])
change = { # change = {
'author': '{} <{}>'.format(commit['author']['name'], # 'author': '{} <{}>'.format(commit['author']['name'],
commit['author']['email']), # commit['author']['email']),
'files': files, # 'files': files,
'comments': commit['message'], # 'comments': commit['message'],
'revision': commit['id'], # 'revision': commit['id'],
'when_timestamp': timestamp, # 'when_timestamp': timestamp,
'branch': branch, # 'branch': branch,
'revlink': commit['url'], # 'revlink': commit['url'],
'repository': repo_url, # 'repository': repo_url,
'project': project, # 'project': project,
'category': event_type, # 'category': event_type,
'properties': { # 'properties': {
'event': event_type, # 'event': event_type,
'repository_name': repository['name'], # 'repository_name': repository['name'],
'owner': repository["owner"]["username"] # 'owner': repository["owner"]["username"]
}, # },
} # }
if codebase is not None: # if codebase is not None:
change['codebase'] = codebase # change['codebase'] = codebase
changes.insert(0, change) # changes.insert(0, change)
return changes # return changes
def process_review_completed(self, payload, event_type, codebase): def process_review_completed(self, payload, transifex_data):
action = payload['action'] pass
if event_type == 'review_completed':
pass
# Only handle potential new stuff, ignore close/.
# Merge itself is handled by the regular branch push message
if action not in ['opened', 'synchronized', 'edited', 'reopened']:
log.msg("Transifex Pull Request event '{}' ignored".format(action))
return []
# pull_request = payload['pull_request']
# if not pull_request['mergeable']:
# log.msg("Transifex Pull Request ignored because it is not mergeable.")
# return []
# if pull_request['merged']:
# log.msg("Transifex Pull Request ignored because it is already merged.")
# return []
# timestamp = dateparse(pull_request['updated_at'])
# base = pull_request['base']
# head = pull_request['head']
repository = payload['repository']
change = {
'author': '{} <{}>'.format(pull_request['user']['full_name'],
pull_request['user']['email']),
# 'comments': 'PR#{}: {}\n\n{}'.format(
# pull_request['number'],
# pull_request['title'],
# pull_request['body']),
'revision': base['sha'],
'when_timestamp': timestamp,
'branch': base['ref'],
# 'revlink': pull_request['html_url'],
'repository': base['repo']['ssh_url'],
'project': repository['full_name'],
'category': event_type,
'properties': {
'event': event_type,
'base_branch': base['ref'],
'base_sha': base['sha'],
'base_repo_id': base['repo_id'],
'base_repository': base['repo']['clone_url'],
'base_git_ssh_url': base['repo']['ssh_url'],
'head_branch': head['ref'],
'head_sha': head['sha'],
'head_repo_id': head['repo_id'],
'head_repository': head['repo']['clone_url'],
'head_git_ssh_url': head['repo']['ssh_url'],
'head_owner': head['repo']['owner']['username'],
'head_reponame': head['repo']['name'],
# 'pr_id': pull_request['id'],
# 'pr_number': pull_request['number'],
'repository_name': repository['name'],
'owner': repository["owner"]["username"],
},
}
if codebase is not None:
change['codebase'] = codebase
return [change]
def _transform_variables(payload):
def _transform_variables(self, payload=transifex_dict):
retval = { retval = {
project: payload.get('project'), 'project': payload[''].get('project'),
repository = [payload.get('resource')], 'repository': [payload.get('resource')],
branch = payload.get('language') 'branch': payload.get('language')
} }
return retval return retval
@defer.inlineCallbacks @defer.inlineCallbacks
def getChanges(self, request): def getChanges(self, request):
secret = None self.secret = None
if isinstance(self.options, dict): if isinstance(self.options, dict):
secret = self.options.get('secret') self.secret = self.options.get('secret')
try: try:
content = request.content.read() content = request.content.read()
content_text = bytes2unicode(content) content_text = bytes2unicode(content)
@ -217,35 +180,16 @@ class TransifexHandler(BaseHookHandler):
except Exception as exception: except Exception as exception:
raise ValueError('Error loading JSON: ' + str(exception)) raise ValueError('Error loading JSON: ' + str(exception))
if self.secret is not None:
if secret is not None:
p = Properties() p = Properties()
p.master = self.master p.master = self.master
rendered_secret = yield p.render(secret) rendered_secret = yield p.render(self.secret)
signature = hmac.new( signature = hmac.new(
unicode2bytes(rendered_secret), unicode2bytes(rendered_secret),
unicode2bytes(content_text.strip()), unicode2bytes(content_text.strip()),
digestmod=hashlib.sha256) digestmod=hashlib.sha256)
header_signature = bytes2unicode( header_signature = bytes2unicode(
request.getHeader(_HEADER_SIGNATURE)) request.getHeader(_HEADER_SIGNATURE))
http_verb = 'POST'
http_url_path = request.headers('X-TX-Url')
http_gmt_date = request.headers('Date')
content_md5 = hashlib.md5(content).hexdigest()
msg = b'\n'.join([
http_verb, http_url_path, http_gmt_date, content_md5
])
tx_signature = base64.b64encode(
hmac.new(
key=rendered_secret,
msg=msg,
digestmod=hashlib.sha256
).digest()
)
if tx_signature() != header_signature:
raise ValueError('Invalid secret')
event_type = bytes2unicode(payload.get(_EVENT_KEY), "None") event_type = bytes2unicode(payload.get(_EVENT_KEY), "None")
log.msg("Received event '{}' from transifex".format(event_type)) log.msg("Received event '{}' from transifex".format(event_type))