WIP[downloadTranslatedLanguage() and updateLocalTranslationDestination()]

buildbot_transifex/webhook.py

@@ -1,8 +1,19 @@
+
+import sys
+
+import os
 import base64
+
+import time
 import json
 import re
 import hmac
+import pprint
 import hashlib
+import requests
+from subprocess import call
+from base64 import b64encode
+
 from buildbot.process.properties import Properties
 from buildbot.util import bytes2unicode, unicode2bytes
 from buildbot.www.hooks.base import BaseHookHandler
@@ -15,220 +26,32 @@ from dateutil.parser import parse as dateparse
 _HEADER_USER_AGENT = 'User-Agent'
 _HEADER_SIGNATURE = 'X-TX-Signature'
 _EVENT_KEY = 'event'
+transifexSecret = ""
+transifexUsername = ""
+transifexPassword = ""
+transifex_dict = {}
+secret = ""
+master = ""
+
+gitHubToken = os.environ('gitHubToken')
 
 class TransifexHandler(BaseHookHandler):
 
-    # def verifyGitHubSignature (environ, payload_body):
+    def __init__(self, master, secret, transifex_dict):
-    #     signature = 'sha1=' + hmac.new(gitHubSecret, payload_body, sha1).hexdigest()
+        self.secret = secret
-    #     try:
+        self.master = master
-    #         if signature != environ.get('X-TX-Signature'):
+        self.transifex_dict = transifex_dict
-    #             return False
-    #         return True
-    #     except:
-    #         return False
-
-    # def verifyTransifexSignature (environ, payload_body):
-    #     signature = b64encode(hmac.new(transifexSecret, payload_body, sha1).digest())
-    #     try:
-    #         debug(signature)
-    #         if signature != environ.get('HTTP_X_TX_SIGNATURE'):
-    #             return False
-    #         return True
-    #     except:
-    #         return False
-
-    # def returnMessage(status = False, message = "Unimplemented"):
-    #     output = json.dumps({"status": "OK" if status else "FAIL", "message": message})
-    #     return [output, [('Content-type', 'text/plain'),
-    #         ('Content-Length', str(len(output)))
-    #     ]]
-
-    # def application(environ, start_response):
-    #     status = '200 OK'
-    #     output = ''
-    #     lockWait()
-    #     length = int(environ.get('CONTENT_LENGTH', '0'))
-    #     body = environ['wsgi.input'].read(length)
-
-    #     if "Transifex" in environ.get("HTTP_USER_AGENT"):
-    # #        debug(environ)
-    # #        debug(body)
-    #         if not verifyTransifexSignature(environ, body):
-    #             debug ("Verify Transifex Signature fail, but fuck them")
-    #         else:
-    #             debug ("Verify Transifex Signature ok")
-    # #            output, responseHeaders = returnMessage(False, "Checksum bad")
-    # #            start_response(status, responseHeaders)
-    # #            unlock()
-    # #            return [output]
-    #         try:
-    # #            debug(body)
-    #             payload = parse_qs(body)
-    # #            debug(payload)
-    #             if 'pybitmessage' in payload['project'] and 'pybitmessage' in payload['resource']:
-    #                 if 'translated' in payload and '100' in payload['translated']:
-    #                     ts = int(time.time())
-    #                     updateLocalTranslationDestination(ts, payload['language'][0].lower())
-    #                     downloadTranslatedLanguage(ts, payload['language'][0])
-    #                     response = commitTranslatedLanguage(ts, payload['language'][0].lower())
-    #                     if response.ok:
-    #                         output, responseHeaders = returnMessage(True, "Processed.")
-    #                     else:
-    #                         output, responseHeaders = returnMessage(False, "Error: %i." % (response.status_code))
-    #                 else:
-    #                     output, responseHeaders = returnMessage(False, "Nothing to do")
-    #             else:
-    #                 output, responseHeaders = returnMessage(False, "Nothing to do")
-    #         except:
-    #             output, responseHeaders = returnMessage(True, "Not processing")
-    #     else:
-    #         debug("Unknown command %s" % (environ.get("HTTP_X_GITHUB_EVENT")))
-    #         output, responseHeaders = returnMessage(True, "Unknown command, ignoring")
-    def __init__(self, transifex_dict=None):
-        super(TransifexHandler, self).__init__(*args, **kwargs)
-
-    def process_translation_completed(self, payload, event_type, codebase):
-        refname = payload["ref"]
-
-        changes = []
-      
-        # We only care about regular heads or tags
-        match = re.match(r"^refs/(heads|tags)/(.+)$", refname)
-        if event_type == 'translation_completed':
-            pass
-
-        if not match:
-            log.msg("Ignoring refname '{}': Not a branch or tag".format(refname))
-            return changes
-
-        branch = match.group(2)
-
-        repository = payload['repository']
-        repo_url = repository['ssh_url']
-        project = repository['full_name']
-
-        commits = payload['commits']
-        if isinstance(self.options, dict) and self.options.get('onlyIncludePushCommit', False):
-            commits = commits[:1]
-
-        for commit in commits:
-            files = []
-            for kind in ('added', 'modified', 'removed'):
-                files.extend(commit.get(kind, []) or [])
-            timestamp = dateparse(commit['timestamp'])
-            change = {
-                'author': '{} <{}>'.format(commit['author']['name'],
-                                           commit['author']['email']),
-                'files': files,
-                'comments': commit['message'],
-                'revision': commit['id'],
-                'when_timestamp': timestamp,
-                'branch': branch,
-                'revlink': commit['url'],
-                'repository': repo_url,
-                'project': project,
-                'category': event_type,
-                'properties': {
-                    'event': event_type,
-                    'repository_name': repository['name'],
-                    'owner': repository["owner"]["username"]
-                },
-            }
-            if codebase is not None:
-                change['codebase'] = codebase
-            changes.insert(0, change)
-        return changes
-
-    def process_review_completed(self, payload, event_type, codebase):
-        action = payload['action']
-        if event_type == 'review_completed':
-            pass
-        # Only handle potential new stuff, ignore close/.
-        # Merge itself is handled by the regular branch push message
-        if action not in ['opened', 'synchronized', 'edited', 'reopened']:
-            log.msg("Transifex Pull Request event '{}' ignored".format(action))
-            return []
-        # pull_request = payload['pull_request']
-        # if not pull_request['mergeable']:
-        #     log.msg("Transifex Pull Request ignored because it is not mergeable.")
-        #     return []
-        # if pull_request['merged']:
-        #     log.msg("Transifex Pull Request ignored because it is already merged.")
-        #     return []
-        # timestamp = dateparse(pull_request['updated_at'])
-        # base = pull_request['base']
-        # head = pull_request['head']
-        repository = payload['repository']
-        change = {
-            'author': '{} <{}>'.format(pull_request['user']['full_name'],
-                                       pull_request['user']['email']),
-            # 'comments': 'PR#{}: {}\n\n{}'.format(
-            #     pull_request['number'],
-            #     pull_request['title'],
-            #     pull_request['body']),
-            'revision': base['sha'],
-            'when_timestamp': timestamp,
-            'branch': base['ref'],
-            # 'revlink': pull_request['html_url'],
-            'repository': base['repo']['ssh_url'],
-            'project': repository['full_name'],
-            'category': event_type,
-            'properties': {
-                'event': event_type,
-                'base_branch': base['ref'],
-                'base_sha': base['sha'],
-                'base_repo_id': base['repo_id'],
-                'base_repository': base['repo']['clone_url'],
-                'base_git_ssh_url': base['repo']['ssh_url'],
-                'head_branch': head['ref'],
-                'head_sha': head['sha'],
-                'head_repo_id': head['repo_id'],
-                'head_repository': head['repo']['clone_url'],
-                'head_git_ssh_url': head['repo']['ssh_url'],
-                'head_owner': head['repo']['owner']['username'],
-                'head_reponame': head['repo']['name'],
-                # 'pr_id': pull_request['id'],
-                # 'pr_number': pull_request['number'],
-                'repository_name': repository['name'],
-                'owner': repository["owner"]["username"],
-            },
-        }
-        if codebase is not None:
-            change['codebase'] = codebase
-        return [change]
-
-    def _transform_variables(payload):
-        retval = {
-            project: payload.get('project'),
-            repository = [payload.get('resource')],
-            branch = payload.get('language')
-        }
-        return retval
-
-    @defer.inlineCallbacks
-    def getChanges(self, request):
-        secret = None
-        if isinstance(self.options, dict):
-            secret = self.options.get('secret')
-        try:
-            content = request.content.read()
-            content_text = bytes2unicode(content)
-            payload = json.loads(content_text)
-        except Exception as exception:
-            raise ValueError('Error loading JSON: ' + str(exception))
 
 
-        if secret is not None:
+    def returnMessage(self, status = False, message = "Unimplemented"):
-            p = Properties()
+        output = json.dumps({"status": "OK" if status else "FAIL", "message": message})
-            p.master = self.master
+        return [output, [('Content-type', 'text/plain'),
-            rendered_secret = yield p.render(secret)
+            ('Content-Length', str(len(output)))
-            signature = hmac.new(
+        ]]
-                unicode2bytes(rendered_secret),
-                unicode2bytes(content_text.strip()),
-                digestmod=hashlib.sha256)
-            header_signature = bytes2unicode(
-                request.getHeader(_HEADER_SIGNATURE))
 
+    def verifyTransifexSignature(
+        self, request, content, rendered_secret, signature, header_signature
+        ):
         http_verb = 'POST'
         http_url_path = request.headers('X-TX-Url')
         http_gmt_date = request.headers('Date')
@@ -246,6 +69,136 @@ class TransifexHandler(BaseHookHandler):
         if tx_signature() != header_signature:
             raise ValueError('Invalid secret')
 
+        try:
+            if signature != os.environ.get('HTTP_X_TX_SIGNATURE'):
+                return False
+            return True
+        except:
+            return False
+
+    def downloadTranslatedLanguage(self, ts, lang):
+        headers = {"Authorization": "Basic " + b64encode(transifexUsername + ":" + transifexPassword)}
+        fname = "bitmessage_" + lang.lower() + ".po"
+        with open("src/translations/" + fname, "wt") as handle:
+            response = requests.get("https://www.transifex.com/api/2/project/pybitmessage/resource/pybitmessage/translation/" + lang + "/",
+                headers=headers)
+            if response.ok:
+                content = json.loads(response.content)["content"]
+                handle.write(content.encode("utf-8"))
+        return response
+
+    def commitTranslatedLanguage(self, ts, lang):
+        call(["kivy", "src/translations/messages.pro"])
+        call(["git", "add", "src/translations/bitmessage_" + lang + ".ts", "src/translations/bitmessage_" + lang + ".qm"])
+        call(["git", "commit", "-q", "-S", "-m", "Auto-updated language %s from transifex" % (lang)])
+        newbranch = "translate_" + lang + "_" + str(ts)
+        call(["git", "push", "-q", "translations", newbranch + ":" + newbranch])
+        branch = transifex_dict['branch']
+
+        request = {
+            "title": "Translation update " + lang,
+            "body": "Auto-updated from transifex",
+            "head": "PyBitmessageTranslations:" + newbranch,
+            "base": branch
+        }
+        headers = {"Authorization": "token " + gitHubToken}
+        response = requests.post("https://api.github.com/repos/Bitmessage/PyBitmessage/pulls",
+            headers=headers, data=json.dumps(request))
+        return response
+
+
+    def process_translation_completed(self, payload, transifex_dict, event_type, codebase):
+        changes = []
+        transifex_response = self._transform_variables(payload, transifex_dict)
+        if 'pybitmessage-test' in transifex_response['project'] and 'messagespot' in transifex_response['resource']:
+            if 'translation_completed' in transifex_response['event'] and 100 in transifex_response['translated']:
+                ts = int(time.time())
+                lang = transifex_response['language']
+                branch = transifex_dict['branch']
+                self.downloadTranslatedLanguage(ts, lang.lower())
+                response = self.commitTranslatedLanguage(ts, lang.lower())
+                if response.ok:
+                    output, responseHeaders = self.returnMessage(True, "Processed.")
+                else:
+                    output, responseHeaders = self.returnMessage(False, "Error: %i." % (response.status_code))
+            else:
+                output, responseHeaders = self.returnMessage(False, "Nothing to do")
+        else:
+            output, responseHeaders = self.returnMessage(False, "Nothing to do")
+
+        # if isinstance(self.options, dict) and self.options.get('onlyIncludePushCommit', False):
+        #     commits = commits[:1]
+
+        # for commit in commits:
+        #     files = []
+        #     for kind in ('added', 'modified', 'removed'):
+        #         files.extend(commit.get(kind, []) or [])
+        #     timestamp = dateparse(commit['timestamp'])
+        #     change = {
+        #         'author': '{} <{}>'.format(commit['author']['name'],
+        #                                    commit['author']['email']),
+        #         'files': files,
+        #         'comments': commit['message'],
+        #         'revision': commit['id'],
+        #         'when_timestamp': timestamp,
+        #         'branch': branch,
+        #         'revlink': commit['url'],
+        #         'repository': repo_url,
+        #         'project': project,
+        #         'category': event_type,
+        #         'properties': {
+        #             'event': event_type,
+        #             'repository_name': repository['name'],
+        #             'owner': repository["owner"]["username"]
+        #         },
+        #     }
+        #     if codebase is not None:
+        #         change['codebase'] = codebase
+        #     changes.insert(0, change)
+        return changes
+
+    def process_review_completed(self, payload, transifex_data):
+        pass
+
+
+    def _transform_variables(self, payload, transifex_dict):
+        transifex_variables = {
+            'project': payload['project'],
+            "translated": payload['translated'],
+            "resource": payload['resource'],
+            "event": payload['event'],
+            "language": payload['language']
+        }
+
+        return transifex_variables
+
+    @defer.inlineCallbacks
+    def getChanges(self, request):
+        self.secret = None
+        if isinstance(self.options, dict):
+            self.secret = self.options.get('secret')
+        try:
+            content = request.content.read()
+            content_text = bytes2unicode(content)
+            payload = json.loads(content_text)
+        except Exception as exception:
+            raise ValueError('Error loading JSON: ' + str(exception))
+
+        if self.secret is not None:
+            p = Properties()
+            p.master = self.master
+            rendered_secret = yield p.render(self.secret)
+            signature = hmac.new(
+                unicode2bytes(rendered_secret),
+                unicode2bytes(content_text.strip()),
+                digestmod=hashlib.sha256)
+            header_signature = bytes2unicode(
+                request.getHeader(_HEADER_SIGNATURE))
+            self.verifyTransifexSignature(
+                request, content, rendered_secret, 
+                signature, header_signature
+            )
+
         event_type = bytes2unicode(payload.get(_EVENT_KEY), "None")
         log.msg("Received event '{}' from transifex".format(event_type))