Merge production changes

- bugfixes, code quality

lib/renderers.py

@@ -1,24 +1,24 @@
 from buildbot.plugins import util
 import re
 
+def _is_build_script_available(props):
+    return props.getProperty("build_script_available", default=False)
 
 @util.renderer
 def is_build_script_available(props):
-    # Actual check will got here
+    return _is_build_script_available(props)
-    return props.getProperty("build_available", default=False)
-
 
 @util.renderer
 def isnt_build_script_available(props):
-    return not is_build_script_available(props)
+    return not _is_build_script_available(props)
 
+def _is_test_script_available(props):
+    return props.getProperty("test_script_available", default=False)
 
 @util.renderer
 def is_test_script_available(props):
-    # Actual check will got here
+    return _is_test_script_available(props)
-    return props.getProperty("test_available", default=False)
-
 
 @util.renderer
 def isnt_test_script_available(props):
-    return not is_test_script_available(props)
+    return not _is_test_script_available(props)

lib/worker_multibuild.py

@@ -1,8 +1,8 @@
-import os
 from os import listdir
-from os.path import exists, isfile, join, islink, isdir
+from os.path import exists, isfile, join, islink
 import requests
 import re
+from subprocess import Popen, PIPE
 
 
 request_data = {
@@ -11,7 +11,7 @@ request_data = {
 }
 
 ty = "/change_hook/base"
-path =".buildbot"
+path = ".buildbot"
 dockerfile_extra_contents = {}
 dockerfile_extra_contents['focal'] = """
 
@@ -51,11 +51,13 @@ ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$
 
 """
 
+
 def get_secret():
-    with open(path.join(os.environ['HOME'], "multibuild_parent_key.key"),'r') as f:
+    with open("multibuild_parent_key.key", 'r') as f:
         data = f.read()
     return data
 
+
 def list_jobs(directory=".buildbot"):
     """
     list jobs found in a directory
@@ -63,39 +65,55 @@ def list_jobs(directory=".buildbot"):
     results = []
     files = ["Dockerfile", "build.sh", "test.sh"]
     for item in listdir(directory):
+        print("checking directory {}".format(item))
         flag = False
-        for file in files:
+        for fname in files:
-            filepath = join(directory, item, file)
+            filepath = join(directory, item, fname)
+            if not exists(filepath):
+                continue
             if islink(filepath) or not isfile(filepath):
                 flag = True
                 break
         if flag:
-            continue    
+            continue
         if (exists(join(directory, item, 'Dockerfile')) and exists(join(directory, item, 'build.sh'))) or exists(join(directory, item, 'test.sh')):
             results.append(item)
     return results
 
 
-def _get_dockerfile_contents(jobname):
+def get_revision():
+    proc = Popen(["git", "rev-parse", "HEAD"], stdout=PIPE)
+    retval = proc.stdout.read().strip()
+    retval = retval.decode('utf-8')
+    return retval
+
+
+def _get_dockerfile_contents(dockerfile):
     """
     Read contents of a Dockerfile and add extra contents for the given os_codename
     """
-    os_codename='bionic'
+    os_codename = 'bionic'
     res = ""
-    with open(join(path + jobname), "r") as file:
+    with open(dockerfile, "r") as file:
-        contents = file.read()
+        contents = file.readlines()
+        has_from = False
         # accept any line containing FROM and RUN keywords
         res = ""
         inside_allowed_command = False
         for line in contents:
-            if re.match(r"(?m)^(FROM|RUN|ENV).*$", line):
+            m = re.match(r"(?m)^(FROM|RUN|ENV).*$", line)
+            if m:
                 inside_allowed_command = True
+                if m.group(1) == "FROM":
+                    has_from = True
             if inside_allowed_command:
                 res += line
-                l = line.strip()
+                ls = line.strip()
-                if not l.endswith("\\"):
+                if not ls.endswith("\\"):
                     inside_allowed_command = False
 
+    if not has_from:
+        return None
     return res + dockerfile_extra_contents[os_codename]
 
 
@@ -104,10 +122,11 @@ def trigger_child_hooks(buildbotUrl: str, repository, branch, directory=".buildb
     # List all jobs in the directory
     jobs = list_jobs(directory)
     request_headers = {
-        "Content-Type": "application/x-www-form-urlencoded",
+        "Content-Type": "application/json",
         "X-Multibuild-Trigger": get_secret(),
         "Accept": "text/plain",
     }
+    revision = get_revision()
 
     # Check if build.sh or test.sh exists in each of the jobs
     for job in jobs:
@@ -119,15 +138,43 @@ def trigger_child_hooks(buildbotUrl: str, repository, branch, directory=".buildb
             test_script_exists = True
 
         # make a post request
+        dockerfile = _get_dockerfile_contents(
+            join(directory, job, "Dockerfile")
+        )
+
+        if not dockerfile:
+            continue
+
+        request_data["branch"] = branch
+        request_data["revision"] = revision
+
         request_data["properties"] = {
-            "dockerfile": _get_dockerfile_contents(
+            "dockerfile": dockerfile,
-                join(directory, job, "Dockerfile")
+            "build_script_available": build_script_exists,
-            ),
+            "test_script_available": test_script_exists,
-            "build_script_available": is_build_script_available(build_script_exists),
-            "test_script_available": is_test_script_available(test_script_exists),
-            "repository": repository,
-            "branch": branch,
             "jobname": job,
         }
+        request_data["changes"] = {
+            "author": "buildbot_multibuild",
+            "repository": repository,
+            "project": "/".join(repository.split("/")[-2:]),
+        }
 
-        requests.post(request_url, headers=request_headers, data=request_data)
+        retval = requests.post(request_url, headers=request_headers, json=request_data)
+        print("Triggered job for {} on {}: {}".format(job, request_url, retval.text))
+
+
+if __name__ == "__main__":
+    # expect jobname, repository, branch, buildbotUrl from command line
+    import sys
+
+    if len(sys.argv) == 4:
+        buildbotUrl = sys.argv[1]
+        repository = sys.argv[2]
+        branch = sys.argv[3]
+
+        trigger_child_hooks(buildbotUrl, repository, branch)
+    else:
+        sys.exit(
+            "Usage: python3 multibuild.py <buildbotUrl> <repository> <branch>"
+        )

multibuild.py

@@ -10,10 +10,8 @@ Requires docker
 # TODO: write hook job, maybe also a dockerfile?
 # TODO: what to do about non-docker jobs
 
-from os import listdir, walk, getenv
+from os import listdir
-from os.path import exists, isfile, join
+from os.path import isfile, join
-import requests
-import re
 from buildbot.plugins import steps, util
 
 from .lib.renderers import *
@@ -35,24 +33,32 @@ def add_parent_step(build_factory):
     """
 
     build_factory.addStep(steps.ShellCommand(
-        name="create directory",
+        name="Update APT cache",
-        command=["mkdir", "-p", join(getenv['HOME'], '.local/bin') ]
+        command=["sudo", "apt", "update"]
     ))
 
     build_factory.addStep(steps.ShellCommand(
-        name="download worker",
+        name="Install dependencies",
-        command=["wget", "-O", "https://git.bitmessage.org/Bitmessage/buildbot_multibuild/raw/branch/master/lib/worker_multibuild.py", join(getenv['HOME'], '.local/bin/worker_multibuild.py')]
+        command=["sudo", "apt", "-y", "install", "python3-requests"]
     ))
-    
+
+    build_factory.addStep(
+        steps.FileDownload(
+            workerdest="worker_multibuild.py",
+            mastersrc="buildbot_multibuild/lib/worker_multibuild.py",
+            mode=0o444
+        )
+    )
+
     build_factory.addStep(
         steps.ShellCommand(
             name="Execute worker script",
             command=[
                 "python3",
-                join(getenv['HOME'], '.local/bin/worker_multibuild.py'),
+                'worker_multibuild.py',
+                util.Property("buildboturl"),
                 util.Property('repository'),
-                util.Property('branch'),
+                util.Property('branch')
-                util.getURLForBuild(util.Property("url"), util.Property("builderid"), util.Property("buildnumber")),
             ],
         )
     )
@@ -80,20 +86,3 @@ def add_child_sh_steps(build_factory, directory=".buildbot"):
             hideStepIf=isnt_test_script_available,
         )
     )
-
-
-if __name__ == "__main__":
-    # expect jobname, repository, branch, buildbotUrl from command line
-    import sys
-
-    if len(sys.argv) == 6:
-        jobname = sys.argv[1]
-        repository = sys.argv[2]
-        branch = sys.argv[3]
-        buildbotUrl = sys.argv[4]
-
-        trigger_child_hooks(buildbotUrl, repository, branch)
-    else:
-        print(
-            "Usage: python3 multibuild.py <buildbotUrl> <repository> <branch>  "
-        )