Add xenial and jammy

- added some security checks so that symlinks can be supported
- also some code quality changes

lib/worker_multibuild.py

@@ -1,5 +1,5 @@
-from os import listdir
-from os.path import exists, isfile, join, islink
+from os import getcwd, listdir
+from os.path import exists, isfile, islink, join, realpath
 import requests
 import re
 from subprocess import Popen, PIPE
@@ -16,8 +16,8 @@ dockerfile_extra_contents = {}
 dockerfile_extra_contents['focal'] = """
 
 # Buildbot
-RUN apt-get install -yq --no-install-suggests --no-install-recommends \
-    buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
+RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
+    python3-buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
     python3-pip dumb-init curl openssh-client wget
 
 # buildbot entrypoint
@@ -35,7 +35,7 @@ ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$
 dockerfile_extra_contents['bionic'] = """
 
 # Buildbot
-RUN apt-get install -yq --no-install-suggests --no-install-recommends \
+RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
     buildbot-slave git subversion python3-dev libffi-dev python3-setuptools \
     python3-pip dumb-init curl openssh-client wget
 
@@ -51,6 +51,44 @@ ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$
 
 """
 
+dockerfile_extra_contents['jammy'] = """
+
+# Buildbot
+RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
+    python3-buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
+    python3-pip dumb-init curl openssh-client wget
+
+# buildbot entrypoint
+RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
+RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
+
+RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+
+USER buildbot
+
+ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
+
+"""
+
+dockerfile_extra_contents['xenial'] = """
+
+# Buildbot
+RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
+    buildbot-slave git subversion python3-dev libffi-dev python3-setuptools \
+    python3-pip dumb-init curl openssh-client wget \
+    python-setuptools python-psutil libssl-dev python-dev libgmp-dev python-virtualenv
+
+# buildbot entrypoint
+RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
+RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
+
+RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+
+USER buildbot
+
+ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
+
+"""
 
 def get_secret():
     with open("multibuild_parent_key.key", 'r') as f:
@@ -69,20 +107,29 @@ def list_jobs(directory=".buildbot"):
         flag = False
         for fname in files:
             filepath = join(directory, item, fname)
+            # must exist
             if not exists(filepath):
                 continue
-            if islink(filepath) or not isfile(filepath):
+            # must be a file
+            if not isfile(filepath):
+                flag = True
+                break
+            # symlink OK as long as it points to files within the repo
+            if islink(filepath) \
+                    and not realpath(filepath).startswith(getcwd()):
                 flag = True
                 break
         if flag:
             continue
-        if (exists(join(directory, item, 'Dockerfile')) and exists(join(directory, item, 'build.sh'))) or exists(join(directory, item, 'test.sh')):
+        if (exists(join(directory, item, 'Dockerfile'))
+                and exists(join(directory, item, 'build.sh'))) \
+                or exists(join(directory, item, 'test.sh')):
             results.append(item)
     return results
 
 
-def get_revision():
-    proc = Popen(["git", "rev-parse", "HEAD"], stdout=PIPE)
+def get_revision(branch):
+    proc = Popen(["git", "rev-parse", branch], stdout=PIPE)
     retval = proc.stdout.read().strip()
     retval = retval.decode('utf-8')
     return retval
@@ -90,7 +137,8 @@ def get_revision():
 
 def _get_dockerfile_contents(dockerfile):
     """
-    Read contents of a Dockerfile and add extra contents for the given os_codename
+    Read contents of a Dockerfile and add buildbot worker bootstrap
+    for a given os_codename
     """
     os_codename = 'bionic'
     res = ""
@@ -105,6 +153,7 @@ def _get_dockerfile_contents(dockerfile):
             if m:
                 inside_allowed_command = True
                 if m.group(1) == "FROM":
+                    os_codename = m.group().split()[1].split(":")[1]
                     has_from = True
             if inside_allowed_command:
                 res += line
@@ -114,10 +163,15 @@ def _get_dockerfile_contents(dockerfile):
 
     if not has_from:
         return None
-    return res + dockerfile_extra_contents[os_codename]
+
+    try:
+        return res + dockerfile_extra_contents[os_codename]
+    except KeyError:
+        return None
 
 
-def trigger_child_hooks(buildbotUrl: str, repository, branch, directory=".buildbot"):
+def trigger_child_hooks(buildbotUrl: str, repository, branch, revision,
+                        directory=".buildbot"):
     request_url = buildbotUrl + ty
     # List all jobs in the directory
     jobs = list_jobs(directory)
@@ -126,7 +180,7 @@ def trigger_child_hooks(buildbotUrl: str, repository, branch, directory=".buildb
         "X-Multibuild-Trigger": get_secret(),
         "Accept": "text/plain",
     }
-    revision = get_revision()
+    # revision = get_revision(branch)
 
     # Check if build.sh or test.sh exists in each of the jobs
     for job in jobs:
@@ -160,21 +214,24 @@ def trigger_child_hooks(buildbotUrl: str, repository, branch, directory=".buildb
             "project": "/".join(repository.split("/")[-2:]),
         }
 
-        retval = requests.post(request_url, headers=request_headers, json=request_data)
-        print("Triggered job for {} on {}: {}".format(job, request_url, retval.text))
+        retval = requests.post(request_url, headers=request_headers,
+                               json=request_data)
+        print("Triggered job for {} on {}: {}".format(job, request_url,
+                                                      retval.text))
 
 
 if __name__ == "__main__":
     # expect jobname, repository, branch, buildbotUrl from command line
     import sys
 
-    if len(sys.argv) == 4:
+    if len(sys.argv) == 5:
         buildbotUrl = sys.argv[1]
         repository = sys.argv[2]
         branch = sys.argv[3]
+        revision = sys.argv[4]
 
-        trigger_child_hooks(buildbotUrl, repository, branch)
+        trigger_child_hooks(buildbotUrl, repository, branch, revision)
     else:
         sys.exit(
-            "Usage: python3 multibuild.py <buildbotUrl> <repository> <branch>"
+            "Usage: python3 multibuild.py <buildbotUrl> <repository> <branch> <revision>"
         )

multibuild.py

@@ -32,33 +32,16 @@ def add_parent_step(build_factory):
     Add a step to the parent build factory that will trigger the child hooks
     """
 
-    build_factory.addStep(steps.ShellCommand(
-        name="Update APT cache",
-        command=["sudo", "apt", "update"]
-    ))
-
-    build_factory.addStep(steps.ShellCommand(
-        name="Install dependencies",
-        command=["sudo", "apt", "-y", "install", "python3-requests"]
-    ))
-
-    build_factory.addStep(
-        steps.FileDownload(
-            workerdest="worker_multibuild.py",
-            mastersrc="buildbot_multibuild/lib/worker_multibuild.py",
-            mode=0o444
-        )
-    )
-
     build_factory.addStep(
         steps.ShellCommand(
             name="Execute worker script",
             command=[
                 "python3",
-                'worker_multibuild.py',
+                '/usr/local/bin/worker_multibuild.py',
                 util.Property("buildboturl"),
                 util.Property('repository'),
-                util.Property('branch')
+                util.Property('branch'),
+                util.Property('revision')
             ],
         )
     )