Downgrade Automat

- python 3.6 compatiblity

lib/renderers.py

@@ -22,3 +22,58 @@ def is_test_script_available(props):
 @util.renderer
 def isnt_test_script_available(props):
     return not _is_test_script_available(props)
+
+def _files_to_upload(props):
+    try:
+        return ','.join(props.getProperty("files_to_upload", default="").rstrip().split("\n"))
+    except AttributeError:
+        return ""
+
+@util.renderer
+def files_to_upload(props):
+    return _files_to_upload(props)
+
+@util.renderer
+def has_files_to_upload(props):
+    return bool(_files_to_upload(props))
+
+@util.renderer
+def no_files_to_upload(props):
+    return not _files_to_upload(props)
+
+def _should_build_sign(props):
+    if props.getProperty('branch') == 'v0.6' \
+        and props.getProperty('jobname') == 'android' \
+        and props.getProperty('repository') in (
+                'git@github.com:Bitmessage/PyBitmessage.git',
+                'https://github.com/Bitmessage/PyBitmessage'
+                ):
+        return True
+    return False
+
+@util.renderer
+def should_build_sign(props):
+    return _is_build_script_available(props) and _should_build_sign(props)
+
+@util.renderer
+def shouldnt_build_sign(props):
+    return _is_build_script_available(props) and not _should_build_sign(props)
+
+@util.renderer
+def build_env(props):
+    default_envs = {
+        "BUILDBOT_REPOSITORY": props.getProperty("repository"),
+        "BUILDBOT_BRANCH": props.getProperty("branch"),
+        "BUILDBOT_JOBNAME": props.getProperty("jobname")
+    }
+    new_envs = {}
+    if props.getProperty("jobname", default="") == "android":
+        new_envs = {
+            "P4A_RELEASE_KEYSTORE": "/var/lib/buildbot/keystore",
+            "P4A_RELEASE_KEYSTORE_PASSWD": util.Secret("bitmessage-keystore"),
+            "P4A_RELEASE_KEYALIAS_PASSWD": util.Secret("bitmessage-keystore"),
+            "P4A_RELEASE_KEYALIAS": "bitmessagetest"
+        }
+    if _should_build_sign(props):
+        return {**default_envs, **new_envs}
+    return default_envs

lib/worker_multibuild.py

@@ -3,6 +3,7 @@ from os.path import exists, isfile, islink, join, realpath
 import requests
 import re
 from subprocess import Popen, PIPE
+from time import sleep
 
 
 request_data = {
@@ -16,38 +17,104 @@ dockerfile_extra_contents = {}
 dockerfile_extra_contents['focal'] = """
 
 # Buildbot
-RUN apt-get install -yq --no-install-suggests --no-install-recommends \
+RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
-    buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
+    python3-buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
     python3-pip dumb-init curl openssh-client wget
 
 # buildbot entrypoint
 RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
 RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
+RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt
+RUN update-ca-certificates
 
 RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
 
 USER buildbot
 
-ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
+ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS"
 
 """
 
 dockerfile_extra_contents['bionic'] = """
 
 # Buildbot
-RUN apt-get install -yq --no-install-suggests --no-install-recommends \
+RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
-    buildbot-slave git subversion python3-dev libffi-dev python3-setuptools \
+    git subversion python3-dev libffi-dev python3-setuptools \
-    python3-pip dumb-init curl openssh-client wget
+    python3-pip dumb-init curl openssh-client wget python3-wheel \
+    pkg-config rustc cargo
+RUN pip3 install setuptools_rust
+RUN pip3 install 'buildbot-worker==3.1.1' \
+                 'cryptography==2.1.4' \
+                 'twisted==17.9.0' \
+                 'Automat==20.2.0' \
+                 'pyopenssl==17.5.0'
+RUN groupadd buildbot
+RUN useradd -d /var/lib/buildbot -m -g buildbot buildbot
 
 # buildbot entrypoint
 RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
 RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
+RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt
+RUN update-ca-certificates
 
 RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
 
 USER buildbot
 
-ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
+ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS"
+
+"""
+
+dockerfile_extra_contents['jammy'] = """
+
+# Buildbot
+RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
+    python3-buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
+    python3-pip dumb-init curl openssh-client wget
+
+# buildbot entrypoint
+RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
+RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
+RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt
+RUN update-ca-certificates
+
+RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+
+USER buildbot
+
+ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS"
+
+"""
+
+dockerfile_extra_contents['xenial'] = """
+
+# Buildbot
+RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
+    git subversion python3-dev libffi-dev python3-setuptools \
+    python3-pip curl openssh-client wget \
+    python-setuptools python-psutil libssl-dev python-dev libgmp-dev \
+    python-virtualenv python3-wheel pkg-config rustc cargo \
+    python3-openssl
+RUN pip3 install setuptools_rust
+RUN pip3 install 'buildbot-worker<3.2.0' 'cryptography<35.0.0'
+RUN groupadd buildbot
+RUN useradd -d /var/lib/buildbot -m -g buildbot buildbot
+
+# dumb-init
+RUN wget https://github.com/Yelp/dumb-init/releases/download/v1.2.5/dumb-init_1.2.5_amd64.deb
+RUN dpkg -i dumb-init_*.deb && rm -f dumb-init_*.deb
+
+# buildbot entrypoint
+RUN [ -f /usr/local/bin/buildbot_entrypoint.sh ] || wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/xenial/entrypoint.sh
+RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
+RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt
+RUN update-ca-certificates
+
+RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+
+USER buildbot
+
+ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS"
 
 """
 
@@ -64,6 +131,8 @@ def list_jobs(directory=".buildbot"):
     """
     results = []
     files = ["Dockerfile", "build.sh", "test.sh"]
+    if not exists(directory):
+        return results
     for item in listdir(directory):
         print("checking directory {}".format(item))
         flag = False
@@ -115,6 +184,7 @@ def _get_dockerfile_contents(dockerfile):
             if m:
                 inside_allowed_command = True
                 if m.group(1) == "FROM":
+                    os_codename = m.group().split()[1].split(":")[1]
                     has_from = True
             if inside_allowed_command:
                 res += line
@@ -124,7 +194,11 @@ def _get_dockerfile_contents(dockerfile):
 
     if not has_from:
         return None
-    return res + dockerfile_extra_contents[os_codename]
+
+    try:
+        return res + dockerfile_extra_contents[os_codename]
+    except KeyError:
+        return None
 
 
 def trigger_child_hooks(buildbotUrl: str, repository, branch, revision,
@@ -168,13 +242,14 @@ def trigger_child_hooks(buildbotUrl: str, repository, branch, revision,
         request_data["changes"] = {
             "author": "buildbot_multibuild",
             "repository": repository,
-            "project": "/".join(repository.split("/")[-2:]),
+            "project": job,
         }
 
         retval = requests.post(request_url, headers=request_headers,
                                json=request_data)
         print("Triggered job for {} on {}: {}".format(job, request_url,
                                                       retval.text))
+        sleep(1)
 
 
 if __name__ == "__main__":

multibuild.py

@@ -13,6 +13,7 @@ Requires docker
 from os import listdir
 from os.path import isfile, join
 from buildbot.plugins import steps, util
+from buildbot.process.results import SUCCESS
 
 from .lib.renderers import *
 
@@ -53,19 +54,89 @@ def add_child_sh_steps(build_factory, directory=".buildbot"):
     """
 
     build_factory.addStep(
-        steps.ShellCommand(
+        steps.FileDownload(
-            name=util.Interpolate("build_%(prop:jobname)s"),
+            name="Upload keystore",
-            command=util.Interpolate("%(kw:directory)s/%(prop:jobname)s/build.sh", directory=directory),
+            workerdest="/var/lib/buildbot/keystore",
-            doStepIf=is_build_script_available,
+            mastersrc=util.Interpolate("keystore/%(prop:jobname)s.keystore"),
-            hideStepIf=isnt_build_script_available,
+            doStepIf=should_build_sign,
+            hideStepIf=True,
+            mode=0o600
         )
     )
 
     build_factory.addStep(
         steps.ShellCommand(
-            name= util.Interpolate("test_%(prop:jobname)s"),
+            name=util.Interpolate("build_%(prop:jobname)s"),
-            command=util.Interpolate("%(kw:directory)s/%(prop:jobname)s/test.sh", directory=directory),
+            command=util.Interpolate("%(kw:directory)s/"
+                                     "%(prop:jobname)s/build.sh",
+                                     directory=directory),
+            env=build_env,
+            doStepIf=shouldnt_build_sign,
+            hideStepIf=isnt_build_script_available,
+            timeout=7200,
+        )
+    )
+
+    build_factory.addStep(
+        steps.ShellCommand(
+            name=util.Interpolate("build_and_sign_%(prop:jobname)s"),
+            command=util.Interpolate("%(kw:directory)s/"
+                                     "%(prop:jobname)s/build.sh",
+                                     directory=directory),
+            env=build_env,
+            doStepIf=should_build_sign,
+            hideStepIf=isnt_build_script_available,
+            timeout=7200,
+        )
+    )
+
+    build_factory.addStep(
+        steps.ShellCommand(
+            name="Delete keystore",
+            command="rm -f /var/lib/buildbot/keystore",
+            doStepIf=should_build_sign,
+            hideStepIf=True,
+        )
+    )
+
+    build_factory.addStep(
+        steps.ShellCommand(
+            name=util.Interpolate("test_%(prop:jobname)s"),
+            command=util.Interpolate("%(kw:directory)s/"
+                                     "%(prop:jobname)s/test.sh",
+                                     directory=directory),
+            haltOnFailure=False,
+            flunkOnFailure=True,
             doStepIf=is_test_script_available,
             hideStepIf=isnt_test_script_available,
         )
     )
+
+    build_factory.addStep(
+        steps.SetPropertyFromCommand(
+            name="Find files to upload",
+            command="find . -maxdepth 1 -mindepth 1 "
+                    "-type f -printf '%P\n'",
+            workdir="out",
+            hideStepIf=True,
+            decodeRC={0: SUCCESS, 1: SUCCESS},
+            property="files_to_upload"
+        )
+    )
+
+    build_factory.addStep(
+        steps.ShellCommand(
+            name="Upload files",
+            workdir="out",
+            doStepIf=has_files_to_upload,
+            hideStepIf=no_files_to_upload,
+            command=util.Interpolate(
+                "curl -T {%s} -u buildbot:%s "
+                "https://artifacts.bitmessage.at/%s/%s/",
+                files_to_upload,
+                util.Secret('artifact_upload'),
+                util.Property('jobname'),
+                util.Property('buildnumber'),
+            )
+        )
+    )