Downgrade Automat

- python 3.6 compatiblity

lib/renderers.py

@@ -1,24 +1,79 @@
 from buildbot.plugins import util
 import re
 
+def _is_build_script_available(props):
+    return props.getProperty("build_script_available", default=False)
 
 @util.renderer
 def is_build_script_available(props):
-    # Actual check will got here
+    return _is_build_script_available(props)
-    return props.getProperty("build_available", default=False)
-
 
 @util.renderer
 def isnt_build_script_available(props):
-    return not is_build_script_available(props)
+    return not _is_build_script_available(props)
 
+def _is_test_script_available(props):
+    return props.getProperty("test_script_available", default=False)
 
 @util.renderer
 def is_test_script_available(props):
-    # Actual check will got here
+    return _is_test_script_available(props)
-    return props.getProperty("test_available", default=False)
-
 
 @util.renderer
 def isnt_test_script_available(props):
-    return not is_test_script_available(props)
+    return not _is_test_script_available(props)
+
+def _files_to_upload(props):
+    try:
+        return ','.join(props.getProperty("files_to_upload", default="").rstrip().split("\n"))
+    except AttributeError:
+        return ""
+
+@util.renderer
+def files_to_upload(props):
+    return _files_to_upload(props)
+
+@util.renderer
+def has_files_to_upload(props):
+    return bool(_files_to_upload(props))
+
+@util.renderer
+def no_files_to_upload(props):
+    return not _files_to_upload(props)
+
+def _should_build_sign(props):
+    if props.getProperty('branch') == 'v0.6' \
+        and props.getProperty('jobname') == 'android' \
+        and props.getProperty('repository') in (
+                'git@github.com:Bitmessage/PyBitmessage.git',
+                'https://github.com/Bitmessage/PyBitmessage'
+                ):
+        return True
+    return False
+
+@util.renderer
+def should_build_sign(props):
+    return _is_build_script_available(props) and _should_build_sign(props)
+
+@util.renderer
+def shouldnt_build_sign(props):
+    return _is_build_script_available(props) and not _should_build_sign(props)
+
+@util.renderer
+def build_env(props):
+    default_envs = {
+        "BUILDBOT_REPOSITORY": props.getProperty("repository"),
+        "BUILDBOT_BRANCH": props.getProperty("branch"),
+        "BUILDBOT_JOBNAME": props.getProperty("jobname")
+    }
+    new_envs = {}
+    if props.getProperty("jobname", default="") == "android":
+        new_envs = {
+            "P4A_RELEASE_KEYSTORE": "/var/lib/buildbot/keystore",
+            "P4A_RELEASE_KEYSTORE_PASSWD": util.Secret("bitmessage-keystore"),
+            "P4A_RELEASE_KEYALIAS_PASSWD": util.Secret("bitmessage-keystore"),
+            "P4A_RELEASE_KEYALIAS": "bitmessagetest"
+        }
+    if _should_build_sign(props):
+        return {**default_envs, **new_envs}
+    return default_envs

lib/worker_multibuild.py

@@ -0,0 +1,269 @@
+from os import getcwd, listdir
+from os.path import exists, isfile, islink, join, realpath
+import requests
+import re
+from subprocess import Popen, PIPE
+from time import sleep
+
+
+request_data = {
+    "project": "testproject",
+    "comments": "testcomment",
+}
+
+ty = "/change_hook/base"
+path = ".buildbot"
+dockerfile_extra_contents = {}
+dockerfile_extra_contents['focal'] = """
+
+# Buildbot
+RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
+    python3-buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
+    python3-pip dumb-init curl openssh-client wget
+
+# buildbot entrypoint
+RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
+RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
+RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt
+RUN update-ca-certificates
+
+RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+
+USER buildbot
+
+ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS"
+
+"""
+
+dockerfile_extra_contents['bionic'] = """
+
+# Buildbot
+RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
+    git subversion python3-dev libffi-dev python3-setuptools \
+    python3-pip dumb-init curl openssh-client wget python3-wheel \
+    pkg-config rustc cargo
+RUN pip3 install setuptools_rust
+RUN pip3 install 'buildbot-worker==3.1.1' \
+                 'cryptography==2.1.4' \
+                 'twisted==17.9.0' \
+                 'Automat==20.2.0' \
+                 'pyopenssl==17.5.0'
+RUN groupadd buildbot
+RUN useradd -d /var/lib/buildbot -m -g buildbot buildbot
+
+# buildbot entrypoint
+RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
+RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
+RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt
+RUN update-ca-certificates
+
+RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+
+USER buildbot
+
+ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS"
+
+"""
+
+dockerfile_extra_contents['jammy'] = """
+
+# Buildbot
+RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
+    python3-buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
+    python3-pip dumb-init curl openssh-client wget
+
+# buildbot entrypoint
+RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
+RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
+RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt
+RUN update-ca-certificates
+
+RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+
+USER buildbot
+
+ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS"
+
+"""
+
+dockerfile_extra_contents['xenial'] = """
+
+# Buildbot
+RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
+    git subversion python3-dev libffi-dev python3-setuptools \
+    python3-pip curl openssh-client wget \
+    python-setuptools python-psutil libssl-dev python-dev libgmp-dev \
+    python-virtualenv python3-wheel pkg-config rustc cargo \
+    python3-openssl
+RUN pip3 install setuptools_rust
+RUN pip3 install 'buildbot-worker<3.2.0' 'cryptography<35.0.0'
+RUN groupadd buildbot
+RUN useradd -d /var/lib/buildbot -m -g buildbot buildbot
+
+# dumb-init
+RUN wget https://github.com/Yelp/dumb-init/releases/download/v1.2.5/dumb-init_1.2.5_amd64.deb
+RUN dpkg -i dumb-init_*.deb && rm -f dumb-init_*.deb
+
+# buildbot entrypoint
+RUN [ -f /usr/local/bin/buildbot_entrypoint.sh ] || wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/xenial/entrypoint.sh
+RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
+RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt
+RUN update-ca-certificates
+
+RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+
+USER buildbot
+
+ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS"
+
+"""
+
+
+def get_secret():
+    with open("multibuild_parent_key.key", 'r') as f:
+        data = f.read()
+    return data
+
+
+def list_jobs(directory=".buildbot"):
+    """
+    list jobs found in a directory
+    """
+    results = []
+    files = ["Dockerfile", "build.sh", "test.sh"]
+    if not exists(directory):
+        return results
+    for item in listdir(directory):
+        print("checking directory {}".format(item))
+        flag = False
+        for fname in files:
+            filepath = join(directory, item, fname)
+            # must exist
+            if not exists(filepath):
+                continue
+            # must be a file
+            if not isfile(filepath):
+                flag = True
+                break
+            # symlink OK as long as it points to files within the repo
+            if islink(filepath) \
+                    and not realpath(filepath).startswith(getcwd()):
+                flag = True
+                break
+        if flag:
+            continue
+        if (exists(join(directory, item, 'Dockerfile'))
+                and exists(join(directory, item, 'build.sh'))) \
+                or exists(join(directory, item, 'test.sh')):
+            results.append(item)
+    return results
+
+
+def get_revision(branch):
+    proc = Popen(["git", "rev-parse", branch], stdout=PIPE)
+    retval = proc.stdout.read().strip()
+    retval = retval.decode('utf-8')
+    return retval
+
+
+def _get_dockerfile_contents(dockerfile):
+    """
+    Read contents of a Dockerfile and add buildbot worker bootstrap
+    for a given os_codename
+    """
+    os_codename = 'bionic'
+    res = ""
+    with open(dockerfile, "r") as file:
+        contents = file.readlines()
+        has_from = False
+        # accept any line containing FROM and RUN keywords
+        res = ""
+        inside_allowed_command = False
+        for line in contents:
+            m = re.match(r"(?m)^(FROM|RUN|ENV).*$", line)
+            if m:
+                inside_allowed_command = True
+                if m.group(1) == "FROM":
+                    os_codename = m.group().split()[1].split(":")[1]
+                    has_from = True
+            if inside_allowed_command:
+                res += line
+                ls = line.strip()
+                if not ls.endswith("\\"):
+                    inside_allowed_command = False
+
+    if not has_from:
+        return None
+
+    try:
+        return res + dockerfile_extra_contents[os_codename]
+    except KeyError:
+        return None
+
+
+def trigger_child_hooks(buildbotUrl: str, repository, branch, revision,
+                        directory=".buildbot"):
+    request_url = buildbotUrl + ty
+    # List all jobs in the directory
+    jobs = list_jobs(directory)
+    request_headers = {
+        "Content-Type": "application/json",
+        "X-Multibuild-Trigger": get_secret(),
+        "Accept": "text/plain",
+    }
+    # revision = get_revision(branch)
+
+    # Check if build.sh or test.sh exists in each of the jobs
+    for job in jobs:
+        build_script_exists = False
+        test_script_exists = False
+        if exists(join(directory, job, "build.sh")):
+            build_script_exists = True
+        if exists(join(directory, job, "test.sh")):
+            test_script_exists = True
+
+        # make a post request
+        dockerfile = _get_dockerfile_contents(
+            join(directory, job, "Dockerfile")
+        )
+
+        if not dockerfile:
+            continue
+
+        request_data["branch"] = branch
+        request_data["revision"] = revision
+
+        request_data["properties"] = {
+            "dockerfile": dockerfile,
+            "build_script_available": build_script_exists,
+            "test_script_available": test_script_exists,
+            "jobname": job,
+        }
+        request_data["changes"] = {
+            "author": "buildbot_multibuild",
+            "repository": repository,
+            "project": job,
+        }
+
+        retval = requests.post(request_url, headers=request_headers,
+                               json=request_data)
+        print("Triggered job for {} on {}: {}".format(job, request_url,
+                                                      retval.text))
+        sleep(1)
+
+
+if __name__ == "__main__":
+    # expect jobname, repository, branch, buildbotUrl from command line
+    import sys
+
+    if len(sys.argv) == 5:
+        buildbotUrl = sys.argv[1]
+        repository = sys.argv[2]
+        branch = sys.argv[3]
+        revision = sys.argv[4]
+
+        trigger_child_hooks(buildbotUrl, repository, branch, revision)
+    else:
+        sys.exit(
+            "Usage: python3 multibuild.py <buildbotUrl> <repository> <branch> <revision>"
+        )

multibuild.py

@@ -10,77 +10,13 @@ Requires docker
 # TODO: write hook job, maybe also a dockerfile?
 # TODO: what to do about non-docker jobs
 
-from os import walk
+from os import listdir
-from os.path import exists, isfile, join, listdir
+from os.path import isfile, join
-import requests
-import re
 from buildbot.plugins import steps, util
+from buildbot.process.results import SUCCESS
 
 from .lib.renderers import *
 
-ty = "/change_hook/base"
-request_headers = {
-    "Content-Type": "application/x-www-form-urlencoded",
-    "Accept": "text/plain",
-}
-request_data = {
-    "project": "testproject",
-    "comments": "testcomment",
-}
-
-dockerfile_extra_contents_focal = """
-
-# Buildbot
-RUN apt-get install -yq --no-install-suggests --no-install-recommends \
-    buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
-    python3-pip dumb-init curl openssh-client wget
-
-# buildbot entrypoint
-RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
-RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
-
-RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
-
-USER buildbot
-
-ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
-
-"""
-
-dockerfile_extra_contents_bionic = """
-
-# Buildbot
-RUN apt-get install -yq --no-install-suggests --no-install-recommends \
-    buildbot-slave git subversion python3-dev libffi-dev python3-setuptools \
-    python3-pip dumb-init curl openssh-client wget
-
-# buildbot entrypoint
-RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
-RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
-
-RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
-
-USER buildbot
-
-ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
-
-"""
-
-
-def list_jobs(directory=".buildbot"):
-    """
-    list jobs found in a directory
-    """
-    results = []
-    for _ in next(walk(directory))[1]:
-        if exists(join(directory, _, "Dockerfile")) and (
-            exists(join(directory, _, "build.sh"))
-            or exists(join(directory, _, "test.sh"))
-        ):
-            results.append(_)
-
-    return results
-
 
 def find_artifacts(directory="out"):
     """
@@ -92,119 +28,115 @@ def find_artifacts(directory="out"):
         return join(directory, _)
 
 
-def get_dockerfile_contents(path, os_codename):
+def add_parent_step(build_factory):
-    """
-    Read contents of a Dockerfile and add extra contents for the given os_codename
-    """
-    with open(path, "r") as file:
-        contents = file.read()
-        # remove any line containing CMD or ENTRYFILE keywords
-        re.sub(r"(?m)^(CMD|ENTRYFILE).*$", "", contents)
-
-    return contents + {
-        "focal": dockerfile_extra_contents_focal,
-        "bionic": dockerfile_extra_contents_bionic,
-    }[os_codename]
-
-
-def trigger_child_hooks(buildbotUrl: str, os_codename: str, repository, branch, jobname, directory=".buildbot"):
-    request_url = buildbotUrl + ty
-
-    # List all jobs in the directory
-    jobs = list_jobs(directory)
-
-    # Check if build.sh or test.sh exists in each of the jobs
-    for job in jobs:
-        build_script_exists = False
-        test_script_exists = False
-        if exists(join(directory, job, "build.sh")):
-            build_script_exists = True
-        if exists(join(directory, job, "test.sh")):
-            test_script_exists = True
-
-        # make a post request
-        request_data["properties"] = {
-            "dockerfile": get_dockerfile_contents(
-                join(directory, job, "Dockerfile"), os_codename
-            ),
-            "build_script_available": is_build_script_available(build_script_exists),
-            "test_script_available": is_test_script_available(test_script_exists),
-            "repository": repository,
-            "branch": branch,
-            "jobname": jobname,
-        }
-        requests.post(request_url, headers=request_headers, data=request_data)
-
-
-def add_parent_step(build_factory, jobname, repository, branch):
     """
     Add a step to the parent build factory that will trigger the child hooks
     """
-    build_factory.addStep(
-        steps.SetPropertyFromCommand(
-            name="Get OS codename",
-            command="grep ^VERSION_CODENAME= /etc/os-release | cut -d= -f2",
-            property="os_codename",
-        )
-    )
 
     build_factory.addStep(
         steps.ShellCommand(
-            name="Execute multibuild script",
+            name="Execute worker script",
             command=[
-                "python",
+                "python3",
-                "multibuild.py",
+                '/usr/local/bin/worker_multibuild.py',
-                jobname,
+                util.Property("buildboturl"),
-                repository,
+                util.Property('repository'),
-                branch,
+                util.Property('branch'),
-                "https://buildbot.bitmessage.org",
+                util.Property('revision')
-                util.Interpolate("%(prop:os_codename)s"),
             ],
         )
     )
 
 
-def add_child_build_sh_step(build_factory, job, directory=".buildbot"):
+def add_child_sh_steps(build_factory, directory=".buildbot"):
     """
-    Add a step to the build factory
+    Add a step to the download, build and test factory
     """
+
+    build_factory.addStep(
+        steps.FileDownload(
+            name="Upload keystore",
+            workerdest="/var/lib/buildbot/keystore",
+            mastersrc=util.Interpolate("keystore/%(prop:jobname)s.keystore"),
+            doStepIf=should_build_sign,
+            hideStepIf=True,
+            mode=0o600
+        )
+    )
+
     build_factory.addStep(
         steps.ShellCommand(
-            name="build_" + job,
+            name=util.Interpolate("build_%(prop:jobname)s"),
-            command=["bash", "-c", join(directory, job, "build.sh")],
+            command=util.Interpolate("%(kw:directory)s/"
-            doStepIf=is_build_script_available,
+                                     "%(prop:jobname)s/build.sh",
+                                     directory=directory),
+            env=build_env,
+            doStepIf=shouldnt_build_sign,
             hideStepIf=isnt_build_script_available,
+            timeout=7200,
         )
     )
 
-
-def add_child_test_sh_step(build_factory, job, directory=".buildbot"):
-    """
-    Add a step to the build factory
-    """
     build_factory.addStep(
         steps.ShellCommand(
-            name="test_" + job,
+            name=util.Interpolate("build_and_sign_%(prop:jobname)s"),
-            command=["bash", "-c", join(directory, job, "test.sh")],
+            command=util.Interpolate("%(kw:directory)s/"
+                                     "%(prop:jobname)s/build.sh",
+                                     directory=directory),
+            env=build_env,
+            doStepIf=should_build_sign,
+            hideStepIf=isnt_build_script_available,
+            timeout=7200,
+        )
+    )
+
+    build_factory.addStep(
+        steps.ShellCommand(
+            name="Delete keystore",
+            command="rm -f /var/lib/buildbot/keystore",
+            doStepIf=should_build_sign,
+            hideStepIf=True,
+        )
+    )
+
+    build_factory.addStep(
+        steps.ShellCommand(
+            name=util.Interpolate("test_%(prop:jobname)s"),
+            command=util.Interpolate("%(kw:directory)s/"
+                                     "%(prop:jobname)s/test.sh",
+                                     directory=directory),
+            haltOnFailure=False,
+            flunkOnFailure=True,
             doStepIf=is_test_script_available,
             hideStepIf=isnt_test_script_available,
         )
     )
 
-
+    build_factory.addStep(
-if __name__ == "__main__":
+        steps.SetPropertyFromCommand(
-    # expect jobname, repository, branch, buildbotUrl, os_codename from command line args
+            name="Find files to upload",
-    import sys
+            command="find . -maxdepth 1 -mindepth 1 "
-
+                    "-type f -printf '%P\n'",
-    if len(sys.argv) == 6:
+            workdir="out",
-        jobname = sys.argv[1]
+            hideStepIf=True,
-        repository = sys.argv[2]
+            decodeRC={0: SUCCESS, 1: SUCCESS},
-        branch = sys.argv[3]
+            property="files_to_upload"
-        buildbotUrl = sys.argv[4]
+        )
-        os_codename = sys.argv[5]
+    )
-
+
-        trigger_child_hooks(buildbotUrl, os_codename, repository, branch, jobname)
+    build_factory.addStep(
-    else:
+        steps.ShellCommand(
-        print(
+            name="Upload files",
-            "Usage: python3 multibuild.py <jobname> <repository> <branch> <buildbotUrl> <os_codename>"
+            workdir="out",
+            doStepIf=has_files_to_upload,
+            hideStepIf=no_files_to_upload,
+            command=util.Interpolate(
+                "curl -T {%s} -u buildbot:%s "
+                "https://artifacts.bitmessage.at/%s/%s/",
+                files_to_upload,
+                util.Secret('artifact_upload'),
+                util.Property('jobname'),
+                util.Property('buildnumber'),
+            )
+        )
     )