lib/renderers.py

@@ -1,79 +1,24 @@
 from buildbot.plugins import util
 import re
 
-def _is_build_script_available(props):
-    return props.getProperty("build_script_available", default=False)
 
 @util.renderer
 def is_build_script_available(props):
-    return _is_build_script_available(props)
+    # Actual check will got here
+    return props.getProperty("build_available", default=False)
+
 
 @util.renderer
 def isnt_build_script_available(props):
-    return not _is_build_script_available(props)
+    return not is_build_script_available(props)
 
-def _is_test_script_available(props):
-    return props.getProperty("test_script_available", default=False)
 
 @util.renderer
 def is_test_script_available(props):
-    return _is_test_script_available(props)
+    # Actual check will got here
+    return props.getProperty("test_available", default=False)
+
 
 @util.renderer
 def isnt_test_script_available(props):
-    return not _is_test_script_available(props)
+    return not is_test_script_available(props)
-
-def _files_to_upload(props):
-    try:
-        return ','.join(props.getProperty("files_to_upload", default="").rstrip().split("\n"))
-    except AttributeError:
-        return ""
-
-@util.renderer
-def files_to_upload(props):
-    return _files_to_upload(props)
-
-@util.renderer
-def has_files_to_upload(props):
-    return bool(_files_to_upload(props))
-
-@util.renderer
-def no_files_to_upload(props):
-    return not _files_to_upload(props)
-
-def _should_build_sign(props):
-    if props.getProperty('branch') == 'v0.6' \
-        and props.getProperty('jobname') == 'android' \
-        and props.getProperty('repository') in (
-                'git@github.com:Bitmessage/PyBitmessage.git',
-                'https://github.com/Bitmessage/PyBitmessage'
-                ):
-        return True
-    return False
-
-@util.renderer
-def should_build_sign(props):
-    return _is_build_script_available(props) and _should_build_sign(props)
-
-@util.renderer
-def shouldnt_build_sign(props):
-    return _is_build_script_available(props) and not _should_build_sign(props)
-
-@util.renderer
-def build_env(props):
-    default_envs = {
-        "BUILDBOT_REPOSITORY": props.getProperty("repository"),
-        "BUILDBOT_BRANCH": props.getProperty("branch"),
-        "BUILDBOT_JOBNAME": props.getProperty("jobname")
-    }
-    new_envs = {}
-    if props.getProperty("jobname", default="") == "android":
-        new_envs = {
-            "P4A_RELEASE_KEYSTORE": "/var/lib/buildbot/keystore",
-            "P4A_RELEASE_KEYSTORE_PASSWD": util.Secret("bitmessage-keystore"),
-            "P4A_RELEASE_KEYALIAS_PASSWD": util.Secret("bitmessage-keystore"),
-            "P4A_RELEASE_KEYALIAS": "bitmessagetest"
-        }
-    if _should_build_sign(props):
-        return {**default_envs, **new_envs}
-    return default_envs

lib/worker_multibuild.py

@@ -1,15 +1,17 @@
-from os import getcwd, listdir
+from os import listdir, walk
-from os.path import exists, isfile, islink, join, realpath
+from os.path import exists, isfile, join, islink, isdir
 import requests
 import re
-from subprocess import Popen, PIPE
-from time import sleep
 
 
 request_data = {
     "project": "testproject",
     "comments": "testcomment",
 }
+request_headers = {
+    "Content-Type": "application/x-www-form-urlencoded",
+    "Accept": "text/plain",
+}
 
 ty = "/change_hook/base"
 path =".buildbot"
@@ -17,201 +19,88 @@ dockerfile_extra_contents = {}
 dockerfile_extra_contents['focal'] = """
 
 # Buildbot
-RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
+RUN apt-get install -yq --no-install-suggests --no-install-recommends \
-    python3-buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
+    buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
     python3-pip dumb-init curl openssh-client wget
 
 # buildbot entrypoint
 RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
 RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
-RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt
-RUN update-ca-certificates
 
 RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
 
 USER buildbot
 
-ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS"
+ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
 
 """
 
 dockerfile_extra_contents['bionic'] = """
 
 # Buildbot
-RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
+RUN apt-get install -yq --no-install-suggests --no-install-recommends \
-    git subversion python3-dev libffi-dev python3-setuptools \
+    buildbot-slave git subversion python3-dev libffi-dev python3-setuptools \
-    python3-pip dumb-init curl openssh-client wget python3-wheel \
-    pkg-config rustc cargo
-RUN pip3 install setuptools_rust
-RUN pip3 install 'buildbot-worker==3.1.1' \
-                 'cryptography==2.1.4' \
-                 'twisted==17.9.0' \
-                 'Automat==20.2.0' \
-                 'pyopenssl==17.5.0'
-RUN groupadd buildbot
-RUN useradd -d /var/lib/buildbot -m -g buildbot buildbot
-
-# buildbot entrypoint
-RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
-RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
-RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt
-RUN update-ca-certificates
-
-RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
-
-USER buildbot
-
-ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS"
-
-"""
-
-dockerfile_extra_contents['jammy'] = """
-
-# Buildbot
-RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
-    python3-buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
     python3-pip dumb-init curl openssh-client wget
 
 # buildbot entrypoint
 RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
 RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
-RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt
-RUN update-ca-certificates
 
 RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
 
 USER buildbot
 
-ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS"
+ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
 
 """
 
-dockerfile_extra_contents['xenial'] = """
-
-# Buildbot
-RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
-    git subversion python3-dev libffi-dev python3-setuptools \
-    python3-pip curl openssh-client wget \
-    python-setuptools python-psutil libssl-dev python-dev libgmp-dev \
-    python-virtualenv python3-wheel pkg-config rustc cargo \
-    python3-openssl
-RUN pip3 install setuptools_rust
-RUN pip3 install 'buildbot-worker<3.2.0' 'cryptography<35.0.0'
-RUN groupadd buildbot
-RUN useradd -d /var/lib/buildbot -m -g buildbot buildbot
-
-# dumb-init
-RUN wget https://github.com/Yelp/dumb-init/releases/download/v1.2.5/dumb-init_1.2.5_amd64.deb
-RUN dpkg -i dumb-init_*.deb && rm -f dumb-init_*.deb
-
-# buildbot entrypoint
-RUN [ -f /usr/local/bin/buildbot_entrypoint.sh ] || wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/xenial/entrypoint.sh
-RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
-RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt
-RUN update-ca-certificates
-
-RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
-
-USER buildbot
-
-ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS"
-
-"""
-
-
-def get_secret():
-    with open("multibuild_parent_key.key", 'r') as f:
-        data = f.read()
-    return data
-
-
 def list_jobs(directory=".buildbot"):
     """
     list jobs found in a directory
     """
     results = []
     files = ["Dockerfile", "build.sh", "test.sh"]
-    if not exists(directory):
-        return results
     for item in listdir(directory):
-        print("checking directory {}".format(item))
         flag = False
-        for fname in files:
+        for file in files:
-            filepath = join(directory, item, fname)
+            filepath = join(directory, item, file)
-            # must exist
+            if islink(filepath) or not isfile(filepath):
-            if not exists(filepath):
-                continue
-            # must be a file
-            if not isfile(filepath):
-                flag = True
-                break
-            # symlink OK as long as it points to files within the repo
-            if islink(filepath) \
-                    and not realpath(filepath).startswith(getcwd()):
                 flag = True
                 break
         if flag:
             continue    
-        if (exists(join(directory, item, 'Dockerfile'))
+        if (exists(join(directory, item, 'Dockerfile')) and exists(join(directory, item, 'build.sh'))) or exists(join(directory, item, 'test.sh')):
-                and exists(join(directory, item, 'build.sh'))) \
-                or exists(join(directory, item, 'test.sh')):
             results.append(item)
     return results
 
 
-def get_revision(branch):
+def _get_dockerfile_contents(jobname):
-    proc = Popen(["git", "rev-parse", branch], stdout=PIPE)
-    retval = proc.stdout.read().strip()
-    retval = retval.decode('utf-8')
-    return retval
-
-
-def _get_dockerfile_contents(dockerfile):
     """
-    Read contents of a Dockerfile and add buildbot worker bootstrap
+    Read contents of a Dockerfile and add extra contents for the given os_codename
-    for a given os_codename
     """
     os_codename='bionic'
     res = ""
-    with open(dockerfile, "r") as file:
+    with open(join(path + jobname), "r") as file:
-        contents = file.readlines()
+        contents = file.read()
-        has_from = False
         # accept any line containing FROM and RUN keywords
         res = ""
         inside_allowed_command = False
         for line in contents:
-            m = re.match(r"(?m)^(FROM|RUN|ENV).*$", line)
+            if re.match(r"(?m)^(FROM|RUN|ENV).*$", line):
-            if m:
                 inside_allowed_command = True
-                if m.group(1) == "FROM":
-                    os_codename = m.group().split()[1].split(":")[1]
-                    has_from = True
             if inside_allowed_command:
                 res += line
-                ls = line.strip()
+                l = line.strip()
-                if not ls.endswith("\\"):
+                if not l.endswith("\\"):
                     inside_allowed_command = False
 
-    if not has_from:
-        return None
-
-    try:
     return res + dockerfile_extra_contents[os_codename]
-    except KeyError:
-        return None
 
 
-def trigger_child_hooks(buildbotUrl: str, repository, branch, revision,
+def trigger_child_hooks(buildbotUrl: str, repository, branch, directory=".buildbot"):
-                        directory=".buildbot"):
     request_url = buildbotUrl + ty
     # List all jobs in the directory
     jobs = list_jobs(directory)
-    request_headers = {
-        "Content-Type": "application/json",
-        "X-Multibuild-Trigger": get_secret(),
-        "Accept": "text/plain",
-    }
-    # revision = get_revision(branch)
 
     # Check if build.sh or test.sh exists in each of the jobs
     for job in jobs:
@@ -223,47 +112,14 @@ def trigger_child_hooks(buildbotUrl: str, repository, branch, revision,
             test_script_exists = True
 
         # make a post request
-        dockerfile = _get_dockerfile_contents(
-            join(directory, job, "Dockerfile")
-        )
-
-        if not dockerfile:
-            continue
-
-        request_data["branch"] = branch
-        request_data["revision"] = revision
-
         request_data["properties"] = {
-            "dockerfile": dockerfile,
+            "dockerfile": _get_dockerfile_contents(
-            "build_script_available": build_script_exists,
+                join(directory, job, "Dockerfile")
-            "test_script_available": test_script_exists,
+            ),
+            "build_script_available": is_build_script_available(build_script_exists),
+            "test_script_available": is_test_script_available(test_script_exists),
+            "repository": repository,
+            "branch": branch,
             "jobname": job,
         }
-        request_data["changes"] = {
+        requests.post(request_url, headers=request_headers, data=request_data)
-            "author": "buildbot_multibuild",
-            "repository": repository,
-            "project": job,
-        }
-
-        retval = requests.post(request_url, headers=request_headers,
-                               json=request_data)
-        print("Triggered job for {} on {}: {}".format(job, request_url,
-                                                      retval.text))
-        sleep(1)
-
-
-if __name__ == "__main__":
-    # expect jobname, repository, branch, buildbotUrl from command line
-    import sys
-
-    if len(sys.argv) == 5:
-        buildbotUrl = sys.argv[1]
-        repository = sys.argv[2]
-        branch = sys.argv[3]
-        revision = sys.argv[4]
-
-        trigger_child_hooks(buildbotUrl, repository, branch, revision)
-    else:
-        sys.exit(
-            "Usage: python3 multibuild.py <buildbotUrl> <repository> <branch> <revision>"
-        )

multibuild.py

@@ -10,10 +10,11 @@ Requires docker
 # TODO: write hook job, maybe also a dockerfile?
 # TODO: what to do about non-docker jobs
 
-from os import listdir
+from os import listdir, walk, getenv
-from os.path import isfile, join
+from os.path import exists, isfile, join
+import requests
+import re
 from buildbot.plugins import steps, util
-from buildbot.process.results import SUCCESS
 
 from .lib.renderers import *
 
@@ -33,16 +34,25 @@ def add_parent_step(build_factory):
     Add a step to the parent build factory that will trigger the child hooks
     """
 
+    build_factory.addStep(steps.ShellCommand(
+        name="create directory",
+        command=["mkdir", "-p", join(getenv['HOME'], '.local/bin') ]
+    ))
+
+    build_factory.addStep(steps.ShellCommand(
+        name="download worker",
+        command=["wget", "-O", "https://git.bitmessage.org/Bitmessage/buildbot_multibuild/raw/branch/master/lib/worker_multibuild.py", join(getenv['HOME'], '.local/bin/worker_multibuild.py')]
+    ))
+    
     build_factory.addStep(
         steps.ShellCommand(
             name="Execute worker script",
             command=[
                 "python3",
-                '/usr/local/bin/worker_multibuild.py',
+                join(getenv['HOME'], '.local/bin/worker_multibuild.py'),
-                util.Property("buildboturl"),
                 util.Property('repository'),
                 util.Property('branch'),
-                util.Property('revision')
+                util.getURLForBuild(util.Property("url"), util.Property("builderid"), util.Property("buildnumber")),
             ],
         )
     )
@@ -53,90 +63,37 @@ def add_child_sh_steps(build_factory, directory=".buildbot"):
     Add a step to the download, build and test factory
     """
 
-    build_factory.addStep(
-        steps.FileDownload(
-            name="Upload keystore",
-            workerdest="/var/lib/buildbot/keystore",
-            mastersrc=util.Interpolate("keystore/%(prop:jobname)s.keystore"),
-            doStepIf=should_build_sign,
-            hideStepIf=True,
-            mode=0o600
-        )
-    )
-
     build_factory.addStep(
         steps.ShellCommand(
             name=util.Interpolate("build_%(prop:jobname)s"),
-            command=util.Interpolate("%(kw:directory)s/"
+            command=util.Interpolate("%(kw:directory)s/%(prop:jobname)s/build.sh", directory=directory),
-                                     "%(prop:jobname)s/build.sh",
+            doStepIf=is_build_script_available,
-                                     directory=directory),
-            env=build_env,
-            doStepIf=shouldnt_build_sign,
             hideStepIf=isnt_build_script_available,
-            timeout=7200,
-        )
-    )
-
-    build_factory.addStep(
-        steps.ShellCommand(
-            name=util.Interpolate("build_and_sign_%(prop:jobname)s"),
-            command=util.Interpolate("%(kw:directory)s/"
-                                     "%(prop:jobname)s/build.sh",
-                                     directory=directory),
-            env=build_env,
-            doStepIf=should_build_sign,
-            hideStepIf=isnt_build_script_available,
-            timeout=7200,
-        )
-    )
-
-    build_factory.addStep(
-        steps.ShellCommand(
-            name="Delete keystore",
-            command="rm -f /var/lib/buildbot/keystore",
-            doStepIf=should_build_sign,
-            hideStepIf=True,
         )
     )
 
     build_factory.addStep(
         steps.ShellCommand(
             name= util.Interpolate("test_%(prop:jobname)s"),
-            command=util.Interpolate("%(kw:directory)s/"
+            command=util.Interpolate("%(kw:directory)s/%(prop:jobname)s/test.sh", directory=directory),
-                                     "%(prop:jobname)s/test.sh",
-                                     directory=directory),
-            haltOnFailure=False,
-            flunkOnFailure=True,
             doStepIf=is_test_script_available,
             hideStepIf=isnt_test_script_available,
         )
     )
 
-    build_factory.addStep(
-        steps.SetPropertyFromCommand(
-            name="Find files to upload",
-            command="find . -maxdepth 1 -mindepth 1 "
-                    "-type f -printf '%P\n'",
-            workdir="out",
-            hideStepIf=True,
-            decodeRC={0: SUCCESS, 1: SUCCESS},
-            property="files_to_upload"
-        )
-    )
 
-    build_factory.addStep(
+if __name__ == "__main__":
-        steps.ShellCommand(
+    # expect jobname, repository, branch, buildbotUrl from command line
-            name="Upload files",
+    import sys
-            workdir="out",
+
-            doStepIf=has_files_to_upload,
+    if len(sys.argv) == 6:
-            hideStepIf=no_files_to_upload,
+        jobname = sys.argv[1]
-            command=util.Interpolate(
+        repository = sys.argv[2]
-                "curl -T {%s} -u buildbot:%s "
+        branch = sys.argv[3]
-                "https://artifacts.bitmessage.at/%s/%s/",
+        buildbotUrl = sys.argv[4]
-                files_to_upload,
+
-                util.Secret('artifact_upload'),
+        trigger_child_hooks(buildbotUrl, repository, branch)
-                util.Property('jobname'),
+    else:
-                util.Property('buildnumber'),
+        print(
-            )
+            "Usage: python3 multibuild.py <buildbotUrl> <repository> <branch>  "
-        )
         )