Split Worker and Master

lib/worker_multibuild.py

@@ -0,0 +1,125 @@
+from os import listdir, walk
+from os.path import exists, isfile, join, islink, isdir
+import requests
+import re
+
+
+request_data = {
+    "project": "testproject",
+    "comments": "testcomment",
+}
+request_headers = {
+    "Content-Type": "application/x-www-form-urlencoded",
+    "Accept": "text/plain",
+}
+
+ty = "/change_hook/base"
+path =".buildbot"
+dockerfile_extra_contents = {}
+dockerfile_extra_contents['focal'] = """
+
+# Buildbot
+RUN apt-get install -yq --no-install-suggests --no-install-recommends \
+    buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
+    python3-pip dumb-init curl openssh-client wget
+
+# buildbot entrypoint
+RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
+RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
+
+RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+
+USER buildbot
+
+ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
+
+"""
+
+dockerfile_extra_contents['bionic'] = """
+
+# Buildbot
+RUN apt-get install -yq --no-install-suggests --no-install-recommends \
+    buildbot-slave git subversion python3-dev libffi-dev python3-setuptools \
+    python3-pip dumb-init curl openssh-client wget
+
+# buildbot entrypoint
+RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
+RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
+
+RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+
+USER buildbot
+
+ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
+
+"""
+
+def list_jobs(directory=".buildbot"):
+    """
+    list jobs found in a directory
+    """
+    results = []
+    files = ["Dockerfile", "build.sh", "test.sh"]
+    for item in listdir(directory):
+        flag = False
+        for file in files:
+            filepath = join(directory, item, file)
+            if islink(filepath) or not isfile(filepath):
+                flag = True
+                break
+        if flag:
+            continue    
+        if (exists(join(directory, item, 'Dockerfile')) and exists(join(directory, item, 'build.sh'))) or exists(join(directory, item, 'test.sh')):
+            results.append(item)
+    return results
+
+
+def _get_dockerfile_contents(jobname):
+    """
+    Read contents of a Dockerfile and add extra contents for the given os_codename
+    """
+    os_codename='bionic'
+    res = ""
+    with open(join(path + jobname), "r") as file:
+        contents = file.read()
+        # accept any line containing FROM and RUN keywords
+        res = ""
+        inside_allowed_command = False
+        for line in contents:
+            if re.match(r"(?m)^(FROM|RUN|ENV).*$", line):
+                inside_allowed_command = True
+            if inside_allowed_command:
+                res += line
+                l = line.strip()
+                if not l.endswith("\\"):
+                    inside_allowed_command = False
+
+    return res + dockerfile_extra_contents[os_codename]
+
+
+def trigger_child_hooks(buildbotUrl: str, repository, branch, directory=".buildbot"):
+    request_url = buildbotUrl + ty
+    # List all jobs in the directory
+    jobs = list_jobs(directory)
+
+    # Check if build.sh or test.sh exists in each of the jobs
+    for job in jobs:
+        build_script_exists = False
+        test_script_exists = False
+        if exists(join(directory, job, "build.sh")):
+            build_script_exists = True
+        if exists(join(directory, job, "test.sh")):
+            test_script_exists = True
+
+        # make a post request
+        request_data["properties"] = {
+            "dockerfile": _get_dockerfile_contents(
+                join(directory, job, "Dockerfile")
+            ),
+            "build_script_available": is_build_script_available(build_script_exists),
+            "test_script_available": is_test_script_available(test_script_exists),
+            "repository": repository,
+            "branch": branch,
+            "jobname": job,
+        }
+        requests.post(request_url, headers=request_headers, data=request_data)

multibuild.py

@@ -10,77 +10,14 @@ Requires docker
 # TODO: write hook job, maybe also a dockerfile?
 # TODO: what to do about non-docker jobs
 
-from os import walk
-from os.path import exists, isfile, join, listdir
+from os import listdir, walk, getenv
+from os.path import exists, isfile, join
 import requests
 import re
 from buildbot.plugins import steps, util
 
 from .lib.renderers import *
 
-ty = "/change_hook/base"
-request_headers = {
-    "Content-Type": "application/x-www-form-urlencoded",
-    "Accept": "text/plain",
-}
-request_data = {
-    "project": "testproject",
-    "comments": "testcomment",
-}
-
-dockerfile_extra_contents_focal = """
-
-# Buildbot
-RUN apt-get install -yq --no-install-suggests --no-install-recommends \
-    buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
-    python3-pip dumb-init curl openssh-client wget
-
-# buildbot entrypoint
-RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
-RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
-
-RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
-
-USER buildbot
-
-ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
-
-"""
-
-dockerfile_extra_contents_bionic = """
-
-# Buildbot
-RUN apt-get install -yq --no-install-suggests --no-install-recommends \
-    buildbot-slave git subversion python3-dev libffi-dev python3-setuptools \
-    python3-pip dumb-init curl openssh-client wget
-
-# buildbot entrypoint
-RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
-RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
-
-RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
-
-USER buildbot
-
-ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
-
-"""
-
-
-def list_jobs(directory=".buildbot"):
-    """
-    list jobs found in a directory
-    """
-    results = []
-    for _ in next(walk(directory))[1]:
-        if exists(join(directory, _, "Dockerfile")) and (
-            exists(join(directory, _, "build.sh"))
-            or exists(join(directory, _, "test.sh"))
-        ):
-            results.append(_)
-
-    return results
-
 
 def find_artifacts(directory="out"):
     """
@@ -92,100 +29,53 @@ def find_artifacts(directory="out"):
         return join(directory, _)
 
 
-def get_dockerfile_contents(path, os_codename):
-    """
-    Read contents of a Dockerfile and add extra contents for the given os_codename
-    """
-    with open(path, "r") as file:
-        contents = file.read()
-        # remove any line containing CMD or ENTRYFILE keywords
-        re.sub(r"(?m)^(CMD|ENTRYFILE).*$", "", contents)
-
-    return contents + {
-        "focal": dockerfile_extra_contents_focal,
-        "bionic": dockerfile_extra_contents_bionic,
-    }[os_codename]
-
-
-def trigger_child_hooks(buildbotUrl: str, os_codename: str, repository, branch, jobname, directory=".buildbot"):
-    request_url = buildbotUrl + ty
-
-    # List all jobs in the directory
-    jobs = list_jobs(directory)
-
-    # Check if build.sh or test.sh exists in each of the jobs
-    for job in jobs:
-        build_script_exists = False
-        test_script_exists = False
-        if exists(join(directory, job, "build.sh")):
-            build_script_exists = True
-        if exists(join(directory, job, "test.sh")):
-            test_script_exists = True
-
-        # make a post request
-        request_data["properties"] = {
-            "dockerfile": get_dockerfile_contents(
-                join(directory, job, "Dockerfile"), os_codename
-            ),
-            "build_script_available": is_build_script_available(build_script_exists),
-            "test_script_available": is_test_script_available(test_script_exists),
-            "repository": repository,
-            "branch": branch,
-            "jobname": jobname,
-        }
-        requests.post(request_url, headers=request_headers, data=request_data)
-
-
-def add_parent_step(build_factory, jobname, repository, branch):
+def add_parent_step(build_factory):
     """
     Add a step to the parent build factory that will trigger the child hooks
     """
-    build_factory.addStep(
-        steps.SetPropertyFromCommand(
-            name="Get OS codename",
-            command="grep ^VERSION_CODENAME= /etc/os-release | cut -d= -f2",
-            property="os_codename",
-        )
-    )
+
+    build_factory.addStep(steps.ShellCommand(
+        name="create directory",
+        command=["mkdir", "-p", join(getenv['HOME'], '.local/bin') ]
+    ))
+
+    build_factory.addStep(steps.ShellCommand(
+        name="download worker",
+        command=["wget", "-O", "https://git.bitmessage.org/Bitmessage/buildbot_multibuild/raw/branch/master/lib/worker_multibuild.py", join(getenv['HOME'], '.local/bin/worker_multibuild.py')]
+    ))
     
     build_factory.addStep(
         steps.ShellCommand(
-            name="Execute multibuild script",
+            name="Execute worker script",
             command=[
-                "python",
-                "multibuild.py",
-                jobname,
-                repository,
-                branch,
-                "https://buildbot.bitmessage.org",
-                util.Interpolate("%(prop:os_codename)s"),
+                "python3",
+                join(getenv['HOME'], '.local/bin/worker_multibuild.py'),
+                util.Property('repository'),
+                util.Property('branch'),
+                util.getURLForBuild(util.Property("url"), util.Property("builderid"), util.Property("buildnumber")),
             ],
         )
     )
 
 
-def add_child_build_sh_step(build_factory, job, directory=".buildbot"):
+def add_child_sh_steps(build_factory, directory=".buildbot"):
     """
-    Add a step to the build factory
+    Add a step to the download, build and test factory
     """
+
     build_factory.addStep(
         steps.ShellCommand(
-            name="build_" + job,
-            command=["bash", "-c", join(directory, job, "build.sh")],
+            name=util.Interpolate("build_%(prop:jobname)s"),
+            command=util.Interpolate("%(kw:directory)s/%(prop:jobname)s/build.sh", directory=directory),
             doStepIf=is_build_script_available,
             hideStepIf=isnt_build_script_available,
         )
     )
 
-
-def add_child_test_sh_step(build_factory, job, directory=".buildbot"):
-    """
-    Add a step to the build factory
-    """
     build_factory.addStep(
         steps.ShellCommand(
-            name="test_" + job,
-            command=["bash", "-c", join(directory, job, "test.sh")],
+            name= util.Interpolate("test_%(prop:jobname)s"),
+            command=util.Interpolate("%(kw:directory)s/%(prop:jobname)s/test.sh", directory=directory),
             doStepIf=is_test_script_available,
             hideStepIf=isnt_test_script_available,
         )
@@ -193,7 +83,7 @@ def add_child_test_sh_step(build_factory, job, directory=".buildbot"):
 
 
 if __name__ == "__main__":
-    # expect jobname, repository, branch, buildbotUrl, os_codename from command line args
+    # expect jobname, repository, branch, buildbotUrl from command line
     import sys
 
     if len(sys.argv) == 6:
@@ -201,10 +91,9 @@ if __name__ == "__main__":
         repository = sys.argv[2]
         branch = sys.argv[3]
         buildbotUrl = sys.argv[4]
-        os_codename = sys.argv[5]
 
-        trigger_child_hooks(buildbotUrl, os_codename, repository, branch, jobname)
+        trigger_child_hooks(buildbotUrl, repository, branch)
     else:
         print(
-            "Usage: python3 multibuild.py <jobname> <repository> <branch> <buildbotUrl> <os_codename>"
+            "Usage: python3 multibuild.py <buildbotUrl> <repository> <branch>  "
         )