#!/usr/bin/python3 """ Allow buildbot to run jobs dynamically defined a a project repo Requires docker """ # TODO: change "ghcontext" in master.cfg to interpolate the job name # TODO: write upload script # TODO: write hook (perhaps the default hook is ok), authentication for hook # TODO: write hook job, maybe also a dockerfile? # TODO: what to do about non-docker jobs from os import listdir, walk from os.path import exists, isfile, join import requests import re from buildbot.plugins import steps, util from .lib.renderers import * ty = "/change_hook/base" request_headers = { "Content-Type": "application/x-www-form-urlencoded", "Accept": "text/plain", } request_data = { "project": "testproject", "comments": "testcomment", } dockerfile_extra_contents_focal = """ # Buildbot RUN apt-get install -yq --no-install-suggests --no-install-recommends \ buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \ python3-pip dumb-init curl openssh-client wget # buildbot entrypoint RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers USER buildbot ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS" """ dockerfile_extra_contents_bionic = """ # Buildbot RUN apt-get install -yq --no-install-suggests --no-install-recommends \ buildbot-slave git subversion python3-dev libffi-dev python3-setuptools \ python3-pip dumb-init curl openssh-client wget # buildbot entrypoint RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers USER buildbot ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS" """ path =".buildbot" def list_jobs(directory=".buildbot"): """ list jobs found in a directory """ results = [] for _ in next(walk(directory))[1]: if exists(join(directory, _, "Dockerfile")) and ( exists(join(directory, _, "build.sh")) or exists(join(directory, _, "test.sh")) ): results.append(_) return results def find_artifacts(directory="out"): """ find artifacts (any file) in a directory """ for _ in listdir(directory): if not isfile(join(directory, _)): continue return join(directory, _) def get_dockerfile_contents(props): """ Read contents of a Dockerfile and add extra contents for the given os_codename """ with open(join(path + props.getProperty('jobname', default=None)), "r") as file: contents = file.read() # remove any line containing CMD or ENTRYFILE keywords re.sub(r"(?m)^(CMD|ENTRYFILE).*$", "", contents) return contents + { "focal": dockerfile_extra_contents_focal, "bionic": dockerfile_extra_contents_bionic, }[util.Interpolate("%(prop:os_codename)s")] def trigger_child_hooks(buildbotUrl: str, os_codename: str, repository, branch, jobname, directory=".buildbot"): request_url = buildbotUrl + ty # List all jobs in the directory jobs = list_jobs(directory) # Check if build.sh or test.sh exists in each of the jobs for job in jobs: build_script_exists = False test_script_exists = False if exists(join(directory, job, "build.sh")): build_script_exists = True if exists(join(directory, job, "test.sh")): test_script_exists = True # make a post request request_data["properties"] = { "dockerfile": get_dockerfile_contents( join(directory, job, "Dockerfile"), os_codename ), "build_script_available": is_build_script_available(build_script_exists), "test_script_available": is_test_script_available(test_script_exists), "repository": repository, "branch": branch, "jobname": jobname, } requests.post(request_url, headers=request_headers, data=request_data) def add_parent_step(build_factory): """ Add a step to the parent build factory that will trigger the child hooks """ build_factory.addStep( steps.SetPropertyFromCommand( name="Get OS codename", command="grep ^VERSION_CODENAME= /etc/os-release | cut -d= -f2", property="os_codename", ) ) build_factory.addStep( steps.ShellCommand( name="Execute multibuild script", command=[ "python", "multibuild.py", util.Interpolate("%(prop:jobname)s"), util.Property('repository'), util.Property('branch'), "https://buildbot.bitmessage.org", util.Interpolate("%(prop:os_codename)s"), ], ) ) def add_child_build_sh_step(build_factory, directory=".buildbot"): """ Add a step to the build factory """ build_factory.addStep( steps.ShellCommand( name=util.Interpolate("build_%(prop:jobname)s"), command=util.Interpolate("%(kw:directory)s/%(prop:jobname)s/test.sh", directory=directory), doStepIf=is_build_script_available, hideStepIf=isnt_build_script_available, ) ) def add_child_test_sh_step(build_factory, directory=".buildbot"): """ Add a step to the build factory """ build_factory.addStep( steps.ShellCommand( name= util.Interpolate("test_%(prop:jobname)s"), command=util.Interpolate("%(kw:directory)s/%(prop:jobname)s/test.sh", directory=directory), doStepIf=is_test_script_available, hideStepIf=isnt_test_script_available, ) ) if __name__ == "__main__": # expect jobname, repository, branch, buildbotUrl, os_codename from command line args import sys if len(sys.argv) == 6: jobname = sys.argv[1] repository = sys.argv[2] branch = sys.argv[3] buildbotUrl = sys.argv[4] os_codename = sys.argv[5] trigger_child_hooks(buildbotUrl, os_codename, repository, branch, jobname) else: print( "Usage: python3 multibuild.py " )