From d847415a418c1568f10564c126f1820df10f0e19 Mon Sep 17 00:00:00 2001 From: Muzahid Date: Fri, 11 Feb 2022 17:20:23 +0530 Subject: [PATCH] Split Worker and Master --- lib/worker_multibuild.py | 125 +++++++++++++++++++++++++++++++ multibuild.py | 156 ++++++--------------------------------- 2 files changed, 147 insertions(+), 134 deletions(-) create mode 100644 lib/worker_multibuild.py diff --git a/lib/worker_multibuild.py b/lib/worker_multibuild.py new file mode 100644 index 0000000..1dfdcbf --- /dev/null +++ b/lib/worker_multibuild.py @@ -0,0 +1,125 @@ +from os import listdir, walk +from os.path import exists, isfile, join, islink, isdir +import requests +import re + + +request_data = { + "project": "testproject", + "comments": "testcomment", +} +request_headers = { + "Content-Type": "application/x-www-form-urlencoded", + "Accept": "text/plain", +} + +ty = "/change_hook/base" +path =".buildbot" +dockerfile_extra_contents = {} +dockerfile_extra_contents['focal'] = """ + +# Buildbot +RUN apt-get install -yq --no-install-suggests --no-install-recommends \ + buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \ + python3-pip dumb-init curl openssh-client wget + +# buildbot entrypoint +RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh +RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh + +RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers + +USER buildbot + +ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS" + +""" + +dockerfile_extra_contents['bionic'] = """ + +# Buildbot +RUN apt-get install -yq --no-install-suggests --no-install-recommends \ + buildbot-slave git subversion python3-dev libffi-dev python3-setuptools \ + python3-pip dumb-init curl openssh-client wget + +# buildbot entrypoint +RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh +RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh + +RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers + +USER buildbot + +ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS" + +""" + +def list_jobs(directory=".buildbot"): + """ + list jobs found in a directory + """ + results = [] + files = ["Dockerfile", "build.sh", "test.sh"] + for item in listdir(directory): + flag = False + for file in files: + filepath = join(directory, item, file) + if islink(filepath) or not isfile(filepath): + flag = True + break + if flag: + continue + if (exists(join(directory, item, 'Dockerfile')) and exists(join(directory, item, 'build.sh'))) or exists(join(directory, item, 'test.sh')): + results.append(item) + return results + + +def _get_dockerfile_contents(jobname): + """ + Read contents of a Dockerfile and add extra contents for the given os_codename + """ + os_codename='bionic' + res = "" + with open(join(path + jobname), "r") as file: + contents = file.read() + # accept any line containing FROM and RUN keywords + res = "" + inside_allowed_command = False + for line in contents: + if re.match(r"(?m)^(FROM|RUN|ENV).*$", line): + inside_allowed_command = True + if inside_allowed_command: + res += line + l = line.strip() + if not l.endswith("\\"): + inside_allowed_command = False + + return res + dockerfile_extra_contents[os_codename] + + +def trigger_child_hooks(buildbotUrl: str, repository, branch, directory=".buildbot"): + request_url = buildbotUrl + ty + # List all jobs in the directory + jobs = list_jobs(directory) + + # Check if build.sh or test.sh exists in each of the jobs + for job in jobs: + build_script_exists = False + test_script_exists = False + if exists(join(directory, job, "build.sh")): + build_script_exists = True + if exists(join(directory, job, "test.sh")): + test_script_exists = True + + # make a post request + request_data["properties"] = { + "dockerfile": _get_dockerfile_contents( + join(directory, job, "Dockerfile") + ), + "build_script_available": is_build_script_available(build_script_exists), + "test_script_available": is_test_script_available(test_script_exists), + "repository": repository, + "branch": branch, + "jobname": job, + } + requests.post(request_url, headers=request_headers, data=request_data) diff --git a/multibuild.py b/multibuild.py index 7442686..8063acf 100644 --- a/multibuild.py +++ b/multibuild.py @@ -10,7 +10,7 @@ Requires docker # TODO: write hook job, maybe also a dockerfile? # TODO: what to do about non-docker jobs -from os import listdir, walk +from os import listdir, walk, getenv from os.path import exists, isfile, join import requests import re @@ -18,70 +18,6 @@ from buildbot.plugins import steps, util from .lib.renderers import * -ty = "/change_hook/base" -request_headers = { - "Content-Type": "application/x-www-form-urlencoded", - "Accept": "text/plain", -} -request_data = { - "project": "testproject", - "comments": "testcomment", -} - -dockerfile_extra_contents_focal = """ - -# Buildbot -RUN apt-get install -yq --no-install-suggests --no-install-recommends \ - buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \ - python3-pip dumb-init curl openssh-client wget - -# buildbot entrypoint -RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh -RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh - -RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers - -USER buildbot - -ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS" - -""" - -dockerfile_extra_contents_bionic = """ - -# Buildbot -RUN apt-get install -yq --no-install-suggests --no-install-recommends \ - buildbot-slave git subversion python3-dev libffi-dev python3-setuptools \ - python3-pip dumb-init curl openssh-client wget - -# buildbot entrypoint -RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh -RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh - -RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers - -USER buildbot - -ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS" - -""" - -path =".buildbot" - -def list_jobs(directory=".buildbot"): - """ - list jobs found in a directory - """ - results = [] - for _ in next(walk(directory))[1]: - if exists(join(directory, _, "Dockerfile")) and ( - exists(join(directory, _, "build.sh")) - or exists(join(directory, _, "test.sh")) - ): - results.append(_) - - return results - def find_artifacts(directory="out"): """ @@ -93,96 +29,49 @@ def find_artifacts(directory="out"): return join(directory, _) -def get_dockerfile_contents(props): - """ - Read contents of a Dockerfile and add extra contents for the given os_codename - """ - with open(join(path + props.getProperty('jobname', default=None)), "r") as file: - contents = file.read() - # remove any line containing CMD or ENTRYFILE keywords - re.sub(r"(?m)^(CMD|ENTRYFILE).*$", "", contents) - - return contents + { - "focal": dockerfile_extra_contents_focal, - "bionic": dockerfile_extra_contents_bionic, - }[util.Interpolate("%(prop:os_codename)s")] - - -def trigger_child_hooks(buildbotUrl: str, os_codename: str, repository, branch, jobname, directory=".buildbot"): - request_url = buildbotUrl + ty - - # List all jobs in the directory - jobs = list_jobs(directory) - - # Check if build.sh or test.sh exists in each of the jobs - for job in jobs: - build_script_exists = False - test_script_exists = False - if exists(join(directory, job, "build.sh")): - build_script_exists = True - if exists(join(directory, job, "test.sh")): - test_script_exists = True - - # make a post request - request_data["properties"] = { - "dockerfile": get_dockerfile_contents( - join(directory, job, "Dockerfile"), os_codename - ), - "build_script_available": is_build_script_available(build_script_exists), - "test_script_available": is_test_script_available(test_script_exists), - "repository": repository, - "branch": branch, - "jobname": jobname, - } - requests.post(request_url, headers=request_headers, data=request_data) - - def add_parent_step(build_factory): """ Add a step to the parent build factory that will trigger the child hooks """ - build_factory.addStep( - steps.SetPropertyFromCommand( - name="Get OS codename", - command="grep ^VERSION_CODENAME= /etc/os-release | cut -d= -f2", - property="os_codename", - ) - ) + build_factory.addStep(steps.ShellCommand( + name="create directory", + command=["mkdir", "-p", join(getenv['HOME'], '.local/bin') ] + )) + + build_factory.addStep(steps.ShellCommand( + name="download worker", + command=["wget", "-O", "https://git.bitmessage.org/Bitmessage/buildbot_multibuild/raw/branch/master/lib/worker_multibuild.py", join(getenv['HOME'], '.local/bin/worker_multibuild.py')] + )) + build_factory.addStep( steps.ShellCommand( - name="Execute multibuild script", + name="Execute worker script", command=[ - "python", - "multibuild.py", - util.Interpolate("%(prop:jobname)s"), + "python3", + join(getenv['HOME'], '.local/bin/worker_multibuild.py'), util.Property('repository'), util.Property('branch'), - "https://buildbot.bitmessage.org", - util.Interpolate("%(prop:os_codename)s"), + util.getURLForBuild(util.Property("url"), util.Property("builderid"), util.Property("buildnumber")), ], ) ) -def add_child_build_sh_step(build_factory, directory=".buildbot"): +def add_child_sh_steps(build_factory, directory=".buildbot"): """ - Add a step to the build factory + Add a step to the download, build and test factory """ + build_factory.addStep( steps.ShellCommand( name=util.Interpolate("build_%(prop:jobname)s"), - command=util.Interpolate("%(kw:directory)s/%(prop:jobname)s/test.sh", directory=directory), + command=util.Interpolate("%(kw:directory)s/%(prop:jobname)s/build.sh", directory=directory), doStepIf=is_build_script_available, hideStepIf=isnt_build_script_available, ) ) - -def add_child_test_sh_step(build_factory, directory=".buildbot"): - """ - Add a step to the build factory - """ build_factory.addStep( steps.ShellCommand( name= util.Interpolate("test_%(prop:jobname)s"), @@ -194,7 +83,7 @@ def add_child_test_sh_step(build_factory, directory=".buildbot"): if __name__ == "__main__": - # expect jobname, repository, branch, buildbotUrl, os_codename from command line args + # expect jobname, repository, branch, buildbotUrl from command line import sys if len(sys.argv) == 6: @@ -202,10 +91,9 @@ if __name__ == "__main__": repository = sys.argv[2] branch = sys.argv[3] buildbotUrl = sys.argv[4] - os_codename = sys.argv[5] - trigger_child_hooks(buildbotUrl, os_codename, repository, branch, jobname) + trigger_child_hooks(buildbotUrl, repository, branch) else: print( - "Usage: python3 multibuild.py " + "Usage: python3 multibuild.py " )