From b020325a089edd9757ae893d00fd95be77b09f7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20=C5=A0urda?= Date: Thu, 7 Apr 2022 20:46:29 +0800 Subject: [PATCH] Merge production changes - bugfixes, code quality --- lib/renderers.py | 16 +++---- lib/worker_multibuild.py | 93 ++++++++++++++++++++++++++++++---------- multibuild.py | 47 ++++++++------------ 3 files changed, 96 insertions(+), 60 deletions(-) diff --git a/lib/renderers.py b/lib/renderers.py index 81ad592..f8632f1 100644 --- a/lib/renderers.py +++ b/lib/renderers.py @@ -1,24 +1,24 @@ from buildbot.plugins import util import re +def _is_build_script_available(props): + return props.getProperty("build_script_available", default=False) @util.renderer def is_build_script_available(props): - # Actual check will got here - return props.getProperty("build_available", default=False) - + return _is_build_script_available(props) @util.renderer def isnt_build_script_available(props): - return not is_build_script_available(props) + return not _is_build_script_available(props) +def _is_test_script_available(props): + return props.getProperty("test_script_available", default=False) @util.renderer def is_test_script_available(props): - # Actual check will got here - return props.getProperty("test_available", default=False) - + return _is_test_script_available(props) @util.renderer def isnt_test_script_available(props): - return not is_test_script_available(props) + return not _is_test_script_available(props) diff --git a/lib/worker_multibuild.py b/lib/worker_multibuild.py index fbd10d5..b672718 100644 --- a/lib/worker_multibuild.py +++ b/lib/worker_multibuild.py @@ -1,8 +1,8 @@ -import os from os import listdir -from os.path import exists, isfile, join, islink, isdir +from os.path import exists, isfile, join, islink import requests import re +from subprocess import Popen, PIPE request_data = { @@ -11,7 +11,7 @@ request_data = { } ty = "/change_hook/base" -path =".buildbot" +path = ".buildbot" dockerfile_extra_contents = {} dockerfile_extra_contents['focal'] = """ @@ -51,11 +51,13 @@ ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$ """ + def get_secret(): - with open(path.join(os.environ['HOME'], "multibuild_parent_key.key"),'r') as f: + with open("multibuild_parent_key.key", 'r') as f: data = f.read() return data + def list_jobs(directory=".buildbot"): """ list jobs found in a directory @@ -63,39 +65,55 @@ def list_jobs(directory=".buildbot"): results = [] files = ["Dockerfile", "build.sh", "test.sh"] for item in listdir(directory): + print("checking directory {}".format(item)) flag = False - for file in files: - filepath = join(directory, item, file) + for fname in files: + filepath = join(directory, item, fname) + if not exists(filepath): + continue if islink(filepath) or not isfile(filepath): flag = True break if flag: - continue + continue if (exists(join(directory, item, 'Dockerfile')) and exists(join(directory, item, 'build.sh'))) or exists(join(directory, item, 'test.sh')): results.append(item) return results -def _get_dockerfile_contents(jobname): +def get_revision(): + proc = Popen(["git", "rev-parse", "HEAD"], stdout=PIPE) + retval = proc.stdout.read().strip() + retval = retval.decode('utf-8') + return retval + + +def _get_dockerfile_contents(dockerfile): """ Read contents of a Dockerfile and add extra contents for the given os_codename """ - os_codename='bionic' + os_codename = 'bionic' res = "" - with open(join(path + jobname), "r") as file: - contents = file.read() + with open(dockerfile, "r") as file: + contents = file.readlines() + has_from = False # accept any line containing FROM and RUN keywords res = "" inside_allowed_command = False for line in contents: - if re.match(r"(?m)^(FROM|RUN|ENV).*$", line): + m = re.match(r"(?m)^(FROM|RUN|ENV).*$", line) + if m: inside_allowed_command = True + if m.group(1) == "FROM": + has_from = True if inside_allowed_command: res += line - l = line.strip() - if not l.endswith("\\"): + ls = line.strip() + if not ls.endswith("\\"): inside_allowed_command = False + if not has_from: + return None return res + dockerfile_extra_contents[os_codename] @@ -104,10 +122,11 @@ def trigger_child_hooks(buildbotUrl: str, repository, branch, directory=".buildb # List all jobs in the directory jobs = list_jobs(directory) request_headers = { - "Content-Type": "application/x-www-form-urlencoded", + "Content-Type": "application/json", "X-Multibuild-Trigger": get_secret(), "Accept": "text/plain", } + revision = get_revision() # Check if build.sh or test.sh exists in each of the jobs for job in jobs: @@ -119,15 +138,43 @@ def trigger_child_hooks(buildbotUrl: str, repository, branch, directory=".buildb test_script_exists = True # make a post request + dockerfile = _get_dockerfile_contents( + join(directory, job, "Dockerfile") + ) + + if not dockerfile: + continue + + request_data["branch"] = branch + request_data["revision"] = revision + request_data["properties"] = { - "dockerfile": _get_dockerfile_contents( - join(directory, job, "Dockerfile") - ), - "build_script_available": is_build_script_available(build_script_exists), - "test_script_available": is_test_script_available(test_script_exists), - "repository": repository, - "branch": branch, + "dockerfile": dockerfile, + "build_script_available": build_script_exists, + "test_script_available": test_script_exists, "jobname": job, } + request_data["changes"] = { + "author": "buildbot_multibuild", + "repository": repository, + "project": "/".join(repository.split("/")[-2:]), + } - requests.post(request_url, headers=request_headers, data=request_data) + retval = requests.post(request_url, headers=request_headers, json=request_data) + print("Triggered job for {} on {}: {}".format(job, request_url, retval.text)) + + +if __name__ == "__main__": + # expect jobname, repository, branch, buildbotUrl from command line + import sys + + if len(sys.argv) == 4: + buildbotUrl = sys.argv[1] + repository = sys.argv[2] + branch = sys.argv[3] + + trigger_child_hooks(buildbotUrl, repository, branch) + else: + sys.exit( + "Usage: python3 multibuild.py " + ) diff --git a/multibuild.py b/multibuild.py index 8063acf..e5b3da8 100644 --- a/multibuild.py +++ b/multibuild.py @@ -10,10 +10,8 @@ Requires docker # TODO: write hook job, maybe also a dockerfile? # TODO: what to do about non-docker jobs -from os import listdir, walk, getenv -from os.path import exists, isfile, join -import requests -import re +from os import listdir +from os.path import isfile, join from buildbot.plugins import steps, util from .lib.renderers import * @@ -35,24 +33,32 @@ def add_parent_step(build_factory): """ build_factory.addStep(steps.ShellCommand( - name="create directory", - command=["mkdir", "-p", join(getenv['HOME'], '.local/bin') ] + name="Update APT cache", + command=["sudo", "apt", "update"] )) build_factory.addStep(steps.ShellCommand( - name="download worker", - command=["wget", "-O", "https://git.bitmessage.org/Bitmessage/buildbot_multibuild/raw/branch/master/lib/worker_multibuild.py", join(getenv['HOME'], '.local/bin/worker_multibuild.py')] + name="Install dependencies", + command=["sudo", "apt", "-y", "install", "python3-requests"] )) - + + build_factory.addStep( + steps.FileDownload( + workerdest="worker_multibuild.py", + mastersrc="buildbot_multibuild/lib/worker_multibuild.py", + mode=0o444 + ) + ) + build_factory.addStep( steps.ShellCommand( name="Execute worker script", command=[ "python3", - join(getenv['HOME'], '.local/bin/worker_multibuild.py'), + 'worker_multibuild.py', + util.Property("buildboturl"), util.Property('repository'), - util.Property('branch'), - util.getURLForBuild(util.Property("url"), util.Property("builderid"), util.Property("buildnumber")), + util.Property('branch') ], ) ) @@ -80,20 +86,3 @@ def add_child_sh_steps(build_factory, directory=".buildbot"): hideStepIf=isnt_test_script_available, ) ) - - -if __name__ == "__main__": - # expect jobname, repository, branch, buildbotUrl from command line - import sys - - if len(sys.argv) == 6: - jobname = sys.argv[1] - repository = sys.argv[2] - branch = sys.argv[3] - buildbotUrl = sys.argv[4] - - trigger_child_hooks(buildbotUrl, repository, branch) - else: - print( - "Usage: python3 multibuild.py " - )