From 27da79a92bf38b0b84a133afeccee24346ca7158 Mon Sep 17 00:00:00 2001 From: Shashi Date: Thu, 23 Dec 2021 13:31:52 +0530 Subject: [PATCH 1/7] Added parent hook and some renderer --- lib/wine.py | 38 +++++++++++++++++++++++++++ multibuild.py | 40 ++++++++++++++++++++++++++++ parent_hook | 73 +++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 151 insertions(+) create mode 100644 lib/wine.py create mode 100644 parent_hook diff --git a/lib/wine.py b/lib/wine.py new file mode 100644 index 0000000..c12d036 --- /dev/null +++ b/lib/wine.py @@ -0,0 +1,38 @@ +from buildbot.plugins import util +import re + +def _is_github(props): + if re.search(r'[/@]github.com', props.getProperty('repository')): + return True + return False + +@util.renderer +def is_github(props): + return _is_github(props) + +@util.renderer +def isnt_github(props): + return not _is_github(props) + +def _is_gitea(props): + if re.search(r'[/@]git.bitmessage.org', props.getProperty('repository'), re.I): + return True + return False + +@util.renderer +def is_gitea(props): + return _is_gitea(props) + +@util.renderer +def isnt_gitea(props): + return not _is_gitea(props) + +@util.renderer +def is_build_script_available(props): + # Actual check will got here + return props.getProperty('jobname', default='') != '' + +@util.renderer +def is_test_script_available(props): + # Actual check will got here + return props.getProperty('jobname', default='') != '' \ No newline at end of file diff --git a/multibuild.py b/multibuild.py index 17c9f32..05eee87 100644 --- a/multibuild.py +++ b/multibuild.py @@ -12,6 +12,23 @@ Requires docker from os import walk from os.path import exists, isfile, join, listdir +import requests + +request_url = "https://buildbot.sysdeploy.org/change_hook/base" +request_headers = { + 'Content-Type': 'application/x-www-form-urlencoded', + 'Accept': 'text/plain' +} +request_data = { + 'repository': 'git@git.bitmessage.org:Sysdeploy/buildbot-scripts.git', + 'project': 'testproject', + 'author': 'cloud-init', + 'comments': 'testcomment', +} + +ty = 'child_hook' +hn = 'buildbot.sysdeploy.org' #hostname here(required?) +addn = ',"build_script_available":"{build_script_available}","test_script_available":"{test_script_available}"' def list_jobs(directory=".buildbot"): @@ -37,3 +54,26 @@ def find_artifacts(directory="out"): if not isfile(join(directory, _)): continue return join(directory, _) + +def trigger_child_hooks(directory=".buildbot"): + + # List all jobs in the directory + jobs = list_jobs(directory) + + # Check if build.sh or test.sh exists in each of the jobs + for job in jobs: + build_script_exists = False + test_script_exists = False + if exists(join(directory, job, "build.sh")): + build_script_exists = True + if exists(join(directory, job, "test.sh")): + test_script_exists = True + + addn = addn.format(build_script_available=build_script_exists, test_script_available=test_script_exists) + + # make a post request + request_data['properties'] = '{"node":{hn},"type":{ty}{addn}"}'.format(hn=hn, ty=ty, addn=addn) + requests.post(request_url, headers=request_headers, data=request_data) + +if __name__ == "__main__": + trigger_child_hooks() \ No newline at end of file diff --git a/parent_hook b/parent_hook new file mode 100644 index 0000000..aace749 --- /dev/null +++ b/parent_hook @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +""" parent webhook """ + +from buildbot.plugins import steps, util, secrets +from .lib.wine import * + +c = BuildmasterConfig = {} + +c['buildbotNetUsageData'] = None + +####### SECRETS + +c['secretsProviders'] = [secrets.SecretInAFile(dirname="/var/lib/buildbot/secrets/bitmessage")] +gitea_known_hosts = util.Secret('gitea_known_hosts') +gitea_privkey = util.Secret('gitea_privkey') + +travis_bash = util.BuildFactory() +travis_bash.addStep( + steps.GitHub( + repourl=util.Property("repository"), + name="github", + doStepIf=is_github, + hideStepIf=isnt_github, + branch=util.Property("branch"), + mode="incremental", + ) +) +travis_bash.addStep( + steps.Gitea( + repourl=util.Property("repository"), + sshPrivateKey=gitea_privkey, + sshKnownHosts=gitea_known_hosts, + name="gitea", + doStepIf=is_gitea, + hideStepIf=isnt_gitea, + branch=util.Property("branch"), + mode="incremental", + ) +) + +# execute an interpolate of '.buildbot/$(prop:jobname)/build.sh' +travis_bash.addStep( + steps.ShellCommand( + name="Execute build script", + command=[ + "bash", + util.Interpolate(".buildbot/$(prop:jobname)/build.sh"), + ], + doStepIf=is_build_script_available, + ) +) + +travis_bash.addStep( + steps.ShellCommand( + name="Execute test script", + command=[ + "bash", + util.Interpolate(".buildbot/$(prop:jobname)/test.sh"), + ], + doStepIf=is_test_script_available, + ) +) + +# execute multibuild.py +travis_bash.addStep( + steps.ShellCommand( + name="Execute multibuild script", + command=[ + "python", + util.Interpolate("multibuild.py"), + ], + ) +) -- 2.40.1 From f38fd1c826293642fe7c61da61b2b86ef0a70466 Mon Sep 17 00:00:00 2001 From: Shashi Date: Thu, 23 Dec 2021 16:56:44 +0530 Subject: [PATCH 2/7] Updated-S --- lib/{wine.py => renderers.py} | 0 multibuild.py | 34 ++++++++++++++++++++++------------ parent_hook | 12 ++++++++---- 3 files changed, 30 insertions(+), 16 deletions(-) rename lib/{wine.py => renderers.py} (100%) diff --git a/lib/wine.py b/lib/renderers.py similarity index 100% rename from lib/wine.py rename to lib/renderers.py diff --git a/multibuild.py b/multibuild.py index 05eee87..3315d2e 100644 --- a/multibuild.py +++ b/multibuild.py @@ -14,22 +14,16 @@ from os import walk from os.path import exists, isfile, join, listdir import requests -request_url = "https://buildbot.sysdeploy.org/change_hook/base" +ty = '/change_hook/base' request_headers = { 'Content-Type': 'application/x-www-form-urlencoded', 'Accept': 'text/plain' } request_data = { - 'repository': 'git@git.bitmessage.org:Sysdeploy/buildbot-scripts.git', 'project': 'testproject', - 'author': 'cloud-init', 'comments': 'testcomment', } -ty = 'child_hook' -hn = 'buildbot.sysdeploy.org' #hostname here(required?) -addn = ',"build_script_available":"{build_script_available}","test_script_available":"{test_script_available}"' - def list_jobs(directory=".buildbot"): """ @@ -55,7 +49,8 @@ def find_artifacts(directory="out"): continue return join(directory, _) -def trigger_child_hooks(directory=".buildbot"): +def trigger_child_hooks(buildbotUrl:str, directory=".buildbot"): + request_url = buildbotUrl + ty # List all jobs in the directory jobs = list_jobs(directory) @@ -69,11 +64,26 @@ def trigger_child_hooks(directory=".buildbot"): if exists(join(directory, job, "test.sh")): test_script_exists = True - addn = addn.format(build_script_available=build_script_exists, test_script_available=test_script_exists) - # make a post request - request_data['properties'] = '{"node":{hn},"type":{ty}{addn}"}'.format(hn=hn, ty=ty, addn=addn) + request_data['dockerfile'] = open(join(directory, job, "Dockerfile"), "r").read() + request_data['build_script_available'] = str(build_script_exists) + request_data['test_script_available'] = str(test_script_exists) requests.post(request_url, headers=request_headers, data=request_data) if __name__ == "__main__": - trigger_child_hooks() \ No newline at end of file + # expect jobname, repository, branch from command line args + import sys + if len(sys.argv) == 5: + jobname = sys.argv[1] + repository = sys.argv[2] + branch = sys.argv[3] + buildbotUrl = sys.argv[4] + + # add these into the request_data + request_data['jobname'] = jobname + request_data['repository'] = repository + request_data['branch'] = branch + + trigger_child_hooks(buildbotUrl) + else: + print("Usage: python3 multibuild.py ") \ No newline at end of file diff --git a/parent_hook b/parent_hook index aace749..4863696 100644 --- a/parent_hook +++ b/parent_hook @@ -2,7 +2,7 @@ """ parent webhook """ from buildbot.plugins import steps, util, secrets -from .lib.wine import * +from .lib.renderers import * c = BuildmasterConfig = {} @@ -44,7 +44,7 @@ travis_bash.addStep( name="Execute build script", command=[ "bash", - util.Interpolate(".buildbot/$(prop:jobname)/build.sh"), + util.Interpolate(".buildbot/%(prop:jobname)s/build.sh"), ], doStepIf=is_build_script_available, ) @@ -55,7 +55,7 @@ travis_bash.addStep( name="Execute test script", command=[ "bash", - util.Interpolate(".buildbot/$(prop:jobname)/test.sh"), + util.Interpolate(".buildbot/%(prop:jobname)s/test.sh"), ], doStepIf=is_test_script_available, ) @@ -67,7 +67,11 @@ travis_bash.addStep( name="Execute multibuild script", command=[ "python", - util.Interpolate("multibuild.py"), + "multibuild.py", + util.Interpolate("%(prop:jobname)s"), + util.Property("repository"), + util.Property("branch"), + "https://buildbot.bitmessage.org", ], ) ) -- 2.40.1 From 1432e510a17a5ae3b03fc23e107e749f62f4128e Mon Sep 17 00:00:00 2001 From: Shashi Date: Fri, 24 Dec 2021 09:36:25 +0530 Subject: [PATCH 3/7] Add extra content in dockerfile based on OS --- multibuild.py | 59 +++++++++++++++++++++++++++++++++++++++++++++------ parent_hook | 6 ++++++ 2 files changed, 59 insertions(+), 6 deletions(-) diff --git a/multibuild.py b/multibuild.py index 3315d2e..909ad57 100644 --- a/multibuild.py +++ b/multibuild.py @@ -13,6 +13,7 @@ Requires docker from os import walk from os.path import exists, isfile, join, listdir import requests +import re ty = '/change_hook/base' request_headers = { @@ -24,6 +25,39 @@ request_data = { 'comments': 'testcomment', } +dockerfile_extra_contents_focal = ''' + +# Buildbot +RUN apt-get install -yq --no-install-suggests --no-install-recommends \ + buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \ + python3-pip dumb-init curl openssh-client wget + +RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers + +USER buildbot + +ENTRYPOINT /entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS" + +''' + +dockerfile_extra_contents_bionic = ''' + +# Buildbot +RUN apt-get install -yq --no-install-suggests --no-install-recommends \ + buildbot-slave git subversion python3-dev libffi-dev python3-setuptools \ + python3-pip dumb-init curl openssh-client + +# buildbot entrypoint +RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh +RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh + +RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers + +USER buildbot + +ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS" + +''' def list_jobs(directory=".buildbot"): """ @@ -49,7 +83,19 @@ def find_artifacts(directory="out"): continue return join(directory, _) -def trigger_child_hooks(buildbotUrl:str, directory=".buildbot"): +def get_dockerfile_contents(path, os_codename): + ''' + Read contents of a Dockerfile and add extra contents for the given os_codename + ''' + with open(path, 'r') as file: + contents = file.read() + # remove any line containing CMD or ENTRYFILE keywords + contents = re.sub(r'(?m)^CMD.*$', '', contents) + contents = re.sub(r'(?m)^ENTRYFILE.*$', '', contents) + + return contents + eval("dockerfile_extra_contents_" + os_codename) + +def trigger_child_hooks(buildbotUrl:str, os_codename:str, directory=".buildbot"): request_url = buildbotUrl + ty # List all jobs in the directory @@ -65,25 +111,26 @@ def trigger_child_hooks(buildbotUrl:str, directory=".buildbot"): test_script_exists = True # make a post request - request_data['dockerfile'] = open(join(directory, job, "Dockerfile"), "r").read() + request_data['dockerfile'] = get_dockerfile_contents(join(directory, job, "Dockerfile"), os_codename) request_data['build_script_available'] = str(build_script_exists) request_data['test_script_available'] = str(test_script_exists) requests.post(request_url, headers=request_headers, data=request_data) if __name__ == "__main__": - # expect jobname, repository, branch from command line args + # expect jobname, repository, branch, buildbotUrl, os_codename from command line args import sys - if len(sys.argv) == 5: + if len(sys.argv) == 6: jobname = sys.argv[1] repository = sys.argv[2] branch = sys.argv[3] buildbotUrl = sys.argv[4] + os_codename = sys.argv[5] # add these into the request_data request_data['jobname'] = jobname request_data['repository'] = repository request_data['branch'] = branch - trigger_child_hooks(buildbotUrl) + trigger_child_hooks(buildbotUrl, os_codename) else: - print("Usage: python3 multibuild.py ") \ No newline at end of file + print("Usage: python3 multibuild.py ") \ No newline at end of file diff --git a/parent_hook b/parent_hook index 4863696..d0af5dd 100644 --- a/parent_hook +++ b/parent_hook @@ -61,6 +61,11 @@ travis_bash.addStep( ) ) +travis_bash.addStep(steps.SetPropertyFromCommand( + command="grep ^VERSION_CODENAME= /etc/os-release | cut -d= -f2", + property="os_codename" +)) + # execute multibuild.py travis_bash.addStep( steps.ShellCommand( @@ -72,6 +77,7 @@ travis_bash.addStep( util.Property("repository"), util.Property("branch"), "https://buildbot.bitmessage.org", + util.Interpolate("%(prop:os_codename)s"), ], ) ) -- 2.40.1 From 9a55687ec6414a8437e6bb44c68e72643bfe0030 Mon Sep 17 00:00:00 2001 From: Shashi Date: Sat, 25 Dec 2021 10:19:17 +0530 Subject: [PATCH 4/7] Added functions to add more steps to a buildFactory --- multibuild.py | 118 ++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 90 insertions(+), 28 deletions(-) diff --git a/multibuild.py b/multibuild.py index 909ad57..94bf1b2 100644 --- a/multibuild.py +++ b/multibuild.py @@ -14,18 +14,19 @@ from os import walk from os.path import exists, isfile, join, listdir import requests import re +from buildbot.plugins import steps, util -ty = '/change_hook/base' +ty = "/change_hook/base" request_headers = { - 'Content-Type': 'application/x-www-form-urlencoded', - 'Accept': 'text/plain' + "Content-Type": "application/x-www-form-urlencoded", + "Accept": "text/plain", } request_data = { - 'project': 'testproject', - 'comments': 'testcomment', + "project": "testproject", + "comments": "testcomment", } -dockerfile_extra_contents_focal = ''' +dockerfile_extra_contents_focal = """ # Buildbot RUN apt-get install -yq --no-install-suggests --no-install-recommends \ @@ -38,9 +39,9 @@ USER buildbot ENTRYPOINT /entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS" -''' +""" -dockerfile_extra_contents_bionic = ''' +dockerfile_extra_contents_bionic = """ # Buildbot RUN apt-get install -yq --no-install-suggests --no-install-recommends \ @@ -57,7 +58,8 @@ USER buildbot ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS" -''' +""" + def list_jobs(directory=".buildbot"): """ @@ -65,10 +67,10 @@ def list_jobs(directory=".buildbot"): """ results = [] for _ in next(walk(directory))[1]: - if exists(join(directory, _, "Dockerfile")) \ - and (exists(join(directory, _, "build.sh")) - or exists(join(directory, _, "test.sh")) - ): + if exists(join(directory, _, "Dockerfile")) and ( + exists(join(directory, _, "build.sh")) + or exists(join(directory, _, "test.sh")) + ): results.append(_) return results @@ -83,21 +85,23 @@ def find_artifacts(directory="out"): continue return join(directory, _) + def get_dockerfile_contents(path, os_codename): - ''' + """ Read contents of a Dockerfile and add extra contents for the given os_codename - ''' - with open(path, 'r') as file: + """ + with open(path, "r") as file: contents = file.read() # remove any line containing CMD or ENTRYFILE keywords - contents = re.sub(r'(?m)^CMD.*$', '', contents) - contents = re.sub(r'(?m)^ENTRYFILE.*$', '', contents) + contents = re.sub(r"(?m)^CMD.*$", "", contents) + contents = re.sub(r"(?m)^ENTRYFILE.*$", "", contents) return contents + eval("dockerfile_extra_contents_" + os_codename) -def trigger_child_hooks(buildbotUrl:str, os_codename:str, directory=".buildbot"): + +def trigger_child_hooks(buildbotUrl: str, os_codename: str, directory=".buildbot"): request_url = buildbotUrl + ty - + # List all jobs in the directory jobs = list_jobs(directory) @@ -109,16 +113,72 @@ def trigger_child_hooks(buildbotUrl:str, os_codename:str, directory=".buildbot") build_script_exists = True if exists(join(directory, job, "test.sh")): test_script_exists = True - + # make a post request - request_data['dockerfile'] = get_dockerfile_contents(join(directory, job, "Dockerfile"), os_codename) - request_data['build_script_available'] = str(build_script_exists) - request_data['test_script_available'] = str(test_script_exists) + request_data["dockerfile"] = get_dockerfile_contents( + join(directory, job, "Dockerfile"), os_codename + ) + request_data["build_script_available"] = str(build_script_exists) + request_data["test_script_available"] = str(test_script_exists) requests.post(request_url, headers=request_headers, data=request_data) + +def add_parent_step(build_factory, jobname, repository, branch): + """ + Add a step to the parent build factory that will trigger the child hooks + """ + build_factory.addStep( + steps.SetPropertyFromCommand( + name="Get OS codename", + command="grep ^VERSION_CODENAME= /etc/os-release | cut -d= -f2", + property="os_codename", + ) + ) + + build_factory.addStep( + steps.ShellCommand( + name="Execute multibuild script", + command=[ + "python", + "multibuild.py", + jobname, + repository, + branch, + "https://buildbot.bitmessage.org", + util.Interpolate("%(prop:os_codename)s"), + ], + ) + ) + + +def add_child_build_sh_step(build_factory, job, directory=".buildbot"): + """ + Add a step to the build factory + """ + build_factory.addStep( + steps.ShellCommand( + name="build_" + job, + command=["bash", "-c", "cd " + join(directory, job) + " && ./build.sh"], + ) + ) + + +def add_child_test_sh_step(build_factory, job, directory=".buildbot"): + """ + Add a step to the build factory + """ + build_factory.addStep( + steps.ShellCommand( + name="test_" + job, + command=["bash", "-c", "cd " + join(directory, job) + " && ./test.sh"], + ) + ) + + if __name__ == "__main__": # expect jobname, repository, branch, buildbotUrl, os_codename from command line args import sys + if len(sys.argv) == 6: jobname = sys.argv[1] repository = sys.argv[2] @@ -127,10 +187,12 @@ if __name__ == "__main__": os_codename = sys.argv[5] # add these into the request_data - request_data['jobname'] = jobname - request_data['repository'] = repository - request_data['branch'] = branch + request_data["jobname"] = jobname + request_data["repository"] = repository + request_data["branch"] = branch trigger_child_hooks(buildbotUrl, os_codename) else: - print("Usage: python3 multibuild.py ") \ No newline at end of file + print( + "Usage: python3 multibuild.py " + ) -- 2.40.1 From bea032dc782686760acac2452121d611f8a4fe5b Mon Sep 17 00:00:00 2001 From: Shashi Date: Sat, 25 Dec 2021 12:27:44 +0530 Subject: [PATCH 5/7] Updated multibuild and renderers --- lib/renderers.py | 40 +++++++++++++--------------------------- multibuild.py | 38 ++++++++++++++++++++++++++------------ 2 files changed, 39 insertions(+), 39 deletions(-) diff --git a/lib/renderers.py b/lib/renderers.py index c12d036..f3aff34 100644 --- a/lib/renderers.py +++ b/lib/renderers.py @@ -1,38 +1,24 @@ from buildbot.plugins import util import re -def _is_github(props): - if re.search(r'[/@]github.com', props.getProperty('repository')): - return True - return False - -@util.renderer -def is_github(props): - return _is_github(props) - -@util.renderer -def isnt_github(props): - return not _is_github(props) - -def _is_gitea(props): - if re.search(r'[/@]git.bitmessage.org', props.getProperty('repository'), re.I): - return True - return False - -@util.renderer -def is_gitea(props): - return _is_gitea(props) - -@util.renderer -def isnt_gitea(props): - return not _is_gitea(props) @util.renderer def is_build_script_available(props): # Actual check will got here - return props.getProperty('jobname', default='') != '' + return props.getProperty("jobname", default="") != "" + + +@util.renderer +def isnt_build_script_available(props): + return not is_build_script_available(props) + @util.renderer def is_test_script_available(props): # Actual check will got here - return props.getProperty('jobname', default='') != '' \ No newline at end of file + return props.getProperty("jobname", default="") != "" + + +@util.renderer +def isnt_test_script_available(props): + return not is_test_script_available(props) diff --git a/multibuild.py b/multibuild.py index 94bf1b2..d1a8f3c 100644 --- a/multibuild.py +++ b/multibuild.py @@ -16,6 +16,8 @@ import requests import re from buildbot.plugins import steps, util +from .lib.renderers import * + ty = "/change_hook/base" request_headers = { "Content-Type": "application/x-www-form-urlencoded", @@ -33,11 +35,15 @@ RUN apt-get install -yq --no-install-suggests --no-install-recommends \ buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \ python3-pip dumb-init curl openssh-client wget +# buildbot entrypoint +RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh +RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh + RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers USER buildbot -ENTRYPOINT /entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS" +ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS" """ @@ -46,7 +52,7 @@ dockerfile_extra_contents_bionic = """ # Buildbot RUN apt-get install -yq --no-install-suggests --no-install-recommends \ buildbot-slave git subversion python3-dev libffi-dev python3-setuptools \ - python3-pip dumb-init curl openssh-client + python3-pip dumb-init curl openssh-client wget # buildbot entrypoint RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh @@ -93,10 +99,12 @@ def get_dockerfile_contents(path, os_codename): with open(path, "r") as file: contents = file.read() # remove any line containing CMD or ENTRYFILE keywords - contents = re.sub(r"(?m)^CMD.*$", "", contents) - contents = re.sub(r"(?m)^ENTRYFILE.*$", "", contents) + re.sub(r"(?m)^(CMD|ENTRYFILE).*$", "", contents) - return contents + eval("dockerfile_extra_contents_" + os_codename) + return contents + { + "focal": dockerfile_extra_contents_focal, + "bionic": dockerfile_extra_contents_bionic, + }[os_codename] def trigger_child_hooks(buildbotUrl: str, os_codename: str, directory=".buildbot"): @@ -115,11 +123,13 @@ def trigger_child_hooks(buildbotUrl: str, os_codename: str, directory=".buildbot test_script_exists = True # make a post request - request_data["dockerfile"] = get_dockerfile_contents( - join(directory, job, "Dockerfile"), os_codename - ) - request_data["build_script_available"] = str(build_script_exists) - request_data["test_script_available"] = str(test_script_exists) + request_data["properties"] = { + "dockerfile": get_dockerfile_contents( + join(directory, job, "Dockerfile"), os_codename + ), + "build_script_available": is_build_script_available(build_script_exists), + "test_script_available": is_test_script_available(test_script_exists), + } requests.post(request_url, headers=request_headers, data=request_data) @@ -158,7 +168,9 @@ def add_child_build_sh_step(build_factory, job, directory=".buildbot"): build_factory.addStep( steps.ShellCommand( name="build_" + job, - command=["bash", "-c", "cd " + join(directory, job) + " && ./build.sh"], + command=["bash", "-c", join(directory, job, "build.sh")], + doStepIf=is_build_script_available, + hideStepIf=isnt_build_script_available, ) ) @@ -170,7 +182,9 @@ def add_child_test_sh_step(build_factory, job, directory=".buildbot"): build_factory.addStep( steps.ShellCommand( name="test_" + job, - command=["bash", "-c", "cd " + join(directory, job) + " && ./test.sh"], + command=["bash", "-c", join(directory, job, "test.sh")], + doStepIf=is_test_script_available, + hideStepIf=isnt_test_script_available, ) ) -- 2.40.1 From c9d1ffa1177db267d499028f9e77261b7b96290c Mon Sep 17 00:00:00 2001 From: Shashi Date: Sat, 25 Dec 2021 16:51:52 +0530 Subject: [PATCH 6/7] Minor updates --- lib/renderers.py | 4 ++-- multibuild.py | 12 +++++------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/lib/renderers.py b/lib/renderers.py index f3aff34..81ad592 100644 --- a/lib/renderers.py +++ b/lib/renderers.py @@ -5,7 +5,7 @@ import re @util.renderer def is_build_script_available(props): # Actual check will got here - return props.getProperty("jobname", default="") != "" + return props.getProperty("build_available", default=False) @util.renderer @@ -16,7 +16,7 @@ def isnt_build_script_available(props): @util.renderer def is_test_script_available(props): # Actual check will got here - return props.getProperty("jobname", default="") != "" + return props.getProperty("test_available", default=False) @util.renderer diff --git a/multibuild.py b/multibuild.py index d1a8f3c..9355421 100644 --- a/multibuild.py +++ b/multibuild.py @@ -107,7 +107,7 @@ def get_dockerfile_contents(path, os_codename): }[os_codename] -def trigger_child_hooks(buildbotUrl: str, os_codename: str, directory=".buildbot"): +def trigger_child_hooks(buildbotUrl: str, os_codename: str, repository, branch, jobname, directory=".buildbot"): request_url = buildbotUrl + ty # List all jobs in the directory @@ -129,6 +129,9 @@ def trigger_child_hooks(buildbotUrl: str, os_codename: str, directory=".buildbot ), "build_script_available": is_build_script_available(build_script_exists), "test_script_available": is_test_script_available(test_script_exists), + "repository": repository, + "branch": branch, + "jobname": jobname, } requests.post(request_url, headers=request_headers, data=request_data) @@ -200,12 +203,7 @@ if __name__ == "__main__": buildbotUrl = sys.argv[4] os_codename = sys.argv[5] - # add these into the request_data - request_data["jobname"] = jobname - request_data["repository"] = repository - request_data["branch"] = branch - - trigger_child_hooks(buildbotUrl, os_codename) + trigger_child_hooks(buildbotUrl, os_codename, repository, branch, jobname) else: print( "Usage: python3 multibuild.py " -- 2.40.1 From 6039d3a39b2d21647b19a784a270d222e907faf0 Mon Sep 17 00:00:00 2001 From: Shashi Date: Sun, 26 Dec 2021 13:00:48 +0530 Subject: [PATCH 7/7] Removed parent hook --- parent_hook | 83 ----------------------------------------------------- 1 file changed, 83 deletions(-) delete mode 100644 parent_hook diff --git a/parent_hook b/parent_hook deleted file mode 100644 index d0af5dd..0000000 --- a/parent_hook +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env python3 -""" parent webhook """ - -from buildbot.plugins import steps, util, secrets -from .lib.renderers import * - -c = BuildmasterConfig = {} - -c['buildbotNetUsageData'] = None - -####### SECRETS - -c['secretsProviders'] = [secrets.SecretInAFile(dirname="/var/lib/buildbot/secrets/bitmessage")] -gitea_known_hosts = util.Secret('gitea_known_hosts') -gitea_privkey = util.Secret('gitea_privkey') - -travis_bash = util.BuildFactory() -travis_bash.addStep( - steps.GitHub( - repourl=util.Property("repository"), - name="github", - doStepIf=is_github, - hideStepIf=isnt_github, - branch=util.Property("branch"), - mode="incremental", - ) -) -travis_bash.addStep( - steps.Gitea( - repourl=util.Property("repository"), - sshPrivateKey=gitea_privkey, - sshKnownHosts=gitea_known_hosts, - name="gitea", - doStepIf=is_gitea, - hideStepIf=isnt_gitea, - branch=util.Property("branch"), - mode="incremental", - ) -) - -# execute an interpolate of '.buildbot/$(prop:jobname)/build.sh' -travis_bash.addStep( - steps.ShellCommand( - name="Execute build script", - command=[ - "bash", - util.Interpolate(".buildbot/%(prop:jobname)s/build.sh"), - ], - doStepIf=is_build_script_available, - ) -) - -travis_bash.addStep( - steps.ShellCommand( - name="Execute test script", - command=[ - "bash", - util.Interpolate(".buildbot/%(prop:jobname)s/test.sh"), - ], - doStepIf=is_test_script_available, - ) -) - -travis_bash.addStep(steps.SetPropertyFromCommand( - command="grep ^VERSION_CODENAME= /etc/os-release | cut -d= -f2", - property="os_codename" -)) - -# execute multibuild.py -travis_bash.addStep( - steps.ShellCommand( - name="Execute multibuild script", - command=[ - "python", - "multibuild.py", - util.Interpolate("%(prop:jobname)s"), - util.Property("repository"), - util.Property("branch"), - "https://buildbot.bitmessage.org", - util.Interpolate("%(prop:os_codename)s"), - ], - ) -) -- 2.40.1