Added parent hook and some renderer #1

Merged
PeterSurda merged 7 commits from Ss_singh/buildbot_multibuild:hooks into master 2021-12-26 09:05:49 +01:00
2 changed files with 199 additions and 4 deletions

24
lib/renderers.py Normal file
View File

@ -0,0 +1,24 @@
from buildbot.plugins import util
import re
@util.renderer
def is_build_script_available(props):
# Actual check will got here
return props.getProperty("build_available", default=False)
@util.renderer
def isnt_build_script_available(props):
return not is_build_script_available(props)
@util.renderer
def is_test_script_available(props):
# Actual check will got here
return props.getProperty("test_available", default=False)
@util.renderer
def isnt_test_script_available(props):
return not is_test_script_available(props)

View File

@ -12,6 +12,59 @@ Requires docker
from os import walk from os import walk
from os.path import exists, isfile, join, listdir from os.path import exists, isfile, join, listdir
import requests
import re
from buildbot.plugins import steps, util
from .lib.renderers import *
ty = "/change_hook/base"
request_headers = {
"Content-Type": "application/x-www-form-urlencoded",
"Accept": "text/plain",
}
request_data = {
"project": "testproject",
"comments": "testcomment",
}
dockerfile_extra_contents_focal = """
# Buildbot
RUN apt-get install -yq --no-install-suggests --no-install-recommends \
buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
python3-pip dumb-init curl openssh-client wget
# buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
"""
dockerfile_extra_contents_bionic = """
# Buildbot
RUN apt-get install -yq --no-install-suggests --no-install-recommends \
buildbot-slave git subversion python3-dev libffi-dev python3-setuptools \
python3-pip dumb-init curl openssh-client wget
# buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
"""
def list_jobs(directory=".buildbot"): def list_jobs(directory=".buildbot"):
@ -20,8 +73,8 @@ def list_jobs(directory=".buildbot"):
""" """
results = [] results = []
for _ in next(walk(directory))[1]: for _ in next(walk(directory))[1]:
if exists(join(directory, _, "Dockerfile")) \ if exists(join(directory, _, "Dockerfile")) and (
and (exists(join(directory, _, "build.sh")) exists(join(directory, _, "build.sh"))
or exists(join(directory, _, "test.sh")) or exists(join(directory, _, "test.sh"))
): ):
results.append(_) results.append(_)
@ -37,3 +90,121 @@ def find_artifacts(directory="out"):
if not isfile(join(directory, _)): if not isfile(join(directory, _)):
continue continue
return join(directory, _) return join(directory, _)
def get_dockerfile_contents(path, os_codename):
"""
Ss_singh marked this conversation as resolved
Review

How about just

contents = re.sub(r"(?m)^(CMD|ENTRYFILE).*$", "", contents)
How about just ``` contents = re.sub(r"(?m)^(CMD|ENTRYFILE).*$", "", contents) ```
Read contents of a Dockerfile and add extra contents for the given os_codename
"""
with open(path, "r") as file:
contents = file.read()
# remove any line containing CMD or ENTRYFILE keywords
re.sub(r"(?m)^(CMD|ENTRYFILE).*$", "", contents)
return contents + {
"focal": dockerfile_extra_contents_focal,
"bionic": dockerfile_extra_contents_bionic,
}[os_codename]
def trigger_child_hooks(buildbotUrl: str, os_codename: str, repository, branch, jobname, directory=".buildbot"):
request_url = buildbotUrl + ty
# List all jobs in the directory
jobs = list_jobs(directory)
# Check if build.sh or test.sh exists in each of the jobs
for job in jobs:
build_script_exists = False
test_script_exists = False
if exists(join(directory, job, "build.sh")):
build_script_exists = True
if exists(join(directory, job, "test.sh")):
test_script_exists = True
Ss_singh marked this conversation as resolved
Review

proxied properties are missing

proxied properties are missing
Review

I think they are still missing. Like repository and branch.

I think they are still missing. Like `repository` and `branch`.
Review

Well, they were already being added before the hook function was getting triggered but anyways, now I've them placed altogether.

Well, they were already being added before the hook function was getting triggered but anyways, now I've them placed altogether.
# make a post request
request_data["properties"] = {
"dockerfile": get_dockerfile_contents(
join(directory, job, "Dockerfile"), os_codename
),
"build_script_available": is_build_script_available(build_script_exists),
"test_script_available": is_test_script_available(test_script_exists),
"repository": repository,
Review

I think repository and branch are a level higher.

I think `repository` and `branch` are a level higher.
"branch": branch,
"jobname": jobname,
}
requests.post(request_url, headers=request_headers, data=request_data)
def add_parent_step(build_factory, jobname, repository, branch):
"""
Add a step to the parent build factory that will trigger the child hooks
"""
build_factory.addStep(
steps.SetPropertyFromCommand(
name="Get OS codename",
command="grep ^VERSION_CODENAME= /etc/os-release | cut -d= -f2",
property="os_codename",
)
)
build_factory.addStep(
steps.ShellCommand(
name="Execute multibuild script",
command=[
"python",
"multibuild.py",
jobname,
repository,
branch,
Ss_singh marked this conversation as resolved
Review

should also have conditionals for build_script_exists. If it doesn't, this step is skipped and hidden.

should also have conditionals for `build_script_exists`. If it doesn't, this step is skipped and hidden.
"https://buildbot.bitmessage.org",
util.Interpolate("%(prop:os_codename)s"),
Ss_singh marked this conversation as resolved
Review

I wouldn't change the directory, but just run it from it, i.e.

command=["bash", "-c", join(directory, job, "build.sh")],
I wouldn't change the directory, but just run it from it, i.e. ``` command=["bash", "-c", join(directory, job, "build.sh")],
],
Review

For now it's probably ok. Over longer term however the os_codename isn't extracted from the parent (parent may be running something else than the child should), but from the child Dockerfile. The Dockerfile will begin with something like FROM ubuntu:bionic AS ..., so that determines the OS.

For now it's probably ok. Over longer term however the `os_codename` isn't extracted from the parent (parent may be running something else than the child should), but from the child `Dockerfile`. The `Dockerfile` will begin with something like `FROM ubuntu:bionic AS ...`, so that determines the OS.
)
)
def add_child_build_sh_step(build_factory, job, directory=".buildbot"):
Review

I'm not 100% sure this will pass properties. A safer solution would be to create a child of ShellCommand and call super with a constructed command.

I'm not 100% sure this will pass properties. A safer solution would be to create a child of `ShellCommand` and call `super` with a constructed `command`.
"""
Add a step to the build factory
"""
build_factory.addStep(
steps.ShellCommand(
name="build_" + job,
command=["bash", "-c", join(directory, job, "build.sh")],
doStepIf=is_build_script_available,
hideStepIf=isnt_build_script_available,
)
)
def add_child_test_sh_step(build_factory, job, directory=".buildbot"):
"""
Add a step to the build factory
"""
build_factory.addStep(
steps.ShellCommand(
name="test_" + job,
command=["bash", "-c", join(directory, job, "test.sh")],
doStepIf=is_test_script_available,
hideStepIf=isnt_test_script_available,
)
)
if __name__ == "__main__":
# expect jobname, repository, branch, buildbotUrl, os_codename from command line args
import sys
if len(sys.argv) == 6:
jobname = sys.argv[1]
repository = sys.argv[2]
branch = sys.argv[3]
buildbotUrl = sys.argv[4]
os_codename = sys.argv[5]
trigger_child_hooks(buildbotUrl, os_codename, repository, branch, jobname)
else:
print(
"Usage: python3 multibuild.py <jobname> <repository> <branch> <buildbotUrl> <os_codename>"
)