Added parent hook and some renderer #1
24
lib/renderers.py
Normal file
|
@ -0,0 +1,24 @@
|
||||||
|
from buildbot.plugins import util
|
||||||
|
import re
|
||||||
|
|
||||||
|
|
||||||
|
@util.renderer
|
||||||
|
def is_build_script_available(props):
|
||||||
|
# Actual check will got here
|
||||||
|
return props.getProperty("build_available", default=False)
|
||||||
|
|
||||||
|
|
||||||
|
@util.renderer
|
||||||
|
def isnt_build_script_available(props):
|
||||||
|
return not is_build_script_available(props)
|
||||||
|
|
||||||
|
|
||||||
|
@util.renderer
|
||||||
|
def is_test_script_available(props):
|
||||||
|
# Actual check will got here
|
||||||
|
return props.getProperty("test_available", default=False)
|
||||||
|
|
||||||
|
|
||||||
|
@util.renderer
|
||||||
|
def isnt_test_script_available(props):
|
||||||
|
return not is_test_script_available(props)
|
175
multibuild.py
|
@ -12,6 +12,59 @@ Requires docker
|
||||||
|
|
||||||
from os import walk
|
from os import walk
|
||||||
from os.path import exists, isfile, join, listdir
|
from os.path import exists, isfile, join, listdir
|
||||||
|
import requests
|
||||||
|
import re
|
||||||
|
from buildbot.plugins import steps, util
|
||||||
|
|
||||||
|
from .lib.renderers import *
|
||||||
|
|
||||||
|
ty = "/change_hook/base"
|
||||||
|
request_headers = {
|
||||||
|
"Content-Type": "application/x-www-form-urlencoded",
|
||||||
|
"Accept": "text/plain",
|
||||||
|
}
|
||||||
|
request_data = {
|
||||||
|
"project": "testproject",
|
||||||
|
"comments": "testcomment",
|
||||||
|
}
|
||||||
|
|
||||||
|
dockerfile_extra_contents_focal = """
|
||||||
|
|
||||||
|
# Buildbot
|
||||||
|
RUN apt-get install -yq --no-install-suggests --no-install-recommends \
|
||||||
|
buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
|
||||||
|
python3-pip dumb-init curl openssh-client wget
|
||||||
|
|
||||||
|
# buildbot entrypoint
|
||||||
|
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
|
||||||
|
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
|
||||||
|
|
||||||
|
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
|
||||||
|
|
||||||
|
USER buildbot
|
||||||
|
|
||||||
|
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
dockerfile_extra_contents_bionic = """
|
||||||
|
|
||||||
|
# Buildbot
|
||||||
|
RUN apt-get install -yq --no-install-suggests --no-install-recommends \
|
||||||
|
buildbot-slave git subversion python3-dev libffi-dev python3-setuptools \
|
||||||
|
python3-pip dumb-init curl openssh-client wget
|
||||||
|
|
||||||
|
# buildbot entrypoint
|
||||||
|
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
|
||||||
|
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
|
||||||
|
|
||||||
|
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
|
||||||
|
|
||||||
|
USER buildbot
|
||||||
|
|
||||||
|
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
def list_jobs(directory=".buildbot"):
|
def list_jobs(directory=".buildbot"):
|
||||||
|
@ -20,8 +73,8 @@ def list_jobs(directory=".buildbot"):
|
||||||
"""
|
"""
|
||||||
results = []
|
results = []
|
||||||
for _ in next(walk(directory))[1]:
|
for _ in next(walk(directory))[1]:
|
||||||
if exists(join(directory, _, "Dockerfile")) \
|
if exists(join(directory, _, "Dockerfile")) and (
|
||||||
and (exists(join(directory, _, "build.sh"))
|
exists(join(directory, _, "build.sh"))
|
||||||
or exists(join(directory, _, "test.sh"))
|
or exists(join(directory, _, "test.sh"))
|
||||||
):
|
):
|
||||||
results.append(_)
|
results.append(_)
|
||||||
|
@ -37,3 +90,121 @@ def find_artifacts(directory="out"):
|
||||||
if not isfile(join(directory, _)):
|
if not isfile(join(directory, _)):
|
||||||
continue
|
continue
|
||||||
return join(directory, _)
|
return join(directory, _)
|
||||||
|
|
||||||
|
|
||||||
|
def get_dockerfile_contents(path, os_codename):
|
||||||
|
"""
|
||||||
Ss_singh marked this conversation as resolved
|
|||||||
|
Read contents of a Dockerfile and add extra contents for the given os_codename
|
||||||
|
"""
|
||||||
|
with open(path, "r") as file:
|
||||||
|
contents = file.read()
|
||||||
|
# remove any line containing CMD or ENTRYFILE keywords
|
||||||
|
re.sub(r"(?m)^(CMD|ENTRYFILE).*$", "", contents)
|
||||||
|
|
||||||
|
return contents + {
|
||||||
|
"focal": dockerfile_extra_contents_focal,
|
||||||
|
"bionic": dockerfile_extra_contents_bionic,
|
||||||
|
}[os_codename]
|
||||||
|
|
||||||
|
|
||||||
|
def trigger_child_hooks(buildbotUrl: str, os_codename: str, repository, branch, jobname, directory=".buildbot"):
|
||||||
|
request_url = buildbotUrl + ty
|
||||||
|
|
||||||
|
# List all jobs in the directory
|
||||||
|
jobs = list_jobs(directory)
|
||||||
|
|
||||||
|
# Check if build.sh or test.sh exists in each of the jobs
|
||||||
|
for job in jobs:
|
||||||
|
build_script_exists = False
|
||||||
|
test_script_exists = False
|
||||||
|
if exists(join(directory, job, "build.sh")):
|
||||||
|
build_script_exists = True
|
||||||
|
if exists(join(directory, job, "test.sh")):
|
||||||
|
test_script_exists = True
|
||||||
Ss_singh marked this conversation as resolved
PeterSurda
commented
proxied properties are missing proxied properties are missing
PeterSurda
commented
I think they are still missing. Like I think they are still missing. Like `repository` and `branch`.
Ss_singh
commented
Well, they were already being added before the hook function was getting triggered but anyways, now I've them placed altogether. Well, they were already being added before the hook function was getting triggered but anyways, now I've them placed altogether.
|
|||||||
|
|
||||||
|
# make a post request
|
||||||
|
request_data["properties"] = {
|
||||||
|
"dockerfile": get_dockerfile_contents(
|
||||||
|
join(directory, job, "Dockerfile"), os_codename
|
||||||
|
),
|
||||||
|
"build_script_available": is_build_script_available(build_script_exists),
|
||||||
|
"test_script_available": is_test_script_available(test_script_exists),
|
||||||
|
"repository": repository,
|
||||||
PeterSurda
commented
I think I think `repository` and `branch` are a level higher.
|
|||||||
|
"branch": branch,
|
||||||
|
"jobname": jobname,
|
||||||
|
}
|
||||||
|
requests.post(request_url, headers=request_headers, data=request_data)
|
||||||
|
|
||||||
|
|
||||||
|
def add_parent_step(build_factory, jobname, repository, branch):
|
||||||
|
"""
|
||||||
|
Add a step to the parent build factory that will trigger the child hooks
|
||||||
|
"""
|
||||||
|
build_factory.addStep(
|
||||||
|
steps.SetPropertyFromCommand(
|
||||||
|
name="Get OS codename",
|
||||||
|
command="grep ^VERSION_CODENAME= /etc/os-release | cut -d= -f2",
|
||||||
|
property="os_codename",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
build_factory.addStep(
|
||||||
|
steps.ShellCommand(
|
||||||
|
name="Execute multibuild script",
|
||||||
|
command=[
|
||||||
|
"python",
|
||||||
|
"multibuild.py",
|
||||||
|
jobname,
|
||||||
|
repository,
|
||||||
|
branch,
|
||||||
Ss_singh marked this conversation as resolved
PeterSurda
commented
should also have conditionals for should also have conditionals for `build_script_exists`. If it doesn't, this step is skipped and hidden.
|
|||||||
|
"https://buildbot.bitmessage.org",
|
||||||
|
util.Interpolate("%(prop:os_codename)s"),
|
||||||
Ss_singh marked this conversation as resolved
PeterSurda
commented
I wouldn't change the directory, but just run it from it, i.e.
I wouldn't change the directory, but just run it from it, i.e.
```
command=["bash", "-c", join(directory, job, "build.sh")],
|
|||||||
|
],
|
||||||
PeterSurda
commented
For now it's probably ok. Over longer term however the For now it's probably ok. Over longer term however the `os_codename` isn't extracted from the parent (parent may be running something else than the child should), but from the child `Dockerfile`. The `Dockerfile` will begin with something like `FROM ubuntu:bionic AS ...`, so that determines the OS.
|
|||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def add_child_build_sh_step(build_factory, job, directory=".buildbot"):
|
||||||
PeterSurda
commented
I'm not 100% sure this will pass properties. A safer solution would be to create a child of I'm not 100% sure this will pass properties. A safer solution would be to create a child of `ShellCommand` and call `super` with a constructed `command`.
|
|||||||
|
"""
|
||||||
|
Add a step to the build factory
|
||||||
|
"""
|
||||||
|
build_factory.addStep(
|
||||||
|
steps.ShellCommand(
|
||||||
|
name="build_" + job,
|
||||||
|
command=["bash", "-c", join(directory, job, "build.sh")],
|
||||||
|
doStepIf=is_build_script_available,
|
||||||
|
hideStepIf=isnt_build_script_available,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def add_child_test_sh_step(build_factory, job, directory=".buildbot"):
|
||||||
|
"""
|
||||||
|
Add a step to the build factory
|
||||||
|
"""
|
||||||
|
build_factory.addStep(
|
||||||
|
steps.ShellCommand(
|
||||||
|
name="test_" + job,
|
||||||
|
command=["bash", "-c", join(directory, job, "test.sh")],
|
||||||
|
doStepIf=is_test_script_available,
|
||||||
|
hideStepIf=isnt_test_script_available,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
# expect jobname, repository, branch, buildbotUrl, os_codename from command line args
|
||||||
|
import sys
|
||||||
|
|
||||||
|
if len(sys.argv) == 6:
|
||||||
|
jobname = sys.argv[1]
|
||||||
|
repository = sys.argv[2]
|
||||||
|
branch = sys.argv[3]
|
||||||
|
buildbotUrl = sys.argv[4]
|
||||||
|
os_codename = sys.argv[5]
|
||||||
|
|
||||||
|
trigger_child_hooks(buildbotUrl, os_codename, repository, branch, jobname)
|
||||||
|
else:
|
||||||
|
print(
|
||||||
|
"Usage: python3 multibuild.py <jobname> <repository> <branch> <buildbotUrl> <os_codename>"
|
||||||
|
)
|
||||||
|
|
How about just