buildbot_multibuild/multibuild.py

213 lines
6.2 KiB
Python

#!/usr/bin/python3
"""
Allow buildbot to run jobs dynamically defined a a project repo
Requires docker
"""
# TODO: change "ghcontext" in master.cfg to interpolate the job name
# TODO: write upload script
# TODO: write hook (perhaps the default hook is ok), authentication for hook
# TODO: write hook job, maybe also a dockerfile?
# TODO: what to do about non-docker jobs
from os import walk
from os.path import exists, isfile, join, listdir
import requests
import re
from buildbot.plugins import steps, util
from .lib.renderers import *
ty = "/change_hook/base"
request_headers = {
"Content-Type": "application/x-www-form-urlencoded",
"Accept": "text/plain",
}
request_data = {
"project": "testproject",
"comments": "testcomment",
}
dockerfile_extra_contents_focal = """
# Buildbot
RUN apt-get install -yq --no-install-suggests --no-install-recommends \
buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
python3-pip dumb-init curl openssh-client wget
# buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
"""
dockerfile_extra_contents_bionic = """
# Buildbot
RUN apt-get install -yq --no-install-suggests --no-install-recommends \
buildbot-slave git subversion python3-dev libffi-dev python3-setuptools \
python3-pip dumb-init curl openssh-client wget
# buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
"""
def list_jobs(directory=".buildbot"):
"""
list jobs found in a directory
"""
results = []
for _ in next(walk(directory))[1]:
if exists(join(directory, _, "Dockerfile")) and (
exists(join(directory, _, "build.sh"))
or exists(join(directory, _, "test.sh"))
):
results.append(_)
return results
def find_artifacts(directory="out"):
"""
find artifacts (any file) in a directory
"""
for _ in listdir(directory):
if not isfile(join(directory, _)):
continue
return join(directory, _)
def get_dockerfile_contents(path, os_codename):
"""
Read contents of a Dockerfile and add extra contents for the given os_codename
"""
with open(path, "r") as file:
contents = file.read()
# remove any line containing CMD or ENTRYFILE keywords
re.sub(r"(?m)^(CMD|ENTRYFILE).*$", "", contents)
return contents + {
"focal": dockerfile_extra_contents_focal,
"bionic": dockerfile_extra_contents_bionic,
}[os_codename]
def trigger_child_hooks(buildbotUrl: str, os_codename: str, directory=".buildbot"):
request_url = buildbotUrl + ty
# List all jobs in the directory
jobs = list_jobs(directory)
# Check if build.sh or test.sh exists in each of the jobs
for job in jobs:
build_script_exists = False
test_script_exists = False
if exists(join(directory, job, "build.sh")):
build_script_exists = True
if exists(join(directory, job, "test.sh")):
test_script_exists = True
# make a post request
request_data["properties"] = {
"dockerfile": get_dockerfile_contents(
join(directory, job, "Dockerfile"), os_codename
),
"build_script_available": is_build_script_available(build_script_exists),
"test_script_available": is_test_script_available(test_script_exists),
}
requests.post(request_url, headers=request_headers, data=request_data)
def add_parent_step(build_factory, jobname, repository, branch):
"""
Add a step to the parent build factory that will trigger the child hooks
"""
build_factory.addStep(
steps.SetPropertyFromCommand(
name="Get OS codename",
command="grep ^VERSION_CODENAME= /etc/os-release | cut -d= -f2",
property="os_codename",
)
)
build_factory.addStep(
steps.ShellCommand(
name="Execute multibuild script",
command=[
"python",
"multibuild.py",
jobname,
repository,
branch,
"https://buildbot.bitmessage.org",
util.Interpolate("%(prop:os_codename)s"),
],
)
)
def add_child_build_sh_step(build_factory, job, directory=".buildbot"):
"""
Add a step to the build factory
"""
build_factory.addStep(
steps.ShellCommand(
name="build_" + job,
command=["bash", "-c", join(directory, job, "build.sh")],
doStepIf=is_build_script_available,
hideStepIf=isnt_build_script_available,
)
)
def add_child_test_sh_step(build_factory, job, directory=".buildbot"):
"""
Add a step to the build factory
"""
build_factory.addStep(
steps.ShellCommand(
name="test_" + job,
command=["bash", "-c", join(directory, job, "test.sh")],
doStepIf=is_test_script_available,
hideStepIf=isnt_test_script_available,
)
)
if __name__ == "__main__":
# expect jobname, repository, branch, buildbotUrl, os_codename from command line args
import sys
if len(sys.argv) == 6:
jobname = sys.argv[1]
repository = sys.argv[2]
branch = sys.argv[3]
buildbotUrl = sys.argv[4]
os_codename = sys.argv[5]
# add these into the request_data
request_data["jobname"] = jobname
request_data["repository"] = repository
request_data["branch"] = branch
trigger_child_hooks(buildbotUrl, os_codename)
else:
print(
"Usage: python3 multibuild.py <jobname> <repository> <branch> <buildbotUrl> <os_codename>"
)