Split Worker and Master

This commit is contained in:
Muzahid 2022-02-11 17:20:23 +05:30
parent a337147f43
commit d847415a41
Signed by untrusted user: cis-muzahid
GPG Key ID: 1DC85E7D3AB613EA
2 changed files with 147 additions and 134 deletions

125
lib/worker_multibuild.py Normal file
View File

@ -0,0 +1,125 @@
from os import listdir, walk
from os.path import exists, isfile, join, islink, isdir
import requests
import re
request_data = {
"project": "testproject",
"comments": "testcomment",
}
request_headers = {
"Content-Type": "application/x-www-form-urlencoded",
"Accept": "text/plain",
}
ty = "/change_hook/base"
path =".buildbot"
dockerfile_extra_contents = {}
dockerfile_extra_contents['focal'] = """
# Buildbot
RUN apt-get install -yq --no-install-suggests --no-install-recommends \
buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
python3-pip dumb-init curl openssh-client wget
# buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
"""
dockerfile_extra_contents['bionic'] = """
# Buildbot
RUN apt-get install -yq --no-install-suggests --no-install-recommends \
buildbot-slave git subversion python3-dev libffi-dev python3-setuptools \
python3-pip dumb-init curl openssh-client wget
# buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
"""
def list_jobs(directory=".buildbot"):
"""
list jobs found in a directory
"""
results = []
files = ["Dockerfile", "build.sh", "test.sh"]
for item in listdir(directory):
flag = False
for file in files:
filepath = join(directory, item, file)
if islink(filepath) or not isfile(filepath):
flag = True
break
if flag:
continue
if (exists(join(directory, item, 'Dockerfile')) and exists(join(directory, item, 'build.sh'))) or exists(join(directory, item, 'test.sh')):
results.append(item)
return results
def _get_dockerfile_contents(jobname):
"""
Read contents of a Dockerfile and add extra contents for the given os_codename
"""
os_codename='bionic'
res = ""
with open(join(path + jobname), "r") as file:
contents = file.read()
# accept any line containing FROM and RUN keywords
res = ""
inside_allowed_command = False
for line in contents:
if re.match(r"(?m)^(FROM|RUN|ENV).*$", line):
inside_allowed_command = True
if inside_allowed_command:
res += line
l = line.strip()
if not l.endswith("\\"):
inside_allowed_command = False
return res + dockerfile_extra_contents[os_codename]
def trigger_child_hooks(buildbotUrl: str, repository, branch, directory=".buildbot"):
request_url = buildbotUrl + ty
# List all jobs in the directory
jobs = list_jobs(directory)
# Check if build.sh or test.sh exists in each of the jobs
for job in jobs:
build_script_exists = False
test_script_exists = False
if exists(join(directory, job, "build.sh")):
build_script_exists = True
if exists(join(directory, job, "test.sh")):
test_script_exists = True
# make a post request
request_data["properties"] = {
"dockerfile": _get_dockerfile_contents(
join(directory, job, "Dockerfile")
),
"build_script_available": is_build_script_available(build_script_exists),
"test_script_available": is_test_script_available(test_script_exists),
"repository": repository,
"branch": branch,
"jobname": job,
}
requests.post(request_url, headers=request_headers, data=request_data)

View File

@ -10,7 +10,7 @@ Requires docker
# TODO: write hook job, maybe also a dockerfile? # TODO: write hook job, maybe also a dockerfile?
# TODO: what to do about non-docker jobs # TODO: what to do about non-docker jobs
from os import listdir, walk from os import listdir, walk, getenv
from os.path import exists, isfile, join from os.path import exists, isfile, join
import requests import requests
import re import re
@ -18,70 +18,6 @@ from buildbot.plugins import steps, util
from .lib.renderers import * from .lib.renderers import *
ty = "/change_hook/base"
request_headers = {
"Content-Type": "application/x-www-form-urlencoded",
"Accept": "text/plain",
}
request_data = {
"project": "testproject",
"comments": "testcomment",
}
dockerfile_extra_contents_focal = """
# Buildbot
RUN apt-get install -yq --no-install-suggests --no-install-recommends \
buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
python3-pip dumb-init curl openssh-client wget
# buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
"""
dockerfile_extra_contents_bionic = """
# Buildbot
RUN apt-get install -yq --no-install-suggests --no-install-recommends \
buildbot-slave git subversion python3-dev libffi-dev python3-setuptools \
python3-pip dumb-init curl openssh-client wget
# buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
"""
path =".buildbot"
def list_jobs(directory=".buildbot"):
"""
list jobs found in a directory
"""
results = []
for _ in next(walk(directory))[1]:
if exists(join(directory, _, "Dockerfile")) and (
exists(join(directory, _, "build.sh"))
or exists(join(directory, _, "test.sh"))
):
results.append(_)
return results
def find_artifacts(directory="out"): def find_artifacts(directory="out"):
""" """
@ -93,96 +29,49 @@ def find_artifacts(directory="out"):
return join(directory, _) return join(directory, _)
def get_dockerfile_contents(props):
"""
Read contents of a Dockerfile and add extra contents for the given os_codename
"""
with open(join(path + props.getProperty('jobname', default=None)), "r") as file:
contents = file.read()
# remove any line containing CMD or ENTRYFILE keywords
re.sub(r"(?m)^(CMD|ENTRYFILE).*$", "", contents)
return contents + {
"focal": dockerfile_extra_contents_focal,
"bionic": dockerfile_extra_contents_bionic,
}[util.Interpolate("%(prop:os_codename)s")]
def trigger_child_hooks(buildbotUrl: str, os_codename: str, repository, branch, jobname, directory=".buildbot"):
request_url = buildbotUrl + ty
# List all jobs in the directory
jobs = list_jobs(directory)
# Check if build.sh or test.sh exists in each of the jobs
for job in jobs:
build_script_exists = False
test_script_exists = False
if exists(join(directory, job, "build.sh")):
build_script_exists = True
if exists(join(directory, job, "test.sh")):
test_script_exists = True
# make a post request
request_data["properties"] = {
"dockerfile": get_dockerfile_contents(
join(directory, job, "Dockerfile"), os_codename
),
"build_script_available": is_build_script_available(build_script_exists),
"test_script_available": is_test_script_available(test_script_exists),
"repository": repository,
"branch": branch,
"jobname": jobname,
}
requests.post(request_url, headers=request_headers, data=request_data)
def add_parent_step(build_factory): def add_parent_step(build_factory):
""" """
Add a step to the parent build factory that will trigger the child hooks Add a step to the parent build factory that will trigger the child hooks
""" """
build_factory.addStep(
steps.SetPropertyFromCommand(
name="Get OS codename",
command="grep ^VERSION_CODENAME= /etc/os-release | cut -d= -f2",
property="os_codename",
)
)
build_factory.addStep(steps.ShellCommand(
name="create directory",
command=["mkdir", "-p", join(getenv['HOME'], '.local/bin') ]
))
build_factory.addStep(steps.ShellCommand(
name="download worker",
command=["wget", "-O", "https://git.bitmessage.org/Bitmessage/buildbot_multibuild/raw/branch/master/lib/worker_multibuild.py", join(getenv['HOME'], '.local/bin/worker_multibuild.py')]
))
build_factory.addStep( build_factory.addStep(
steps.ShellCommand( steps.ShellCommand(
name="Execute multibuild script", name="Execute worker script",
command=[ command=[
"python", "python3",
"multibuild.py", join(getenv['HOME'], '.local/bin/worker_multibuild.py'),
util.Interpolate("%(prop:jobname)s"),
util.Property('repository'), util.Property('repository'),
util.Property('branch'), util.Property('branch'),
"https://buildbot.bitmessage.org", util.getURLForBuild(util.Property("url"), util.Property("builderid"), util.Property("buildnumber")),
util.Interpolate("%(prop:os_codename)s"),
], ],
) )
) )
def add_child_build_sh_step(build_factory, directory=".buildbot"): def add_child_sh_steps(build_factory, directory=".buildbot"):
""" """
Add a step to the build factory Add a step to the download, build and test factory
""" """
build_factory.addStep( build_factory.addStep(
steps.ShellCommand( steps.ShellCommand(
name=util.Interpolate("build_%(prop:jobname)s"), name=util.Interpolate("build_%(prop:jobname)s"),
command=util.Interpolate("%(kw:directory)s/%(prop:jobname)s/test.sh", directory=directory), command=util.Interpolate("%(kw:directory)s/%(prop:jobname)s/build.sh", directory=directory),
doStepIf=is_build_script_available, doStepIf=is_build_script_available,
hideStepIf=isnt_build_script_available, hideStepIf=isnt_build_script_available,
) )
) )
def add_child_test_sh_step(build_factory, directory=".buildbot"):
"""
Add a step to the build factory
"""
build_factory.addStep( build_factory.addStep(
steps.ShellCommand( steps.ShellCommand(
name= util.Interpolate("test_%(prop:jobname)s"), name= util.Interpolate("test_%(prop:jobname)s"),
@ -194,7 +83,7 @@ def add_child_test_sh_step(build_factory, directory=".buildbot"):
if __name__ == "__main__": if __name__ == "__main__":
# expect jobname, repository, branch, buildbotUrl, os_codename from command line args # expect jobname, repository, branch, buildbotUrl from command line
import sys import sys
if len(sys.argv) == 6: if len(sys.argv) == 6:
@ -202,10 +91,9 @@ if __name__ == "__main__":
repository = sys.argv[2] repository = sys.argv[2]
branch = sys.argv[3] branch = sys.argv[3]
buildbotUrl = sys.argv[4] buildbotUrl = sys.argv[4]
os_codename = sys.argv[5]
trigger_child_hooks(buildbotUrl, os_codename, repository, branch, jobname) trigger_child_hooks(buildbotUrl, repository, branch)
else: else:
print( print(
"Usage: python3 multibuild.py <jobname> <repository> <branch> <buildbotUrl> <os_codename>" "Usage: python3 multibuild.py <buildbotUrl> <repository> <branch> "
) )