Split Worker and Master

This commit is contained in:
Muzahid 2022-02-11 17:20:23 +05:30
parent a337147f43
commit d847415a41
Signed by untrusted user: cis-muzahid
GPG Key ID: 1DC85E7D3AB613EA
2 changed files with 147 additions and 134 deletions

125
lib/worker_multibuild.py Normal file
View File

@ -0,0 +1,125 @@
from os import listdir, walk
from os.path import exists, isfile, join, islink, isdir
import requests
import re
request_data = {
"project": "testproject",
"comments": "testcomment",
}
request_headers = {
"Content-Type": "application/x-www-form-urlencoded",
"Accept": "text/plain",
}
ty = "/change_hook/base"
path =".buildbot"
dockerfile_extra_contents = {}
dockerfile_extra_contents['focal'] = """
# Buildbot
RUN apt-get install -yq --no-install-suggests --no-install-recommends \
buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
python3-pip dumb-init curl openssh-client wget
# buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
"""
dockerfile_extra_contents['bionic'] = """
# Buildbot
RUN apt-get install -yq --no-install-suggests --no-install-recommends \
buildbot-slave git subversion python3-dev libffi-dev python3-setuptools \
python3-pip dumb-init curl openssh-client wget
# buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
"""
def list_jobs(directory=".buildbot"):
"""
list jobs found in a directory
"""
results = []
files = ["Dockerfile", "build.sh", "test.sh"]
for item in listdir(directory):
flag = False
for file in files:
filepath = join(directory, item, file)
if islink(filepath) or not isfile(filepath):
flag = True
break
if flag:
continue
if (exists(join(directory, item, 'Dockerfile')) and exists(join(directory, item, 'build.sh'))) or exists(join(directory, item, 'test.sh')):
results.append(item)
return results
def _get_dockerfile_contents(jobname):
"""
Read contents of a Dockerfile and add extra contents for the given os_codename
"""
os_codename='bionic'
res = ""
with open(join(path + jobname), "r") as file:
contents = file.read()
# accept any line containing FROM and RUN keywords
res = ""
inside_allowed_command = False
for line in contents:
if re.match(r"(?m)^(FROM|RUN|ENV).*$", line):
inside_allowed_command = True
if inside_allowed_command:
res += line
l = line.strip()
if not l.endswith("\\"):
inside_allowed_command = False
return res + dockerfile_extra_contents[os_codename]
def trigger_child_hooks(buildbotUrl: str, repository, branch, directory=".buildbot"):
request_url = buildbotUrl + ty
# List all jobs in the directory
jobs = list_jobs(directory)
# Check if build.sh or test.sh exists in each of the jobs
for job in jobs:
build_script_exists = False
test_script_exists = False
if exists(join(directory, job, "build.sh")):
build_script_exists = True
if exists(join(directory, job, "test.sh")):
test_script_exists = True
# make a post request
request_data["properties"] = {
"dockerfile": _get_dockerfile_contents(
join(directory, job, "Dockerfile")
),
"build_script_available": is_build_script_available(build_script_exists),
"test_script_available": is_test_script_available(test_script_exists),
"repository": repository,
"branch": branch,
"jobname": job,
}
requests.post(request_url, headers=request_headers, data=request_data)

View File

@ -10,7 +10,7 @@ Requires docker
# TODO: write hook job, maybe also a dockerfile?
# TODO: what to do about non-docker jobs
from os import listdir, walk
from os import listdir, walk, getenv
from os.path import exists, isfile, join
import requests
import re
@ -18,70 +18,6 @@ from buildbot.plugins import steps, util
from .lib.renderers import *
ty = "/change_hook/base"
request_headers = {
"Content-Type": "application/x-www-form-urlencoded",
"Accept": "text/plain",
}
request_data = {
"project": "testproject",
"comments": "testcomment",
}
dockerfile_extra_contents_focal = """
# Buildbot
RUN apt-get install -yq --no-install-suggests --no-install-recommends \
buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
python3-pip dumb-init curl openssh-client wget
# buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
"""
dockerfile_extra_contents_bionic = """
# Buildbot
RUN apt-get install -yq --no-install-suggests --no-install-recommends \
buildbot-slave git subversion python3-dev libffi-dev python3-setuptools \
python3-pip dumb-init curl openssh-client wget
# buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
"""
path =".buildbot"
def list_jobs(directory=".buildbot"):
"""
list jobs found in a directory
"""
results = []
for _ in next(walk(directory))[1]:
if exists(join(directory, _, "Dockerfile")) and (
exists(join(directory, _, "build.sh"))
or exists(join(directory, _, "test.sh"))
):
results.append(_)
return results
def find_artifacts(directory="out"):
"""
@ -93,96 +29,49 @@ def find_artifacts(directory="out"):
return join(directory, _)
def get_dockerfile_contents(props):
"""
Read contents of a Dockerfile and add extra contents for the given os_codename
"""
with open(join(path + props.getProperty('jobname', default=None)), "r") as file:
contents = file.read()
# remove any line containing CMD or ENTRYFILE keywords
re.sub(r"(?m)^(CMD|ENTRYFILE).*$", "", contents)
return contents + {
"focal": dockerfile_extra_contents_focal,
"bionic": dockerfile_extra_contents_bionic,
}[util.Interpolate("%(prop:os_codename)s")]
def trigger_child_hooks(buildbotUrl: str, os_codename: str, repository, branch, jobname, directory=".buildbot"):
request_url = buildbotUrl + ty
# List all jobs in the directory
jobs = list_jobs(directory)
# Check if build.sh or test.sh exists in each of the jobs
for job in jobs:
build_script_exists = False
test_script_exists = False
if exists(join(directory, job, "build.sh")):
build_script_exists = True
if exists(join(directory, job, "test.sh")):
test_script_exists = True
# make a post request
request_data["properties"] = {
"dockerfile": get_dockerfile_contents(
join(directory, job, "Dockerfile"), os_codename
),
"build_script_available": is_build_script_available(build_script_exists),
"test_script_available": is_test_script_available(test_script_exists),
"repository": repository,
"branch": branch,
"jobname": jobname,
}
requests.post(request_url, headers=request_headers, data=request_data)
def add_parent_step(build_factory):
"""
Add a step to the parent build factory that will trigger the child hooks
"""
build_factory.addStep(
steps.SetPropertyFromCommand(
name="Get OS codename",
command="grep ^VERSION_CODENAME= /etc/os-release | cut -d= -f2",
property="os_codename",
)
)
build_factory.addStep(steps.ShellCommand(
name="create directory",
command=["mkdir", "-p", join(getenv['HOME'], '.local/bin') ]
))
build_factory.addStep(steps.ShellCommand(
name="download worker",
command=["wget", "-O", "https://git.bitmessage.org/Bitmessage/buildbot_multibuild/raw/branch/master/lib/worker_multibuild.py", join(getenv['HOME'], '.local/bin/worker_multibuild.py')]
))
build_factory.addStep(
steps.ShellCommand(
name="Execute multibuild script",
name="Execute worker script",
command=[
"python",
"multibuild.py",
util.Interpolate("%(prop:jobname)s"),
"python3",
join(getenv['HOME'], '.local/bin/worker_multibuild.py'),
util.Property('repository'),
util.Property('branch'),
"https://buildbot.bitmessage.org",
util.Interpolate("%(prop:os_codename)s"),
util.getURLForBuild(util.Property("url"), util.Property("builderid"), util.Property("buildnumber")),
],
)
)
def add_child_build_sh_step(build_factory, directory=".buildbot"):
def add_child_sh_steps(build_factory, directory=".buildbot"):
"""
Add a step to the build factory
Add a step to the download, build and test factory
"""
build_factory.addStep(
steps.ShellCommand(
name=util.Interpolate("build_%(prop:jobname)s"),
command=util.Interpolate("%(kw:directory)s/%(prop:jobname)s/test.sh", directory=directory),
command=util.Interpolate("%(kw:directory)s/%(prop:jobname)s/build.sh", directory=directory),
doStepIf=is_build_script_available,
hideStepIf=isnt_build_script_available,
)
)
def add_child_test_sh_step(build_factory, directory=".buildbot"):
"""
Add a step to the build factory
"""
build_factory.addStep(
steps.ShellCommand(
name= util.Interpolate("test_%(prop:jobname)s"),
@ -194,7 +83,7 @@ def add_child_test_sh_step(build_factory, directory=".buildbot"):
if __name__ == "__main__":
# expect jobname, repository, branch, buildbotUrl, os_codename from command line args
# expect jobname, repository, branch, buildbotUrl from command line
import sys
if len(sys.argv) == 6:
@ -202,10 +91,9 @@ if __name__ == "__main__":
repository = sys.argv[2]
branch = sys.argv[3]
buildbotUrl = sys.argv[4]
os_codename = sys.argv[5]
trigger_child_hooks(buildbotUrl, os_codename, repository, branch, jobname)
trigger_child_hooks(buildbotUrl, repository, branch)
else:
print(
"Usage: python3 multibuild.py <jobname> <repository> <branch> <buildbotUrl> <os_codename>"
"Usage: python3 multibuild.py <buildbotUrl> <repository> <branch> "
)