Split Worker and Master
This commit is contained in:
parent
a337147f43
commit
d847415a41
GPG Key ID:
1DC85E7D3AB613EA
125
lib/worker_multibuild.py
Normal file
125
lib/worker_multibuild.py
Normal file
|
|
@ -0,0 +1,125 @@
|
||||||
|
from os import listdir, walk
|
||||||
|
from os.path import exists, isfile, join, islink, isdir
|
||||||
|
import requests
|
||||||
|
import re
|
||||||
|
|
||||||
|
|
||||||
|
request_data = {
|
||||||
|
"project": "testproject",
|
||||||
|
"comments": "testcomment",
|
||||||
|
}
|
||||||
|
request_headers = {
|
||||||
|
"Content-Type": "application/x-www-form-urlencoded",
|
||||||
|
"Accept": "text/plain",
|
||||||
|
}
|
||||||
|
|
||||||
|
ty = "/change_hook/base"
|
||||||
|
path =".buildbot"
|
||||||
|
dockerfile_extra_contents = {}
|
||||||
|
dockerfile_extra_contents['focal'] = """
|
||||||
|
|
||||||
|
# Buildbot
|
||||||
|
RUN apt-get install -yq --no-install-suggests --no-install-recommends \
|
||||||
|
buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
|
||||||
|
python3-pip dumb-init curl openssh-client wget
|
||||||
|
|
||||||
|
# buildbot entrypoint
|
||||||
|
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
|
||||||
|
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
|
||||||
|
|
||||||
|
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
|
||||||
|
|
||||||
|
USER buildbot
|
||||||
|
|
||||||
|
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
dockerfile_extra_contents['bionic'] = """
|
||||||
|
|
||||||
|
# Buildbot
|
||||||
|
RUN apt-get install -yq --no-install-suggests --no-install-recommends \
|
||||||
|
buildbot-slave git subversion python3-dev libffi-dev python3-setuptools \
|
||||||
|
python3-pip dumb-init curl openssh-client wget
|
||||||
|
|
||||||
|
# buildbot entrypoint
|
||||||
|
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
|
||||||
|
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
|
||||||
|
|
||||||
|
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
|
||||||
|
|
||||||
|
USER buildbot
|
||||||
|
|
||||||
|
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
def list_jobs(directory=".buildbot"):
|
||||||
|
"""
|
||||||
|
list jobs found in a directory
|
||||||
|
"""
|
||||||
|
results = []
|
||||||
|
files = ["Dockerfile", "build.sh", "test.sh"]
|
||||||
|
for item in listdir(directory):
|
||||||
|
flag = False
|
||||||
|
for file in files:
|
||||||
|
filepath = join(directory, item, file)
|
||||||
|
if islink(filepath) or not isfile(filepath):
|
||||||
|
flag = True
|
||||||
|
break
|
||||||
|
if flag:
|
||||||
|
continue
|
||||||
|
if (exists(join(directory, item, 'Dockerfile')) and exists(join(directory, item, 'build.sh'))) or exists(join(directory, item, 'test.sh')):
|
||||||
|
results.append(item)
|
||||||
|
return results
|
||||||
|
|
||||||
|
|
||||||
|
def _get_dockerfile_contents(jobname):
|
||||||
|
"""
|
||||||
|
Read contents of a Dockerfile and add extra contents for the given os_codename
|
||||||
|
"""
|
||||||
|
os_codename='bionic'
|
||||||
|
res = ""
|
||||||
|
with open(join(path + jobname), "r") as file:
|
||||||
|
contents = file.read()
|
||||||
|
# accept any line containing FROM and RUN keywords
|
||||||
|
res = ""
|
||||||
|
inside_allowed_command = False
|
||||||
|
for line in contents:
|
||||||
|
if re.match(r"(?m)^(FROM|RUN|ENV).*$", line):
|
||||||
|
inside_allowed_command = True
|
||||||
|
if inside_allowed_command:
|
||||||
|
res += line
|
||||||
|
l = line.strip()
|
||||||
|
if not l.endswith("\\"):
|
||||||
|
inside_allowed_command = False
|
||||||
|
|
||||||
|
return res + dockerfile_extra_contents[os_codename]
|
||||||
|
|
||||||
|
|
||||||
|
def trigger_child_hooks(buildbotUrl: str, repository, branch, directory=".buildbot"):
|
||||||
|
request_url = buildbotUrl + ty
|
||||||
|
# List all jobs in the directory
|
||||||
|
jobs = list_jobs(directory)
|
||||||
|
|
||||||
|
# Check if build.sh or test.sh exists in each of the jobs
|
||||||
|
for job in jobs:
|
||||||
|
build_script_exists = False
|
||||||
|
test_script_exists = False
|
||||||
|
if exists(join(directory, job, "build.sh")):
|
||||||
|
build_script_exists = True
|
||||||
|
if exists(join(directory, job, "test.sh")):
|
||||||
|
test_script_exists = True
|
||||||
|
|
||||||
|
# make a post request
|
||||||
|
request_data["properties"] = {
|
||||||
|
"dockerfile": _get_dockerfile_contents(
|
||||||
|
join(directory, job, "Dockerfile")
|
||||||
|
),
|
||||||
|
"build_script_available": is_build_script_available(build_script_exists),
|
||||||
|
"test_script_available": is_test_script_available(test_script_exists),
|
||||||
|
"repository": repository,
|
||||||
|
"branch": branch,
|
||||||
|
"jobname": job,
|
||||||
|
}
|
||||||
|
requests.post(request_url, headers=request_headers, data=request_data)
|
||||||
156
multibuild.py
156
multibuild.py
|
|
@ -10,7 +10,7 @@ Requires docker
|
||||||
# TODO: write hook job, maybe also a dockerfile?
|
# TODO: write hook job, maybe also a dockerfile?
|
||||||
# TODO: what to do about non-docker jobs
|
# TODO: what to do about non-docker jobs
|
||||||
|
|
||||||
from os import listdir, walk
|
from os import listdir, walk, getenv
|
||||||
from os.path import exists, isfile, join
|
from os.path import exists, isfile, join
|
||||||
import requests
|
import requests
|
||||||
import re
|
import re
|
||||||
|
|
@ -18,70 +18,6 @@ from buildbot.plugins import steps, util
|
||||||
|
|
||||||
from .lib.renderers import *
|
from .lib.renderers import *
|
||||||
|
|
||||||
ty = "/change_hook/base"
|
|
||||||
request_headers = {
|
|
||||||
"Content-Type": "application/x-www-form-urlencoded",
|
|
||||||
"Accept": "text/plain",
|
|
||||||
}
|
|
||||||
request_data = {
|
|
||||||
"project": "testproject",
|
|
||||||
"comments": "testcomment",
|
|
||||||
}
|
|
||||||
|
|
||||||
dockerfile_extra_contents_focal = """
|
|
||||||
|
|
||||||
# Buildbot
|
|
||||||
RUN apt-get install -yq --no-install-suggests --no-install-recommends \
|
|
||||||
buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
|
|
||||||
python3-pip dumb-init curl openssh-client wget
|
|
||||||
|
|
||||||
# buildbot entrypoint
|
|
||||||
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
|
|
||||||
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
|
|
||||||
|
|
||||||
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
|
|
||||||
|
|
||||||
USER buildbot
|
|
||||||
|
|
||||||
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
|
|
||||||
|
|
||||||
"""
|
|
||||||
|
|
||||||
dockerfile_extra_contents_bionic = """
|
|
||||||
|
|
||||||
# Buildbot
|
|
||||||
RUN apt-get install -yq --no-install-suggests --no-install-recommends \
|
|
||||||
buildbot-slave git subversion python3-dev libffi-dev python3-setuptools \
|
|
||||||
python3-pip dumb-init curl openssh-client wget
|
|
||||||
|
|
||||||
# buildbot entrypoint
|
|
||||||
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
|
|
||||||
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
|
|
||||||
|
|
||||||
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
|
|
||||||
|
|
||||||
USER buildbot
|
|
||||||
|
|
||||||
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
|
|
||||||
|
|
||||||
"""
|
|
||||||
|
|
||||||
path =".buildbot"
|
|
||||||
|
|
||||||
def list_jobs(directory=".buildbot"):
|
|
||||||
"""
|
|
||||||
list jobs found in a directory
|
|
||||||
"""
|
|
||||||
results = []
|
|
||||||
for _ in next(walk(directory))[1]:
|
|
||||||
if exists(join(directory, _, "Dockerfile")) and (
|
|
||||||
exists(join(directory, _, "build.sh"))
|
|
||||||
or exists(join(directory, _, "test.sh"))
|
|
||||||
):
|
|
||||||
results.append(_)
|
|
||||||
|
|
||||||
return results
|
|
||||||
|
|
||||||
|
|
||||||
def find_artifacts(directory="out"):
|
def find_artifacts(directory="out"):
|
||||||
"""
|
"""
|
||||||
|
|
@ -93,96 +29,49 @@ def find_artifacts(directory="out"):
|
||||||
return join(directory, _)
|
return join(directory, _)
|
||||||
|
|
||||||
|
|
||||||
def get_dockerfile_contents(props):
|
|
||||||
"""
|
|
||||||
Read contents of a Dockerfile and add extra contents for the given os_codename
|
|
||||||
"""
|
|
||||||
with open(join(path + props.getProperty('jobname', default=None)), "r") as file:
|
|
||||||
contents = file.read()
|
|
||||||
# remove any line containing CMD or ENTRYFILE keywords
|
|
||||||
re.sub(r"(?m)^(CMD|ENTRYFILE).*$", "", contents)
|
|
||||||
|
|
||||||
return contents + {
|
|
||||||
"focal": dockerfile_extra_contents_focal,
|
|
||||||
"bionic": dockerfile_extra_contents_bionic,
|
|
||||||
}[util.Interpolate("%(prop:os_codename)s")]
|
|
||||||
|
|
||||||
|
|
||||||
def trigger_child_hooks(buildbotUrl: str, os_codename: str, repository, branch, jobname, directory=".buildbot"):
|
|
||||||
request_url = buildbotUrl + ty
|
|
||||||
|
|
||||||
# List all jobs in the directory
|
|
||||||
jobs = list_jobs(directory)
|
|
||||||
|
|
||||||
# Check if build.sh or test.sh exists in each of the jobs
|
|
||||||
for job in jobs:
|
|
||||||
build_script_exists = False
|
|
||||||
test_script_exists = False
|
|
||||||
if exists(join(directory, job, "build.sh")):
|
|
||||||
build_script_exists = True
|
|
||||||
if exists(join(directory, job, "test.sh")):
|
|
||||||
test_script_exists = True
|
|
||||||
|
|
||||||
# make a post request
|
|
||||||
request_data["properties"] = {
|
|
||||||
"dockerfile": get_dockerfile_contents(
|
|
||||||
join(directory, job, "Dockerfile"), os_codename
|
|
||||||
),
|
|
||||||
"build_script_available": is_build_script_available(build_script_exists),
|
|
||||||
"test_script_available": is_test_script_available(test_script_exists),
|
|
||||||
"repository": repository,
|
|
||||||
"branch": branch,
|
|
||||||
"jobname": jobname,
|
|
||||||
}
|
|
||||||
requests.post(request_url, headers=request_headers, data=request_data)
|
|
||||||
|
|
||||||
|
|
||||||
def add_parent_step(build_factory):
|
def add_parent_step(build_factory):
|
||||||
"""
|
"""
|
||||||
Add a step to the parent build factory that will trigger the child hooks
|
Add a step to the parent build factory that will trigger the child hooks
|
||||||
"""
|
"""
|
||||||
build_factory.addStep(
|
|
||||||
steps.SetPropertyFromCommand(
|
|
||||||
name="Get OS codename",
|
|
||||||
command="grep ^VERSION_CODENAME= /etc/os-release | cut -d= -f2",
|
|
||||||
property="os_codename",
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
|
build_factory.addStep(steps.ShellCommand(
|
||||||
|
name="create directory",
|
||||||
|
command=["mkdir", "-p", join(getenv['HOME'], '.local/bin') ]
|
||||||
|
))
|
||||||
|
|
||||||
|
build_factory.addStep(steps.ShellCommand(
|
||||||
|
name="download worker",
|
||||||
|
command=["wget", "-O", "https://git.bitmessage.org/Bitmessage/buildbot_multibuild/raw/branch/master/lib/worker_multibuild.py", join(getenv['HOME'], '.local/bin/worker_multibuild.py')]
|
||||||
|
))
|
||||||
|
|
||||||
build_factory.addStep(
|
build_factory.addStep(
|
||||||
steps.ShellCommand(
|
steps.ShellCommand(
|
||||||
name="Execute multibuild script",
|
name="Execute worker script",
|
||||||
command=[
|
command=[
|
||||||
"python",
|
"python3",
|
||||||
"multibuild.py",
|
join(getenv['HOME'], '.local/bin/worker_multibuild.py'),
|
||||||
util.Interpolate("%(prop:jobname)s"),
|
|
||||||
util.Property('repository'),
|
util.Property('repository'),
|
||||||
util.Property('branch'),
|
util.Property('branch'),
|
||||||
"https://buildbot.bitmessage.org",
|
util.getURLForBuild(util.Property("url"), util.Property("builderid"), util.Property("buildnumber")),
|
||||||
util.Interpolate("%(prop:os_codename)s"),
|
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def add_child_build_sh_step(build_factory, directory=".buildbot"):
|
def add_child_sh_steps(build_factory, directory=".buildbot"):
|
||||||
"""
|
"""
|
||||||
Add a step to the build factory
|
Add a step to the download, build and test factory
|
||||||
"""
|
"""
|
||||||
|
|
||||||
build_factory.addStep(
|
build_factory.addStep(
|
||||||
steps.ShellCommand(
|
steps.ShellCommand(
|
||||||
name=util.Interpolate("build_%(prop:jobname)s"),
|
name=util.Interpolate("build_%(prop:jobname)s"),
|
||||||
command=util.Interpolate("%(kw:directory)s/%(prop:jobname)s/test.sh", directory=directory),
|
command=util.Interpolate("%(kw:directory)s/%(prop:jobname)s/build.sh", directory=directory),
|
||||||
doStepIf=is_build_script_available,
|
doStepIf=is_build_script_available,
|
||||||
hideStepIf=isnt_build_script_available,
|
hideStepIf=isnt_build_script_available,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def add_child_test_sh_step(build_factory, directory=".buildbot"):
|
|
||||||
"""
|
|
||||||
Add a step to the build factory
|
|
||||||
"""
|
|
||||||
build_factory.addStep(
|
build_factory.addStep(
|
||||||
steps.ShellCommand(
|
steps.ShellCommand(
|
||||||
name= util.Interpolate("test_%(prop:jobname)s"),
|
name= util.Interpolate("test_%(prop:jobname)s"),
|
||||||
|
|
@ -194,7 +83,7 @@ def add_child_test_sh_step(build_factory, directory=".buildbot"):
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
# expect jobname, repository, branch, buildbotUrl, os_codename from command line args
|
# expect jobname, repository, branch, buildbotUrl from command line
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
if len(sys.argv) == 6:
|
if len(sys.argv) == 6:
|
||||||
|
|
@ -202,10 +91,9 @@ if __name__ == "__main__":
|
||||||
repository = sys.argv[2]
|
repository = sys.argv[2]
|
||||||
branch = sys.argv[3]
|
branch = sys.argv[3]
|
||||||
buildbotUrl = sys.argv[4]
|
buildbotUrl = sys.argv[4]
|
||||||
os_codename = sys.argv[5]
|
|
||||||
|
|
||||||
trigger_child_hooks(buildbotUrl, os_codename, repository, branch, jobname)
|
trigger_child_hooks(buildbotUrl, repository, branch)
|
||||||
else:
|
else:
|
||||||
print(
|
print(
|
||||||
"Usage: python3 multibuild.py <jobname> <repository> <branch> <buildbotUrl> <os_codename>"
|
"Usage: python3 multibuild.py <buildbotUrl> <repository> <branch> "
|
||||||
)
|
)
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue
Block a user