Compare commits

..

No commits in common. "d847415a418c1568f10564c126f1820df10f0e19" and "6039d3a39b2d21647b19a784a270d222e907faf0" have entirely different histories.

2 changed files with 140 additions and 154 deletions

View File

@ -1,125 +0,0 @@
from os import listdir, walk
from os.path import exists, isfile, join, islink, isdir
import requests
import re
request_data = {
"project": "testproject",
"comments": "testcomment",
}
request_headers = {
"Content-Type": "application/x-www-form-urlencoded",
"Accept": "text/plain",
}
ty = "/change_hook/base"
path =".buildbot"
dockerfile_extra_contents = {}
dockerfile_extra_contents['focal'] = """
# Buildbot
RUN apt-get install -yq --no-install-suggests --no-install-recommends \
buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
python3-pip dumb-init curl openssh-client wget
# buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
"""
dockerfile_extra_contents['bionic'] = """
# Buildbot
RUN apt-get install -yq --no-install-suggests --no-install-recommends \
buildbot-slave git subversion python3-dev libffi-dev python3-setuptools \
python3-pip dumb-init curl openssh-client wget
# buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
"""
def list_jobs(directory=".buildbot"):
"""
list jobs found in a directory
"""
results = []
files = ["Dockerfile", "build.sh", "test.sh"]
for item in listdir(directory):
flag = False
for file in files:
filepath = join(directory, item, file)
if islink(filepath) or not isfile(filepath):
flag = True
break
if flag:
continue
if (exists(join(directory, item, 'Dockerfile')) and exists(join(directory, item, 'build.sh'))) or exists(join(directory, item, 'test.sh')):
results.append(item)
return results
def _get_dockerfile_contents(jobname):
"""
Read contents of a Dockerfile and add extra contents for the given os_codename
"""
os_codename='bionic'
res = ""
with open(join(path + jobname), "r") as file:
contents = file.read()
# accept any line containing FROM and RUN keywords
res = ""
inside_allowed_command = False
for line in contents:
if re.match(r"(?m)^(FROM|RUN|ENV).*$", line):
inside_allowed_command = True
if inside_allowed_command:
res += line
l = line.strip()
if not l.endswith("\\"):
inside_allowed_command = False
return res + dockerfile_extra_contents[os_codename]
def trigger_child_hooks(buildbotUrl: str, repository, branch, directory=".buildbot"):
request_url = buildbotUrl + ty
# List all jobs in the directory
jobs = list_jobs(directory)
# Check if build.sh or test.sh exists in each of the jobs
for job in jobs:
build_script_exists = False
test_script_exists = False
if exists(join(directory, job, "build.sh")):
build_script_exists = True
if exists(join(directory, job, "test.sh")):
test_script_exists = True
# make a post request
request_data["properties"] = {
"dockerfile": _get_dockerfile_contents(
join(directory, job, "Dockerfile")
),
"build_script_available": is_build_script_available(build_script_exists),
"test_script_available": is_test_script_available(test_script_exists),
"repository": repository,
"branch": branch,
"jobname": job,
}
requests.post(request_url, headers=request_headers, data=request_data)

View File

@ -10,14 +10,77 @@ Requires docker
# TODO: write hook job, maybe also a dockerfile?
# TODO: what to do about non-docker jobs
from os import listdir, walk, getenv
from os.path import exists, isfile, join
from os import walk
from os.path import exists, isfile, join, listdir
import requests
import re
from buildbot.plugins import steps, util
from .lib.renderers import *
ty = "/change_hook/base"
request_headers = {
"Content-Type": "application/x-www-form-urlencoded",
"Accept": "text/plain",
}
request_data = {
"project": "testproject",
"comments": "testcomment",
}
dockerfile_extra_contents_focal = """
# Buildbot
RUN apt-get install -yq --no-install-suggests --no-install-recommends \
buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
python3-pip dumb-init curl openssh-client wget
# buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
"""
dockerfile_extra_contents_bionic = """
# Buildbot
RUN apt-get install -yq --no-install-suggests --no-install-recommends \
buildbot-slave git subversion python3-dev libffi-dev python3-setuptools \
python3-pip dumb-init curl openssh-client wget
# buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
"""
def list_jobs(directory=".buildbot"):
"""
list jobs found in a directory
"""
results = []
for _ in next(walk(directory))[1]:
if exists(join(directory, _, "Dockerfile")) and (
exists(join(directory, _, "build.sh"))
or exists(join(directory, _, "test.sh"))
):
results.append(_)
return results
def find_artifacts(directory="out"):
"""
@ -29,53 +92,100 @@ def find_artifacts(directory="out"):
return join(directory, _)
def add_parent_step(build_factory):
def get_dockerfile_contents(path, os_codename):
"""
Read contents of a Dockerfile and add extra contents for the given os_codename
"""
with open(path, "r") as file:
contents = file.read()
# remove any line containing CMD or ENTRYFILE keywords
re.sub(r"(?m)^(CMD|ENTRYFILE).*$", "", contents)
return contents + {
"focal": dockerfile_extra_contents_focal,
"bionic": dockerfile_extra_contents_bionic,
}[os_codename]
def trigger_child_hooks(buildbotUrl: str, os_codename: str, repository, branch, jobname, directory=".buildbot"):
request_url = buildbotUrl + ty
# List all jobs in the directory
jobs = list_jobs(directory)
# Check if build.sh or test.sh exists in each of the jobs
for job in jobs:
build_script_exists = False
test_script_exists = False
if exists(join(directory, job, "build.sh")):
build_script_exists = True
if exists(join(directory, job, "test.sh")):
test_script_exists = True
# make a post request
request_data["properties"] = {
"dockerfile": get_dockerfile_contents(
join(directory, job, "Dockerfile"), os_codename
),
"build_script_available": is_build_script_available(build_script_exists),
"test_script_available": is_test_script_available(test_script_exists),
"repository": repository,
"branch": branch,
"jobname": jobname,
}
requests.post(request_url, headers=request_headers, data=request_data)
def add_parent_step(build_factory, jobname, repository, branch):
"""
Add a step to the parent build factory that will trigger the child hooks
"""
build_factory.addStep(
steps.SetPropertyFromCommand(
name="Get OS codename",
command="grep ^VERSION_CODENAME= /etc/os-release | cut -d= -f2",
property="os_codename",
)
)
build_factory.addStep(steps.ShellCommand(
name="create directory",
command=["mkdir", "-p", join(getenv['HOME'], '.local/bin') ]
))
build_factory.addStep(steps.ShellCommand(
name="download worker",
command=["wget", "-O", "https://git.bitmessage.org/Bitmessage/buildbot_multibuild/raw/branch/master/lib/worker_multibuild.py", join(getenv['HOME'], '.local/bin/worker_multibuild.py')]
))
build_factory.addStep(
steps.ShellCommand(
name="Execute worker script",
name="Execute multibuild script",
command=[
"python3",
join(getenv['HOME'], '.local/bin/worker_multibuild.py'),
util.Property('repository'),
util.Property('branch'),
util.getURLForBuild(util.Property("url"), util.Property("builderid"), util.Property("buildnumber")),
"python",
"multibuild.py",
jobname,
repository,
branch,
"https://buildbot.bitmessage.org",
util.Interpolate("%(prop:os_codename)s"),
],
)
)
def add_child_sh_steps(build_factory, directory=".buildbot"):
def add_child_build_sh_step(build_factory, job, directory=".buildbot"):
"""
Add a step to the download, build and test factory
Add a step to the build factory
"""
build_factory.addStep(
steps.ShellCommand(
name=util.Interpolate("build_%(prop:jobname)s"),
command=util.Interpolate("%(kw:directory)s/%(prop:jobname)s/build.sh", directory=directory),
name="build_" + job,
command=["bash", "-c", join(directory, job, "build.sh")],
doStepIf=is_build_script_available,
hideStepIf=isnt_build_script_available,
)
)
def add_child_test_sh_step(build_factory, job, directory=".buildbot"):
"""
Add a step to the build factory
"""
build_factory.addStep(
steps.ShellCommand(
name= util.Interpolate("test_%(prop:jobname)s"),
command=util.Interpolate("%(kw:directory)s/%(prop:jobname)s/test.sh", directory=directory),
name="test_" + job,
command=["bash", "-c", join(directory, job, "test.sh")],
doStepIf=is_test_script_available,
hideStepIf=isnt_test_script_available,
)
@ -83,7 +193,7 @@ def add_child_sh_steps(build_factory, directory=".buildbot"):
if __name__ == "__main__":
# expect jobname, repository, branch, buildbotUrl from command line
# expect jobname, repository, branch, buildbotUrl, os_codename from command line args
import sys
if len(sys.argv) == 6:
@ -91,9 +201,10 @@ if __name__ == "__main__":
repository = sys.argv[2]
branch = sys.argv[3]
buildbotUrl = sys.argv[4]
os_codename = sys.argv[5]
trigger_child_hooks(buildbotUrl, repository, branch)
trigger_child_hooks(buildbotUrl, os_codename, repository, branch, jobname)
else:
print(
"Usage: python3 multibuild.py <buildbotUrl> <repository> <branch> "
"Usage: python3 multibuild.py <jobname> <repository> <branch> <buildbotUrl> <os_codename>"
)