Compare commits

..

4 Commits

Author SHA1 Message Date
ff87ae8ec5
Add xenial and jammy
Some checks failed
buildbot/travis_bionic Build done.
buildbot/multibuild_parent Build done.
2022-04-27 20:02:11 +05:30
9d8b6f41b4
Symlink support
Some checks failed
buildbot/multibuild_parent Build done.
buildbot/travis_bionic Build done.
- added some security checks so that symlinks can be supported
- also some code quality changes
2022-04-12 14:30:33 +08:00
f2cb5fd8d3
Fix revision handling (should fix github/gitea reporting)
Some checks failed
buildbot/multibuild_parent Build done.
buildbot/travis_bionic Build done.
2022-04-11 16:53:39 +08:00
53fcd7e7b9
Optimize steps
Some checks failed
buildbot/travis_bionic Build done.
2022-04-08 11:20:50 +08:00
2 changed files with 78 additions and 38 deletions

View File

@ -1,5 +1,5 @@
from os import listdir
from os.path import exists, isfile, join, islink
from os import getcwd, listdir
from os.path import exists, isfile, islink, join, realpath
import requests
import re
from subprocess import Popen, PIPE
@ -16,8 +16,8 @@ dockerfile_extra_contents = {}
dockerfile_extra_contents['focal'] = """
# Buildbot
RUN apt-get install -yq --no-install-suggests --no-install-recommends \
buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
python3-buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
python3-pip dumb-init curl openssh-client wget
# buildbot entrypoint
@ -35,7 +35,7 @@ ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$
dockerfile_extra_contents['bionic'] = """
# Buildbot
RUN apt-get install -yq --no-install-suggests --no-install-recommends \
RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
buildbot-slave git subversion python3-dev libffi-dev python3-setuptools \
python3-pip dumb-init curl openssh-client wget
@ -51,6 +51,44 @@ ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$
"""
dockerfile_extra_contents['jammy'] = """
# Buildbot
RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
python3-buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
python3-pip dumb-init curl openssh-client wget
# buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
"""
dockerfile_extra_contents['xenial'] = """
# Buildbot
RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
buildbot-slave git subversion python3-dev libffi-dev python3-setuptools \
python3-pip dumb-init curl openssh-client wget \
python-setuptools python-psutil libssl-dev python-dev libgmp-dev python-virtualenv
# buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
"""
def get_secret():
with open("multibuild_parent_key.key", 'r') as f:
@ -69,20 +107,29 @@ def list_jobs(directory=".buildbot"):
flag = False
for fname in files:
filepath = join(directory, item, fname)
# must exist
if not exists(filepath):
continue
if islink(filepath) or not isfile(filepath):
# must be a file
if not isfile(filepath):
flag = True
break
# symlink OK as long as it points to files within the repo
if islink(filepath) \
and not realpath(filepath).startswith(getcwd()):
flag = True
break
if flag:
continue
if (exists(join(directory, item, 'Dockerfile')) and exists(join(directory, item, 'build.sh'))) or exists(join(directory, item, 'test.sh')):
if (exists(join(directory, item, 'Dockerfile'))
and exists(join(directory, item, 'build.sh'))) \
or exists(join(directory, item, 'test.sh')):
results.append(item)
return results
def get_revision():
proc = Popen(["git", "rev-parse", "HEAD"], stdout=PIPE)
def get_revision(branch):
proc = Popen(["git", "rev-parse", branch], stdout=PIPE)
retval = proc.stdout.read().strip()
retval = retval.decode('utf-8')
return retval
@ -90,7 +137,8 @@ def get_revision():
def _get_dockerfile_contents(dockerfile):
"""
Read contents of a Dockerfile and add extra contents for the given os_codename
Read contents of a Dockerfile and add buildbot worker bootstrap
for a given os_codename
"""
os_codename = 'bionic'
res = ""
@ -105,6 +153,7 @@ def _get_dockerfile_contents(dockerfile):
if m:
inside_allowed_command = True
if m.group(1) == "FROM":
os_codename = m.group().split()[1].split(":")[1]
has_from = True
if inside_allowed_command:
res += line
@ -114,10 +163,15 @@ def _get_dockerfile_contents(dockerfile):
if not has_from:
return None
try:
return res + dockerfile_extra_contents[os_codename]
except KeyError:
return None
def trigger_child_hooks(buildbotUrl: str, repository, branch, directory=".buildbot"):
def trigger_child_hooks(buildbotUrl: str, repository, branch, revision,
directory=".buildbot"):
request_url = buildbotUrl + ty
# List all jobs in the directory
jobs = list_jobs(directory)
@ -126,7 +180,7 @@ def trigger_child_hooks(buildbotUrl: str, repository, branch, directory=".buildb
"X-Multibuild-Trigger": get_secret(),
"Accept": "text/plain",
}
revision = get_revision()
# revision = get_revision(branch)
# Check if build.sh or test.sh exists in each of the jobs
for job in jobs:
@ -160,21 +214,24 @@ def trigger_child_hooks(buildbotUrl: str, repository, branch, directory=".buildb
"project": "/".join(repository.split("/")[-2:]),
}
retval = requests.post(request_url, headers=request_headers, json=request_data)
print("Triggered job for {} on {}: {}".format(job, request_url, retval.text))
retval = requests.post(request_url, headers=request_headers,
json=request_data)
print("Triggered job for {} on {}: {}".format(job, request_url,
retval.text))
if __name__ == "__main__":
# expect jobname, repository, branch, buildbotUrl from command line
import sys
if len(sys.argv) == 4:
if len(sys.argv) == 5:
buildbotUrl = sys.argv[1]
repository = sys.argv[2]
branch = sys.argv[3]
revision = sys.argv[4]
trigger_child_hooks(buildbotUrl, repository, branch)
trigger_child_hooks(buildbotUrl, repository, branch, revision)
else:
sys.exit(
"Usage: python3 multibuild.py <buildbotUrl> <repository> <branch>"
"Usage: python3 multibuild.py <buildbotUrl> <repository> <branch> <revision>"
)

View File

@ -32,33 +32,16 @@ def add_parent_step(build_factory):
Add a step to the parent build factory that will trigger the child hooks
"""
build_factory.addStep(steps.ShellCommand(
name="Update APT cache",
command=["sudo", "apt", "update"]
))
build_factory.addStep(steps.ShellCommand(
name="Install dependencies",
command=["sudo", "apt", "-y", "install", "python3-requests"]
))
build_factory.addStep(
steps.FileDownload(
workerdest="worker_multibuild.py",
mastersrc="buildbot_multibuild/lib/worker_multibuild.py",
mode=0o444
)
)
build_factory.addStep(
steps.ShellCommand(
name="Execute worker script",
command=[
"python3",
'worker_multibuild.py',
'/usr/local/bin/worker_multibuild.py',
util.Property("buildboturl"),
util.Property('repository'),
util.Property('branch')
util.Property('branch'),
util.Property('revision')
],
)
)