Bitmessage/buildbot_multibuild
6
0
Fork 5
Code Issues 4 Pull Requests Projects Releases Wiki Activity

Compare commits

base: Bitmessage:master
Branches Tags
Bitmessage:master
PeterSurda:master
..
compare: PeterSurda:master
Branches Tags
PeterSurda:master
Bitmessage:master

No commits in common. "master" and "master" have entirely different histories.
master ... master

3 changed files with 46 additions and 244 deletions
Show Stats Expand all files Collapse all files

55
lib/renderers.py
View File

@ -22,58 +22,3 @@ def is_test_script_available(props):
@util.renderer @util.renderer
def isnt_test_script_available(props): def isnt_test_script_available(props):
return not _is_test_script_available(props) return not _is_test_script_available(props)
def _files_to_upload(props):
try:
return ','.join(props.getProperty("files_to_upload", default="").rstrip().split("\n"))
except AttributeError:
return ""
@util.renderer
def files_to_upload(props):
return _files_to_upload(props)
@util.renderer
def has_files_to_upload(props):
return bool(_files_to_upload(props))
@util.renderer
def no_files_to_upload(props):
return not _files_to_upload(props)
def _should_build_sign(props):
if props.getProperty('branch') == 'v0.6' \
and props.getProperty('jobname') == 'android' \
and props.getProperty('repository') in (
'git@github.com:Bitmessage/PyBitmessage.git',
'https://github.com/Bitmessage/PyBitmessage'
):
return True
return False
@util.renderer
def should_build_sign(props):
return _is_build_script_available(props) and _should_build_sign(props)
@util.renderer
def shouldnt_build_sign(props):
return _is_build_script_available(props) and not _should_build_sign(props)
@util.renderer
def build_env(props):
default_envs = {
"BUILDBOT_REPOSITORY": props.getProperty("repository"),
"BUILDBOT_BRANCH": props.getProperty("branch"),
"BUILDBOT_JOBNAME": props.getProperty("jobname")
}
new_envs = {}
if props.getProperty("jobname", default="") == "android":
new_envs = {
"P4A_RELEASE_KEYSTORE": "/var/lib/buildbot/keystore",
"P4A_RELEASE_KEYSTORE_PASSWD": util.Secret("bitmessage-keystore"),
"P4A_RELEASE_KEYALIAS_PASSWD": util.Secret("bitmessage-keystore"),
"P4A_RELEASE_KEYALIAS": "bitmessagetest"
}
if _should_build_sign(props):
return {**default_envs, **new_envs}
return default_envs

131
lib/worker_multibuild.py
View File

@ -1,9 +1,8 @@
from os import getcwd, listdir from os import listdir
from os.path import exists, isfile, islink, join, realpath from os.path import exists, isfile, join, islink
import requests import requests
import re import re
from subprocess import Popen, PIPE from subprocess import Popen, PIPE
from time import sleep
request_data = { request_data = {
@ -17,104 +16,38 @@ dockerfile_extra_contents = {}
dockerfile_extra_contents['focal'] = """ dockerfile_extra_contents['focal'] = """
# Buildbot # Buildbot
RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \ RUN apt-get install -yq --no-install-suggests --no-install-recommends \
python3-buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \ buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
python3-pip dumb-init curl openssh-client wget python3-pip dumb-init curl openssh-client wget
# buildbot entrypoint # buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt
RUN update-ca-certificates
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS" ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
""" """
dockerfile_extra_contents['bionic'] = """ dockerfile_extra_contents['bionic'] = """
# Buildbot # Buildbot
RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \ RUN apt-get install -yq --no-install-suggests --no-install-recommends \
git subversion python3-dev libffi-dev python3-setuptools \ buildbot-slave git subversion python3-dev libffi-dev python3-setuptools \
python3-pip dumb-init curl openssh-client wget python3-wheel \
pkg-config rustc cargo
RUN pip3 install setuptools_rust
RUN pip3 install 'buildbot-worker==3.1.1' \
'cryptography==2.1.4' \
'twisted==17.9.0' \
'Automat==20.2.0' \
'pyopenssl==17.5.0'
RUN groupadd buildbot
RUN useradd -d /var/lib/buildbot -m -g buildbot buildbot
# buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt
RUN update-ca-certificates
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS"
"""
dockerfile_extra_contents['jammy'] = """
# Buildbot
RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
python3-buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
python3-pip dumb-init curl openssh-client wget python3-pip dumb-init curl openssh-client wget
# buildbot entrypoint # buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt
RUN update-ca-certificates
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS" ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
"""
dockerfile_extra_contents['xenial'] = """
# Buildbot
RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
git subversion python3-dev libffi-dev python3-setuptools \
python3-pip curl openssh-client wget \
python-setuptools python-psutil libssl-dev python-dev libgmp-dev \
python-virtualenv python3-wheel pkg-config rustc cargo \
python3-openssl
RUN pip3 install setuptools_rust
RUN pip3 install 'buildbot-worker<3.2.0' 'cryptography<35.0.0'
RUN groupadd buildbot
RUN useradd -d /var/lib/buildbot -m -g buildbot buildbot
# dumb-init
RUN wget https://github.com/Yelp/dumb-init/releases/download/v1.2.5/dumb-init_1.2.5_amd64.deb
RUN dpkg -i dumb-init_*.deb && rm -f dumb-init_*.deb
# buildbot entrypoint
RUN [ -f /usr/local/bin/buildbot_entrypoint.sh ] || wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/xenial/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt
RUN update-ca-certificates
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS"
""" """
@ -131,36 +64,25 @@ def list_jobs(directory=".buildbot"):
""" """
results = [] results = []
files = ["Dockerfile", "build.sh", "test.sh"] files = ["Dockerfile", "build.sh", "test.sh"]
if not exists(directory):
return results
for item in listdir(directory): for item in listdir(directory):
print("checking directory {}".format(item)) print("checking directory {}".format(item))
flag = False flag = False
for fname in files: for fname in files:
filepath = join(directory, item, fname) filepath = join(directory, item, fname)
# must exist
if not exists(filepath): if not exists(filepath):
continue continue
# must be a file if islink(filepath) or not isfile(filepath):
if not isfile(filepath):
flag = True
break
# symlink OK as long as it points to files within the repo
if islink(filepath) \
and not realpath(filepath).startswith(getcwd()):
flag = True flag = True
break break
if flag: if flag:
continue continue
if (exists(join(directory, item, 'Dockerfile')) if (exists(join(directory, item, 'Dockerfile')) and exists(join(directory, item, 'build.sh'))) or exists(join(directory, item, 'test.sh')):
and exists(join(directory, item, 'build.sh'))) \
or exists(join(directory, item, 'test.sh')):
results.append(item) results.append(item)
return results return results
def get_revision(branch): def get_revision():
proc = Popen(["git", "rev-parse", branch], stdout=PIPE) proc = Popen(["git", "rev-parse", "HEAD"], stdout=PIPE)
retval = proc.stdout.read().strip() retval = proc.stdout.read().strip()
retval = retval.decode('utf-8') retval = retval.decode('utf-8')
return retval return retval
@ -168,8 +90,7 @@ def get_revision(branch):
def _get_dockerfile_contents(dockerfile): def _get_dockerfile_contents(dockerfile):
""" """
Read contents of a Dockerfile and add buildbot worker bootstrap Read contents of a Dockerfile and add extra contents for the given os_codename
for a given os_codename
""" """
os_codename = 'bionic' os_codename = 'bionic'
res = "" res = ""
@ -184,7 +105,6 @@ def _get_dockerfile_contents(dockerfile):
if m: if m:
inside_allowed_command = True inside_allowed_command = True
if m.group(1) == "FROM": if m.group(1) == "FROM":
os_codename = m.group().split()[1].split(":")[1]
has_from = True has_from = True
if inside_allowed_command: if inside_allowed_command:
res += line res += line
@ -194,15 +114,10 @@ def _get_dockerfile_contents(dockerfile):
if not has_from: if not has_from:
return None return None
try:
return res + dockerfile_extra_contents[os_codename] return res + dockerfile_extra_contents[os_codename]
except KeyError:
return None
def trigger_child_hooks(buildbotUrl: str, repository, branch, revision, def trigger_child_hooks(buildbotUrl: str, repository, branch, directory=".buildbot"):
directory=".buildbot"):
request_url = buildbotUrl + ty request_url = buildbotUrl + ty
# List all jobs in the directory # List all jobs in the directory
jobs = list_jobs(directory) jobs = list_jobs(directory)
@ -211,7 +126,7 @@ def trigger_child_hooks(buildbotUrl: str, repository, branch, revision,
"X-Multibuild-Trigger": get_secret(), "X-Multibuild-Trigger": get_secret(),
"Accept": "text/plain", "Accept": "text/plain",
} }
# revision = get_revision(branch) revision = get_revision()
# Check if build.sh or test.sh exists in each of the jobs # Check if build.sh or test.sh exists in each of the jobs
for job in jobs: for job in jobs:
@ -242,28 +157,24 @@ def trigger_child_hooks(buildbotUrl: str, repository, branch, revision,
request_data["changes"] = { request_data["changes"] = {
"author": "buildbot_multibuild", "author": "buildbot_multibuild",
"repository": repository, "repository": repository,
"project": job, "project": "/".join(repository.split("/")[-2:]),
} }
retval = requests.post(request_url, headers=request_headers, retval = requests.post(request_url, headers=request_headers, json=request_data)
json=request_data) print("Triggered job for {} on {}: {}".format(job, request_url, retval.text))
print("Triggered job for {} on {}: {}".format(job, request_url,
retval.text))
sleep(1)
if __name__ == "__main__": if __name__ == "__main__":
# expect jobname, repository, branch, buildbotUrl from command line # expect jobname, repository, branch, buildbotUrl from command line
import sys import sys
if len(sys.argv) == 5: if len(sys.argv) == 4:
buildbotUrl = sys.argv[1] buildbotUrl = sys.argv[1]
repository = sys.argv[2] repository = sys.argv[2]
branch = sys.argv[3] branch = sys.argv[3]
revision = sys.argv[4]
trigger_child_hooks(buildbotUrl, repository, branch, revision) trigger_child_hooks(buildbotUrl, repository, branch)
else: else:
sys.exit( sys.exit(
"Usage: python3 multibuild.py <buildbotUrl> <repository> <branch> <revision>" "Usage: python3 multibuild.py <buildbotUrl> <repository> <branch>"
) )

100
multibuild.py
View File

@ -13,7 +13,6 @@ Requires docker
from os import listdir from os import listdir
from os.path import isfile, join from os.path import isfile, join
from buildbot.plugins import steps, util from buildbot.plugins import steps, util
from buildbot.process.results import SUCCESS
from .lib.renderers import * from .lib.renderers import *
@ -33,16 +32,33 @@ def add_parent_step(build_factory):
Add a step to the parent build factory that will trigger the child hooks Add a step to the parent build factory that will trigger the child hooks
""" """
build_factory.addStep(steps.ShellCommand(
name="Update APT cache",
command=["sudo", "apt", "update"]
))
build_factory.addStep(steps.ShellCommand(
name="Install dependencies",
command=["sudo", "apt", "-y", "install", "python3-requests"]
))
build_factory.addStep(
steps.FileDownload(
workerdest="worker_multibuild.py",
mastersrc="buildbot_multibuild/lib/worker_multibuild.py",
mode=0o444
)
)
build_factory.addStep( build_factory.addStep(
steps.ShellCommand( steps.ShellCommand(
name="Execute worker script", name="Execute worker script",
command=[ command=[
"python3", "python3",
'/usr/local/bin/worker_multibuild.py', 'worker_multibuild.py',
util.Property("buildboturl"), util.Property("buildboturl"),
util.Property('repository'), util.Property('repository'),
util.Property('branch'), util.Property('branch')
util.Property('revision')
], ],
) )
) )
@ -53,90 +69,20 @@ def add_child_sh_steps(build_factory, directory=".buildbot"):
Add a step to the download, build and test factory Add a step to the download, build and test factory
""" """
build_factory.addStep(
steps.FileDownload(
name="Upload keystore",
workerdest="/var/lib/buildbot/keystore",
mastersrc=util.Interpolate("keystore/%(prop:jobname)s.keystore"),
doStepIf=should_build_sign,
hideStepIf=True,
mode=0o600
)
)
build_factory.addStep( build_factory.addStep(
steps.ShellCommand( steps.ShellCommand(
name=util.Interpolate("build_%(prop:jobname)s"), name=util.Interpolate("build_%(prop:jobname)s"),
command=util.Interpolate("%(kw:directory)s/" command=util.Interpolate("%(kw:directory)s/%(prop:jobname)s/build.sh", directory=directory),
"%(prop:jobname)s/build.sh", doStepIf=is_build_script_available,
directory=directory),
env=build_env,
doStepIf=shouldnt_build_sign,
hideStepIf=isnt_build_script_available, hideStepIf=isnt_build_script_available,
timeout=7200,
)
)
build_factory.addStep(
steps.ShellCommand(
name=util.Interpolate("build_and_sign_%(prop:jobname)s"),
command=util.Interpolate("%(kw:directory)s/"
"%(prop:jobname)s/build.sh",
directory=directory),
env=build_env,
doStepIf=should_build_sign,
hideStepIf=isnt_build_script_available,
timeout=7200,
)
)
build_factory.addStep(
steps.ShellCommand(
name="Delete keystore",
command="rm -f /var/lib/buildbot/keystore",
doStepIf=should_build_sign,
hideStepIf=True,
) )
) )
build_factory.addStep( build_factory.addStep(
steps.ShellCommand( steps.ShellCommand(
name= util.Interpolate("test_%(prop:jobname)s"), name= util.Interpolate("test_%(prop:jobname)s"),
command=util.Interpolate("%(kw:directory)s/" command=util.Interpolate("%(kw:directory)s/%(prop:jobname)s/test.sh", directory=directory),
"%(prop:jobname)s/test.sh",
directory=directory),
haltOnFailure=False,
flunkOnFailure=True,
doStepIf=is_test_script_available, doStepIf=is_test_script_available,
hideStepIf=isnt_test_script_available, hideStepIf=isnt_test_script_available,
) )
) )
build_factory.addStep(
steps.SetPropertyFromCommand(
name="Find files to upload",
command="find . -maxdepth 1 -mindepth 1 "
"-type f -printf '%P\n'",
workdir="out",
hideStepIf=True,
decodeRC={0: SUCCESS, 1: SUCCESS},
property="files_to_upload"
)
)
build_factory.addStep(
steps.ShellCommand(
name="Upload files",
workdir="out",
doStepIf=has_files_to_upload,
hideStepIf=no_files_to_upload,
command=util.Interpolate(
"curl -T {%s} -u buildbot:%s "
"https://artifacts.bitmessage.at/%s/%s/",
files_to_upload,
util.Secret('artifact_upload'),
util.Property('jobname'),
util.Property('buildnumber'),
)
)
)