Bitmessage/buildbot_multibuild
6
0
Fork 5
Code Issues 4 Pull Requests Projects Releases Wiki Activity

Compare commits

base: Bitmessage:master
Branches Tags
Bitmessage:master
PeterSurda:master
..
compare: PeterSurda:master
Branches Tags
PeterSurda:master
Bitmessage:master

No commits in common. "master" and "master" have entirely different histories.
master ... master

3 changed files with 46 additions and 244 deletions
Show Stats Expand all files Collapse all files

55
lib/renderers.py
View File

@ -22,58 +22,3 @@ def is_test_script_available(props):
@util.renderer
def isnt_test_script_available(props):
return not _is_test_script_available(props)
def _files_to_upload(props):
try:
return ','.join(props.getProperty("files_to_upload", default="").rstrip().split("\n"))
except AttributeError:
return ""
@util.renderer
def files_to_upload(props):
return _files_to_upload(props)
@util.renderer
def has_files_to_upload(props):
return bool(_files_to_upload(props))
@util.renderer
def no_files_to_upload(props):
return not _files_to_upload(props)
def _should_build_sign(props):
if props.getProperty('branch') == 'v0.6' \
and props.getProperty('jobname') == 'android' \
and props.getProperty('repository') in (
'git@github.com:Bitmessage/PyBitmessage.git',
'https://github.com/Bitmessage/PyBitmessage'
):
return True
return False
@util.renderer
def should_build_sign(props):
return _is_build_script_available(props) and _should_build_sign(props)
@util.renderer
def shouldnt_build_sign(props):
return _is_build_script_available(props) and not _should_build_sign(props)
@util.renderer
def build_env(props):
default_envs = {
"BUILDBOT_REPOSITORY": props.getProperty("repository"),
"BUILDBOT_BRANCH": props.getProperty("branch"),
"BUILDBOT_JOBNAME": props.getProperty("jobname")
}
new_envs = {}
if props.getProperty("jobname", default="") == "android":
new_envs = {
"P4A_RELEASE_KEYSTORE": "/var/lib/buildbot/keystore",
"P4A_RELEASE_KEYSTORE_PASSWD": util.Secret("bitmessage-keystore"),
"P4A_RELEASE_KEYALIAS_PASSWD": util.Secret("bitmessage-keystore"),
"P4A_RELEASE_KEYALIAS": "bitmessagetest"
}
if _should_build_sign(props):
return {**default_envs, **new_envs}
return default_envs

131
lib/worker_multibuild.py
View File

@ -1,9 +1,8 @@
from os import getcwd, listdir
from os.path import exists, isfile, islink, join, realpath
from os import listdir
from os.path import exists, isfile, join, islink
import requests
import re
from subprocess import Popen, PIPE
from time import sleep
request_data = {
@ -17,104 +16,38 @@ dockerfile_extra_contents = {}
dockerfile_extra_contents['focal'] = """
# Buildbot
RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
python3-buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
RUN apt-get install -yq --no-install-suggests --no-install-recommends \
buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
python3-pip dumb-init curl openssh-client wget
# buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt
RUN update-ca-certificates
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS"
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
"""
dockerfile_extra_contents['bionic'] = """
# Buildbot
RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
git subversion python3-dev libffi-dev python3-setuptools \
python3-pip dumb-init curl openssh-client wget python3-wheel \
pkg-config rustc cargo
RUN pip3 install setuptools_rust
RUN pip3 install 'buildbot-worker==3.1.1' \
'cryptography==2.1.4' \
'twisted==17.9.0' \
'Automat==20.2.0' \
'pyopenssl==17.5.0'
RUN groupadd buildbot
RUN useradd -d /var/lib/buildbot -m -g buildbot buildbot
# buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt
RUN update-ca-certificates
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS"
"""
dockerfile_extra_contents['jammy'] = """
# Buildbot
RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
python3-buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
RUN apt-get install -yq --no-install-suggests --no-install-recommends \
buildbot-slave git subversion python3-dev libffi-dev python3-setuptools \
python3-pip dumb-init curl openssh-client wget
# buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt
RUN update-ca-certificates
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS"
"""
dockerfile_extra_contents['xenial'] = """
# Buildbot
RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
git subversion python3-dev libffi-dev python3-setuptools \
python3-pip curl openssh-client wget \
python-setuptools python-psutil libssl-dev python-dev libgmp-dev \
python-virtualenv python3-wheel pkg-config rustc cargo \
python3-openssl
RUN pip3 install setuptools_rust
RUN pip3 install 'buildbot-worker<3.2.0' 'cryptography<35.0.0'
RUN groupadd buildbot
RUN useradd -d /var/lib/buildbot -m -g buildbot buildbot
# dumb-init
RUN wget https://github.com/Yelp/dumb-init/releases/download/v1.2.5/dumb-init_1.2.5_amd64.deb
RUN dpkg -i dumb-init_*.deb && rm -f dumb-init_*.deb
# buildbot entrypoint
RUN [ -f /usr/local/bin/buildbot_entrypoint.sh ] || wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/xenial/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt
RUN update-ca-certificates
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS"
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
"""
@ -131,36 +64,25 @@ def list_jobs(directory=".buildbot"):
"""
results = []
files = ["Dockerfile", "build.sh", "test.sh"]
if not exists(directory):
return results
for item in listdir(directory):
print("checking directory {}".format(item))
flag = False
for fname in files:
filepath = join(directory, item, fname)
# must exist
if not exists(filepath):
continue
# must be a file
if not isfile(filepath):
flag = True
break
# symlink OK as long as it points to files within the repo
if islink(filepath) \
and not realpath(filepath).startswith(getcwd()):
if islink(filepath) or not isfile(filepath):
flag = True
break
if flag:
continue
if (exists(join(directory, item, 'Dockerfile'))
and exists(join(directory, item, 'build.sh'))) \
or exists(join(directory, item, 'test.sh')):
if (exists(join(directory, item, 'Dockerfile')) and exists(join(directory, item, 'build.sh'))) or exists(join(directory, item, 'test.sh')):
results.append(item)
return results
def get_revision(branch):
proc = Popen(["git", "rev-parse", branch], stdout=PIPE)
def get_revision():
proc = Popen(["git", "rev-parse", "HEAD"], stdout=PIPE)
retval = proc.stdout.read().strip()
retval = retval.decode('utf-8')
return retval
@ -168,8 +90,7 @@ def get_revision(branch):
def _get_dockerfile_contents(dockerfile):
"""
Read contents of a Dockerfile and add buildbot worker bootstrap
for a given os_codename
Read contents of a Dockerfile and add extra contents for the given os_codename
"""
os_codename = 'bionic'
res = ""
@ -184,7 +105,6 @@ def _get_dockerfile_contents(dockerfile):
if m:
inside_allowed_command = True
if m.group(1) == "FROM":
os_codename = m.group().split()[1].split(":")[1]
has_from = True
if inside_allowed_command:
res += line
@ -194,15 +114,10 @@ def _get_dockerfile_contents(dockerfile):
if not has_from:
return None
try:
return res + dockerfile_extra_contents[os_codename]
except KeyError:
return None
def trigger_child_hooks(buildbotUrl: str, repository, branch, revision,
directory=".buildbot"):
def trigger_child_hooks(buildbotUrl: str, repository, branch, directory=".buildbot"):
request_url = buildbotUrl + ty
# List all jobs in the directory
jobs = list_jobs(directory)
@ -211,7 +126,7 @@ def trigger_child_hooks(buildbotUrl: str, repository, branch, revision,
"X-Multibuild-Trigger": get_secret(),
"Accept": "text/plain",
}
# revision = get_revision(branch)
revision = get_revision()
# Check if build.sh or test.sh exists in each of the jobs
for job in jobs:
@ -242,28 +157,24 @@ def trigger_child_hooks(buildbotUrl: str, repository, branch, revision,
request_data["changes"] = {
"author": "buildbot_multibuild",
"repository": repository,
"project": job,
"project": "/".join(repository.split("/")[-2:]),
}
retval = requests.post(request_url, headers=request_headers,
json=request_data)
print("Triggered job for {} on {}: {}".format(job, request_url,
retval.text))
sleep(1)
retval = requests.post(request_url, headers=request_headers, json=request_data)
print("Triggered job for {} on {}: {}".format(job, request_url, retval.text))
if __name__ == "__main__":
# expect jobname, repository, branch, buildbotUrl from command line
import sys
if len(sys.argv) == 5:
if len(sys.argv) == 4:
buildbotUrl = sys.argv[1]
repository = sys.argv[2]
branch = sys.argv[3]
revision = sys.argv[4]
trigger_child_hooks(buildbotUrl, repository, branch, revision)
trigger_child_hooks(buildbotUrl, repository, branch)
else:
sys.exit(
"Usage: python3 multibuild.py <buildbotUrl> <repository> <branch> <revision>"
"Usage: python3 multibuild.py <buildbotUrl> <repository> <branch>"
)

102
multibuild.py
View File

@ -13,7 +13,6 @@ Requires docker
from os import listdir
from os.path import isfile, join
from buildbot.plugins import steps, util
from buildbot.process.results import SUCCESS
from .lib.renderers import *
@ -33,16 +32,33 @@ def add_parent_step(build_factory):
Add a step to the parent build factory that will trigger the child hooks
"""
build_factory.addStep(steps.ShellCommand(
name="Update APT cache",
command=["sudo", "apt", "update"]
))
build_factory.addStep(steps.ShellCommand(
name="Install dependencies",
command=["sudo", "apt", "-y", "install", "python3-requests"]
))
build_factory.addStep(
steps.FileDownload(
workerdest="worker_multibuild.py",
mastersrc="buildbot_multibuild/lib/worker_multibuild.py",
mode=0o444
)
)
build_factory.addStep(
steps.ShellCommand(
name="Execute worker script",
command=[
"python3",
'/usr/local/bin/worker_multibuild.py',
'worker_multibuild.py',
util.Property("buildboturl"),
util.Property('repository'),
util.Property('branch'),
util.Property('revision')
util.Property('branch')
],
)
)
@ -53,90 +69,20 @@ def add_child_sh_steps(build_factory, directory=".buildbot"):
Add a step to the download, build and test factory
"""
build_factory.addStep(
steps.FileDownload(
name="Upload keystore",
workerdest="/var/lib/buildbot/keystore",
mastersrc=util.Interpolate("keystore/%(prop:jobname)s.keystore"),
doStepIf=should_build_sign,
hideStepIf=True,
mode=0o600
)
)
build_factory.addStep(
steps.ShellCommand(
name=util.Interpolate("build_%(prop:jobname)s"),
command=util.Interpolate("%(kw:directory)s/"
"%(prop:jobname)s/build.sh",
directory=directory),
env=build_env,
doStepIf=shouldnt_build_sign,
command=util.Interpolate("%(kw:directory)s/%(prop:jobname)s/build.sh", directory=directory),
doStepIf=is_build_script_available,
hideStepIf=isnt_build_script_available,
timeout=7200,
)
)
build_factory.addStep(
steps.ShellCommand(
name=util.Interpolate("build_and_sign_%(prop:jobname)s"),
command=util.Interpolate("%(kw:directory)s/"
"%(prop:jobname)s/build.sh",
directory=directory),
env=build_env,
doStepIf=should_build_sign,
hideStepIf=isnt_build_script_available,
timeout=7200,
)
)
build_factory.addStep(
steps.ShellCommand(
name="Delete keystore",
command="rm -f /var/lib/buildbot/keystore",
doStepIf=should_build_sign,
hideStepIf=True,
)
)
build_factory.addStep(
steps.ShellCommand(
name=util.Interpolate("test_%(prop:jobname)s"),
command=util.Interpolate("%(kw:directory)s/"
"%(prop:jobname)s/test.sh",
directory=directory),
haltOnFailure=False,
flunkOnFailure=True,
name= util.Interpolate("test_%(prop:jobname)s"),
command=util.Interpolate("%(kw:directory)s/%(prop:jobname)s/test.sh", directory=directory),
doStepIf=is_test_script_available,
hideStepIf=isnt_test_script_available,
)
)
build_factory.addStep(
steps.SetPropertyFromCommand(
name="Find files to upload",
command="find . -maxdepth 1 -mindepth 1 "
"-type f -printf '%P\n'",
workdir="out",
hideStepIf=True,
decodeRC={0: SUCCESS, 1: SUCCESS},
property="files_to_upload"
)
)
build_factory.addStep(
steps.ShellCommand(
name="Upload files",
workdir="out",
doStepIf=has_files_to_upload,
hideStepIf=no_files_to_upload,
command=util.Interpolate(
"curl -T {%s} -u buildbot:%s "
"https://artifacts.bitmessage.at/%s/%s/",
files_to_upload,
util.Secret('artifact_upload'),
util.Property('jobname'),
util.Property('buildnumber'),
)
)
)