Compare commits
No commits in common. "master" and "master" have entirely different histories.
|
|
@ -1,79 +0,0 @@
|
|||
from buildbot.plugins import util
|
||||
import re
|
||||
|
||||
def _is_build_script_available(props):
|
||||
return props.getProperty("build_script_available", default=False)
|
||||
|
||||
@util.renderer
|
||||
def is_build_script_available(props):
|
||||
return _is_build_script_available(props)
|
||||
|
||||
@util.renderer
|
||||
def isnt_build_script_available(props):
|
||||
return not _is_build_script_available(props)
|
||||
|
||||
def _is_test_script_available(props):
|
||||
return props.getProperty("test_script_available", default=False)
|
||||
|
||||
@util.renderer
|
||||
def is_test_script_available(props):
|
||||
return _is_test_script_available(props)
|
||||
|
||||
@util.renderer
|
||||
def isnt_test_script_available(props):
|
||||
return not _is_test_script_available(props)
|
||||
|
||||
def _files_to_upload(props):
|
||||
try:
|
||||
return ','.join(props.getProperty("files_to_upload", default="").rstrip().split("\n"))
|
||||
except AttributeError:
|
||||
return ""
|
||||
|
||||
@util.renderer
|
||||
def files_to_upload(props):
|
||||
return _files_to_upload(props)
|
||||
|
||||
@util.renderer
|
||||
def has_files_to_upload(props):
|
||||
return bool(_files_to_upload(props))
|
||||
|
||||
@util.renderer
|
||||
def no_files_to_upload(props):
|
||||
return not _files_to_upload(props)
|
||||
|
||||
def _should_build_sign(props):
|
||||
if props.getProperty('branch') == 'v0.6' \
|
||||
and props.getProperty('jobname') == 'android' \
|
||||
and props.getProperty('repository') in (
|
||||
'git@github.com:Bitmessage/PyBitmessage.git',
|
||||
'https://github.com/Bitmessage/PyBitmessage'
|
||||
):
|
||||
return True
|
||||
return False
|
||||
|
||||
@util.renderer
|
||||
def should_build_sign(props):
|
||||
return _is_build_script_available(props) and _should_build_sign(props)
|
||||
|
||||
@util.renderer
|
||||
def shouldnt_build_sign(props):
|
||||
return _is_build_script_available(props) and not _should_build_sign(props)
|
||||
|
||||
@util.renderer
|
||||
def build_env(props):
|
||||
default_envs = {
|
||||
"BUILDBOT_REPOSITORY": props.getProperty("repository"),
|
||||
"BUILDBOT_BRANCH": props.getProperty("branch"),
|
||||
"BUILDBOT_JOBNAME": props.getProperty("jobname")
|
||||
}
|
||||
new_envs = {}
|
||||
if props.getProperty("jobname", default="") == "android":
|
||||
new_envs = {
|
||||
"P4A_RELEASE_KEYSTORE": "/var/lib/buildbot/keystore",
|
||||
"P4A_RELEASE_KEYSTORE_PASSWD": util.Secret("bitmessage-keystore"),
|
||||
"P4A_RELEASE_KEYALIAS_PASSWD": util.Secret("bitmessage-keystore"),
|
||||
"P4A_RELEASE_KEYALIAS": "bitmessagetest"
|
||||
}
|
||||
if _should_build_sign(props):
|
||||
return {**default_envs, **new_envs}
|
||||
return default_envs
|
||||
|
|
@ -1,268 +0,0 @@
|
|||
from os import getcwd, listdir
|
||||
from os.path import exists, isfile, islink, join, realpath
|
||||
import requests
|
||||
import re
|
||||
from subprocess import Popen, PIPE
|
||||
from time import sleep
|
||||
|
||||
|
||||
request_data = {
|
||||
"project": "testproject",
|
||||
"comments": "testcomment",
|
||||
}
|
||||
|
||||
ty = "/change_hook/base"
|
||||
path = ".buildbot"
|
||||
dockerfile_extra_contents = {}
|
||||
dockerfile_extra_contents['focal'] = """
|
||||
|
||||
# Buildbot
|
||||
RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
|
||||
python3-buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
|
||||
python3-pip dumb-init curl openssh-client wget
|
||||
|
||||
# buildbot entrypoint
|
||||
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
|
||||
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
|
||||
RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt
|
||||
RUN update-ca-certificates
|
||||
|
||||
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
|
||||
|
||||
USER buildbot
|
||||
|
||||
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS"
|
||||
|
||||
"""
|
||||
|
||||
dockerfile_extra_contents['bionic'] = """
|
||||
|
||||
# Buildbot
|
||||
RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
|
||||
git subversion python3-dev libffi-dev python3-setuptools \
|
||||
python3-pip dumb-init curl openssh-client wget python3-wheel \
|
||||
pkg-config rustc cargo
|
||||
RUN pip3 install setuptools_rust
|
||||
RUN pip3 install 'buildbot-worker==3.1.1' \
|
||||
'cryptography==2.1.4' \
|
||||
'twisted==17.9.0' \
|
||||
'pyopenssl==17.5.0'
|
||||
RUN groupadd buildbot
|
||||
RUN useradd -d /var/lib/buildbot -m -g buildbot buildbot
|
||||
|
||||
# buildbot entrypoint
|
||||
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
|
||||
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
|
||||
RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt
|
||||
RUN update-ca-certificates
|
||||
|
||||
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
|
||||
|
||||
USER buildbot
|
||||
|
||||
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS"
|
||||
|
||||
"""
|
||||
|
||||
dockerfile_extra_contents['jammy'] = """
|
||||
|
||||
# Buildbot
|
||||
RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
|
||||
python3-buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
|
||||
python3-pip dumb-init curl openssh-client wget
|
||||
|
||||
# buildbot entrypoint
|
||||
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
|
||||
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
|
||||
RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt
|
||||
RUN update-ca-certificates
|
||||
|
||||
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
|
||||
|
||||
USER buildbot
|
||||
|
||||
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS"
|
||||
|
||||
"""
|
||||
|
||||
dockerfile_extra_contents['xenial'] = """
|
||||
|
||||
# Buildbot
|
||||
RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
|
||||
git subversion python3-dev libffi-dev python3-setuptools \
|
||||
python3-pip curl openssh-client wget \
|
||||
python-setuptools python-psutil libssl-dev python-dev libgmp-dev \
|
||||
python-virtualenv python3-wheel pkg-config rustc cargo \
|
||||
python3-openssl
|
||||
RUN pip3 install setuptools_rust
|
||||
RUN pip3 install 'buildbot-worker<3.2.0' 'cryptography<35.0.0'
|
||||
RUN groupadd buildbot
|
||||
RUN useradd -d /var/lib/buildbot -m -g buildbot buildbot
|
||||
|
||||
# dumb-init
|
||||
RUN wget https://github.com/Yelp/dumb-init/releases/download/v1.2.5/dumb-init_1.2.5_amd64.deb
|
||||
RUN dpkg -i dumb-init_*.deb && rm -f dumb-init_*.deb
|
||||
|
||||
# buildbot entrypoint
|
||||
RUN [ -f /usr/local/bin/buildbot_entrypoint.sh ] || wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/xenial/entrypoint.sh
|
||||
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
|
||||
RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt
|
||||
RUN update-ca-certificates
|
||||
|
||||
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
|
||||
|
||||
USER buildbot
|
||||
|
||||
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS"
|
||||
|
||||
"""
|
||||
|
||||
|
||||
def get_secret():
|
||||
with open("multibuild_parent_key.key", 'r') as f:
|
||||
data = f.read()
|
||||
return data
|
||||
|
||||
|
||||
def list_jobs(directory=".buildbot"):
|
||||
"""
|
||||
list jobs found in a directory
|
||||
"""
|
||||
results = []
|
||||
files = ["Dockerfile", "build.sh", "test.sh"]
|
||||
if not exists(directory):
|
||||
return results
|
||||
for item in listdir(directory):
|
||||
print("checking directory {}".format(item))
|
||||
flag = False
|
||||
for fname in files:
|
||||
filepath = join(directory, item, fname)
|
||||
# must exist
|
||||
if not exists(filepath):
|
||||
continue
|
||||
# must be a file
|
||||
if not isfile(filepath):
|
||||
flag = True
|
||||
break
|
||||
# symlink OK as long as it points to files within the repo
|
||||
if islink(filepath) \
|
||||
and not realpath(filepath).startswith(getcwd()):
|
||||
flag = True
|
||||
break
|
||||
if flag:
|
||||
continue
|
||||
if (exists(join(directory, item, 'Dockerfile'))
|
||||
and exists(join(directory, item, 'build.sh'))) \
|
||||
or exists(join(directory, item, 'test.sh')):
|
||||
results.append(item)
|
||||
return results
|
||||
|
||||
|
||||
def get_revision(branch):
|
||||
proc = Popen(["git", "rev-parse", branch], stdout=PIPE)
|
||||
retval = proc.stdout.read().strip()
|
||||
retval = retval.decode('utf-8')
|
||||
return retval
|
||||
|
||||
|
||||
def _get_dockerfile_contents(dockerfile):
|
||||
"""
|
||||
Read contents of a Dockerfile and add buildbot worker bootstrap
|
||||
for a given os_codename
|
||||
"""
|
||||
os_codename = 'bionic'
|
||||
res = ""
|
||||
with open(dockerfile, "r") as file:
|
||||
contents = file.readlines()
|
||||
has_from = False
|
||||
# accept any line containing FROM and RUN keywords
|
||||
res = ""
|
||||
inside_allowed_command = False
|
||||
for line in contents:
|
||||
m = re.match(r"(?m)^(FROM|RUN|ENV).*$", line)
|
||||
if m:
|
||||
inside_allowed_command = True
|
||||
if m.group(1) == "FROM":
|
||||
os_codename = m.group().split()[1].split(":")[1]
|
||||
has_from = True
|
||||
if inside_allowed_command:
|
||||
res += line
|
||||
ls = line.strip()
|
||||
if not ls.endswith("\\"):
|
||||
inside_allowed_command = False
|
||||
|
||||
if not has_from:
|
||||
return None
|
||||
|
||||
try:
|
||||
return res + dockerfile_extra_contents[os_codename]
|
||||
except KeyError:
|
||||
return None
|
||||
|
||||
|
||||
def trigger_child_hooks(buildbotUrl: str, repository, branch, revision,
|
||||
directory=".buildbot"):
|
||||
request_url = buildbotUrl + ty
|
||||
# List all jobs in the directory
|
||||
jobs = list_jobs(directory)
|
||||
request_headers = {
|
||||
"Content-Type": "application/json",
|
||||
"X-Multibuild-Trigger": get_secret(),
|
||||
"Accept": "text/plain",
|
||||
}
|
||||
# revision = get_revision(branch)
|
||||
|
||||
# Check if build.sh or test.sh exists in each of the jobs
|
||||
for job in jobs:
|
||||
build_script_exists = False
|
||||
test_script_exists = False
|
||||
if exists(join(directory, job, "build.sh")):
|
||||
build_script_exists = True
|
||||
if exists(join(directory, job, "test.sh")):
|
||||
test_script_exists = True
|
||||
|
||||
# make a post request
|
||||
dockerfile = _get_dockerfile_contents(
|
||||
join(directory, job, "Dockerfile")
|
||||
)
|
||||
|
||||
if not dockerfile:
|
||||
continue
|
||||
|
||||
request_data["branch"] = branch
|
||||
request_data["revision"] = revision
|
||||
|
||||
request_data["properties"] = {
|
||||
"dockerfile": dockerfile,
|
||||
"build_script_available": build_script_exists,
|
||||
"test_script_available": test_script_exists,
|
||||
"jobname": job,
|
||||
}
|
||||
request_data["changes"] = {
|
||||
"author": "buildbot_multibuild",
|
||||
"repository": repository,
|
||||
"project": job,
|
||||
}
|
||||
|
||||
retval = requests.post(request_url, headers=request_headers,
|
||||
json=request_data)
|
||||
print("Triggered job for {} on {}: {}".format(job, request_url,
|
||||
retval.text))
|
||||
sleep(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# expect jobname, repository, branch, buildbotUrl from command line
|
||||
import sys
|
||||
|
||||
if len(sys.argv) == 5:
|
||||
buildbotUrl = sys.argv[1]
|
||||
repository = sys.argv[2]
|
||||
branch = sys.argv[3]
|
||||
revision = sys.argv[4]
|
||||
|
||||
trigger_child_hooks(buildbotUrl, repository, branch, revision)
|
||||
else:
|
||||
sys.exit(
|
||||
"Usage: python3 multibuild.py <buildbotUrl> <repository> <branch> <revision>"
|
||||
)
|
||||
135
multibuild.py
135
multibuild.py
|
|
@ -10,12 +10,23 @@ Requires docker
|
|||
# TODO: write hook job, maybe also a dockerfile?
|
||||
# TODO: what to do about non-docker jobs
|
||||
|
||||
from os import listdir
|
||||
from os.path import isfile, join
|
||||
from buildbot.plugins import steps, util
|
||||
from buildbot.process.results import SUCCESS
|
||||
from os import walk
|
||||
from os.path import exists, isfile, join, listdir
|
||||
|
||||
from .lib.renderers import *
|
||||
|
||||
def list_jobs(directory=".buildbot"):
|
||||
"""
|
||||
list jobs found in a directory
|
||||
"""
|
||||
results = []
|
||||
for _ in next(walk(directory))[1]:
|
||||
if exists(join(directory, _, "Dockerfile")) \
|
||||
and (exists(join(directory, _, "build.sh"))
|
||||
or exists(join(directory, _, "test.sh"))
|
||||
):
|
||||
results.append(_)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def find_artifacts(directory="out"):
|
||||
|
|
@ -26,117 +37,3 @@ def find_artifacts(directory="out"):
|
|||
if not isfile(join(directory, _)):
|
||||
continue
|
||||
return join(directory, _)
|
||||
|
||||
|
||||
def add_parent_step(build_factory):
|
||||
"""
|
||||
Add a step to the parent build factory that will trigger the child hooks
|
||||
"""
|
||||
|
||||
build_factory.addStep(
|
||||
steps.ShellCommand(
|
||||
name="Execute worker script",
|
||||
command=[
|
||||
"python3",
|
||||
'/usr/local/bin/worker_multibuild.py',
|
||||
util.Property("buildboturl"),
|
||||
util.Property('repository'),
|
||||
util.Property('branch'),
|
||||
util.Property('revision')
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def add_child_sh_steps(build_factory, directory=".buildbot"):
|
||||
"""
|
||||
Add a step to the download, build and test factory
|
||||
"""
|
||||
|
||||
build_factory.addStep(
|
||||
steps.FileDownload(
|
||||
name="Upload keystore",
|
||||
workerdest="/var/lib/buildbot/keystore",
|
||||
mastersrc=util.Interpolate("keystore/%(prop:jobname)s.keystore"),
|
||||
doStepIf=should_build_sign,
|
||||
hideStepIf=True,
|
||||
mode=0o600
|
||||
)
|
||||
)
|
||||
|
||||
build_factory.addStep(
|
||||
steps.ShellCommand(
|
||||
name=util.Interpolate("build_%(prop:jobname)s"),
|
||||
command=util.Interpolate("%(kw:directory)s/"
|
||||
"%(prop:jobname)s/build.sh",
|
||||
directory=directory),
|
||||
env=build_env,
|
||||
doStepIf=shouldnt_build_sign,
|
||||
hideStepIf=isnt_build_script_available,
|
||||
timeout=7200,
|
||||
)
|
||||
)
|
||||
|
||||
build_factory.addStep(
|
||||
steps.ShellCommand(
|
||||
name=util.Interpolate("build_and_sign_%(prop:jobname)s"),
|
||||
command=util.Interpolate("%(kw:directory)s/"
|
||||
"%(prop:jobname)s/build.sh",
|
||||
directory=directory),
|
||||
env=build_env,
|
||||
doStepIf=should_build_sign,
|
||||
hideStepIf=isnt_build_script_available,
|
||||
timeout=7200,
|
||||
)
|
||||
)
|
||||
|
||||
build_factory.addStep(
|
||||
steps.ShellCommand(
|
||||
name="Delete keystore",
|
||||
command="rm -f /var/lib/buildbot/keystore",
|
||||
doStepIf=should_build_sign,
|
||||
hideStepIf=True,
|
||||
)
|
||||
)
|
||||
|
||||
build_factory.addStep(
|
||||
steps.ShellCommand(
|
||||
name=util.Interpolate("test_%(prop:jobname)s"),
|
||||
command=util.Interpolate("%(kw:directory)s/"
|
||||
"%(prop:jobname)s/test.sh",
|
||||
directory=directory),
|
||||
haltOnFailure=False,
|
||||
flunkOnFailure=True,
|
||||
doStepIf=is_test_script_available,
|
||||
hideStepIf=isnt_test_script_available,
|
||||
)
|
||||
)
|
||||
|
||||
build_factory.addStep(
|
||||
steps.SetPropertyFromCommand(
|
||||
name="Find files to upload",
|
||||
command="find . -maxdepth 1 -mindepth 1 "
|
||||
"-type f -printf '%P\n'",
|
||||
workdir="out",
|
||||
hideStepIf=True,
|
||||
decodeRC={0: SUCCESS, 1: SUCCESS},
|
||||
property="files_to_upload"
|
||||
)
|
||||
)
|
||||
|
||||
build_factory.addStep(
|
||||
steps.ShellCommand(
|
||||
name="Upload files",
|
||||
workdir="out",
|
||||
doStepIf=has_files_to_upload,
|
||||
hideStepIf=no_files_to_upload,
|
||||
command=util.Interpolate(
|
||||
"curl -T {%s} -u buildbot:%s "
|
||||
"https://artifacts.bitmessage.at/%s/%s/",
|
||||
files_to_upload,
|
||||
util.Secret('artifact_upload'),
|
||||
util.Property('jobname'),
|
||||
util.Property('buildnumber'),
|
||||
)
|
||||
)
|
||||
)
|
||||
|
|
|
|||
Loading…
Reference in New Issue