Compare commits

..

No commits in common. "master" and "github_hook" have entirely different histories.

3 changed files with 77 additions and 311 deletions

View File

@ -1,79 +1,24 @@
from buildbot.plugins import util
import re
def _is_build_script_available(props):
return props.getProperty("build_script_available", default=False)
@util.renderer
def is_build_script_available(props):
return _is_build_script_available(props)
# Actual check will got here
return props.getProperty("build_available", default=False)
@util.renderer
def isnt_build_script_available(props):
return not _is_build_script_available(props)
return not is_build_script_available(props)
def _is_test_script_available(props):
return props.getProperty("test_script_available", default=False)
@util.renderer
def is_test_script_available(props):
return _is_test_script_available(props)
# Actual check will got here
return props.getProperty("test_available", default=False)
@util.renderer
def isnt_test_script_available(props):
return not _is_test_script_available(props)
def _files_to_upload(props):
try:
return ','.join(props.getProperty("files_to_upload", default="").rstrip().split("\n"))
except AttributeError:
return ""
@util.renderer
def files_to_upload(props):
return _files_to_upload(props)
@util.renderer
def has_files_to_upload(props):
return bool(_files_to_upload(props))
@util.renderer
def no_files_to_upload(props):
return not _files_to_upload(props)
def _should_build_sign(props):
if props.getProperty('branch') == 'v0.6' \
and props.getProperty('jobname') == 'android' \
and props.getProperty('repository') in (
'git@github.com:Bitmessage/PyBitmessage.git',
'https://github.com/Bitmessage/PyBitmessage'
):
return True
return False
@util.renderer
def should_build_sign(props):
return _is_build_script_available(props) and _should_build_sign(props)
@util.renderer
def shouldnt_build_sign(props):
return _is_build_script_available(props) and not _should_build_sign(props)
@util.renderer
def build_env(props):
default_envs = {
"BUILDBOT_REPOSITORY": props.getProperty("repository"),
"BUILDBOT_BRANCH": props.getProperty("branch"),
"BUILDBOT_JOBNAME": props.getProperty("jobname")
}
new_envs = {}
if props.getProperty("jobname", default="") == "android":
new_envs = {
"P4A_RELEASE_KEYSTORE": "/var/lib/buildbot/keystore",
"P4A_RELEASE_KEYSTORE_PASSWD": util.Secret("bitmessage-keystore"),
"P4A_RELEASE_KEYALIAS_PASSWD": util.Secret("bitmessage-keystore"),
"P4A_RELEASE_KEYALIAS": "bitmessagetest"
}
if _should_build_sign(props):
return {**default_envs, **new_envs}
return default_envs
return not is_test_script_available(props)

View File

@ -1,9 +1,8 @@
from os import getcwd, listdir
from os.path import exists, isfile, islink, join, realpath
import os
from os import listdir
from os.path import exists, isfile, join, islink, isdir
import requests
import re
from subprocess import Popen, PIPE
from time import sleep
request_data = {
@ -12,206 +11,103 @@ request_data = {
}
ty = "/change_hook/base"
path = ".buildbot"
path =".buildbot"
dockerfile_extra_contents = {}
dockerfile_extra_contents['focal'] = """
# Buildbot
RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
python3-buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
RUN apt-get install -yq --no-install-suggests --no-install-recommends \
buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
python3-pip dumb-init curl openssh-client wget
# buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt
RUN update-ca-certificates
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS"
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
"""
dockerfile_extra_contents['bionic'] = """
# Buildbot
RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
git subversion python3-dev libffi-dev python3-setuptools \
python3-pip dumb-init curl openssh-client wget python3-wheel \
pkg-config rustc cargo
RUN pip3 install setuptools_rust
RUN pip3 install 'buildbot-worker==3.1.1' \
'cryptography==2.1.4' \
'twisted==17.9.0' \
'Automat==20.2.0' \
'pyopenssl==17.5.0'
RUN groupadd buildbot
RUN useradd -d /var/lib/buildbot -m -g buildbot buildbot
# buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt
RUN update-ca-certificates
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS"
"""
dockerfile_extra_contents['jammy'] = """
# Buildbot
RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
python3-buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
RUN apt-get install -yq --no-install-suggests --no-install-recommends \
buildbot-slave git subversion python3-dev libffi-dev python3-setuptools \
python3-pip dumb-init curl openssh-client wget
# buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt
RUN update-ca-certificates
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS"
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
"""
dockerfile_extra_contents['xenial'] = """
# Buildbot
RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
git subversion python3-dev libffi-dev python3-setuptools \
python3-pip curl openssh-client wget \
python-setuptools python-psutil libssl-dev python-dev libgmp-dev \
python-virtualenv python3-wheel pkg-config rustc cargo \
python3-openssl
RUN pip3 install setuptools_rust
RUN pip3 install 'buildbot-worker<3.2.0' 'cryptography<35.0.0'
RUN groupadd buildbot
RUN useradd -d /var/lib/buildbot -m -g buildbot buildbot
# dumb-init
RUN wget https://github.com/Yelp/dumb-init/releases/download/v1.2.5/dumb-init_1.2.5_amd64.deb
RUN dpkg -i dumb-init_*.deb && rm -f dumb-init_*.deb
# buildbot entrypoint
RUN [ -f /usr/local/bin/buildbot_entrypoint.sh ] || wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/xenial/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt
RUN update-ca-certificates
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS"
"""
def get_secret():
with open("multibuild_parent_key.key", 'r') as f:
with open(path.join(os.environ['HOME'], "multibuild_parent_key.key"),'r') as f:
data = f.read()
return data
def list_jobs(directory=".buildbot"):
"""
list jobs found in a directory
"""
results = []
files = ["Dockerfile", "build.sh", "test.sh"]
if not exists(directory):
return results
for item in listdir(directory):
print("checking directory {}".format(item))
flag = False
for fname in files:
filepath = join(directory, item, fname)
# must exist
if not exists(filepath):
continue
# must be a file
if not isfile(filepath):
flag = True
break
# symlink OK as long as it points to files within the repo
if islink(filepath) \
and not realpath(filepath).startswith(getcwd()):
for file in files:
filepath = join(directory, item, file)
if islink(filepath) or not isfile(filepath):
flag = True
break
if flag:
continue
if (exists(join(directory, item, 'Dockerfile'))
and exists(join(directory, item, 'build.sh'))) \
or exists(join(directory, item, 'test.sh')):
continue
if (exists(join(directory, item, 'Dockerfile')) and exists(join(directory, item, 'build.sh'))) or exists(join(directory, item, 'test.sh')):
results.append(item)
return results
def get_revision(branch):
proc = Popen(["git", "rev-parse", branch], stdout=PIPE)
retval = proc.stdout.read().strip()
retval = retval.decode('utf-8')
return retval
def _get_dockerfile_contents(dockerfile):
def _get_dockerfile_contents(jobname):
"""
Read contents of a Dockerfile and add buildbot worker bootstrap
for a given os_codename
Read contents of a Dockerfile and add extra contents for the given os_codename
"""
os_codename = 'bionic'
os_codename='bionic'
res = ""
with open(dockerfile, "r") as file:
contents = file.readlines()
has_from = False
with open(join(path + jobname), "r") as file:
contents = file.read()
# accept any line containing FROM and RUN keywords
res = ""
inside_allowed_command = False
for line in contents:
m = re.match(r"(?m)^(FROM|RUN|ENV).*$", line)
if m:
if re.match(r"(?m)^(FROM|RUN|ENV).*$", line):
inside_allowed_command = True
if m.group(1) == "FROM":
os_codename = m.group().split()[1].split(":")[1]
has_from = True
if inside_allowed_command:
res += line
ls = line.strip()
if not ls.endswith("\\"):
l = line.strip()
if not l.endswith("\\"):
inside_allowed_command = False
if not has_from:
return None
try:
return res + dockerfile_extra_contents[os_codename]
except KeyError:
return None
return res + dockerfile_extra_contents[os_codename]
def trigger_child_hooks(buildbotUrl: str, repository, branch, revision,
directory=".buildbot"):
def trigger_child_hooks(buildbotUrl: str, repository, branch, directory=".buildbot"):
request_url = buildbotUrl + ty
# List all jobs in the directory
jobs = list_jobs(directory)
request_headers = {
"Content-Type": "application/json",
"Content-Type": "application/x-www-form-urlencoded",
"X-Multibuild-Trigger": get_secret(),
"Accept": "text/plain",
}
# revision = get_revision(branch)
# Check if build.sh or test.sh exists in each of the jobs
for job in jobs:
@ -223,47 +119,15 @@ def trigger_child_hooks(buildbotUrl: str, repository, branch, revision,
test_script_exists = True
# make a post request
dockerfile = _get_dockerfile_contents(
join(directory, job, "Dockerfile")
)
if not dockerfile:
continue
request_data["branch"] = branch
request_data["revision"] = revision
request_data["properties"] = {
"dockerfile": dockerfile,
"build_script_available": build_script_exists,
"test_script_available": test_script_exists,
"dockerfile": _get_dockerfile_contents(
join(directory, job, "Dockerfile")
),
"build_script_available": is_build_script_available(build_script_exists),
"test_script_available": is_test_script_available(test_script_exists),
"repository": repository,
"branch": branch,
"jobname": job,
}
request_data["changes"] = {
"author": "buildbot_multibuild",
"repository": repository,
"project": job,
}
retval = requests.post(request_url, headers=request_headers,
json=request_data)
print("Triggered job for {} on {}: {}".format(job, request_url,
retval.text))
sleep(1)
if __name__ == "__main__":
# expect jobname, repository, branch, buildbotUrl from command line
import sys
if len(sys.argv) == 5:
buildbotUrl = sys.argv[1]
repository = sys.argv[2]
branch = sys.argv[3]
revision = sys.argv[4]
trigger_child_hooks(buildbotUrl, repository, branch, revision)
else:
sys.exit(
"Usage: python3 multibuild.py <buildbotUrl> <repository> <branch> <revision>"
)
requests.post(request_url, headers=request_headers, data=request_data)

View File

@ -10,10 +10,11 @@ Requires docker
# TODO: write hook job, maybe also a dockerfile?
# TODO: what to do about non-docker jobs
from os import listdir
from os.path import isfile, join
from os import listdir, walk, getenv
from os.path import exists, isfile, join
import requests
import re
from buildbot.plugins import steps, util
from buildbot.process.results import SUCCESS
from .lib.renderers import *
@ -33,16 +34,25 @@ def add_parent_step(build_factory):
Add a step to the parent build factory that will trigger the child hooks
"""
build_factory.addStep(steps.ShellCommand(
name="create directory",
command=["mkdir", "-p", join(getenv['HOME'], '.local/bin') ]
))
build_factory.addStep(steps.ShellCommand(
name="download worker",
command=["wget", "-O", "https://git.bitmessage.org/Bitmessage/buildbot_multibuild/raw/branch/master/lib/worker_multibuild.py", join(getenv['HOME'], '.local/bin/worker_multibuild.py')]
))
build_factory.addStep(
steps.ShellCommand(
name="Execute worker script",
command=[
"python3",
'/usr/local/bin/worker_multibuild.py',
util.Property("buildboturl"),
join(getenv['HOME'], '.local/bin/worker_multibuild.py'),
util.Property('repository'),
util.Property('branch'),
util.Property('revision')
util.getURLForBuild(util.Property("url"), util.Property("builderid"), util.Property("buildnumber")),
],
)
)
@ -53,90 +63,37 @@ def add_child_sh_steps(build_factory, directory=".buildbot"):
Add a step to the download, build and test factory
"""
build_factory.addStep(
steps.FileDownload(
name="Upload keystore",
workerdest="/var/lib/buildbot/keystore",
mastersrc=util.Interpolate("keystore/%(prop:jobname)s.keystore"),
doStepIf=should_build_sign,
hideStepIf=True,
mode=0o600
)
)
build_factory.addStep(
steps.ShellCommand(
name=util.Interpolate("build_%(prop:jobname)s"),
command=util.Interpolate("%(kw:directory)s/"
"%(prop:jobname)s/build.sh",
directory=directory),
env=build_env,
doStepIf=shouldnt_build_sign,
command=util.Interpolate("%(kw:directory)s/%(prop:jobname)s/build.sh", directory=directory),
doStepIf=is_build_script_available,
hideStepIf=isnt_build_script_available,
timeout=7200,
)
)
build_factory.addStep(
steps.ShellCommand(
name=util.Interpolate("build_and_sign_%(prop:jobname)s"),
command=util.Interpolate("%(kw:directory)s/"
"%(prop:jobname)s/build.sh",
directory=directory),
env=build_env,
doStepIf=should_build_sign,
hideStepIf=isnt_build_script_available,
timeout=7200,
)
)
build_factory.addStep(
steps.ShellCommand(
name="Delete keystore",
command="rm -f /var/lib/buildbot/keystore",
doStepIf=should_build_sign,
hideStepIf=True,
)
)
build_factory.addStep(
steps.ShellCommand(
name=util.Interpolate("test_%(prop:jobname)s"),
command=util.Interpolate("%(kw:directory)s/"
"%(prop:jobname)s/test.sh",
directory=directory),
haltOnFailure=False,
flunkOnFailure=True,
name= util.Interpolate("test_%(prop:jobname)s"),
command=util.Interpolate("%(kw:directory)s/%(prop:jobname)s/test.sh", directory=directory),
doStepIf=is_test_script_available,
hideStepIf=isnt_test_script_available,
)
)
build_factory.addStep(
steps.SetPropertyFromCommand(
name="Find files to upload",
command="find . -maxdepth 1 -mindepth 1 "
"-type f -printf '%P\n'",
workdir="out",
hideStepIf=True,
decodeRC={0: SUCCESS, 1: SUCCESS},
property="files_to_upload"
)
)
build_factory.addStep(
steps.ShellCommand(
name="Upload files",
workdir="out",
doStepIf=has_files_to_upload,
hideStepIf=no_files_to_upload,
command=util.Interpolate(
"curl -T {%s} -u buildbot:%s "
"https://artifacts.bitmessage.at/%s/%s/",
files_to_upload,
util.Secret('artifact_upload'),
util.Property('jobname'),
util.Property('buildnumber'),
)
if __name__ == "__main__":
# expect jobname, repository, branch, buildbotUrl from command line
import sys
if len(sys.argv) == 6:
jobname = sys.argv[1]
repository = sys.argv[2]
branch = sys.argv[3]
buildbotUrl = sys.argv[4]
trigger_child_hooks(buildbotUrl, repository, branch)
else:
print(
"Usage: python3 multibuild.py <buildbotUrl> <repository> <branch> "
)
)