Bitmessage/buildbot_multibuild
6
0
Fork 5
Code Issues 4 Pull Requests Projects Releases Wiki Activity

Compare commits

base: Bitmessage:master
Branches Tags
Bitmessage:master
shekhar-cis:refactor/buildbot_multibuild
shekhar-cis:master
..
compare: shekhar-cis:master
Branches Tags
shekhar-cis:refactor/buildbot_multibuild
shekhar-cis:master
Bitmessage:master

No commits in common. "master" and "master" have entirely different histories.
master ... master

3 changed files with 12 additions and 213 deletions
Show Stats Expand all files Collapse all files

55
lib/renderers.py
View File

@ -22,58 +22,3 @@ def is_test_script_available(props):
@util.renderer @util.renderer
def isnt_test_script_available(props): def isnt_test_script_available(props):
return not _is_test_script_available(props) return not _is_test_script_available(props)
def _files_to_upload(props):
try:
return ','.join(props.getProperty("files_to_upload", default="").rstrip().split("\n"))
except AttributeError:
return ""
@util.renderer
def files_to_upload(props):
return _files_to_upload(props)
@util.renderer
def has_files_to_upload(props):
return bool(_files_to_upload(props))
@util.renderer
def no_files_to_upload(props):
return not _files_to_upload(props)
def _should_build_sign(props):
if props.getProperty('branch') == 'v0.6' \
and props.getProperty('jobname') == 'android' \
and props.getProperty('repository') in (
'git@github.com:Bitmessage/PyBitmessage.git',
'https://github.com/Bitmessage/PyBitmessage'
):
return True
return False
@util.renderer
def should_build_sign(props):
return _is_build_script_available(props) and _should_build_sign(props)
@util.renderer
def shouldnt_build_sign(props):
return _is_build_script_available(props) and not _should_build_sign(props)
@util.renderer
def build_env(props):
default_envs = {
"BUILDBOT_REPOSITORY": props.getProperty("repository"),
"BUILDBOT_BRANCH": props.getProperty("branch"),
"BUILDBOT_JOBNAME": props.getProperty("jobname")
}
new_envs = {}
if props.getProperty("jobname", default="") == "android":
new_envs = {
"P4A_RELEASE_KEYSTORE": "/var/lib/buildbot/keystore",
"P4A_RELEASE_KEYSTORE_PASSWD": util.Secret("bitmessage-keystore"),
"P4A_RELEASE_KEYALIAS_PASSWD": util.Secret("bitmessage-keystore"),
"P4A_RELEASE_KEYALIAS": "bitmessagetest"
}
if _should_build_sign(props):
return {**default_envs, **new_envs}
return default_envs

91
lib/worker_multibuild.py
View File

@ -3,7 +3,6 @@ from os.path import exists, isfile, islink, join, realpath
import requests import requests
import re import re
from subprocess import Popen, PIPE from subprocess import Popen, PIPE
from time import sleep
request_data = { request_data = {
@ -17,104 +16,38 @@ dockerfile_extra_contents = {}
dockerfile_extra_contents['focal'] = """ dockerfile_extra_contents['focal'] = """
# Buildbot # Buildbot
RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \ RUN apt-get install -yq --no-install-suggests --no-install-recommends \
python3-buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \ buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
python3-pip dumb-init curl openssh-client wget python3-pip dumb-init curl openssh-client wget
# buildbot entrypoint # buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt
RUN update-ca-certificates
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS" ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
""" """
dockerfile_extra_contents['bionic'] = """ dockerfile_extra_contents['bionic'] = """
# Buildbot # Buildbot
RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \ RUN apt-get install -yq --no-install-suggests --no-install-recommends \
git subversion python3-dev libffi-dev python3-setuptools \ buildbot-slave git subversion python3-dev libffi-dev python3-setuptools \
python3-pip dumb-init curl openssh-client wget python3-wheel \
pkg-config rustc cargo
RUN pip3 install setuptools_rust
RUN pip3 install 'buildbot-worker==3.1.1' \
'cryptography==2.1.4' \
'twisted==17.9.0' \
'Automat==20.2.0' \
'pyopenssl==17.5.0'
RUN groupadd buildbot
RUN useradd -d /var/lib/buildbot -m -g buildbot buildbot
# buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt
RUN update-ca-certificates
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS"
"""
dockerfile_extra_contents['jammy'] = """
# Buildbot
RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
python3-buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
python3-pip dumb-init curl openssh-client wget python3-pip dumb-init curl openssh-client wget
# buildbot entrypoint # buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt
RUN update-ca-certificates
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS" ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
"""
dockerfile_extra_contents['xenial'] = """
# Buildbot
RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
git subversion python3-dev libffi-dev python3-setuptools \
python3-pip curl openssh-client wget \
python-setuptools python-psutil libssl-dev python-dev libgmp-dev \
python-virtualenv python3-wheel pkg-config rustc cargo \
python3-openssl
RUN pip3 install setuptools_rust
RUN pip3 install 'buildbot-worker<3.2.0' 'cryptography<35.0.0'
RUN groupadd buildbot
RUN useradd -d /var/lib/buildbot -m -g buildbot buildbot
# dumb-init
RUN wget https://github.com/Yelp/dumb-init/releases/download/v1.2.5/dumb-init_1.2.5_amd64.deb
RUN dpkg -i dumb-init_*.deb && rm -f dumb-init_*.deb
# buildbot entrypoint
RUN [ -f /usr/local/bin/buildbot_entrypoint.sh ] || wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/xenial/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt
RUN update-ca-certificates
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS"
""" """
@ -131,8 +64,6 @@ def list_jobs(directory=".buildbot"):
""" """
results = [] results = []
files = ["Dockerfile", "build.sh", "test.sh"] files = ["Dockerfile", "build.sh", "test.sh"]
if not exists(directory):
return results
for item in listdir(directory): for item in listdir(directory):
print("checking directory {}".format(item)) print("checking directory {}".format(item))
flag = False flag = False
@ -184,7 +115,6 @@ def _get_dockerfile_contents(dockerfile):
if m: if m:
inside_allowed_command = True inside_allowed_command = True
if m.group(1) == "FROM": if m.group(1) == "FROM":
os_codename = m.group().split()[1].split(":")[1]
has_from = True has_from = True
if inside_allowed_command: if inside_allowed_command:
res += line res += line
@ -194,11 +124,7 @@ def _get_dockerfile_contents(dockerfile):
if not has_from: if not has_from:
return None return None
return res + dockerfile_extra_contents[os_codename]
try:
return res + dockerfile_extra_contents[os_codename]
except KeyError:
return None
def trigger_child_hooks(buildbotUrl: str, repository, branch, revision, def trigger_child_hooks(buildbotUrl: str, repository, branch, revision,
@ -242,14 +168,13 @@ def trigger_child_hooks(buildbotUrl: str, repository, branch, revision,
request_data["changes"] = { request_data["changes"] = {
"author": "buildbot_multibuild", "author": "buildbot_multibuild",
"repository": repository, "repository": repository,
"project": job, "project": "/".join(repository.split("/")[-2:]),
} }
retval = requests.post(request_url, headers=request_headers, retval = requests.post(request_url, headers=request_headers,
json=request_data) json=request_data)
print("Triggered job for {} on {}: {}".format(job, request_url, print("Triggered job for {} on {}: {}".format(job, request_url,
retval.text)) retval.text))
sleep(1)
if __name__ == "__main__": if __name__ == "__main__":

79
multibuild.py
View File

@ -13,7 +13,6 @@ Requires docker
from os import listdir from os import listdir
from os.path import isfile, join from os.path import isfile, join
from buildbot.plugins import steps, util from buildbot.plugins import steps, util
from buildbot.process.results import SUCCESS
from .lib.renderers import * from .lib.renderers import *
@ -53,90 +52,20 @@ def add_child_sh_steps(build_factory, directory=".buildbot"):
Add a step to the download, build and test factory Add a step to the download, build and test factory
""" """
build_factory.addStep(
steps.FileDownload(
name="Upload keystore",
workerdest="/var/lib/buildbot/keystore",
mastersrc=util.Interpolate("keystore/%(prop:jobname)s.keystore"),
doStepIf=should_build_sign,
hideStepIf=True,
mode=0o600
)
)
build_factory.addStep( build_factory.addStep(
steps.ShellCommand( steps.ShellCommand(
name=util.Interpolate("build_%(prop:jobname)s"), name=util.Interpolate("build_%(prop:jobname)s"),
command=util.Interpolate("%(kw:directory)s/" command=util.Interpolate("%(kw:directory)s/%(prop:jobname)s/build.sh", directory=directory),
"%(prop:jobname)s/build.sh", doStepIf=is_build_script_available,
directory=directory),
env=build_env,
doStepIf=shouldnt_build_sign,
hideStepIf=isnt_build_script_available, hideStepIf=isnt_build_script_available,
timeout=7200,
) )
) )
build_factory.addStep( build_factory.addStep(
steps.ShellCommand( steps.ShellCommand(
name=util.Interpolate("build_and_sign_%(prop:jobname)s"), name= util.Interpolate("test_%(prop:jobname)s"),
command=util.Interpolate("%(kw:directory)s/" command=util.Interpolate("%(kw:directory)s/%(prop:jobname)s/test.sh", directory=directory),
"%(prop:jobname)s/build.sh",
directory=directory),
env=build_env,
doStepIf=should_build_sign,
hideStepIf=isnt_build_script_available,
timeout=7200,
)
)
build_factory.addStep(
steps.ShellCommand(
name="Delete keystore",
command="rm -f /var/lib/buildbot/keystore",
doStepIf=should_build_sign,
hideStepIf=True,
)
)
build_factory.addStep(
steps.ShellCommand(
name=util.Interpolate("test_%(prop:jobname)s"),
command=util.Interpolate("%(kw:directory)s/"
"%(prop:jobname)s/test.sh",
directory=directory),
haltOnFailure=False,
flunkOnFailure=True,
doStepIf=is_test_script_available, doStepIf=is_test_script_available,
hideStepIf=isnt_test_script_available, hideStepIf=isnt_test_script_available,
) )
) )
build_factory.addStep(
steps.SetPropertyFromCommand(
name="Find files to upload",
command="find . -maxdepth 1 -mindepth 1 "
"-type f -printf '%P\n'",
workdir="out",
hideStepIf=True,
decodeRC={0: SUCCESS, 1: SUCCESS},
property="files_to_upload"
)
)
build_factory.addStep(
steps.ShellCommand(
name="Upload files",
workdir="out",
doStepIf=has_files_to_upload,
hideStepIf=no_files_to_upload,
command=util.Interpolate(
"curl -T {%s} -u buildbot:%s "
"https://artifacts.bitmessage.at/%s/%s/",
files_to_upload,
util.Secret('artifact_upload'),
util.Property('jobname'),
util.Property('buildnumber'),
)
)
)