buildbot_multibuild/lib/worker_multibuild.py

265 lines
8.9 KiB
Python

from os import getcwd, listdir
from os.path import exists, isfile, islink, join, realpath
import requests
import re
from subprocess import Popen, PIPE
from time import sleep
request_data = {
"project": "testproject",
"comments": "testcomment",
}
ty = "/change_hook/base"
path = ".buildbot"
dockerfile_extra_contents = {}
dockerfile_extra_contents['focal'] = """
# Buildbot
RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
python3-buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
python3-pip dumb-init curl openssh-client wget
# buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt
RUN update-ca-certificates
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS"
"""
dockerfile_extra_contents['bionic'] = """
# Buildbot
RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
git subversion python3-dev libffi-dev python3-setuptools \
python3-pip dumb-init curl openssh-client wget python3-wheel \
pkg-config rustc cargo
RUN pip3 install setuptools_rust
RUN pip3 install buildbot-worker pyOpenSSL
RUN groupadd buildbot
RUN useradd -d /var/lib/buildbot -m -g buildbot buildbot
# buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt
RUN update-ca-certificates
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS"
"""
dockerfile_extra_contents['jammy'] = """
# Buildbot
RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
python3-buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
python3-pip dumb-init curl openssh-client wget
# buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt
RUN update-ca-certificates
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS"
"""
dockerfile_extra_contents['xenial'] = """
# Buildbot
RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
git subversion python3-dev libffi-dev python3-setuptools \
python3-pip curl openssh-client wget \
python-setuptools python-psutil libssl-dev python-dev libgmp-dev \
python-virtualenv python3-wheel pkg-config rustc cargo
RUN pip3 install setuptools_rust
RUN pip3 install buildbot-worker pyOpenSSL
RUN groupadd buildbot
RUN useradd -d /var/lib/buildbot -m -g buildbot buildbot
# dumb-init
RUN wget https://github.com/Yelp/dumb-init/releases/download/v1.2.5/dumb-init_1.2.5_amd64.deb
RUN dpkg -i dumb-init_*.deb && rm -f dumb-init_*.deb
# buildbot entrypoint
RUN [ -f /usr/local/bin/buildbot_entrypoint.sh ] || wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/xenial/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt
RUN update-ca-certificates
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS"
"""
def get_secret():
with open("multibuild_parent_key.key", 'r') as f:
data = f.read()
return data
def list_jobs(directory=".buildbot"):
"""
list jobs found in a directory
"""
results = []
files = ["Dockerfile", "build.sh", "test.sh"]
if not exists(directory):
return results
for item in listdir(directory):
print("checking directory {}".format(item))
flag = False
for fname in files:
filepath = join(directory, item, fname)
# must exist
if not exists(filepath):
continue
# must be a file
if not isfile(filepath):
flag = True
break
# symlink OK as long as it points to files within the repo
if islink(filepath) \
and not realpath(filepath).startswith(getcwd()):
flag = True
break
if flag:
continue
if (exists(join(directory, item, 'Dockerfile'))
and exists(join(directory, item, 'build.sh'))) \
or exists(join(directory, item, 'test.sh')):
results.append(item)
return results
def get_revision(branch):
proc = Popen(["git", "rev-parse", branch], stdout=PIPE)
retval = proc.stdout.read().strip()
retval = retval.decode('utf-8')
return retval
def _get_dockerfile_contents(dockerfile):
"""
Read contents of a Dockerfile and add buildbot worker bootstrap
for a given os_codename
"""
os_codename = 'bionic'
res = ""
with open(dockerfile, "r") as file:
contents = file.readlines()
has_from = False
# accept any line containing FROM and RUN keywords
res = ""
inside_allowed_command = False
for line in contents:
m = re.match(r"(?m)^(FROM|RUN|ENV).*$", line)
if m:
inside_allowed_command = True
if m.group(1) == "FROM":
os_codename = m.group().split()[1].split(":")[1]
has_from = True
if inside_allowed_command:
res += line
ls = line.strip()
if not ls.endswith("\\"):
inside_allowed_command = False
if not has_from:
return None
try:
return res + dockerfile_extra_contents[os_codename]
except KeyError:
return None
def trigger_child_hooks(buildbotUrl: str, repository, branch, revision,
directory=".buildbot"):
request_url = buildbotUrl + ty
# List all jobs in the directory
jobs = list_jobs(directory)
request_headers = {
"Content-Type": "application/json",
"X-Multibuild-Trigger": get_secret(),
"Accept": "text/plain",
}
# revision = get_revision(branch)
# Check if build.sh or test.sh exists in each of the jobs
for job in jobs:
build_script_exists = False
test_script_exists = False
if exists(join(directory, job, "build.sh")):
build_script_exists = True
if exists(join(directory, job, "test.sh")):
test_script_exists = True
# make a post request
dockerfile = _get_dockerfile_contents(
join(directory, job, "Dockerfile")
)
if not dockerfile:
continue
request_data["branch"] = branch
request_data["revision"] = revision
request_data["properties"] = {
"dockerfile": dockerfile,
"build_script_available": build_script_exists,
"test_script_available": test_script_exists,
"jobname": job,
}
request_data["changes"] = {
"author": "buildbot_multibuild",
"repository": repository,
"project": job,
}
retval = requests.post(request_url, headers=request_headers,
json=request_data)
print("Triggered job for {} on {}: {}".format(job, request_url,
retval.text))
sleep(1)
if __name__ == "__main__":
# expect jobname, repository, branch, buildbotUrl from command line
import sys
if len(sys.argv) == 5:
buildbotUrl = sys.argv[1]
repository = sys.argv[2]
branch = sys.argv[3]
revision = sys.argv[4]
trigger_child_hooks(buildbotUrl, repository, branch, revision)
else:
sys.exit(
"Usage: python3 multibuild.py <buildbotUrl> <repository> <branch> <revision>"
)