buildbot_multibuild/lib/worker_multibuild.py

270 lines
9.1 KiB
Python
Raw Normal View History

from os import getcwd, listdir
from os.path import exists, isfile, islink, join, realpath
2022-02-11 12:50:23 +01:00
import requests
import re
from subprocess import Popen, PIPE
2022-06-02 15:24:21 +02:00
from time import sleep
2022-02-11 12:50:23 +01:00
request_data = {
"project": "testproject",
"comments": "testcomment",
}
ty = "/change_hook/base"
path = ".buildbot"
2022-02-11 12:50:23 +01:00
dockerfile_extra_contents = {}
dockerfile_extra_contents['focal'] = """
# Buildbot
2022-04-26 14:38:46 +02:00
RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
python3-buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
2022-02-11 12:50:23 +01:00
python3-pip dumb-init curl openssh-client wget
# buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt
RUN update-ca-certificates
2022-02-11 12:50:23 +01:00
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS"
2022-02-11 12:50:23 +01:00
"""
dockerfile_extra_contents['bionic'] = """
# Buildbot
2022-04-26 14:38:46 +02:00
RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
2024-03-03 15:49:05 +01:00
git subversion python3-dev libffi-dev python3-setuptools \
2024-03-04 00:09:22 +01:00
python3-pip dumb-init curl openssh-client wget python3-wheel \
2024-03-06 00:57:44 +01:00
pkg-config rustc cargo
2024-03-04 00:55:46 +01:00
RUN pip3 install setuptools_rust
2024-03-06 00:14:31 +01:00
RUN pip3 install 'buildbot-worker==3.1.1' \
'cryptography==2.1.4' \
'twisted==17.9.0' \
'Automat==20.2.0' \
2024-03-06 00:14:31 +01:00
'pyopenssl==17.5.0'
2024-03-03 16:28:31 +01:00
RUN groupadd buildbot
RUN useradd -d /var/lib/buildbot -m -g buildbot buildbot
2024-03-03 15:49:05 +01:00
2022-02-11 12:50:23 +01:00
# buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt
RUN update-ca-certificates
2022-02-11 12:50:23 +01:00
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
2024-03-05 00:10:04 +01:00
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS"
2022-02-11 12:50:23 +01:00
"""
2022-04-26 14:38:46 +02:00
dockerfile_extra_contents['jammy'] = """
# Buildbot
RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
python3-buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
python3-pip dumb-init curl openssh-client wget
# buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt
RUN update-ca-certificates
2022-04-26 14:38:46 +02:00
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
2024-03-05 00:10:04 +01:00
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS"
2022-04-26 14:38:46 +02:00
"""
dockerfile_extra_contents['xenial'] = """
# Buildbot
RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
2024-03-03 15:49:05 +01:00
git subversion python3-dev libffi-dev python3-setuptools \
2022-05-08 05:21:54 +02:00
python3-pip curl openssh-client wget \
2024-03-04 00:09:22 +01:00
python-setuptools python-psutil libssl-dev python-dev libgmp-dev \
python-virtualenv python3-wheel pkg-config rustc cargo \
python3-openssl
2024-03-04 00:55:46 +01:00
RUN pip3 install setuptools_rust
RUN pip3 install 'buildbot-worker<3.2.0' 'cryptography<35.0.0'
2024-03-03 16:28:31 +01:00
RUN groupadd buildbot
RUN useradd -d /var/lib/buildbot -m -g buildbot buildbot
2024-03-03 15:49:05 +01:00
2022-05-08 05:21:54 +02:00
# dumb-init
RUN wget https://github.com/Yelp/dumb-init/releases/download/v1.2.5/dumb-init_1.2.5_amd64.deb
RUN dpkg -i dumb-init_*.deb && rm -f dumb-init_*.deb
2022-04-26 14:38:46 +02:00
# buildbot entrypoint
RUN [ -f /usr/local/bin/buildbot_entrypoint.sh ] || wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/xenial/entrypoint.sh
2022-04-26 14:38:46 +02:00
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt
RUN update-ca-certificates
2022-04-26 14:38:46 +02:00
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
2024-03-05 00:10:04 +01:00
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS"
2022-04-26 14:38:46 +02:00
"""
2022-06-02 15:24:21 +02:00
2022-03-11 16:51:16 +01:00
def get_secret():
with open("multibuild_parent_key.key", 'r') as f:
2022-03-11 16:51:16 +01:00
data = f.read()
return data
2022-02-11 12:50:23 +01:00
def list_jobs(directory=".buildbot"):
"""
list jobs found in a directory
"""
results = []
files = ["Dockerfile", "build.sh", "test.sh"]
if not exists(directory):
return results
2022-02-11 12:50:23 +01:00
for item in listdir(directory):
print("checking directory {}".format(item))
2022-02-11 12:50:23 +01:00
flag = False
for fname in files:
filepath = join(directory, item, fname)
# must exist
if not exists(filepath):
continue
# must be a file
if not isfile(filepath):
flag = True
break
# symlink OK as long as it points to files within the repo
if islink(filepath) \
and not realpath(filepath).startswith(getcwd()):
2022-02-11 12:50:23 +01:00
flag = True
break
if flag:
continue
if (exists(join(directory, item, 'Dockerfile'))
and exists(join(directory, item, 'build.sh'))) \
or exists(join(directory, item, 'test.sh')):
2022-02-11 12:50:23 +01:00
results.append(item)
return results
def get_revision(branch):
proc = Popen(["git", "rev-parse", branch], stdout=PIPE)
retval = proc.stdout.read().strip()
retval = retval.decode('utf-8')
return retval
def _get_dockerfile_contents(dockerfile):
2022-02-11 12:50:23 +01:00
"""
Read contents of a Dockerfile and add buildbot worker bootstrap
for a given os_codename
2022-02-11 12:50:23 +01:00
"""
os_codename = 'bionic'
2022-02-11 12:50:23 +01:00
res = ""
with open(dockerfile, "r") as file:
contents = file.readlines()
has_from = False
2022-02-11 12:50:23 +01:00
# accept any line containing FROM and RUN keywords
res = ""
inside_allowed_command = False
for line in contents:
m = re.match(r"(?m)^(FROM|RUN|ENV).*$", line)
if m:
2022-02-11 12:50:23 +01:00
inside_allowed_command = True
if m.group(1) == "FROM":
2022-04-26 14:38:46 +02:00
os_codename = m.group().split()[1].split(":")[1]
has_from = True
2022-02-11 12:50:23 +01:00
if inside_allowed_command:
res += line
ls = line.strip()
if not ls.endswith("\\"):
2022-02-11 12:50:23 +01:00
inside_allowed_command = False
if not has_from:
return None
2022-04-26 14:38:46 +02:00
try:
return res + dockerfile_extra_contents[os_codename]
except KeyError:
return None
2022-02-11 12:50:23 +01:00
def trigger_child_hooks(buildbotUrl: str, repository, branch, revision,
directory=".buildbot"):
2022-02-11 12:50:23 +01:00
request_url = buildbotUrl + ty
# List all jobs in the directory
jobs = list_jobs(directory)
2022-03-11 16:51:16 +01:00
request_headers = {
"Content-Type": "application/json",
2022-03-11 16:51:16 +01:00
"X-Multibuild-Trigger": get_secret(),
"Accept": "text/plain",
}
# revision = get_revision(branch)
2022-02-11 12:50:23 +01:00
# Check if build.sh or test.sh exists in each of the jobs
for job in jobs:
build_script_exists = False
test_script_exists = False
if exists(join(directory, job, "build.sh")):
build_script_exists = True
if exists(join(directory, job, "test.sh")):
test_script_exists = True
# make a post request
dockerfile = _get_dockerfile_contents(
join(directory, job, "Dockerfile")
)
if not dockerfile:
continue
request_data["branch"] = branch
request_data["revision"] = revision
2022-02-11 12:50:23 +01:00
request_data["properties"] = {
"dockerfile": dockerfile,
"build_script_available": build_script_exists,
"test_script_available": test_script_exists,
2022-02-11 12:50:23 +01:00
"jobname": job,
}
request_data["changes"] = {
"author": "buildbot_multibuild",
"repository": repository,
"project": job,
}
retval = requests.post(request_url, headers=request_headers,
json=request_data)
print("Triggered job for {} on {}: {}".format(job, request_url,
retval.text))
2022-06-02 15:24:21 +02:00
sleep(1)
if __name__ == "__main__":
# expect jobname, repository, branch, buildbotUrl from command line
import sys
if len(sys.argv) == 5:
buildbotUrl = sys.argv[1]
repository = sys.argv[2]
branch = sys.argv[3]
revision = sys.argv[4]
2022-03-11 16:51:16 +01:00
trigger_child_hooks(buildbotUrl, repository, branch, revision)
else:
sys.exit(
"Usage: python3 multibuild.py <buildbotUrl> <repository> <branch> <revision>"
)