269 lines
9.1 KiB
Python
269 lines
9.1 KiB
Python
from os import getcwd, listdir
|
|
from os.path import exists, isfile, islink, join, realpath
|
|
import requests
|
|
import re
|
|
from subprocess import Popen, PIPE
|
|
from time import sleep
|
|
|
|
|
|
request_data = {
|
|
"project": "testproject",
|
|
"comments": "testcomment",
|
|
}
|
|
|
|
ty = "/change_hook/base"
|
|
path = ".buildbot"
|
|
dockerfile_extra_contents = {}
|
|
dockerfile_extra_contents['focal'] = """
|
|
|
|
# Buildbot
|
|
RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
|
|
python3-buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
|
|
python3-pip dumb-init curl openssh-client wget
|
|
|
|
# buildbot entrypoint
|
|
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
|
|
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
|
|
RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt
|
|
RUN update-ca-certificates
|
|
|
|
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
|
|
|
|
USER buildbot
|
|
|
|
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS"
|
|
|
|
"""
|
|
|
|
dockerfile_extra_contents['bionic'] = """
|
|
|
|
# Buildbot
|
|
RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
|
|
git subversion python3-dev libffi-dev python3-setuptools \
|
|
python3-pip dumb-init curl openssh-client wget python3-wheel \
|
|
pkg-config rustc cargo
|
|
RUN pip3 install setuptools_rust
|
|
RUN pip3 install 'buildbot-worker==3.1.1' \
|
|
'cryptography==2.1.4' \
|
|
'twisted==17.9.0' \
|
|
'pyopenssl==17.5.0'
|
|
RUN groupadd buildbot
|
|
RUN useradd -d /var/lib/buildbot -m -g buildbot buildbot
|
|
|
|
# buildbot entrypoint
|
|
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
|
|
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
|
|
RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt
|
|
RUN update-ca-certificates
|
|
|
|
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
|
|
|
|
USER buildbot
|
|
|
|
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS"
|
|
|
|
"""
|
|
|
|
dockerfile_extra_contents['jammy'] = """
|
|
|
|
# Buildbot
|
|
RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
|
|
python3-buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
|
|
python3-pip dumb-init curl openssh-client wget
|
|
|
|
# buildbot entrypoint
|
|
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
|
|
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
|
|
RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt
|
|
RUN update-ca-certificates
|
|
|
|
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
|
|
|
|
USER buildbot
|
|
|
|
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS"
|
|
|
|
"""
|
|
|
|
dockerfile_extra_contents['xenial'] = """
|
|
|
|
# Buildbot
|
|
RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
|
|
git subversion python3-dev libffi-dev python3-setuptools \
|
|
python3-pip curl openssh-client wget \
|
|
python-setuptools python-psutil libssl-dev python-dev libgmp-dev \
|
|
python-virtualenv python3-wheel pkg-config rustc cargo \
|
|
python3-openssl
|
|
RUN pip3 install setuptools_rust
|
|
RUN pip3 install 'buildbot-worker<3.2.0' 'cryptography<35.0.0'
|
|
RUN groupadd buildbot
|
|
RUN useradd -d /var/lib/buildbot -m -g buildbot buildbot
|
|
|
|
# dumb-init
|
|
RUN wget https://github.com/Yelp/dumb-init/releases/download/v1.2.5/dumb-init_1.2.5_amd64.deb
|
|
RUN dpkg -i dumb-init_*.deb && rm -f dumb-init_*.deb
|
|
|
|
# buildbot entrypoint
|
|
RUN [ -f /usr/local/bin/buildbot_entrypoint.sh ] || wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/xenial/entrypoint.sh
|
|
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
|
|
RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt
|
|
RUN update-ca-certificates
|
|
|
|
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
|
|
|
|
USER buildbot
|
|
|
|
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS"
|
|
|
|
"""
|
|
|
|
|
|
def get_secret():
|
|
with open("multibuild_parent_key.key", 'r') as f:
|
|
data = f.read()
|
|
return data
|
|
|
|
|
|
def list_jobs(directory=".buildbot"):
|
|
"""
|
|
list jobs found in a directory
|
|
"""
|
|
results = []
|
|
files = ["Dockerfile", "build.sh", "test.sh"]
|
|
if not exists(directory):
|
|
return results
|
|
for item in listdir(directory):
|
|
print("checking directory {}".format(item))
|
|
flag = False
|
|
for fname in files:
|
|
filepath = join(directory, item, fname)
|
|
# must exist
|
|
if not exists(filepath):
|
|
continue
|
|
# must be a file
|
|
if not isfile(filepath):
|
|
flag = True
|
|
break
|
|
# symlink OK as long as it points to files within the repo
|
|
if islink(filepath) \
|
|
and not realpath(filepath).startswith(getcwd()):
|
|
flag = True
|
|
break
|
|
if flag:
|
|
continue
|
|
if (exists(join(directory, item, 'Dockerfile'))
|
|
and exists(join(directory, item, 'build.sh'))) \
|
|
or exists(join(directory, item, 'test.sh')):
|
|
results.append(item)
|
|
return results
|
|
|
|
|
|
def get_revision(branch):
|
|
proc = Popen(["git", "rev-parse", branch], stdout=PIPE)
|
|
retval = proc.stdout.read().strip()
|
|
retval = retval.decode('utf-8')
|
|
return retval
|
|
|
|
|
|
def _get_dockerfile_contents(dockerfile):
|
|
"""
|
|
Read contents of a Dockerfile and add buildbot worker bootstrap
|
|
for a given os_codename
|
|
"""
|
|
os_codename = 'bionic'
|
|
res = ""
|
|
with open(dockerfile, "r") as file:
|
|
contents = file.readlines()
|
|
has_from = False
|
|
# accept any line containing FROM and RUN keywords
|
|
res = ""
|
|
inside_allowed_command = False
|
|
for line in contents:
|
|
m = re.match(r"(?m)^(FROM|RUN|ENV).*$", line)
|
|
if m:
|
|
inside_allowed_command = True
|
|
if m.group(1) == "FROM":
|
|
os_codename = m.group().split()[1].split(":")[1]
|
|
has_from = True
|
|
if inside_allowed_command:
|
|
res += line
|
|
ls = line.strip()
|
|
if not ls.endswith("\\"):
|
|
inside_allowed_command = False
|
|
|
|
if not has_from:
|
|
return None
|
|
|
|
try:
|
|
return res + dockerfile_extra_contents[os_codename]
|
|
except KeyError:
|
|
return None
|
|
|
|
|
|
def trigger_child_hooks(buildbotUrl: str, repository, branch, revision,
|
|
directory=".buildbot"):
|
|
request_url = buildbotUrl + ty
|
|
# List all jobs in the directory
|
|
jobs = list_jobs(directory)
|
|
request_headers = {
|
|
"Content-Type": "application/json",
|
|
"X-Multibuild-Trigger": get_secret(),
|
|
"Accept": "text/plain",
|
|
}
|
|
# revision = get_revision(branch)
|
|
|
|
# Check if build.sh or test.sh exists in each of the jobs
|
|
for job in jobs:
|
|
build_script_exists = False
|
|
test_script_exists = False
|
|
if exists(join(directory, job, "build.sh")):
|
|
build_script_exists = True
|
|
if exists(join(directory, job, "test.sh")):
|
|
test_script_exists = True
|
|
|
|
# make a post request
|
|
dockerfile = _get_dockerfile_contents(
|
|
join(directory, job, "Dockerfile")
|
|
)
|
|
|
|
if not dockerfile:
|
|
continue
|
|
|
|
request_data["branch"] = branch
|
|
request_data["revision"] = revision
|
|
|
|
request_data["properties"] = {
|
|
"dockerfile": dockerfile,
|
|
"build_script_available": build_script_exists,
|
|
"test_script_available": test_script_exists,
|
|
"jobname": job,
|
|
}
|
|
request_data["changes"] = {
|
|
"author": "buildbot_multibuild",
|
|
"repository": repository,
|
|
"project": job,
|
|
}
|
|
|
|
retval = requests.post(request_url, headers=request_headers,
|
|
json=request_data)
|
|
print("Triggered job for {} on {}: {}".format(job, request_url,
|
|
retval.text))
|
|
sleep(1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
# expect jobname, repository, branch, buildbotUrl from command line
|
|
import sys
|
|
|
|
if len(sys.argv) == 5:
|
|
buildbotUrl = sys.argv[1]
|
|
repository = sys.argv[2]
|
|
branch = sys.argv[3]
|
|
revision = sys.argv[4]
|
|
|
|
trigger_child_hooks(buildbotUrl, repository, branch, revision)
|
|
else:
|
|
sys.exit(
|
|
"Usage: python3 multibuild.py <buildbotUrl> <repository> <branch> <revision>"
|
|
)
|