from os import getcwd, listdir from os.path import exists, isfile, islink, join, realpath import requests import re from subprocess import Popen, PIPE from time import sleep request_data = { "project": "testproject", "comments": "testcomment", } ty = "/change_hook/base" path = ".buildbot" dockerfile_extra_contents = {} dockerfile_extra_contents['focal'] = """ # Buildbot RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \ python3-buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \ python3-pip dumb-init curl openssh-client wget # buildbot entrypoint RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt RUN update-ca-certificates RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers USER buildbot ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS" """ dockerfile_extra_contents['bionic'] = """ # Buildbot RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \ git subversion python3-dev libffi-dev python3-setuptools \ python3-pip dumb-init curl openssh-client wget python3-wheel \ pkg-config rustc cargo RUN pip3 install setuptools_rust RUN pip3 install 'buildbot-worker==3.1.1' \ 'cryptography==2.1.4' \ 'twisted==17.9.0' \ 'Automat==20.2.0' \ 'pyopenssl==17.5.0' RUN groupadd buildbot RUN useradd -d /var/lib/buildbot -m -g buildbot buildbot # buildbot entrypoint RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt RUN update-ca-certificates RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers USER buildbot ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS" """ dockerfile_extra_contents['jammy'] = """ # Buildbot RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \ python3-buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \ python3-pip dumb-init curl openssh-client wget # buildbot entrypoint RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt RUN update-ca-certificates RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers USER buildbot ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS" """ dockerfile_extra_contents['xenial'] = """ # Buildbot RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \ git subversion python3-dev libffi-dev python3-setuptools \ python3-pip curl openssh-client wget \ python-setuptools python-psutil libssl-dev python-dev libgmp-dev \ python-virtualenv python3-wheel pkg-config rustc cargo \ python3-openssl RUN pip3 install setuptools_rust RUN pip3 install 'buildbot-worker<3.2.0' 'cryptography<35.0.0' RUN groupadd buildbot RUN useradd -d /var/lib/buildbot -m -g buildbot buildbot # dumb-init RUN wget https://github.com/Yelp/dumb-init/releases/download/v1.2.5/dumb-init_1.2.5_amd64.deb RUN dpkg -i dumb-init_*.deb && rm -f dumb-init_*.deb # buildbot entrypoint RUN [ -f /usr/local/bin/buildbot_entrypoint.sh ] || wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/xenial/entrypoint.sh RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt RUN update-ca-certificates RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers USER buildbot ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS" """ def get_secret(): with open("multibuild_parent_key.key", 'r') as f: data = f.read() return data def list_jobs(directory=".buildbot"): """ list jobs found in a directory """ results = [] files = ["Dockerfile", "build.sh", "test.sh"] if not exists(directory): return results for item in listdir(directory): print("checking directory {}".format(item)) flag = False for fname in files: filepath = join(directory, item, fname) # must exist if not exists(filepath): continue # must be a file if not isfile(filepath): flag = True break # symlink OK as long as it points to files within the repo if islink(filepath) \ and not realpath(filepath).startswith(getcwd()): flag = True break if flag: continue if (exists(join(directory, item, 'Dockerfile')) and exists(join(directory, item, 'build.sh'))) \ or exists(join(directory, item, 'test.sh')): results.append(item) return results def get_revision(branch): proc = Popen(["git", "rev-parse", branch], stdout=PIPE) retval = proc.stdout.read().strip() retval = retval.decode('utf-8') return retval def _get_dockerfile_contents(dockerfile): """ Read contents of a Dockerfile and add buildbot worker bootstrap for a given os_codename """ os_codename = 'bionic' res = "" with open(dockerfile, "r") as file: contents = file.readlines() has_from = False # accept any line containing FROM and RUN keywords res = "" inside_allowed_command = False for line in contents: m = re.match(r"(?m)^(FROM|RUN|ENV).*$", line) if m: inside_allowed_command = True if m.group(1) == "FROM": os_codename = m.group().split()[1].split(":")[1] has_from = True if inside_allowed_command: res += line ls = line.strip() if not ls.endswith("\\"): inside_allowed_command = False if not has_from: return None try: return res + dockerfile_extra_contents[os_codename] except KeyError: return None def trigger_child_hooks(buildbotUrl: str, repository, branch, revision, directory=".buildbot"): request_url = buildbotUrl + ty # List all jobs in the directory jobs = list_jobs(directory) request_headers = { "Content-Type": "application/json", "X-Multibuild-Trigger": get_secret(), "Accept": "text/plain", } # revision = get_revision(branch) # Check if build.sh or test.sh exists in each of the jobs for job in jobs: build_script_exists = False test_script_exists = False if exists(join(directory, job, "build.sh")): build_script_exists = True if exists(join(directory, job, "test.sh")): test_script_exists = True # make a post request dockerfile = _get_dockerfile_contents( join(directory, job, "Dockerfile") ) if not dockerfile: continue request_data["branch"] = branch request_data["revision"] = revision request_data["properties"] = { "dockerfile": dockerfile, "build_script_available": build_script_exists, "test_script_available": test_script_exists, "jobname": job, } request_data["changes"] = { "author": "buildbot_multibuild", "repository": repository, "project": job, } retval = requests.post(request_url, headers=request_headers, json=request_data) print("Triggered job for {} on {}: {}".format(job, request_url, retval.text)) sleep(1) if __name__ == "__main__": # expect jobname, repository, branch, buildbotUrl from command line import sys if len(sys.argv) == 5: buildbotUrl = sys.argv[1] repository = sys.argv[2] branch = sys.argv[3] revision = sys.argv[4] trigger_child_hooks(buildbotUrl, repository, branch, revision) else: sys.exit( "Usage: python3 multibuild.py " )