buildbot_multibuild/lib/worker_multibuild.py

from os import getcwd, listdir
from os.path import exists, isfile, islink, join, realpath
import requests
import re
from subprocess import Popen, PIPE
from time import sleep


request_data = {
    "project": "testproject",
    "comments": "testcomment",
}

ty = "/change_hook/base"
path = ".buildbot"
dockerfile_extra_contents = {}
dockerfile_extra_contents['focal'] = """

# Buildbot
RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
    python3-buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
    python3-pip dumb-init curl openssh-client wget

# buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt
RUN update-ca-certificates

RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers

USER buildbot

ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS"

"""

dockerfile_extra_contents['bionic'] = """

# Buildbot
RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
    git subversion python3-dev libffi-dev python3-setuptools \
    python3-pip dumb-init curl openssh-client wget python3-wheel \
    pkg-config rustc cargo
RUN pip3 install setuptools_rust
RUN pip3 install 'buildbot-worker==3.1.1' \
                 'cryptography==2.1.4' \
                 'twisted==17.9.0' \
                 'pyopenssl==17.5.0'
RUN groupadd buildbot
RUN useradd -d /var/lib/buildbot -m -g buildbot buildbot

# buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt
RUN update-ca-certificates

RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers

USER buildbot

ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS"

"""

dockerfile_extra_contents['jammy'] = """

# Buildbot
RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
    python3-buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
    python3-pip dumb-init curl openssh-client wget

# buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt
RUN update-ca-certificates

RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers

USER buildbot

ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS"

"""

dockerfile_extra_contents['xenial'] = """

# Buildbot
RUN apt-get update -y && apt-get install -yq --no-install-suggests --no-install-recommends \
    git subversion python3-dev libffi-dev python3-setuptools \
    python3-pip curl openssh-client wget \
    python-setuptools python-psutil libssl-dev python-dev libgmp-dev \
    python-virtualenv python3-wheel pkg-config rustc cargo \
    python3-openssl
RUN pip3 install setuptools_rust
RUN pip3 install 'buildbot-worker<3.2.0' 'cryptography<35.0.0'
RUN groupadd buildbot
RUN useradd -d /var/lib/buildbot -m -g buildbot buildbot

# dumb-init
RUN wget https://github.com/Yelp/dumb-init/releases/download/v1.2.5/dumb-init_1.2.5_amd64.deb
RUN dpkg -i dumb-init_*.deb && rm -f dumb-init_*.deb

# buildbot entrypoint
RUN [ -f /usr/local/bin/buildbot_entrypoint.sh ] || wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/xenial/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN wget -O /usr/local/share/ca-certificates/buildbot-ca.crt https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/buildbot-ca.crt
RUN update-ca-certificates

RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers

USER buildbot

ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS"

"""


def get_secret():
    with open("multibuild_parent_key.key", 'r') as f:
        data = f.read()
    return data


def list_jobs(directory=".buildbot"):
    """
    list jobs found in a directory
    """
    results = []
    files = ["Dockerfile", "build.sh", "test.sh"]
    if not exists(directory):
        return results
    for item in listdir(directory):
        print("checking directory {}".format(item))
        flag = False
        for fname in files:
            filepath = join(directory, item, fname)
            # must exist
            if not exists(filepath):
                continue
            # must be a file
            if not isfile(filepath):
                flag = True
                break
            # symlink OK as long as it points to files within the repo
            if islink(filepath) \
                    and not realpath(filepath).startswith(getcwd()):
                flag = True
                break
        if flag:
            continue
        if (exists(join(directory, item, 'Dockerfile'))
                and exists(join(directory, item, 'build.sh'))) \
                or exists(join(directory, item, 'test.sh')):
            results.append(item)
    return results


def get_revision(branch):
    proc = Popen(["git", "rev-parse", branch], stdout=PIPE)
    retval = proc.stdout.read().strip()
    retval = retval.decode('utf-8')
    return retval


def _get_dockerfile_contents(dockerfile):
    """
    Read contents of a Dockerfile and add buildbot worker bootstrap
    for a given os_codename
    """
    os_codename = 'bionic'
    res = ""
    with open(dockerfile, "r") as file:
        contents = file.readlines()
        has_from = False
        # accept any line containing FROM and RUN keywords
        res = ""
        inside_allowed_command = False
        for line in contents:
            m = re.match(r"(?m)^(FROM|RUN|ENV).*$", line)
            if m:
                inside_allowed_command = True
                if m.group(1) == "FROM":
                    os_codename = m.group().split()[1].split(":")[1]
                    has_from = True
            if inside_allowed_command:
                res += line
                ls = line.strip()
                if not ls.endswith("\\"):
                    inside_allowed_command = False

    if not has_from:
        return None

    try:
        return res + dockerfile_extra_contents[os_codename]
    except KeyError:
        return None


def trigger_child_hooks(buildbotUrl: str, repository, branch, revision,
                        directory=".buildbot"):
    request_url = buildbotUrl + ty
    # List all jobs in the directory
    jobs = list_jobs(directory)
    request_headers = {
        "Content-Type": "application/json",
        "X-Multibuild-Trigger": get_secret(),
        "Accept": "text/plain",
    }
    # revision = get_revision(branch)

    # Check if build.sh or test.sh exists in each of the jobs
    for job in jobs:
        build_script_exists = False
        test_script_exists = False
        if exists(join(directory, job, "build.sh")):
            build_script_exists = True
        if exists(join(directory, job, "test.sh")):
            test_script_exists = True

        # make a post request
        dockerfile = _get_dockerfile_contents(
            join(directory, job, "Dockerfile")
        )

        if not dockerfile:
            continue

        request_data["branch"] = branch
        request_data["revision"] = revision

        request_data["properties"] = {
            "dockerfile": dockerfile,
            "build_script_available": build_script_exists,
            "test_script_available": test_script_exists,
            "jobname": job,
        }
        request_data["changes"] = {
            "author": "buildbot_multibuild",
            "repository": repository,
            "project": job,
        }

        retval = requests.post(request_url, headers=request_headers,
                               json=request_data)
        print("Triggered job for {} on {}: {}".format(job, request_url,
                                                      retval.text))
        sleep(1)


if __name__ == "__main__":
    # expect jobname, repository, branch, buildbotUrl from command line
    import sys

    if len(sys.argv) == 5:
        buildbotUrl = sys.argv[1]
        repository = sys.argv[2]
        branch = sys.argv[3]
        revision = sys.argv[4]

        trigger_child_hooks(buildbotUrl, repository, branch, revision)
    else:
        sys.exit(
            "Usage: python3 multibuild.py <buildbotUrl> <repository> <branch> <revision>"
        )