buildbot_multibuild/multibuild.py

#!/usr/bin/python3
"""
Allow buildbot to run jobs dynamically defined a a project repo
Requires docker
"""

# TODO: change "ghcontext" in master.cfg to interpolate the job name
# TODO: write upload script
# TODO: write hook (perhaps the default hook is ok), authentication for hook
# TODO: write hook job, maybe also a dockerfile?
# TODO: what to do about non-docker jobs

from os import walk
from os.path import exists, isfile, join, listdir
import requests
import re
from buildbot.plugins import steps, util

from .lib.renderers import *

ty = "/change_hook/base"
request_headers = {
    "Content-Type": "application/x-www-form-urlencoded",
    "Accept": "text/plain",
}
request_data = {
    "project": "testproject",
    "comments": "testcomment",
}

dockerfile_extra_contents_focal = """

# Buildbot
RUN apt-get install -yq --no-install-suggests --no-install-recommends \
    buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
    python3-pip dumb-init curl openssh-client wget

# buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh

RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers

USER buildbot

ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"

"""

dockerfile_extra_contents_bionic = """

# Buildbot
RUN apt-get install -yq --no-install-suggests --no-install-recommends \
    buildbot-slave git subversion python3-dev libffi-dev python3-setuptools \
    python3-pip dumb-init curl openssh-client wget

# buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh

RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers

USER buildbot

ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"

"""


def list_jobs(directory=".buildbot"):
    """
    list jobs found in a directory
    """
    results = []
    for _ in next(walk(directory))[1]:
        if exists(join(directory, _, "Dockerfile")) and (
            exists(join(directory, _, "build.sh"))
            or exists(join(directory, _, "test.sh"))
        ):
            results.append(_)

    return results


def find_artifacts(directory="out"):
    """
    find artifacts (any file) in a directory
    """
    for _ in listdir(directory):
        if not isfile(join(directory, _)):
            continue
        return join(directory, _)


def get_dockerfile_contents(path, os_codename):
    """
    Read contents of a Dockerfile and add extra contents for the given os_codename
    """
    with open(path, "r") as file:
        contents = file.read()
        # remove any line containing CMD or ENTRYFILE keywords
        re.sub(r"(?m)^(CMD|ENTRYFILE).*$", "", contents)

    return contents + {
        "focal": dockerfile_extra_contents_focal,
        "bionic": dockerfile_extra_contents_bionic,
    }[os_codename]


def trigger_child_hooks(buildbotUrl: str, os_codename: str, directory=".buildbot"):
    request_url = buildbotUrl + ty

    # List all jobs in the directory
    jobs = list_jobs(directory)

    # Check if build.sh or test.sh exists in each of the jobs
    for job in jobs:
        build_script_exists = False
        test_script_exists = False
        if exists(join(directory, job, "build.sh")):
            build_script_exists = True
        if exists(join(directory, job, "test.sh")):
            test_script_exists = True

        # make a post request
        request_data["properties"] = {
            "dockerfile": get_dockerfile_contents(
                join(directory, job, "Dockerfile"), os_codename
            ),
            "build_script_available": is_build_script_available(build_script_exists),
            "test_script_available": is_test_script_available(test_script_exists),
        }
        requests.post(request_url, headers=request_headers, data=request_data)


def add_parent_step(build_factory, jobname, repository, branch):
    """
    Add a step to the parent build factory that will trigger the child hooks
    """
    build_factory.addStep(
        steps.SetPropertyFromCommand(
            name="Get OS codename",
            command="grep ^VERSION_CODENAME= /etc/os-release | cut -d= -f2",
            property="os_codename",
        )
    )

    build_factory.addStep(
        steps.ShellCommand(
            name="Execute multibuild script",
            command=[
                "python",
                "multibuild.py",
                jobname,
                repository,
                branch,
                "https://buildbot.bitmessage.org",
                util.Interpolate("%(prop:os_codename)s"),
            ],
        )
    )


def add_child_build_sh_step(build_factory, job, directory=".buildbot"):
    """
    Add a step to the build factory
    """
    build_factory.addStep(
        steps.ShellCommand(
            name="build_" + job,
            command=["bash", "-c", join(directory, job, "build.sh")],
            doStepIf=is_build_script_available,
            hideStepIf=isnt_build_script_available,
        )
    )


def add_child_test_sh_step(build_factory, job, directory=".buildbot"):
    """
    Add a step to the build factory
    """
    build_factory.addStep(
        steps.ShellCommand(
            name="test_" + job,
            command=["bash", "-c", join(directory, job, "test.sh")],
            doStepIf=is_test_script_available,
            hideStepIf=isnt_test_script_available,
        )
    )


if __name__ == "__main__":
    # expect jobname, repository, branch, buildbotUrl, os_codename from command line args
    import sys

    if len(sys.argv) == 6:
        jobname = sys.argv[1]
        repository = sys.argv[2]
        branch = sys.argv[3]
        buildbotUrl = sys.argv[4]
        os_codename = sys.argv[5]

        # add these into the request_data
        request_data["jobname"] = jobname
        request_data["repository"] = repository
        request_data["branch"] = branch

        trigger_child_hooks(buildbotUrl, os_codename)
    else:
        print(
            "Usage: python3 multibuild.py <jobname> <repository> <branch> <buildbotUrl> <os_codename>"
        )