buildbot_multibuild/multibuild.py

#!/usr/bin/python3
"""
Allow buildbot to run jobs dynamically defined a a project repo
Requires docker
"""

# TODO: change "ghcontext" in master.cfg to interpolate the job name
# TODO: write upload script
# TODO: write hook (perhaps the default hook is ok), authentication for hook
# TODO: write hook job, maybe also a dockerfile?
# TODO: what to do about non-docker jobs

from os import listdir
from os.path import isfile, join
from buildbot.plugins import steps, util
from buildbot.process.results import SUCCESS

from .lib.renderers import *


def find_artifacts(directory="out"):
    """
    find artifacts (any file) in a directory
    """
    for _ in listdir(directory):
        if not isfile(join(directory, _)):
            continue
        return join(directory, _)


def add_parent_step(build_factory):
    """
    Add a step to the parent build factory that will trigger the child hooks
    """

    build_factory.addStep(
        steps.ShellCommand(
            name="Execute worker script",
            command=[
                "python3",
                '/usr/local/bin/worker_multibuild.py',
                util.Property("buildboturl"),
                util.Property('repository'),
                util.Property('branch'),
                util.Property('revision')
            ],
        )
    )


def add_child_sh_steps(build_factory, directory=".buildbot"):
    """
    Add a step to the download, build and test factory
    """

    build_factory.addStep(
        steps.FileDownload(
            name="Upload keystore",
            workerdest="/var/lib/buildbot/keystore",
            mastersrc=util.Interpolate("keystore/%(prop:jobname)s.keystore")
            doStepIf=should_build_sign,
            hideStepIf=True,
            mode=0o600
        )
    )

    build_factory.addStep(
        steps.ShellCommand(
            name=util.Interpolate("build_%(prop:jobname)s"),
            command=util.Interpolate("%(kw:directory)s/"
                                     "%(prop:jobname)s/build.sh",
                                     directory=directory),
            env=build_env,
            doStepIf=should_build_sign,
            hideStepIf=isnt_build_script_available,
            timeout=7200,
        )
    )

    build_factory.addStep(
        steps.ShellCommand(
            name=util.Interpolate("build_and_sign_%(prop:jobname)s"),
            command=util.Interpolate("%(kw:directory)s/"
                                     "%(prop:jobname)s/build.sh",
                                     directory=directory),
            env=build_env,
            doStepIf=should_build_sign,
            hideStepIf=isnt_build_script_available,
            timeout=7200,
        )
    )

    build_factory.addStep(
        steps.ShellCommand(
            name="Delete keystore",
            command="rm -f /var/lib/buildbot/keystore",
            doStepIf=should_build_sign,
            hideStepIf=True,
        )
    )

    build_factory.addStep(
        steps.ShellCommand(
            name=util.Interpolate("test_%(prop:jobname)s"),
            command=util.Interpolate("%(kw:directory)s/"
                                     "%(prop:jobname)s/test.sh",
                                     directory=directory),
            haltOnFailure=False,
            flunkOnFailure=True,
            doStepIf=is_test_script_available,
            hideStepIf=isnt_test_script_available,
        )
    )

    build_factory.addStep(
        steps.SetPropertyFromCommand(
            name="Find files to upload",
            command="find . -maxdepth 1 -mindepth 1 "
                    "-type f -printf '%P\n'",
            workdir="out",
            hideStepIf=True,
            decodeRC={0: SUCCESS, 1: SUCCESS},
            property="files_to_upload"
        )
    )

    build_factory.addStep(
        steps.ShellCommand(
            name="Upload files",
            workdir="out",
            doStepIf=has_files_to_upload,
            hideStepIf=no_files_to_upload,
            command=util.Interpolate(
                "curl -T {%s} -u buildbot:%s "
                "https://artifacts.bitmessage.at/%s/%s/",
                files_to_upload,
                util.Secret('artifact_upload'),
                util.Property('jobname'),
                util.Property('buildnumber'),
            )
        )
    )