change to call docker function

This commit is contained in:
Muzahid 2022-02-11 17:20:23 +05:30
parent a337147f43
commit c151641ebe
Signed by: cis-muzahid
GPG Key ID: 1DC85E7D3AB613EA
2 changed files with 142 additions and 123 deletions

120
lib/worker_multibuild.py Normal file
View File

@ -0,0 +1,120 @@
from os import listdir, walk
from os.path import exists, isfile, join
import requests
import re
request_data = {
"project": "testproject",
"comments": "testcomment",
}
request_headers = {
"Content-Type": "application/x-www-form-urlencoded",
"Accept": "text/plain",
}
ty = "/change_hook/base"
path =".buildbot"
dockerfile_extra_contents = {}
dockerfile_extra_contents['focal'] = """
# Buildbot
RUN apt-get install -yq --no-install-suggests --no-install-recommends \
buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
python3-pip dumb-init curl openssh-client wget
# buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
"""
dockerfile_extra_contents['bionic'] = """
# Buildbot
RUN apt-get install -yq --no-install-suggests --no-install-recommends \
buildbot-slave git subversion python3-dev libffi-dev python3-setuptools \
python3-pip dumb-init curl openssh-client wget
# buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
"""
def list_jobs(directory=".buildbot"):
"""
list jobs found in a directory
"""
results = []
for _ in next(walk(directory))[1]:
if exists(join(directory, _, "Dockerfile")) and (
exists(join(directory, _, "build.sh"))
or exists(join(directory, _, "test.sh"))
):
results.append(_)
return results
def _get_dockerfile_contents(jobname, os_codename='bionic'):
"""
Read contents of a Dockerfile and add extra contents for the given os_codename
"""
res = ""
with open(join(path + jobname), "r") as file:
contents = file.read()
# accept any line containing FROM and RUN keywords
res = ""
inside_allowed_command = False
for line in contents:
if re.match(r"(?m)^(FROM|RUN).*$", line):
inside_allowed_command = True
if inside_allowed_command:
res += line
l = line.strip()
if not l.endswith("\\"):
inside_allowed_command = False
return res + dockerfile_extra_contents[os_codename]
def trigger_child_hooks(buildbotUrl: str, os_codename: str, repository, branch, jobname, directory=".buildbot", is_build_script_available, is_test_script_available):
request_url = buildbotUrl + ty
# List all jobs in the directory
jobs = list_jobs(directory)
# Check if build.sh or test.sh exists in each of the jobs
for job in jobs:
build_script_exists = False
test_script_exists = False
if exists(join(directory, job, "build.sh")):
build_script_exists = True
if exists(join(directory, job, "test.sh")):
test_script_exists = True
# make a post request
request_data["properties"] = {
"dockerfile": _get_dockerfile_contents(
join(directory, job, "Dockerfile"), os_codename
),
"build_script_available": is_build_script_available(build_script_exists),
"test_script_available": is_test_script_available(test_script_exists),
"repository": repository,
"branch": branch,
"jobname": jobname,
}
requests.post(request_url, headers=request_headers, data=request_data)

View File

@ -10,7 +10,7 @@ Requires docker
# TODO: write hook job, maybe also a dockerfile? # TODO: write hook job, maybe also a dockerfile?
# TODO: what to do about non-docker jobs # TODO: what to do about non-docker jobs
from os import listdir, walk from os import listdir, walk, getenv
from os.path import exists, isfile, join from os.path import exists, isfile, join
import requests import requests
import re import re
@ -18,70 +18,6 @@ from buildbot.plugins import steps, util
from .lib.renderers import * from .lib.renderers import *
ty = "/change_hook/base"
request_headers = {
"Content-Type": "application/x-www-form-urlencoded",
"Accept": "text/plain",
}
request_data = {
"project": "testproject",
"comments": "testcomment",
}
dockerfile_extra_contents_focal = """
# Buildbot
RUN apt-get install -yq --no-install-suggests --no-install-recommends \
buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
python3-pip dumb-init curl openssh-client wget
# buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
"""
dockerfile_extra_contents_bionic = """
# Buildbot
RUN apt-get install -yq --no-install-suggests --no-install-recommends \
buildbot-slave git subversion python3-dev libffi-dev python3-setuptools \
python3-pip dumb-init curl openssh-client wget
# buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
"""
path =".buildbot"
def list_jobs(directory=".buildbot"):
"""
list jobs found in a directory
"""
results = []
for _ in next(walk(directory))[1]:
if exists(join(directory, _, "Dockerfile")) and (
exists(join(directory, _, "build.sh"))
or exists(join(directory, _, "test.sh"))
):
results.append(_)
return results
def find_artifacts(directory="out"): def find_artifacts(directory="out"):
""" """
@ -93,61 +29,28 @@ def find_artifacts(directory="out"):
return join(directory, _) return join(directory, _)
def get_dockerfile_contents(props): # @util.renderer
""" # def get_dockerfile_contents(props):
Read contents of a Dockerfile and add extra contents for the given os_codename # return _get_dockerfile_contents(props.getProperty('jobname', default=None), util.Interpolate("%(prop:os_codename)s"))
"""
with open(join(path + props.getProperty('jobname', default=None)), "r") as file:
contents = file.read()
# remove any line containing CMD or ENTRYFILE keywords
re.sub(r"(?m)^(CMD|ENTRYFILE).*$", "", contents)
return contents + {
"focal": dockerfile_extra_contents_focal,
"bionic": dockerfile_extra_contents_bionic,
}[util.Interpolate("%(prop:os_codename)s")]
def trigger_child_hooks(buildbotUrl: str, os_codename: str, repository, branch, jobname, directory=".buildbot"):
request_url = buildbotUrl + ty
# List all jobs in the directory
jobs = list_jobs(directory)
# Check if build.sh or test.sh exists in each of the jobs
for job in jobs:
build_script_exists = False
test_script_exists = False
if exists(join(directory, job, "build.sh")):
build_script_exists = True
if exists(join(directory, job, "test.sh")):
test_script_exists = True
# make a post request
request_data["properties"] = {
"dockerfile": get_dockerfile_contents(
join(directory, job, "Dockerfile"), os_codename
),
"build_script_available": is_build_script_available(build_script_exists),
"test_script_available": is_test_script_available(test_script_exists),
"repository": repository,
"branch": branch,
"jobname": jobname,
}
requests.post(request_url, headers=request_headers, data=request_data)
def add_parent_step(build_factory): def add_parent_step(build_factory):
""" """
Add a step to the parent build factory that will trigger the child hooks Add a step to the parent build factory that will trigger the child hooks
""" """
build_factory.addStep(
steps.SetPropertyFromCommand( build_factory.addStep(steps.ShellCommand(
name="Get OS codename", name="download worker",
command="grep ^VERSION_CODENAME= /etc/os-release | cut -d= -f2", command=["wget", "-O", "https://git.bitmessage.org/Bitmessage/buildbot_multibuild/raw/branch/master/lib/worker_multibuild.py", join(getenv['HOME'], '.local/bin/worker_multibuild.py')]
property="os_codename", ))
)
) # build_factory.addStep(
# steps.SetPropertyFromCommand(
# name="Get OS codename",
# command="grep ^VERSION_CODENAME= /etc/os-release | cut -d= -f2",
# property="os_codename",
# )
# )
build_factory.addStep( build_factory.addStep(
steps.ShellCommand( steps.ShellCommand(
@ -165,24 +68,20 @@ def add_parent_step(build_factory):
) )
def add_child_build_sh_step(build_factory, directory=".buildbot"): def add_child_sh_steps(build_factory, directory=".buildbot"):
""" """
Add a step to the build factory Add a step to the download, build and test factory
""" """
build_factory.addStep( build_factory.addStep(
steps.ShellCommand( steps.ShellCommand(
name=util.Interpolate("build_%(prop:jobname)s"), name=util.Interpolate("build_%(prop:jobname)s"),
command=util.Interpolate("%(kw:directory)s/%(prop:jobname)s/test.sh", directory=directory), command=util.Interpolate("%(kw:directory)s/%(prop:jobname)s/build.sh", directory=directory),
doStepIf=is_build_script_available, doStepIf=is_build_script_available,
hideStepIf=isnt_build_script_available, hideStepIf=isnt_build_script_available,
) )
) )
def add_child_test_sh_step(build_factory, directory=".buildbot"):
"""
Add a step to the build factory
"""
build_factory.addStep( build_factory.addStep(
steps.ShellCommand( steps.ShellCommand(
name= util.Interpolate("test_%(prop:jobname)s"), name= util.Interpolate("test_%(prop:jobname)s"),
@ -204,7 +103,7 @@ if __name__ == "__main__":
buildbotUrl = sys.argv[4] buildbotUrl = sys.argv[4]
os_codename = sys.argv[5] os_codename = sys.argv[5]
trigger_child_hooks(buildbotUrl, os_codename, repository, branch, jobname) trigger_child_hooks(buildbotUrl, os_codename, repository, branch, jobname, ".buildbot", is_build_script_available, is_test_script_available)
else: else:
print( print(
"Usage: python3 multibuild.py <jobname> <repository> <branch> <buildbotUrl> <os_codename>" "Usage: python3 multibuild.py <jobname> <repository> <branch> <buildbotUrl> <os_codename>"