Bitmessage/buildbot_multibuild
6
0
Fork 5
Code Issues 4 Pull Requests Projects Releases Wiki Activity

worker and master splited #8

Closed
cis-muzahid wants to merge 1 commits from cis-muzahid/buildbot_multibuild:kivy-test into master
pull from: cis-muzahid/buildbot_multibuild:kivy-test
merge into: Bitmessage:master
Conversation 0 Commits 1 Files Changed 2 +142 -123
2 changed files with 142 additions and 123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

120
lib/worker_multibuild.py Normal file
View File

@ -0,0 +1,120 @@
from os import listdir, walk
from os.path import exists, isfile, join
import requests
import re
request_data = {
"project": "testproject",
"comments": "testcomment",
}
request_headers = {
"Content-Type": "application/x-www-form-urlencoded",
"Accept": "text/plain",
}
ty = "/change_hook/base"
path =".buildbot"
dockerfile_extra_contents = {}
dockerfile_extra_contents['focal'] = """
# Buildbot
RUN apt-get install -yq --no-install-suggests --no-install-recommends \
buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
python3-pip dumb-init curl openssh-client wget
# buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
"""
dockerfile_extra_contents['bionic'] = """
# Buildbot
RUN apt-get install -yq --no-install-suggests --no-install-recommends \
buildbot-slave git subversion python3-dev libffi-dev python3-setuptools \
python3-pip dumb-init curl openssh-client wget
# buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
"""
def list_jobs(directory=".buildbot"):
"""
list jobs found in a directory
"""
results = []
for _ in next(walk(directory))[1]:
if exists(join(directory, _, "Dockerfile")) and (
exists(join(directory, _, "build.sh"))
or exists(join(directory, _, "test.sh"))
):
results.append(_)
return results
def _get_dockerfile_contents(jobname, os_codename='bionic'):
"""
Read contents of a Dockerfile and add extra contents for the given os_codename
"""
res = ""
with open(join(path + jobname), "r") as file:
contents = file.read()
# accept any line containing FROM and RUN keywords
res = ""
inside_allowed_command = False
for line in contents:
if re.match(r"(?m)^(FROM|RUN).*$", line):
inside_allowed_command = True
if inside_allowed_command:
res += line
l = line.strip()
if not l.endswith("\\"):
inside_allowed_command = False
return res + dockerfile_extra_contents[os_codename]
def trigger_child_hooks(buildbotUrl: str, os_codename: str, repository, branch, jobname, directory=".buildbot", is_build_script_available, is_test_script_available):
request_url = buildbotUrl + ty
# List all jobs in the directory
jobs = list_jobs(directory)
# Check if build.sh or test.sh exists in each of the jobs
for job in jobs:
build_script_exists = False
test_script_exists = False
if exists(join(directory, job, "build.sh")):
build_script_exists = True
if exists(join(directory, job, "test.sh")):
test_script_exists = True
# make a post request
request_data["properties"] = {
"dockerfile": _get_dockerfile_contents(
join(directory, job, "Dockerfile"), os_codename
),
"build_script_available": is_build_script_available(build_script_exists),
"test_script_available": is_test_script_available(test_script_exists),
"repository": repository,
"branch": branch,
"jobname": jobname,
}
requests.post(request_url, headers=request_headers, data=request_data)

145
multibuild.py
View File

@ -10,7 +10,7 @@ Requires docker
# TODO: write hook job, maybe also a dockerfile? # TODO: write hook job, maybe also a dockerfile?
# TODO: what to do about non-docker jobs # TODO: what to do about non-docker jobs
from os import listdir, walk from os import listdir, walk, getenv
from os.path import exists, isfile, join from os.path import exists, isfile, join
import requests import requests
import re import re
@ -18,70 +18,6 @@ from buildbot.plugins import steps, util
from .lib.renderers import * from .lib.renderers import *
ty = "/change_hook/base"
request_headers = {
"Content-Type": "application/x-www-form-urlencoded",
"Accept": "text/plain",
}
request_data = {
"project": "testproject",
"comments": "testcomment",
}
dockerfile_extra_contents_focal = """
# Buildbot
RUN apt-get install -yq --no-install-suggests --no-install-recommends \
buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
python3-pip dumb-init curl openssh-client wget
# buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
"""
dockerfile_extra_contents_bionic = """
# Buildbot
RUN apt-get install -yq --no-install-suggests --no-install-recommends \
buildbot-slave git subversion python3-dev libffi-dev python3-setuptools \
python3-pip dumb-init curl openssh-client wget
# buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
"""
path =".buildbot"
def list_jobs(directory=".buildbot"):
"""
list jobs found in a directory
"""
results = []
for _ in next(walk(directory))[1]:
if exists(join(directory, _, "Dockerfile")) and (
exists(join(directory, _, "build.sh"))
or exists(join(directory, _, "test.sh"))
):
results.append(_)
return results
def find_artifacts(directory="out"): def find_artifacts(directory="out"):
""" """
@ -93,61 +29,28 @@ def find_artifacts(directory="out"):
return join(directory, _) return join(directory, _)
def get_dockerfile_contents(props): # @util.renderer
PeterSurda marked this conversation as resolved
PeterSurda commented 2022-02-28 07:39:39 +01:00
Review
Copy Link

remove

remove
👍 1
""" # def get_dockerfile_contents(props):
Read contents of a Dockerfile and add extra contents for the given os_codename # return _get_dockerfile_contents(props.getProperty('jobname', default=None), util.Interpolate("%(prop:os_codename)s"))
"""
with open(join(path + props.getProperty('jobname', default=None)), "r") as file:
contents = file.read()
# remove any line containing CMD or ENTRYFILE keywords
re.sub(r"(?m)^(CMD|ENTRYFILE).*$", "", contents)
return contents + {
"focal": dockerfile_extra_contents_focal,
"bionic": dockerfile_extra_contents_bionic,
}[util.Interpolate("%(prop:os_codename)s")]
def trigger_child_hooks(buildbotUrl: str, os_codename: str, repository, branch, jobname, directory=".buildbot"):
request_url = buildbotUrl + ty
# List all jobs in the directory
jobs = list_jobs(directory)
# Check if build.sh or test.sh exists in each of the jobs
for job in jobs:
build_script_exists = False
test_script_exists = False
if exists(join(directory, job, "build.sh")):
build_script_exists = True
if exists(join(directory, job, "test.sh")):
test_script_exists = True
# make a post request
request_data["properties"] = {
"dockerfile": get_dockerfile_contents(
join(directory, job, "Dockerfile"), os_codename
),
"build_script_available": is_build_script_available(build_script_exists),
"test_script_available": is_test_script_available(test_script_exists),
"repository": repository,
"branch": branch,
"jobname": jobname,
}
requests.post(request_url, headers=request_headers, data=request_data)
def add_parent_step(build_factory): def add_parent_step(build_factory):
""" """
Add a step to the parent build factory that will trigger the child hooks Add a step to the parent build factory that will trigger the child hooks
""" """
build_factory.addStep(
steps.SetPropertyFromCommand( build_factory.addStep(steps.ShellCommand(
name="Get OS codename", name="download worker",
command="grep ^VERSION_CODENAME= /etc/os-release | cut -d= -f2", command=["wget", "-O", "https://git.bitmessage.org/Bitmessage/buildbot_multibuild/raw/branch/master/lib/worker_multibuild.py", join(getenv['HOME'], '.local/bin/worker_multibuild.py')]
property="os_codename", ))
)
) # build_factory.addStep(
PeterSurda marked this conversation as resolved
PeterSurda commented 2022-02-28 07:43:55 +01:00
Review
Copy Link

remove

remove
👍 1
# steps.SetPropertyFromCommand(
# name="Get OS codename",
# command="grep ^VERSION_CODENAME= /etc/os-release | cut -d= -f2",
# property="os_codename",
# )
# )
build_factory.addStep( build_factory.addStep(
steps.ShellCommand( steps.ShellCommand(
@ -165,24 +68,20 @@ def add_parent_step(build_factory):
) )
def add_child_build_sh_step(build_factory, directory=".buildbot"): def add_child_sh_steps(build_factory, directory=".buildbot"):
""" """
Add a step to the build factory Add a step to the download, build and test factory
""" """
build_factory.addStep( build_factory.addStep(
steps.ShellCommand( steps.ShellCommand(
name=util.Interpolate("build_%(prop:jobname)s"), name=util.Interpolate("build_%(prop:jobname)s"),
command=util.Interpolate("%(kw:directory)s/%(prop:jobname)s/test.sh", directory=directory), command=util.Interpolate("%(kw:directory)s/%(prop:jobname)s/build.sh", directory=directory),
doStepIf=is_build_script_available, doStepIf=is_build_script_available,
hideStepIf=isnt_build_script_available, hideStepIf=isnt_build_script_available,
) )
) )
def add_child_test_sh_step(build_factory, directory=".buildbot"):
"""
Add a step to the build factory
"""
build_factory.addStep( build_factory.addStep(
steps.ShellCommand( steps.ShellCommand(
name= util.Interpolate("test_%(prop:jobname)s"), name= util.Interpolate("test_%(prop:jobname)s"),
@ -204,7 +103,7 @@ if __name__ == "__main__":
buildbotUrl = sys.argv[4] buildbotUrl = sys.argv[4]
os_codename = sys.argv[5] os_codename = sys.argv[5]
trigger_child_hooks(buildbotUrl, os_codename, repository, branch, jobname) trigger_child_hooks(buildbotUrl, os_codename, repository, branch, jobname, ".buildbot", is_build_script_available, is_test_script_available)
else: else:
print( print(
"Usage: python3 multibuild.py <jobname> <repository> <branch> <buildbotUrl> <os_codename>" "Usage: python3 multibuild.py <jobname> <repository> <branch> <buildbotUrl> <os_codename>"