Bitmessage/buildbot_multibuild
5
0
Fork 5
Code Issues 4 Pull Requests Projects Releases Wiki Activity

Added parent hook and some renderer #1

Merged
PeterSurda merged 7 commits from Ss_singh/buildbot_multibuild:hooks into master 2021-12-26 08:05:49 +00:00
Conversation 2 Commits 7 Files Changed 2 +90 -28
1 changed files with 90 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 9a55687ec6 - Show all commits

114
multibuild.py
View File

@ -14,18 +14,19 @@ from os import walk
from os.path import exists, isfile, join, listdir
import requests
import re
from buildbot.plugins import steps, util
ty = '/change_hook/base'
ty = "/change_hook/base"
request_headers = {
'Content-Type': 'application/x-www-form-urlencoded',
'Accept': 'text/plain'
"Content-Type": "application/x-www-form-urlencoded",
"Accept": "text/plain",
}
request_data = {
'project': 'testproject',
'comments': 'testcomment',
"project": "testproject",
"comments": "testcomment",
}
dockerfile_extra_contents_focal = '''
dockerfile_extra_contents_focal = """
# Buildbot
RUN apt-get install -yq --no-install-suggests --no-install-recommends \
@ -38,9 +39,9 @@ USER buildbot
ENTRYPOINT /entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
'''
"""
dockerfile_extra_contents_bionic = '''
dockerfile_extra_contents_bionic = """
# Buildbot
RUN apt-get install -yq --no-install-suggests --no-install-recommends \
@ -57,7 +58,8 @@ USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
'''
"""
def list_jobs(directory=".buildbot"):
"""
@ -65,10 +67,10 @@ def list_jobs(directory=".buildbot"):
"""
results = []
for _ in next(walk(directory))[1]:
if exists(join(directory, _, "Dockerfile")) \
and (exists(join(directory, _, "build.sh"))
or exists(join(directory, _, "test.sh"))
):
if exists(join(directory, _, "Dockerfile")) and (
exists(join(directory, _, "build.sh"))
or exists(join(directory, _, "test.sh"))
):
results.append(_)
return results
@ -83,19 +85,21 @@ def find_artifacts(directory="out"):
continue
return join(directory, _)
def get_dockerfile_contents(path, os_codename):
'''
"""
Read contents of a Dockerfile and add extra contents for the given os_codename
'''
with open(path, 'r') as file:
"""
with open(path, "r") as file:
contents = file.read()
# remove any line containing CMD or ENTRYFILE keywords
contents = re.sub(r'(?m)^CMD.*$', '', contents)
contents = re.sub(r'(?m)^ENTRYFILE.*$', '', contents)
contents = re.sub(r"(?m)^CMD.*$", "", contents)
Ss_singh marked this conversation as resolved
PeterSurda commented 2021-12-25 05:49:38 +00:00
Review
Copy Link

How about just

contents = re.sub(r"(?m)^(CMD|ENTRYFILE).*$", "", contents)
How about just ``` contents = re.sub(r"(?m)^(CMD|ENTRYFILE).*$", "", contents) ```
contents = re.sub(r"(?m)^ENTRYFILE.*$", "", contents)
return contents + eval("dockerfile_extra_contents_" + os_codename)
def trigger_child_hooks(buildbotUrl:str, os_codename:str, directory=".buildbot"):
def trigger_child_hooks(buildbotUrl: str, os_codename: str, directory=".buildbot"):
request_url = buildbotUrl + ty
# List all jobs in the directory
@ -111,14 +115,70 @@ def trigger_child_hooks(buildbotUrl:str, os_codename:str, directory=".buildbot")
test_script_exists = True
# make a post request
request_data['dockerfile'] = get_dockerfile_contents(join(directory, job, "Dockerfile"), os_codename)
request_data['build_script_available'] = str(build_script_exists)
request_data['test_script_available'] = str(test_script_exists)
request_data["dockerfile"] = get_dockerfile_contents(
join(directory, job, "Dockerfile"), os_codename
)
request_data["build_script_available"] = str(build_script_exists)
request_data["test_script_available"] = str(test_script_exists)
requests.post(request_url, headers=request_headers, data=request_data)
Ss_singh marked this conversation as resolved
PeterSurda commented 2021-12-25 05:53:05 +00:00
Review
Copy Link

proxied properties are missing

proxied properties are missing
PeterSurda commented 2021-12-25 08:18:04 +00:00
Review
Copy Link

I think they are still missing. Like repository and branch.

I think they are still missing. Like `repository` and `branch`.
Ss_singh commented 2021-12-25 11:25:56 +00:00
Review
Copy Link

Well, they were already being added before the hook function was getting triggered but anyways, now I've them placed altogether.

Well, they were already being added before the hook function was getting triggered but anyways, now I've them placed altogether.
def add_parent_step(build_factory, jobname, repository, branch):
"""
Add a step to the parent build factory that will trigger the child hooks
"""
build_factory.addStep(
steps.SetPropertyFromCommand(
name="Get OS codename",
PeterSurda commented 2021-12-26 01:59:39 +00:00
Review
Copy Link

I think repository and branch are a level higher.

I think `repository` and `branch` are a level higher.
command="grep ^VERSION_CODENAME= /etc/os-release | cut -d= -f2",
property="os_codename",
)
)
build_factory.addStep(
steps.ShellCommand(
name="Execute multibuild script",
command=[
"python",
"multibuild.py",
jobname,
repository,
branch,
"https://buildbot.bitmessage.org",
util.Interpolate("%(prop:os_codename)s"),
],
)
)
def add_child_build_sh_step(build_factory, job, directory=".buildbot"):
"""
Add a step to the build factory
"""
build_factory.addStep(
steps.ShellCommand(
Ss_singh marked this conversation as resolved
PeterSurda commented 2021-12-25 05:55:06 +00:00
Review
Copy Link

should also have conditionals for build_script_exists. If it doesn't, this step is skipped and hidden.

should also have conditionals for `build_script_exists`. If it doesn't, this step is skipped and hidden.
name="build_" + job,
command=["bash", "-c", "cd " + join(directory, job) + " && ./build.sh"],
Ss_singh marked this conversation as resolved
PeterSurda commented 2021-12-25 05:56:16 +00:00
Review
Copy Link

I wouldn't change the directory, but just run it from it, i.e.

command=["bash", "-c", join(directory, job, "build.sh")],
I wouldn't change the directory, but just run it from it, i.e. ``` command=["bash", "-c", join(directory, job, "build.sh")],
)
PeterSurda commented 2021-12-26 02:02:19 +00:00
Review
Copy Link

For now it's probably ok. Over longer term however the os_codename isn't extracted from the parent (parent may be running something else than the child should), but from the child Dockerfile. The Dockerfile will begin with something like FROM ubuntu:bionic AS ..., so that determines the OS.

For now it's probably ok. Over longer term however the `os_codename` isn't extracted from the parent (parent may be running something else than the child should), but from the child `Dockerfile`. The `Dockerfile` will begin with something like `FROM ubuntu:bionic AS ...`, so that determines the OS.
)
def add_child_test_sh_step(build_factory, job, directory=".buildbot"):
"""
PeterSurda commented 2021-12-26 02:05:59 +00:00
Review
Copy Link

I'm not 100% sure this will pass properties. A safer solution would be to create a child of ShellCommand and call super with a constructed command.

I'm not 100% sure this will pass properties. A safer solution would be to create a child of `ShellCommand` and call `super` with a constructed `command`.
Add a step to the build factory
"""
build_factory.addStep(
steps.ShellCommand(
name="test_" + job,
command=["bash", "-c", "cd " + join(directory, job) + " && ./test.sh"],
)
)
if __name__ == "__main__":
# expect jobname, repository, branch, buildbotUrl, os_codename from command line args
import sys
if len(sys.argv) == 6:
jobname = sys.argv[1]
repository = sys.argv[2]
@ -127,10 +187,12 @@ if __name__ == "__main__":
os_codename = sys.argv[5]
# add these into the request_data
request_data['jobname'] = jobname
request_data['repository'] = repository
request_data['branch'] = branch
request_data["jobname"] = jobname
request_data["repository"] = repository
request_data["branch"] = branch
trigger_child_hooks(buildbotUrl, os_codename)
else:
print("Usage: python3 multibuild.py <jobname> <repository> <branch> <buildbotUrl> <os_codename>")
print(
"Usage: python3 multibuild.py <jobname> <repository> <branch> <buildbotUrl> <os_codename>"
)