Ss_singh/buildbot_multibuild
1
0
Fork 0
forked from Bitmessage/buildbot_multibuild
Code Issues Pull Requests Projects Releases Wiki Activity

Merge production changes

Browse Source 
- bugfixes, code quality
This commit is contained in:
Peter Šurda 2022-04-07 20:46:29 +08:00
parent 83444728f3
commit b020325a08
Signed by untrusted user: PeterSurda
GPG Key ID: 3E47497CF67ABB95
3 changed files with 96 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
lib/renderers.py
View File

@ -1,24 +1,24 @@
from buildbot.plugins import util
import re
def _is_build_script_available(props):
return props.getProperty("build_script_available", default=False)
@util.renderer
def is_build_script_available(props):
# Actual check will got here
return props.getProperty("build_available", default=False)
return _is_build_script_available(props)
@util.renderer
def isnt_build_script_available(props):
return not is_build_script_available(props)
return not _is_build_script_available(props)
def _is_test_script_available(props):
return props.getProperty("test_script_available", default=False)
@util.renderer
def is_test_script_available(props):
# Actual check will got here
return props.getProperty("test_available", default=False)
return _is_test_script_available(props)
@util.renderer
def isnt_test_script_available(props):
return not is_test_script_available(props)
return not _is_test_script_available(props)

91
lib/worker_multibuild.py
View File

@ -1,8 +1,8 @@
import os
from os import listdir
from os.path import exists, isfile, join, islink, isdir
from os.path import exists, isfile, join, islink
import requests
import re
from subprocess import Popen, PIPE
request_data = {
@ -11,7 +11,7 @@ request_data = {
}
ty = "/change_hook/base"
path =".buildbot"
path = ".buildbot"
dockerfile_extra_contents = {}
dockerfile_extra_contents['focal'] = """
@ -51,11 +51,13 @@ ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$
"""
def get_secret():
with open(path.join(os.environ['HOME'], "multibuild_parent_key.key"),'r') as f:
with open("multibuild_parent_key.key", 'r') as f:
data = f.read()
return data
def list_jobs(directory=".buildbot"):
"""
list jobs found in a directory
@ -63,9 +65,12 @@ def list_jobs(directory=".buildbot"):
results = []
files = ["Dockerfile", "build.sh", "test.sh"]
for item in listdir(directory):
print("checking directory {}".format(item))
flag = False
for file in files:
filepath = join(directory, item, file)
for fname in files:
filepath = join(directory, item, fname)
if not exists(filepath):
continue
if islink(filepath) or not isfile(filepath):
flag = True
break
@ -76,26 +81,39 @@ def list_jobs(directory=".buildbot"):
return results
def _get_dockerfile_contents(jobname):
def get_revision():
proc = Popen(["git", "rev-parse", "HEAD"], stdout=PIPE)
retval = proc.stdout.read().strip()
retval = retval.decode('utf-8')
return retval
def _get_dockerfile_contents(dockerfile):
"""
Read contents of a Dockerfile and add extra contents for the given os_codename
"""
os_codename='bionic'
os_codename = 'bionic'
res = ""
with open(join(path + jobname), "r") as file:
contents = file.read()
with open(dockerfile, "r") as file:
contents = file.readlines()
has_from = False
# accept any line containing FROM and RUN keywords
res = ""
inside_allowed_command = False
for line in contents:
if re.match(r"(?m)^(FROM|RUN|ENV).*$", line):
m = re.match(r"(?m)^(FROM|RUN|ENV).*$", line)
if m:
inside_allowed_command = True
if m.group(1) == "FROM":
has_from = True
if inside_allowed_command:
res += line
l = line.strip()
if not l.endswith("\\"):
ls = line.strip()
if not ls.endswith("\\"):
inside_allowed_command = False
if not has_from:
return None
return res + dockerfile_extra_contents[os_codename]
@ -104,10 +122,11 @@ def trigger_child_hooks(buildbotUrl: str, repository, branch, directory=".buildb
# List all jobs in the directory
jobs = list_jobs(directory)
request_headers = {
"Content-Type": "application/x-www-form-urlencoded",
"Content-Type": "application/json",
"X-Multibuild-Trigger": get_secret(),
"Accept": "text/plain",
}
revision = get_revision()
# Check if build.sh or test.sh exists in each of the jobs
for job in jobs:
@ -119,15 +138,43 @@ def trigger_child_hooks(buildbotUrl: str, repository, branch, directory=".buildb
test_script_exists = True
# make a post request
request_data["properties"] = {
"dockerfile": _get_dockerfile_contents(
dockerfile = _get_dockerfile_contents(
join(directory, job, "Dockerfile")
),
"build_script_available": is_build_script_available(build_script_exists),
"test_script_available": is_test_script_available(test_script_exists),
"repository": repository,
"branch": branch,
)
if not dockerfile:
continue
request_data["branch"] = branch
request_data["revision"] = revision
request_data["properties"] = {
"dockerfile": dockerfile,
"build_script_available": build_script_exists,
"test_script_available": test_script_exists,
"jobname": job,
}
request_data["changes"] = {
"author": "buildbot_multibuild",
"repository": repository,
"project": "/".join(repository.split("/")[-2:]),
}
requests.post(request_url, headers=request_headers, data=request_data)
retval = requests.post(request_url, headers=request_headers, json=request_data)
print("Triggered job for {} on {}: {}".format(job, request_url, retval.text))
if __name__ == "__main__":
# expect jobname, repository, branch, buildbotUrl from command line
import sys
if len(sys.argv) == 4:
buildbotUrl = sys.argv[1]
repository = sys.argv[2]
branch = sys.argv[3]
trigger_child_hooks(buildbotUrl, repository, branch)
else:
sys.exit(
"Usage: python3 multibuild.py <buildbotUrl> <repository> <branch>"
)

45
multibuild.py
View File

@ -10,10 +10,8 @@ Requires docker
# TODO: write hook job, maybe also a dockerfile?
# TODO: what to do about non-docker jobs
from os import listdir, walk, getenv
from os.path import exists, isfile, join
import requests
import re
from os import listdir
from os.path import isfile, join
from buildbot.plugins import steps, util
from .lib.renderers import *
@ -35,24 +33,32 @@ def add_parent_step(build_factory):
"""
build_factory.addStep(steps.ShellCommand(
name="create directory",
command=["mkdir", "-p", join(getenv['HOME'], '.local/bin') ]
name="Update APT cache",
command=["sudo", "apt", "update"]
))
build_factory.addStep(steps.ShellCommand(
name="download worker",
command=["wget", "-O", "https://git.bitmessage.org/Bitmessage/buildbot_multibuild/raw/branch/master/lib/worker_multibuild.py", join(getenv['HOME'], '.local/bin/worker_multibuild.py')]
name="Install dependencies",
command=["sudo", "apt", "-y", "install", "python3-requests"]
))
build_factory.addStep(
steps.FileDownload(
workerdest="worker_multibuild.py",
mastersrc="buildbot_multibuild/lib/worker_multibuild.py",
mode=0o444
)
)
build_factory.addStep(
steps.ShellCommand(
name="Execute worker script",
command=[
"python3",
join(getenv['HOME'], '.local/bin/worker_multibuild.py'),
'worker_multibuild.py',
util.Property("buildboturl"),
util.Property('repository'),
util.Property('branch'),
util.getURLForBuild(util.Property("url"), util.Property("builderid"), util.Property("buildnumber")),
util.Property('branch')
],
)
)
@ -80,20 +86,3 @@ def add_child_sh_steps(build_factory, directory=".buildbot"):
hideStepIf=isnt_test_script_available,
)
)
if __name__ == "__main__":
# expect jobname, repository, branch, buildbotUrl from command line
import sys
if len(sys.argv) == 6:
jobname = sys.argv[1]
repository = sys.argv[2]
branch = sys.argv[3]
buildbotUrl = sys.argv[4]
trigger_child_hooks(buildbotUrl, repository, branch)
else:
print(
"Usage: python3 multibuild.py <buildbotUrl> <repository> <branch> "
)