Merge production changes
- bugfixes, code quality
This commit is contained in:
parent
83444728f3
commit
b020325a08
|
@ -1,24 +1,24 @@
|
||||||
from buildbot.plugins import util
|
from buildbot.plugins import util
|
||||||
import re
|
import re
|
||||||
|
|
||||||
|
def _is_build_script_available(props):
|
||||||
|
return props.getProperty("build_script_available", default=False)
|
||||||
|
|
||||||
@util.renderer
|
@util.renderer
|
||||||
def is_build_script_available(props):
|
def is_build_script_available(props):
|
||||||
# Actual check will got here
|
return _is_build_script_available(props)
|
||||||
return props.getProperty("build_available", default=False)
|
|
||||||
|
|
||||||
|
|
||||||
@util.renderer
|
@util.renderer
|
||||||
def isnt_build_script_available(props):
|
def isnt_build_script_available(props):
|
||||||
return not is_build_script_available(props)
|
return not _is_build_script_available(props)
|
||||||
|
|
||||||
|
def _is_test_script_available(props):
|
||||||
|
return props.getProperty("test_script_available", default=False)
|
||||||
|
|
||||||
@util.renderer
|
@util.renderer
|
||||||
def is_test_script_available(props):
|
def is_test_script_available(props):
|
||||||
# Actual check will got here
|
return _is_test_script_available(props)
|
||||||
return props.getProperty("test_available", default=False)
|
|
||||||
|
|
||||||
|
|
||||||
@util.renderer
|
@util.renderer
|
||||||
def isnt_test_script_available(props):
|
def isnt_test_script_available(props):
|
||||||
return not is_test_script_available(props)
|
return not _is_test_script_available(props)
|
||||||
|
|
|
@ -1,8 +1,8 @@
|
||||||
import os
|
|
||||||
from os import listdir
|
from os import listdir
|
||||||
from os.path import exists, isfile, join, islink, isdir
|
from os.path import exists, isfile, join, islink
|
||||||
import requests
|
import requests
|
||||||
import re
|
import re
|
||||||
|
from subprocess import Popen, PIPE
|
||||||
|
|
||||||
|
|
||||||
request_data = {
|
request_data = {
|
||||||
|
@ -11,7 +11,7 @@ request_data = {
|
||||||
}
|
}
|
||||||
|
|
||||||
ty = "/change_hook/base"
|
ty = "/change_hook/base"
|
||||||
path =".buildbot"
|
path = ".buildbot"
|
||||||
dockerfile_extra_contents = {}
|
dockerfile_extra_contents = {}
|
||||||
dockerfile_extra_contents['focal'] = """
|
dockerfile_extra_contents['focal'] = """
|
||||||
|
|
||||||
|
@ -51,11 +51,13 @@ ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
|
||||||
def get_secret():
|
def get_secret():
|
||||||
with open(path.join(os.environ['HOME'], "multibuild_parent_key.key"),'r') as f:
|
with open("multibuild_parent_key.key", 'r') as f:
|
||||||
data = f.read()
|
data = f.read()
|
||||||
return data
|
return data
|
||||||
|
|
||||||
|
|
||||||
def list_jobs(directory=".buildbot"):
|
def list_jobs(directory=".buildbot"):
|
||||||
"""
|
"""
|
||||||
list jobs found in a directory
|
list jobs found in a directory
|
||||||
|
@ -63,9 +65,12 @@ def list_jobs(directory=".buildbot"):
|
||||||
results = []
|
results = []
|
||||||
files = ["Dockerfile", "build.sh", "test.sh"]
|
files = ["Dockerfile", "build.sh", "test.sh"]
|
||||||
for item in listdir(directory):
|
for item in listdir(directory):
|
||||||
|
print("checking directory {}".format(item))
|
||||||
flag = False
|
flag = False
|
||||||
for file in files:
|
for fname in files:
|
||||||
filepath = join(directory, item, file)
|
filepath = join(directory, item, fname)
|
||||||
|
if not exists(filepath):
|
||||||
|
continue
|
||||||
if islink(filepath) or not isfile(filepath):
|
if islink(filepath) or not isfile(filepath):
|
||||||
flag = True
|
flag = True
|
||||||
break
|
break
|
||||||
|
@ -76,26 +81,39 @@ def list_jobs(directory=".buildbot"):
|
||||||
return results
|
return results
|
||||||
|
|
||||||
|
|
||||||
def _get_dockerfile_contents(jobname):
|
def get_revision():
|
||||||
|
proc = Popen(["git", "rev-parse", "HEAD"], stdout=PIPE)
|
||||||
|
retval = proc.stdout.read().strip()
|
||||||
|
retval = retval.decode('utf-8')
|
||||||
|
return retval
|
||||||
|
|
||||||
|
|
||||||
|
def _get_dockerfile_contents(dockerfile):
|
||||||
"""
|
"""
|
||||||
Read contents of a Dockerfile and add extra contents for the given os_codename
|
Read contents of a Dockerfile and add extra contents for the given os_codename
|
||||||
"""
|
"""
|
||||||
os_codename='bionic'
|
os_codename = 'bionic'
|
||||||
res = ""
|
res = ""
|
||||||
with open(join(path + jobname), "r") as file:
|
with open(dockerfile, "r") as file:
|
||||||
contents = file.read()
|
contents = file.readlines()
|
||||||
|
has_from = False
|
||||||
# accept any line containing FROM and RUN keywords
|
# accept any line containing FROM and RUN keywords
|
||||||
res = ""
|
res = ""
|
||||||
inside_allowed_command = False
|
inside_allowed_command = False
|
||||||
for line in contents:
|
for line in contents:
|
||||||
if re.match(r"(?m)^(FROM|RUN|ENV).*$", line):
|
m = re.match(r"(?m)^(FROM|RUN|ENV).*$", line)
|
||||||
|
if m:
|
||||||
inside_allowed_command = True
|
inside_allowed_command = True
|
||||||
|
if m.group(1) == "FROM":
|
||||||
|
has_from = True
|
||||||
if inside_allowed_command:
|
if inside_allowed_command:
|
||||||
res += line
|
res += line
|
||||||
l = line.strip()
|
ls = line.strip()
|
||||||
if not l.endswith("\\"):
|
if not ls.endswith("\\"):
|
||||||
inside_allowed_command = False
|
inside_allowed_command = False
|
||||||
|
|
||||||
|
if not has_from:
|
||||||
|
return None
|
||||||
return res + dockerfile_extra_contents[os_codename]
|
return res + dockerfile_extra_contents[os_codename]
|
||||||
|
|
||||||
|
|
||||||
|
@ -104,10 +122,11 @@ def trigger_child_hooks(buildbotUrl: str, repository, branch, directory=".buildb
|
||||||
# List all jobs in the directory
|
# List all jobs in the directory
|
||||||
jobs = list_jobs(directory)
|
jobs = list_jobs(directory)
|
||||||
request_headers = {
|
request_headers = {
|
||||||
"Content-Type": "application/x-www-form-urlencoded",
|
"Content-Type": "application/json",
|
||||||
"X-Multibuild-Trigger": get_secret(),
|
"X-Multibuild-Trigger": get_secret(),
|
||||||
"Accept": "text/plain",
|
"Accept": "text/plain",
|
||||||
}
|
}
|
||||||
|
revision = get_revision()
|
||||||
|
|
||||||
# Check if build.sh or test.sh exists in each of the jobs
|
# Check if build.sh or test.sh exists in each of the jobs
|
||||||
for job in jobs:
|
for job in jobs:
|
||||||
|
@ -119,15 +138,43 @@ def trigger_child_hooks(buildbotUrl: str, repository, branch, directory=".buildb
|
||||||
test_script_exists = True
|
test_script_exists = True
|
||||||
|
|
||||||
# make a post request
|
# make a post request
|
||||||
|
dockerfile = _get_dockerfile_contents(
|
||||||
|
join(directory, job, "Dockerfile")
|
||||||
|
)
|
||||||
|
|
||||||
|
if not dockerfile:
|
||||||
|
continue
|
||||||
|
|
||||||
|
request_data["branch"] = branch
|
||||||
|
request_data["revision"] = revision
|
||||||
|
|
||||||
request_data["properties"] = {
|
request_data["properties"] = {
|
||||||
"dockerfile": _get_dockerfile_contents(
|
"dockerfile": dockerfile,
|
||||||
join(directory, job, "Dockerfile")
|
"build_script_available": build_script_exists,
|
||||||
),
|
"test_script_available": test_script_exists,
|
||||||
"build_script_available": is_build_script_available(build_script_exists),
|
|
||||||
"test_script_available": is_test_script_available(test_script_exists),
|
|
||||||
"repository": repository,
|
|
||||||
"branch": branch,
|
|
||||||
"jobname": job,
|
"jobname": job,
|
||||||
}
|
}
|
||||||
|
request_data["changes"] = {
|
||||||
|
"author": "buildbot_multibuild",
|
||||||
|
"repository": repository,
|
||||||
|
"project": "/".join(repository.split("/")[-2:]),
|
||||||
|
}
|
||||||
|
|
||||||
requests.post(request_url, headers=request_headers, data=request_data)
|
retval = requests.post(request_url, headers=request_headers, json=request_data)
|
||||||
|
print("Triggered job for {} on {}: {}".format(job, request_url, retval.text))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
# expect jobname, repository, branch, buildbotUrl from command line
|
||||||
|
import sys
|
||||||
|
|
||||||
|
if len(sys.argv) == 4:
|
||||||
|
buildbotUrl = sys.argv[1]
|
||||||
|
repository = sys.argv[2]
|
||||||
|
branch = sys.argv[3]
|
||||||
|
|
||||||
|
trigger_child_hooks(buildbotUrl, repository, branch)
|
||||||
|
else:
|
||||||
|
sys.exit(
|
||||||
|
"Usage: python3 multibuild.py <buildbotUrl> <repository> <branch>"
|
||||||
|
)
|
||||||
|
|
|
@ -10,10 +10,8 @@ Requires docker
|
||||||
# TODO: write hook job, maybe also a dockerfile?
|
# TODO: write hook job, maybe also a dockerfile?
|
||||||
# TODO: what to do about non-docker jobs
|
# TODO: what to do about non-docker jobs
|
||||||
|
|
||||||
from os import listdir, walk, getenv
|
from os import listdir
|
||||||
from os.path import exists, isfile, join
|
from os.path import isfile, join
|
||||||
import requests
|
|
||||||
import re
|
|
||||||
from buildbot.plugins import steps, util
|
from buildbot.plugins import steps, util
|
||||||
|
|
||||||
from .lib.renderers import *
|
from .lib.renderers import *
|
||||||
|
@ -35,24 +33,32 @@ def add_parent_step(build_factory):
|
||||||
"""
|
"""
|
||||||
|
|
||||||
build_factory.addStep(steps.ShellCommand(
|
build_factory.addStep(steps.ShellCommand(
|
||||||
name="create directory",
|
name="Update APT cache",
|
||||||
command=["mkdir", "-p", join(getenv['HOME'], '.local/bin') ]
|
command=["sudo", "apt", "update"]
|
||||||
))
|
))
|
||||||
|
|
||||||
build_factory.addStep(steps.ShellCommand(
|
build_factory.addStep(steps.ShellCommand(
|
||||||
name="download worker",
|
name="Install dependencies",
|
||||||
command=["wget", "-O", "https://git.bitmessage.org/Bitmessage/buildbot_multibuild/raw/branch/master/lib/worker_multibuild.py", join(getenv['HOME'], '.local/bin/worker_multibuild.py')]
|
command=["sudo", "apt", "-y", "install", "python3-requests"]
|
||||||
))
|
))
|
||||||
|
|
||||||
|
build_factory.addStep(
|
||||||
|
steps.FileDownload(
|
||||||
|
workerdest="worker_multibuild.py",
|
||||||
|
mastersrc="buildbot_multibuild/lib/worker_multibuild.py",
|
||||||
|
mode=0o444
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
build_factory.addStep(
|
build_factory.addStep(
|
||||||
steps.ShellCommand(
|
steps.ShellCommand(
|
||||||
name="Execute worker script",
|
name="Execute worker script",
|
||||||
command=[
|
command=[
|
||||||
"python3",
|
"python3",
|
||||||
join(getenv['HOME'], '.local/bin/worker_multibuild.py'),
|
'worker_multibuild.py',
|
||||||
|
util.Property("buildboturl"),
|
||||||
util.Property('repository'),
|
util.Property('repository'),
|
||||||
util.Property('branch'),
|
util.Property('branch')
|
||||||
util.getURLForBuild(util.Property("url"), util.Property("builderid"), util.Property("buildnumber")),
|
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
@ -80,20 +86,3 @@ def add_child_sh_steps(build_factory, directory=".buildbot"):
|
||||||
hideStepIf=isnt_test_script_available,
|
hideStepIf=isnt_test_script_available,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
# expect jobname, repository, branch, buildbotUrl from command line
|
|
||||||
import sys
|
|
||||||
|
|
||||||
if len(sys.argv) == 6:
|
|
||||||
jobname = sys.argv[1]
|
|
||||||
repository = sys.argv[2]
|
|
||||||
branch = sys.argv[3]
|
|
||||||
buildbotUrl = sys.argv[4]
|
|
||||||
|
|
||||||
trigger_child_hooks(buildbotUrl, repository, branch)
|
|
||||||
else:
|
|
||||||
print(
|
|
||||||
"Usage: python3 multibuild.py <buildbotUrl> <repository> <branch> "
|
|
||||||
)
|
|
||||||
|
|
Loading…
Reference in New Issue
Block a user