Added parent hook and some renderer #1
38
lib/wine.py
Normal file
|
@ -0,0 +1,38 @@
|
|||
from buildbot.plugins import util
|
||||
import re
|
||||
|
||||
def _is_github(props):
|
||||
if re.search(r'[/@]github.com', props.getProperty('repository')):
|
||||
return True
|
||||
return False
|
||||
|
||||
@util.renderer
|
||||
def is_github(props):
|
||||
return _is_github(props)
|
||||
|
||||
@util.renderer
|
||||
def isnt_github(props):
|
||||
return not _is_github(props)
|
||||
|
||||
def _is_gitea(props):
|
||||
if re.search(r'[/@]git.bitmessage.org', props.getProperty('repository'), re.I):
|
||||
return True
|
||||
return False
|
||||
|
||||
@util.renderer
|
||||
def is_gitea(props):
|
||||
return _is_gitea(props)
|
||||
|
||||
@util.renderer
|
||||
def isnt_gitea(props):
|
||||
return not _is_gitea(props)
|
||||
|
||||
@util.renderer
|
||||
def is_build_script_available(props):
|
||||
# Actual check will got here
|
||||
return props.getProperty('jobname', default='') != ''
|
||||
|
||||
@util.renderer
|
||||
def is_test_script_available(props):
|
||||
# Actual check will got here
|
||||
return props.getProperty('jobname', default='') != ''
|
|
@ -12,6 +12,23 @@ Requires docker
|
|||
|
||||
from os import walk
|
||||
from os.path import exists, isfile, join, listdir
|
||||
import requests
|
||||
|
||||
request_url = "https://buildbot.sysdeploy.org/change_hook/base"
|
||||
Ss_singh marked this conversation as resolved
Outdated
|
||||
request_headers = {
|
||||
'Content-Type': 'application/x-www-form-urlencoded',
|
||||
'Accept': 'text/plain'
|
||||
}
|
||||
request_data = {
|
||||
'repository': 'git@git.bitmessage.org:Sysdeploy/buildbot-scripts.git',
|
||||
Ss_singh marked this conversation as resolved
Outdated
PeterSurda
commented
this should be extracted from the current properties. this should be extracted from the current properties.
Ss_singh
commented
You mean it should come from You mean it should come from `util.Property("repository")`? Do I need to save this repo url in some property?
Like, `property="repo_url"` in buildFactory step and use that later when executing the multibuild.py script?
PeterSurda
commented
Yes.
You should pass it as it is. Similar with "branch". I don't think others need passing.
If I understand you correctly, yes. These two properties you're basically just proxying from the parent to the child. But you're also adding more properties, mainly "jobname" and "dockerfile". > You mean it should come from util.Property("repository")?
Yes.
> Do I need to save this repo url in some property?
You should pass it as it is. Similar with "branch". I don't think others need passing.
> Like, property="repo_url" in buildFactory step and use that later when executing the multibuild.py script?
If I understand you correctly, yes. These two properties you're basically just proxying from the parent to the child. But you're also adding more properties, mainly "jobname" and "dockerfile".
|
||||
'project': 'testproject',
|
||||
'author': 'cloud-init',
|
||||
'comments': 'testcomment',
|
||||
}
|
||||
|
||||
ty = 'child_hook'
|
||||
Ss_singh marked this conversation as resolved
Outdated
PeterSurda
commented
this is fixed, derived from the base hook class. this is fixed, derived from the base hook class.
Ss_singh
commented
Since we don't need Since we don't need `type`, I'd just remove it
PeterSurda
commented
We still need to construct the full URL, which consists of the URL of the buildbot server and the path for the base webhook. The latter part is statoc, I don't even think it's configurable in buildbot, it's derived from the hook class. We still need to construct the full URL, which consists of the URL of the buildbot server and the path for the base webhook. The latter part is statoc, I don't even think it's configurable in buildbot, it's derived from the hook class.
Ss_singh
commented
So what this 'base webhook' should actually be? So what this 'base webhook' should actually be?
PeterSurda
commented
I think it's fixed at I think it's fixed at `/change_hook/base`
|
||||
hn = 'buildbot.sysdeploy.org' #hostname here(required?)
|
||||
Ss_singh marked this conversation as resolved
Outdated
PeterSurda
commented
this can be extracted from global variables this can be extracted from global variables
Ss_singh
commented
Similarly we don't need Similarly we don't need `node` here so it can be removed
|
||||
addn = ',"build_script_available":"{build_script_available}","test_script_available":"{test_script_available}"'
|
||||
|
||||
|
||||
def list_jobs(directory=".buildbot"):
|
||||
|
@ -37,3 +54,26 @@ def find_artifacts(directory="out"):
|
|||
if not isfile(join(directory, _)):
|
||||
continue
|
||||
return join(directory, _)
|
||||
|
||||
def trigger_child_hooks(directory=".buildbot"):
|
||||
|
||||
# List all jobs in the directory
|
||||
jobs = list_jobs(directory)
|
||||
|
||||
# Check if build.sh or test.sh exists in each of the jobs
|
||||
for job in jobs:
|
||||
build_script_exists = False
|
||||
test_script_exists = False
|
||||
if exists(join(directory, job, "build.sh")):
|
||||
build_script_exists = True
|
||||
Ss_singh marked this conversation as resolved
Outdated
PeterSurda
commented
There should be additional code for modifying the dockerfile. At the very least, it needs to install buildbot-worker and its dependencies, setup the default worker and run it in entrypoint. At the moment we can assume "apt", but in the future it should be able to detect the distro and adjust commands based on that. For security it should also filter the file to remove CMD or ENTRYFILE lines. There should be additional code for modifying the dockerfile. At the very least, it needs to install buildbot-worker and its dependencies, setup the default worker and run it in entrypoint. At the moment we can assume "apt", but in the future it should be able to detect the distro and adjust commands based on that.
For security it should also filter the file to remove CMD or ENTRYFILE lines.
Ss_singh
commented
Okay it's like take the contents of already available Dockerfile and append it with extra commands based on the distro. Okay it's like take the contents of already available Dockerfile and append it with extra commands based on the distro.
PeterSurda
commented
Yes something like that, but it should be only the buildbot-specific stuff and not the whole existing Yes something like that, but it should be only the buildbot-specific stuff and not the whole existing `Dockerfile`
Ss_singh
commented
I've pushed an update resolving this. Kindly have a look I've pushed an update resolving this. Kindly have a look
|
||||
if exists(join(directory, job, "test.sh")):
|
||||
test_script_exists = True
|
||||
|
||||
addn = addn.format(build_script_available=build_script_exists, test_script_available=test_script_exists)
|
||||
|
||||
PeterSurda
commented
I realised a problem. The code needs to be split between master and worker parts. Master code doesn't see the directories and files, and worker code doesn't have access to properties. So my suggestion would be to add a I realised a problem. The code needs to be split between master and worker parts. Master code doesn't see the directories and files, and worker code doesn't have access to properties.
So my suggestion would be to add a `BuildStep` that constructs a JSON string (i.e. a renderer) and uploads it into the worker as a file. The next step will instruct the worker to run a script that will fetch the props from this JSON files, combines them with the per-job props and calls the child hooks.
|
||||
# make a post request
|
||||
request_data['properties'] = '{"node":{hn},"type":{ty}{addn}"}'.format(hn=hn, ty=ty, addn=addn)
|
||||
PeterSurda
commented
`node` and `type` we don't need, that's for sysdeploy. we should put all the properties (build_script_available, test_script_available, dockerfile, ...) into this dict, there shouldn't be a sub-dict.
PeterSurda
commented
I think it would look something like this (still may need fine tuning):
I think it would look something like this (still may need fine tuning):
```
request_data = {
'repository': Util.Property('repository'),
'branch': Util.Property('branch'),
'properties': {
'build_script_available': build_script_exists,
'test_script_available': test_script_exists,
'dockerfile': dockerfile
}
}
requests.post(request_url, headers=request_headers, data=request_data)
```
Ss_singh
commented
Can I access this Can I access this `Util` in multibuid.py as well? I just passed the 'repository' and 'branch' as command line args while executing multibuild.py
PeterSurda
commented
`multibuild.py` will have to be loaded from the buildbot config, just like now it's importing `lib/wine.py`, for example. But it can be made into a separate pip module, for example, so it can be developed and used independently.
|
||||
requests.post(request_url, headers=request_headers, data=request_data)
|
||||
PeterSurda
commented
we also need to add the repo information, i.e. the pre-existing properties. we also need to add the repo information, i.e. the pre-existing properties.
Ss_singh
commented
And those are what? 'repository', 'branch', 'author', these things? And those are what? 'repository', 'branch', 'author', these things?
PeterSurda
commented
yes. I think for an MVP we only need yes. I think for an MVP we only need `repository` and `branch`, maybe `revision`. For later, probably `owner` and `owners` (the way I have it setup now, it would allow people to re-run jobs for their own PRs manually, without these two only admin can re-run job). I don't think `author` is needed, that's just something I borrowed from somewhere else.
|
||||
|
||||
if __name__ == "__main__":
|
||||
trigger_child_hooks()
|
73
parent_hook
Normal file
|
@ -0,0 +1,73 @@
|
|||
#!/usr/bin/env python3
|
||||
PeterSurda
commented
I think we can remove this file. I think we can remove this file.
|
||||
""" parent webhook """
|
||||
|
||||
from buildbot.plugins import steps, util, secrets
|
||||
from .lib.wine import *
|
||||
|
||||
c = BuildmasterConfig = {}
|
||||
|
||||
c['buildbotNetUsageData'] = None
|
||||
|
||||
####### SECRETS
|
||||
|
||||
c['secretsProviders'] = [secrets.SecretInAFile(dirname="/var/lib/buildbot/secrets/bitmessage")]
|
||||
gitea_known_hosts = util.Secret('gitea_known_hosts')
|
||||
gitea_privkey = util.Secret('gitea_privkey')
|
||||
|
||||
travis_bash = util.BuildFactory()
|
||||
travis_bash.addStep(
|
||||
PeterSurda marked this conversation as resolved
Outdated
PeterSurda
commented
I think something more like the Then we can just have an example file to show how to use it, instead of a full buildbot config needing to reside in this repo. I think something more like the `brew_package_steps` from the config repo. One for adding steps for the parent `BuildFactory`, one for the children.
Then we can just have an example file to show how to use it, instead of a full buildbot config needing to reside in this repo.
|
||||
steps.GitHub(
|
||||
repourl=util.Property("repository"),
|
||||
name="github",
|
||||
doStepIf=is_github,
|
||||
hideStepIf=isnt_github,
|
||||
branch=util.Property("branch"),
|
||||
mode="incremental",
|
||||
)
|
||||
)
|
||||
travis_bash.addStep(
|
||||
steps.Gitea(
|
||||
repourl=util.Property("repository"),
|
||||
sshPrivateKey=gitea_privkey,
|
||||
sshKnownHosts=gitea_known_hosts,
|
||||
name="gitea",
|
||||
doStepIf=is_gitea,
|
||||
hideStepIf=isnt_gitea,
|
||||
branch=util.Property("branch"),
|
||||
mode="incremental",
|
||||
)
|
||||
)
|
||||
|
||||
# execute an interpolate of '.buildbot/$(prop:jobname)/build.sh'
|
||||
travis_bash.addStep(
|
||||
steps.ShellCommand(
|
||||
name="Execute build script",
|
||||
command=[
|
||||
"bash",
|
||||
util.Interpolate(".buildbot/$(prop:jobname)/build.sh"),
|
||||
Ss_singh marked this conversation as resolved
Outdated
PeterSurda
commented
typo (I forgot the correct syntax but at the very least typo (I forgot the correct syntax but at the very least `s` is missing after `)`)
Ss_singh
commented
Got it. It should be Got it. It should be `%(prop:<propname>)s`
|
||||
],
|
||||
doStepIf=is_build_script_available,
|
||||
)
|
||||
)
|
||||
|
||||
travis_bash.addStep(
|
||||
steps.ShellCommand(
|
||||
name="Execute test script",
|
||||
command=[
|
||||
"bash",
|
||||
util.Interpolate(".buildbot/$(prop:jobname)/test.sh"),
|
||||
],
|
||||
doStepIf=is_test_script_available,
|
||||
)
|
||||
)
|
||||
|
||||
# execute multibuild.py
|
||||
PeterSurda
commented
this will be a separate this will be a separate `BuildFactory`. There will be a child one (what you have above), and a parent one (what you have below)
|
||||
travis_bash.addStep(
|
||||
steps.ShellCommand(
|
||||
name="Execute multibuild script",
|
||||
command=[
|
||||
"python",
|
||||
util.Interpolate("multibuild.py"),
|
||||
],
|
||||
)
|
||||
)
|
The hostname part of the
request_url
should be dynamic. However since it doesn't depend on a specific job, it doesn't need to use interpolate.What's the hostname part again?
https://git.bitmessage.org/Bitmessage/buildbot-config/src/branch/master/master.cfg#L1435
You said it should be dynamic, where should this come from then?
Something like
the
buildbotURL
could be passed as an argument totrigger_child_hooks
.