Added parent hook and some renderer #1

Merged
PeterSurda merged 7 commits from Ss_singh/buildbot_multibuild:hooks into master 2021-12-26 09:05:49 +01:00
3 changed files with 151 additions and 0 deletions
Showing only changes of commit 27da79a92b - Show all commits

38
lib/wine.py Normal file
View File

@ -0,0 +1,38 @@
from buildbot.plugins import util
import re
def _is_github(props):
if re.search(r'[/@]github.com', props.getProperty('repository')):
return True
return False
@util.renderer
def is_github(props):
return _is_github(props)
@util.renderer
def isnt_github(props):
return not _is_github(props)
def _is_gitea(props):
if re.search(r'[/@]git.bitmessage.org', props.getProperty('repository'), re.I):
return True
return False
@util.renderer
def is_gitea(props):
return _is_gitea(props)
@util.renderer
def isnt_gitea(props):
return not _is_gitea(props)
@util.renderer
def is_build_script_available(props):
# Actual check will got here
return props.getProperty('jobname', default='') != ''
@util.renderer
def is_test_script_available(props):
# Actual check will got here
return props.getProperty('jobname', default='') != ''

View File

@ -12,6 +12,23 @@ Requires docker
from os import walk
from os.path import exists, isfile, join, listdir
import requests
request_url = "https://buildbot.sysdeploy.org/change_hook/base"
Ss_singh marked this conversation as resolved Outdated

The hostname part of the request_url should be dynamic. However since it doesn't depend on a specific job, it doesn't need to use interpolate.

The hostname part of the `request_url` should be dynamic. However since it doesn't depend on a specific job, it doesn't need to use interpolate.

What's the hostname part again?

What's the hostname part again?
https://git.bitmessage.org/Bitmessage/buildbot-config/src/branch/master/master.cfg#L1435

You said it should be dynamic, where should this come from then?

You said it should be dynamic, where should this come from then?

Something like

c['buildbotURL'] + '/change_hook/base'

the buildbotURL could be passed as an argument to trigger_child_hooks.

Something like ``` c['buildbotURL'] + '/change_hook/base' ``` the `buildbotURL` could be passed as an argument to `trigger_child_hooks`.
request_headers = {
'Content-Type': 'application/x-www-form-urlencoded',
'Accept': 'text/plain'
}
request_data = {
'repository': 'git@git.bitmessage.org:Sysdeploy/buildbot-scripts.git',
Ss_singh marked this conversation as resolved Outdated

this should be extracted from the current properties.

this should be extracted from the current properties.

You mean it should come from util.Property("repository")? Do I need to save this repo url in some property?
Like, property="repo_url" in buildFactory step and use that later when executing the multibuild.py script?

You mean it should come from `util.Property("repository")`? Do I need to save this repo url in some property? Like, `property="repo_url"` in buildFactory step and use that later when executing the multibuild.py script?

You mean it should come from util.Property("repository")?

Yes.

Do I need to save this repo url in some property?

You should pass it as it is. Similar with "branch". I don't think others need passing.

Like, property="repo_url" in buildFactory step and use that later when executing the multibuild.py script?

If I understand you correctly, yes. These two properties you're basically just proxying from the parent to the child. But you're also adding more properties, mainly "jobname" and "dockerfile".

> You mean it should come from util.Property("repository")? Yes. > Do I need to save this repo url in some property? You should pass it as it is. Similar with "branch". I don't think others need passing. > Like, property="repo_url" in buildFactory step and use that later when executing the multibuild.py script? If I understand you correctly, yes. These two properties you're basically just proxying from the parent to the child. But you're also adding more properties, mainly "jobname" and "dockerfile".
'project': 'testproject',
'author': 'cloud-init',
'comments': 'testcomment',
}
ty = 'child_hook'
Ss_singh marked this conversation as resolved Outdated

this is fixed, derived from the base hook class.

this is fixed, derived from the base hook class.

Since we don't need type, I'd just remove it

Since we don't need `type`, I'd just remove it

We still need to construct the full URL, which consists of the URL of the buildbot server and the path for the base webhook. The latter part is statoc, I don't even think it's configurable in buildbot, it's derived from the hook class.

We still need to construct the full URL, which consists of the URL of the buildbot server and the path for the base webhook. The latter part is statoc, I don't even think it's configurable in buildbot, it's derived from the hook class.

So what this 'base webhook' should actually be?

So what this 'base webhook' should actually be?

I think it's fixed at /change_hook/base

I think it's fixed at `/change_hook/base`
hn = 'buildbot.sysdeploy.org' #hostname here(required?)
Ss_singh marked this conversation as resolved Outdated

this can be extracted from global variables

this can be extracted from global variables

Similarly we don't need node here so it can be removed

Similarly we don't need `node` here so it can be removed
addn = ',"build_script_available":"{build_script_available}","test_script_available":"{test_script_available}"'
def list_jobs(directory=".buildbot"):
@ -37,3 +54,26 @@ def find_artifacts(directory="out"):
if not isfile(join(directory, _)):
continue
return join(directory, _)
def trigger_child_hooks(directory=".buildbot"):
# List all jobs in the directory
jobs = list_jobs(directory)
# Check if build.sh or test.sh exists in each of the jobs
for job in jobs:
build_script_exists = False
test_script_exists = False
if exists(join(directory, job, "build.sh")):
build_script_exists = True
Ss_singh marked this conversation as resolved Outdated

There should be additional code for modifying the dockerfile. At the very least, it needs to install buildbot-worker and its dependencies, setup the default worker and run it in entrypoint. At the moment we can assume "apt", but in the future it should be able to detect the distro and adjust commands based on that.

For security it should also filter the file to remove CMD or ENTRYFILE lines.

There should be additional code for modifying the dockerfile. At the very least, it needs to install buildbot-worker and its dependencies, setup the default worker and run it in entrypoint. At the moment we can assume "apt", but in the future it should be able to detect the distro and adjust commands based on that. For security it should also filter the file to remove CMD or ENTRYFILE lines.

Okay it's like take the contents of already available Dockerfile and append it with extra commands based on the distro.

Okay it's like take the contents of already available Dockerfile and append it with extra commands based on the distro.

Yes something like that, but it should be only the buildbot-specific stuff and not the whole existing Dockerfile

Yes something like that, but it should be only the buildbot-specific stuff and not the whole existing `Dockerfile`

I've pushed an update resolving this. Kindly have a look

I've pushed an update resolving this. Kindly have a look
if exists(join(directory, job, "test.sh")):
test_script_exists = True
addn = addn.format(build_script_available=build_script_exists, test_script_available=test_script_exists)

I realised a problem. The code needs to be split between master and worker parts. Master code doesn't see the directories and files, and worker code doesn't have access to properties.

So my suggestion would be to add a BuildStep that constructs a JSON string (i.e. a renderer) and uploads it into the worker as a file. The next step will instruct the worker to run a script that will fetch the props from this JSON files, combines them with the per-job props and calls the child hooks.

I realised a problem. The code needs to be split between master and worker parts. Master code doesn't see the directories and files, and worker code doesn't have access to properties. So my suggestion would be to add a `BuildStep` that constructs a JSON string (i.e. a renderer) and uploads it into the worker as a file. The next step will instruct the worker to run a script that will fetch the props from this JSON files, combines them with the per-job props and calls the child hooks.
# make a post request
request_data['properties'] = '{"node":{hn},"type":{ty}{addn}"}'.format(hn=hn, ty=ty, addn=addn)

node and type we don't need, that's for sysdeploy. we should put all the properties (build_script_available, test_script_available, dockerfile, ...) into this dict, there shouldn't be a sub-dict.

`node` and `type` we don't need, that's for sysdeploy. we should put all the properties (build_script_available, test_script_available, dockerfile, ...) into this dict, there shouldn't be a sub-dict.

I think it would look something like this (still may need fine tuning):

request_data = {
    'repository': Util.Property('repository'),
    'branch': Util.Property('branch'),
    'properties': {
                  'build_script_available': build_script_exists,
                  'test_script_available': test_script_exists,
                  'dockerfile': dockerfile
                  }
               }

requests.post(request_url, headers=request_headers, data=request_data)
I think it would look something like this (still may need fine tuning): ``` request_data = { 'repository': Util.Property('repository'), 'branch': Util.Property('branch'), 'properties': { 'build_script_available': build_script_exists, 'test_script_available': test_script_exists, 'dockerfile': dockerfile } } requests.post(request_url, headers=request_headers, data=request_data) ```

Can I access this Util in multibuid.py as well? I just passed the 'repository' and 'branch' as command line args while executing multibuild.py

Can I access this `Util` in multibuid.py as well? I just passed the 'repository' and 'branch' as command line args while executing multibuild.py

multibuild.py will have to be loaded from the buildbot config, just like now it's importing lib/wine.py, for example. But it can be made into a separate pip module, for example, so it can be developed and used independently.

`multibuild.py` will have to be loaded from the buildbot config, just like now it's importing `lib/wine.py`, for example. But it can be made into a separate pip module, for example, so it can be developed and used independently.
requests.post(request_url, headers=request_headers, data=request_data)

we also need to add the repo information, i.e. the pre-existing properties.

we also need to add the repo information, i.e. the pre-existing properties.

And those are what? 'repository', 'branch', 'author', these things?

And those are what? 'repository', 'branch', 'author', these things?

yes. I think for an MVP we only need repository and branch, maybe revision. For later, probably owner and owners (the way I have it setup now, it would allow people to re-run jobs for their own PRs manually, without these two only admin can re-run job). I don't think author is needed, that's just something I borrowed from somewhere else.

yes. I think for an MVP we only need `repository` and `branch`, maybe `revision`. For later, probably `owner` and `owners` (the way I have it setup now, it would allow people to re-run jobs for their own PRs manually, without these two only admin can re-run job). I don't think `author` is needed, that's just something I borrowed from somewhere else.
if __name__ == "__main__":
trigger_child_hooks()

73
parent_hook Normal file
View File

@ -0,0 +1,73 @@
#!/usr/bin/env python3

I think we can remove this file.

I think we can remove this file.
""" parent webhook """
from buildbot.plugins import steps, util, secrets
from .lib.wine import *
c = BuildmasterConfig = {}
c['buildbotNetUsageData'] = None
####### SECRETS
c['secretsProviders'] = [secrets.SecretInAFile(dirname="/var/lib/buildbot/secrets/bitmessage")]
gitea_known_hosts = util.Secret('gitea_known_hosts')
gitea_privkey = util.Secret('gitea_privkey')
travis_bash = util.BuildFactory()
travis_bash.addStep(
PeterSurda marked this conversation as resolved Outdated

I think something more like the brew_package_steps from the config repo. One for adding steps for the parent BuildFactory, one for the children.

Then we can just have an example file to show how to use it, instead of a full buildbot config needing to reside in this repo.

I think something more like the `brew_package_steps` from the config repo. One for adding steps for the parent `BuildFactory`, one for the children. Then we can just have an example file to show how to use it, instead of a full buildbot config needing to reside in this repo.
steps.GitHub(
repourl=util.Property("repository"),
name="github",
doStepIf=is_github,
hideStepIf=isnt_github,
branch=util.Property("branch"),
mode="incremental",
)
)
travis_bash.addStep(
steps.Gitea(
repourl=util.Property("repository"),
sshPrivateKey=gitea_privkey,
sshKnownHosts=gitea_known_hosts,
name="gitea",
doStepIf=is_gitea,
hideStepIf=isnt_gitea,
branch=util.Property("branch"),
mode="incremental",
)
)
# execute an interpolate of '.buildbot/$(prop:jobname)/build.sh'
travis_bash.addStep(
steps.ShellCommand(
name="Execute build script",
command=[
"bash",
util.Interpolate(".buildbot/$(prop:jobname)/build.sh"),
Ss_singh marked this conversation as resolved Outdated

typo (I forgot the correct syntax but at the very least s is missing after ))

typo (I forgot the correct syntax but at the very least `s` is missing after `)`)

Got it. It should be %(prop:<propname>)s

Got it. It should be `%(prop:<propname>)s`
],
doStepIf=is_build_script_available,
)
)
travis_bash.addStep(
steps.ShellCommand(
name="Execute test script",
command=[
"bash",
util.Interpolate(".buildbot/$(prop:jobname)/test.sh"),
],
doStepIf=is_test_script_available,
)
)
# execute multibuild.py

this will be a separate BuildFactory. There will be a child one (what you have above), and a parent one (what you have below)

this will be a separate `BuildFactory`. There will be a child one (what you have above), and a parent one (what you have below)
travis_bash.addStep(
steps.ShellCommand(
name="Execute multibuild script",
command=[
"python",
util.Interpolate("multibuild.py"),
],
)
)