Bitmessage/buildbot_multibuild
6
0
Fork 5
Code Issues 4 Pull Requests Projects Releases Wiki Activity

worker and master splited #8

Closed
cis-muzahid wants to merge 1 commits from cis-muzahid/buildbot_multibuild:kivy-test into master
pull from: cis-muzahid/buildbot_multibuild:kivy-test
merge into: Bitmessage:master
Conversation 0 Commits 1 Files Changed 2 +142 -123
2 changed files with 142 additions and 123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

120
lib/worker_multibuild.py Normal file
View File

@ -0,0 +1,120 @@
from os import listdir, walk
from os.path import exists, isfile, join
import requests
import re
PeterSurda marked this conversation as resolved Outdated
PeterSurda commented 2022-02-17 08:00:11 +01:00
Outdated
Review
Copy Link

this is probably here only because of @util.renderer, which shouldn't be on the worker

this is probably here only because of `@util.renderer`, which shouldn't be on the worker
cis-muzahid commented 2022-02-17 11:10:37 +01:00
Outdated
Review
Copy Link

Removed from worker

Removed from worker
PeterSurda commented 2022-02-18 08:36:07 +01:00
Outdated
Review
Copy Link

also remove from .renderers import *

also remove `from .renderers import *`
request_data = {
"project": "testproject",
"comments": "testcomment",
}
request_headers = {
"Content-Type": "application/x-www-form-urlencoded",
"Accept": "text/plain",
}
ty = "/change_hook/base"
path =".buildbot"
dockerfile_extra_contents = {}
dockerfile_extra_contents['focal'] = """
PeterSurda marked this conversation as resolved Outdated
PeterSurda commented 2022-02-22 08:17:47 +01:00
Outdated
Review
Copy Link

make this into a dict

make this into a dict
# Buildbot
RUN apt-get install -yq --no-install-suggests --no-install-recommends \
buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
python3-pip dumb-init curl openssh-client wget
# buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
"""
dockerfile_extra_contents['bionic'] = """
# Buildbot
RUN apt-get install -yq --no-install-suggests --no-install-recommends \
buildbot-slave git subversion python3-dev libffi-dev python3-setuptools \
python3-pip dumb-init curl openssh-client wget
# buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
"""
def list_jobs(directory=".buildbot"):
"""
list jobs found in a directory
"""
results = []
for _ in next(walk(directory))[1]:
if exists(join(directory, _, "Dockerfile")) and (
exists(join(directory, _, "build.sh"))
or exists(join(directory, _, "test.sh"))
):
results.append(_)
return results
def _get_dockerfile_contents(jobname, os_codename='bionic'):
PeterSurda marked this conversation as resolved Outdated
PeterSurda commented 2022-02-21 07:36:53 +01:00
Outdated
Review
Copy Link

no props

no `props`
"""
Read contents of a Dockerfile and add extra contents for the given os_codename
PeterSurda marked this conversation as resolved Outdated
PeterSurda commented 2022-02-17 07:59:41 +01:00
Outdated
Review
Copy Link

can't access props, instead the jobname variable needs to be passed from master

can't access `props`, instead the `jobname` variable needs to be passed from master
cis-muzahid commented 2022-02-17 11:14:01 +01:00
Outdated
Review
Copy Link

Also passed os_codename instead of util.Interpolate("%(prop:os_codename)s")

Also passed os_codename instead of `util.Interpolate("%(prop:os_codename)s")`
"""
res = ""
with open(join(path + jobname), "r") as file:
contents = file.read()
# accept any line containing FROM and RUN keywords
PeterSurda marked this conversation as resolved Outdated
PeterSurda commented 2022-02-23 07:48:07 +01:00
Outdated
Review
Copy Link

the check should be reversed. Instead of removing lines containing FROM or RUN, it should remove the other lines.

the check should be reversed. Instead of removing lines containing FROM or RUN, it should remove the other lines.
res = ""
inside_allowed_command = False
for line in contents:
PeterSurda marked this conversation as resolved Outdated
PeterSurda commented 2022-02-24 08:04:18 +01:00
Outdated
Review
Copy Link
inside_allowed_command = False
for line in contents:
    # maybe add newlines when appending line
    if re.match(r"(?m)^(FROM|RUN).*$", line):
        inside_allowed_command = True
    if inside_allowed_command:
        ret += line
        if not re.match(r"\\\w*$", line):
            inside_allowed_command = False
``` inside_allowed_command = False for line in contents: # maybe add newlines when appending line if re.match(r"(?m)^(FROM|RUN).*$", line): inside_allowed_command = True if inside_allowed_command: ret += line if not re.match(r"\\\w*$", line): inside_allowed_command = False ```
if re.match(r"(?m)^(FROM|RUN).*$", line):
inside_allowed_command = True
if inside_allowed_command:
res += line
l = line.strip()
if not l.endswith("\\"):
PeterSurda marked this conversation as resolved Outdated
PeterSurda commented 2022-02-25 07:41:02 +01:00
Outdated
Review
Copy Link
  • \n probably not needed due to strip
  • remove continue and revert conditional
- `\n` probably not needed due to `strip` - remove `continue` and revert conditional
inside_allowed_command = False
PeterSurda marked this conversation as resolved Outdated
PeterSurda commented 2022-02-17 07:57:52 +01:00
Outdated
Review
Copy Link

this can't be on a worker, because of the decorator
however, it needs to worker differently. It needs to return getProperty('dockerfile') or soemthing like that.

this can't be on a worker, because of the decorator however, it needs to worker differently. It needs to return `getProperty('dockerfile')` or soemthing like that.
return res + dockerfile_extra_contents[os_codename]
def trigger_child_hooks(buildbotUrl: str, os_codename: str, repository, branch, jobname, directory=".buildbot", is_build_script_available, is_test_script_available):
PeterSurda marked this conversation as resolved Outdated
PeterSurda commented 2022-02-17 07:55:59 +01:00
Outdated
Review
Copy Link
  • buildbotUrl: not sure, perhaps needs to be passed from master
  • os_codename: probably there is a python library that does this
  • repository, branch, jobname: master needs to pass this data to worker
  • direcotry: can remain as it is as it's only for testing, we don't really need this
- buildbotUrl: not sure, perhaps needs to be passed from master - os_codename: probably there is a python library that does this - repository, branch, jobname: master needs to pass this data to worker - direcotry: can remain as it is as it's only for testing, we don't really need this
request_url = buildbotUrl + ty
# List all jobs in the directory
jobs = list_jobs(directory)
# Check if build.sh or test.sh exists in each of the jobs
for job in jobs:
build_script_exists = False
test_script_exists = False
if exists(join(directory, job, "build.sh")):
build_script_exists = True
if exists(join(directory, job, "test.sh")):
test_script_exists = True
# make a post request
request_data["properties"] = {
"dockerfile": _get_dockerfile_contents(
PeterSurda marked this conversation as resolved Outdated
PeterSurda commented 2022-02-17 07:58:12 +01:00
Outdated
Review
Copy Link

"dockerfile": _get_dockerfile_contents(

`"dockerfile": _get_dockerfile_contents(`
join(directory, job, "Dockerfile"), os_codename
),
"build_script_available": is_build_script_available(build_script_exists),
"test_script_available": is_test_script_available(test_script_exists),
"repository": repository,
"branch": branch,
"jobname": jobname,
}
requests.post(request_url, headers=request_headers, data=request_data)

145
multibuild.py
View File

@ -10,7 +10,7 @@ Requires docker
# TODO: write hook job, maybe also a dockerfile? # TODO: write hook job, maybe also a dockerfile?
# TODO: what to do about non-docker jobs # TODO: what to do about non-docker jobs
from os import listdir, walk from os import listdir, walk, getenv
from os.path import exists, isfile, join from os.path import exists, isfile, join
import requests import requests
import re import re
@ -18,70 +18,6 @@ from buildbot.plugins import steps, util
from .lib.renderers import * from .lib.renderers import *
PeterSurda marked this conversation as resolved Outdated
PeterSurda commented 2022-02-17 07:56:32 +01:00
Outdated
Review
Copy Link

also only worker

also only worker
ty = "/change_hook/base"
request_headers = {
"Content-Type": "application/x-www-form-urlencoded",
"Accept": "text/plain",
}
request_data = {
"project": "testproject",
"comments": "testcomment",
}
dockerfile_extra_contents_focal = """
# Buildbot
RUN apt-get install -yq --no-install-suggests --no-install-recommends \
buildbot-worker git subversion python3-dev libffi-dev python3-setuptools \
python3-pip dumb-init curl openssh-client wget
# buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
"""
dockerfile_extra_contents_bionic = """
# Buildbot
RUN apt-get install -yq --no-install-suggests --no-install-recommends \
buildbot-slave git subversion python3-dev libffi-dev python3-setuptools \
python3-pip dumb-init curl openssh-client wget
# buildbot entrypoint
RUN wget -O /usr/local/bin/buildbot_entrypoint.sh https://git.bitmessage.org/Bitmessage/buildbot-scripts/raw/branch/master/docker/bionic/entrypoint.sh
RUN chmod +x /usr/local/bin/buildbot_entrypoint.sh
RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER buildbot
ENTRYPOINT /usr/local/bin/buildbot_entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS"
"""
path =".buildbot"
def list_jobs(directory=".buildbot"):
"""
list jobs found in a directory
"""
results = []
for _ in next(walk(directory))[1]:
if exists(join(directory, _, "Dockerfile")) and (
exists(join(directory, _, "build.sh"))
or exists(join(directory, _, "test.sh"))
):
results.append(_)
return results
PeterSurda marked this conversation as resolved Outdated
PeterSurda commented 2022-02-18 08:34:18 +01:00
Outdated
Review
Copy Link

ok but different location (inside add_child_step)

ok but different location (inside `add_child_step`)
def find_artifacts(directory="out"): def find_artifacts(directory="out"):
""" """
PeterSurda marked this conversation as resolved Outdated
PeterSurda commented 2022-02-17 07:56:18 +01:00
Outdated
Review
Copy Link

needs to be on the worker

needs to be on the worker
@ -93,62 +29,29 @@ def find_artifacts(directory="out"):
return join(directory, _) return join(directory, _)
def get_dockerfile_contents(props): # @util.renderer
PeterSurda marked this conversation as resolved Outdated
PeterSurda commented 2022-02-22 08:31:06 +01:00
Outdated
Review
Copy Link

we don't need this function

we don't need this function
PeterSurda commented 2022-02-28 07:39:39 +01:00
Review
Copy Link

remove

remove
👍 1
""" # def get_dockerfile_contents(props):
Read contents of a Dockerfile and add extra contents for the given os_codename # return _get_dockerfile_contents(props.getProperty('jobname', default=None), util.Interpolate("%(prop:os_codename)s"))
PeterSurda marked this conversation as resolved Outdated
PeterSurda commented 2022-02-21 07:37:52 +01:00
Outdated
Review
Copy Link

no calling _get_dockerfile_contents, just access props

no calling `_get_dockerfile_contents`, just access `props`
PeterSurda commented 2022-02-22 08:21:05 +01:00
Outdated
Review
Copy Link

return props.getProperty('dockerfile')

We already have this information.

`return props.getProperty('dockerfile')` We already have this information.
"""
with open(join(path + props.getProperty('jobname', default=None)), "r") as file:
contents = file.read()
# remove any line containing CMD or ENTRYFILE keywords
re.sub(r"(?m)^(CMD|ENTRYFILE).*$", "", contents)
return contents + {
"focal": dockerfile_extra_contents_focal,
"bionic": dockerfile_extra_contents_bionic,
}[util.Interpolate("%(prop:os_codename)s")]
def trigger_child_hooks(buildbotUrl: str, os_codename: str, repository, branch, jobname, directory=".buildbot"):
request_url = buildbotUrl + ty
# List all jobs in the directory
jobs = list_jobs(directory)
# Check if build.sh or test.sh exists in each of the jobs
for job in jobs:
build_script_exists = False
test_script_exists = False
if exists(join(directory, job, "build.sh")):
build_script_exists = True
if exists(join(directory, job, "test.sh")):
test_script_exists = True
# make a post request
request_data["properties"] = {
"dockerfile": get_dockerfile_contents(
join(directory, job, "Dockerfile"), os_codename
),
"build_script_available": is_build_script_available(build_script_exists),
"test_script_available": is_test_script_available(test_script_exists),
"repository": repository,
"branch": branch,
"jobname": jobname,
}
requests.post(request_url, headers=request_headers, data=request_data)
def add_parent_step(build_factory): def add_parent_step(build_factory):
""" """
Add a step to the parent build factory that will trigger the child hooks Add a step to the parent build factory that will trigger the child hooks
PeterSurda marked this conversation as resolved Outdated
PeterSurda commented 2022-02-18 08:33:02 +01:00
Outdated
Review
Copy Link

return props.getProperty('dockerfile')

`return props.getProperty('dockerfile')`
""" """
build_factory.addStep(
steps.SetPropertyFromCommand(
name="Get OS codename",
command="grep ^VERSION_CODENAME= /etc/os-release | cut -d= -f2",
property="os_codename",
)
)
build_factory.addStep(steps.ShellCommand(
name="download worker",
command=["wget", "-O", "https://git.bitmessage.org/Bitmessage/buildbot_multibuild/raw/branch/master/lib/worker_multibuild.py", join(getenv['HOME'], '.local/bin/worker_multibuild.py')]
PeterSurda marked this conversation as resolved Outdated
PeterSurda commented 2022-02-23 07:49:36 +01:00
Outdated
Review
Copy Link

no need for sudo, however we need a different directory

no need for sudo, however we need a different directory
PeterSurda commented 2022-02-24 08:07:45 +01:00
Outdated
Review
Copy Link

maybe into ~/.local/bin/ (also needs to create the directory if not existant)

maybe into `~/.local/bin/` (also needs to create the directory if not existant)
PeterSurda commented 2022-02-25 07:41:49 +01:00
Outdated
Review
Copy Link

no sudo

no `sudo`
PeterSurda commented 2022-02-25 07:42:58 +01:00
Outdated
Review
Copy Link

also maybe
os.path.join(os.getenv['HOME'], '.local/bin/worker_multibuild.py')

also maybe `os.path.join(os.getenv['HOME'], '.local/bin/worker_multibuild.py')`
cis-muzahid commented 2022-02-28 13:54:05 +01:00
Outdated
Review
Copy Link

I am using --force-directories for crete folder if not exists

I am using ```--force-directories``` for crete folder if not exists
))
# build_factory.addStep(
PeterSurda marked this conversation as resolved Outdated
PeterSurda commented 2022-02-22 08:15:48 +01:00
Outdated
Review
Copy Link

os_codename should be per-Dockerfile. We don't really need os_codename in the parent, only in the child.

The Dockerfiles will begin with, e.g. FROM ubuntu:bionic .... So the parent will then extract the codename from that, and based on that append the corresponding buildbot's setup and entrypoint.

`os_codename` should be per-Dockerfile. We don't really need `os_codename` in the parent, only in the child. The `Dockerfile`s will begin with, e.g. `FROM ubuntu:bionic ...`. So the parent will then extract the codename from that, and based on that append the corresponding buildbot's setup and entrypoint.
PeterSurda commented 2022-02-28 07:43:55 +01:00
Review
Copy Link

remove

remove
👍 1
# steps.SetPropertyFromCommand(
# name="Get OS codename",
# command="grep ^VERSION_CODENAME= /etc/os-release | cut -d= -f2",
# property="os_codename",
# )
# )
build_factory.addStep( build_factory.addStep(
steps.ShellCommand( steps.ShellCommand(
name="Execute multibuild script", name="Execute multibuild script",
@ -165,24 +68,20 @@ def add_parent_step(build_factory):
) )
PeterSurda marked this conversation as resolved Outdated
PeterSurda commented 2022-02-21 07:38:31 +01:00
Outdated
Review
Copy Link

upon reflection, this BuildStep needs to go to the parent, not child

upon reflection, this `BuildStep` needs to go to the parent, not child
def add_child_build_sh_step(build_factory, directory=".buildbot"): def add_child_sh_steps(build_factory, directory=".buildbot"):
""" """
Add a step to the build factory Add a step to the download, build and test factory
PeterSurda marked this conversation as resolved Outdated
PeterSurda commented 2022-02-25 07:44:40 +01:00
Outdated
Review
Copy Link

please same indent as elsewhere

please same indent as elsewhere
""" """
build_factory.addStep( build_factory.addStep(
steps.ShellCommand( steps.ShellCommand(
name=util.Interpolate("build_%(prop:jobname)s"), name=util.Interpolate("build_%(prop:jobname)s"),
command=util.Interpolate("%(kw:directory)s/%(prop:jobname)s/test.sh", directory=directory), command=util.Interpolate("%(kw:directory)s/%(prop:jobname)s/build.sh", directory=directory),
doStepIf=is_build_script_available, doStepIf=is_build_script_available,
hideStepIf=isnt_build_script_available, hideStepIf=isnt_build_script_available,
) )
) )
def add_child_test_sh_step(build_factory, directory=".buildbot"):
"""
Add a step to the build factory
"""
build_factory.addStep( build_factory.addStep(
steps.ShellCommand( steps.ShellCommand(
name= util.Interpolate("test_%(prop:jobname)s"), name= util.Interpolate("test_%(prop:jobname)s"),
@ -204,7 +103,7 @@ if __name__ == "__main__":
buildbotUrl = sys.argv[4] buildbotUrl = sys.argv[4]
os_codename = sys.argv[5] os_codename = sys.argv[5]
trigger_child_hooks(buildbotUrl, os_codename, repository, branch, jobname) trigger_child_hooks(buildbotUrl, os_codename, repository, branch, jobname, ".buildbot", is_build_script_available, is_test_script_available)
else: else:
print( print(
"Usage: python3 multibuild.py <jobname> <repository> <branch> <buildbotUrl> <os_codename>" "Usage: python3 multibuild.py <jobname> <repository> <branch> <buildbotUrl> <os_codename>"