worker and master splited #9
No reviewers
Labels
No Label
bug
duplicate
enhancement
help wanted
invalid
question
wontfix
No Milestone
No project
No Assignees
2 Participants
Notifications
Due Date
No due date set.
Dependencies
No dependencies set.
Reference: Bitmessage/buildbot_multibuild#9
Loading…
Reference in New Issue
Block a user
No description provided.
Delete Branch "cis-muzahid/buildbot_multibuild:kivy-test"
Deleting a branch is permanent. Although the deleted branch may continue to exist for a short time before it actually gets removed, it CANNOT be undone in most cases. Continue?
Split master and worker file,
New PR behalf of #8 which is deleted bymistake.
66a413f4d5
to6fb2fdfa34
5aaad5b9b6
to5a9465a02d
--force-directories
doesn't do what you think it does.@ -0,0 +109,4 @@
# make a post request
request_data["properties"] = {
"dockerfile": _get_dockerfile_contents(
join(directory, job, "Dockerfile"), os_codename
remove
os_codename
from arguments@ -0,0 +69,4 @@
return results
def _get_dockerfile_contents(jobname, os_codename='bionic'):
remove
os_codename
, make it a variable, not an argument@ -151,0 +36,4 @@
build_factory.addStep(steps.ShellCommand(
name="download worker",
command=["wget", "--force-directories", "-O", "https://git.bitmessage.org/Bitmessage/buildbot_multibuild/raw/branch/master/lib/worker_multibuild.py", join(getenv['HOME'], '.local/bin/worker_multibuild.py')]
add
mkdir -p
@ -204,7 +90,7 @@ if __name__ == "__main__":
buildbotUrl = sys.argv[4]
os_codename = sys.argv[5]
no
os_codename
@ -205,3 +91,3 @@
os_codename = sys.argv[5]
trigger_child_hooks(buildbotUrl, os_codename, repository, branch, jobname)
trigger_child_hooks(buildbotUrl, os_codename, repository, branch, jobname, ".buildbot", is_build_script_available, is_test_script_available)
also here
01e2caa8f5
tof98a503e2b
@ -0,0 +59,4 @@
list jobs found in a directory
"""
results = []
for _ in next(walk(directory))[1]:
I wouldn't use walk, we only need one level. Also, we need to add a check that
Dockerfile
,build.sh
andtest.sh
are regular files and not symlinks (probably two checks needed per file so another function should be written for that)@ -151,0 +36,4 @@
build_factory.addStep(steps.ShellCommand(
name="download worker",
command=["mkdir", "-p", join(getenv['HOME'], '.local/bin'), "&&", "wget", "-O", "https://git.bitmessage.org/Bitmessage/buildbot_multibuild/raw/branch/master/lib/worker_multibuild.py", join(getenv['HOME'], '.local/bin/worker_multibuild.py')]
should be two separate commands.
@ -0,0 +60,4 @@
"""
results = []
files = ["Dockerfile", "build.sh", "test.sh"]
for file in files:
I think I didn't explain it well. It should look for subdirectories, but only at one level, and should ignore symlinked directories
results = []
files = ["Dockerfile", "build.sh", "test.sh"]
for item in listdir(directory):
print(item)
if isdir(join(directory, item)) and not islink(join(directory, item)):
for file in listdir(join(directory, item)):
if file in files and isfile(join(directory, item, file)) and not islink(join(directory, item, file)):
results.append(file)
return results
Is this logic fine? I haven't remove file name from files while it found in path
@ -0,0 +63,4 @@
for file in files:
print(join(directory, file))
print(exists(join(directory, file)))
if exists(join(directory, file)) and not islink(join(directory, file)):
this is incomplete (doesn't filter directories, sockets, devices, etc). Better would be
isfile and not islink
@ -154,3 +49,3 @@
name="Execute multibuild script",
name="Execute worker script",
command=[
"python",
python3
@ -0,0 +62,4 @@
files = ["Dockerfile", "build.sh", "test.sh"]
for item in listdir(directory):
if isdir(join(directory, item)) and not islink(join(directory, item)):
for file in listdir(join(directory, item)):
With the following condition
if exists(Dockerfile) and (exists(build.sh) or exists(test.sh)):
If we have
Dockerfile
as a normal file andbuilds.sh
as symlink so its skipsDockerfile
as it does not include the directory.So its clear we have to keep both
Dockerfile and builds.sh
at same placein the directoryThese files are all per directory. And only if all the files in the directory pass checks, will the directory be permitted as a job.
@ -0,0 +64,4 @@
if isdir(join(directory, item)) and not islink(join(directory, item)):
for file in listdir(join(directory, item)):
if file in files and isfile(join(directory, item, file)) and not islink(join(directory, item, file)):
results.append(file)
we should add
item
, notfile
to the return array (should be fixed as described in the pseudocode above)@ -0,0 +63,4 @@
for item in listdir(directory):
for file in files:
filepath = join(directory, item, file)
if islink(filepath) and not exists(filepath):
@ -0,0 +65,4 @@
filepath = join(directory, item, file)
if islink(filepath) and not exists(filepath):
continue
if exists(join(directory, item, 'Dockerfile')) or exists(join(directory, item, 'build.sh')) or exists(join(directory, item, 'test.sh')):
indent one level up
also first
or
shoud beand
and missing brackets@ -0,0 +82,4 @@
res = ""
inside_allowed_command = False
for line in contents:
if re.match(r"(?m)^(FROM|RUN).*$", line):
ENV
also is allowed@ -157,2 +51,3 @@
"multibuild.py",
"python3",
join(getenv['HOME'], '.local/bin/worker_multibuild.py'),
util.Interpolate("%(prop:jobname)s"),
no
jobname
in parent. Parent inserts the jobnames into buildbot, doesn't read them from buildbot. Child contains a jobname as a property then.same for
os_codename
. Parent inserts os_codename into buildbot.perhaps insteaad of URL, but it may not be possible at least without workarounds
maybe also
@ -0,0 +97,4 @@
return res + dockerfile_extra_contents[os_codename]
def trigger_child_hooks(buildbotUrl: str, repository, branch, jobname, directory=".buildbot", is_build_script_available, is_test_script_available):
no
jobname
no
is_build_script_available
no
is_test_script_available
These are all per-jobs variables, extracted from the filesystem in each loop.
@ -0,0 +121,4 @@
"test_script_available": is_test_script_available(test_script_exists),
"repository": repository,
"branch": branch,
"jobname": jobname,
"jobname": job,
@ -160,3 +57,2 @@
util.Property('branch'),
"https://buildbot.bitmessage.org",
util.Interpolate("%(prop:os_codename)s"),
util.getURLForBuild(util, util.Property("builderid"), util.Property("buildnumber")),
try:
util.Property('url')
@ -205,3 +98,2 @@
os_codename = sys.argv[5]
trigger_child_hooks(buildbotUrl, os_codename, repository, branch, jobname)
trigger_child_hooks(buildbotUrl, repository, branch, jobname, ".buildbot", is_build_script_available, is_test_script_available)
again remove
jobname
,is_build_script_available
,is_test_script_available
@ -81,3 +22,1 @@
results.append(_)
return results
os_codename='bionic'
can be removed
@ -202,10 +94,9 @@ if __name__ == "__main__":
repository = sys.argv[2]
branch = sys.argv[3]
buildbotUrl = sys.argv[4]
remove
buildbotUrl
Sorry, I think
buildbotUrl
need to be hereI think you're right.
@ -205,3 +97,2 @@
os_codename = sys.argv[5]
trigger_child_hooks(buildbotUrl, os_codename, repository, branch, jobname)
trigger_child_hooks(buildbotUrl, repository, branch, ".buildbot")
remove
".buildbot"
d501ce26a0
to2b3963b9ae
@ -208,3 +96,3 @@
else:
print(
"Usage: python3 multibuild.py <jobname> <repository> <branch> <buildbotUrl> <os_codename>"
"Usage: python3 multibuild.py <jobname> <repository> <branch> <buildbotUrl> "
This help message needs to be updated to reflect the arguments correctly.
08999a7a91
tod847415a41