Compare commits

..

18 Commits

Author SHA1 Message Date
Peter Šurda 6238736bff
Improve kvm performance 2024-03-16 13:32:57 +08:00
Peter Šurda 405cafe1c7
Update buildbot PB CA 2024-03-04 06:58:04 +08:00
Peter Šurda a4891458e3
Revert last commit 2024-03-04 02:07:27 +08:00
Peter Šurda 24bcc34915
Add missing argument 2024-03-04 01:42:59 +08:00
Peter Šurda 0cdd912945
Trying connection string 2024-03-04 01:36:08 +08:00
Peter Šurda ae9f18c287
Create worker parente directory if it doesn't exist 2024-03-03 23:39:16 +08:00
Peter Šurda 7a0392ec76
Fix argument order 2024-03-03 23:22:02 +08:00
Peter Šurda e36b4b986e
Change entrypoint TLS setting 2024-03-03 21:37:48 +08:00
Peter Šurda d9356bbd1e
Change BUILDMASTER to CONNECTIONSTRING 2024-03-03 20:18:34 +08:00
Peter Šurda 9d1a28c49f
Add PB CA cert 2024-03-03 19:49:02 +08:00
Peter Šurda 5ef9d31387
Add jammy 2024-03-01 23:22:13 +08:00
Peter Šurda 0a9a5d0968
Add transifex webhook handler 2022-12-13 14:34:16 +08:00
Peter Šurda a7076d3b65
Add deployment of libvirt user-data 2022-12-05 12:40:34 +08:00
Peter Šurda fd0967a53d
Disable elcapitan 2022-12-05 11:44:28 +08:00
Peter Šurda ba0a697b4f
Don't fail if config file missing
- seems to better fit into the workflow
2022-11-16 11:37:53 +08:00
Peter Šurda 2736785d7d
xenial entrypoint fix
buildbot/multibuild_parent Build done. Details
buildbot/travis_bionic Build done. Details
2022-05-08 12:50:47 +08:00
Peter Šurda 9ee854b6f7
Xenial entrypoint
buildbot/multibuild_parent Build done. Details
buildbot/travis_bionic Build done. Details
2022-05-08 12:26:36 +08:00
Peter Šurda 9c955b591c
Python package source workaround
buildbot/multibuild_parent Build done. Details
buildbot/travis_bionic Build done. Details
- system pythohn packages have priority over deadsnakes PPA
2022-04-25 16:08:59 +08:00
6 changed files with 362 additions and 7 deletions

View File

@ -0,0 +1,32 @@
-----BEGIN CERTIFICATE-----
MIIFczCCA1ugAwIBAgIUDGtpHV1pcNVBXVt69fRXvCwBdc0wDQYJKoZIhvcNAQEL
BQAwSTELMAkGA1UEBhMCQVQxDzANBgNVBAgMBlZpZW5uYTEPMA0GA1UEBwwGVmll
bm5hMRgwFgYDVQQKDA9CaXRtZXNzYWdlIEdtYkgwHhcNMjQwMzAzMTgyNzI5WhcN
MjYxMTI4MTgyNzI5WjBJMQswCQYDVQQGEwJBVDEPMA0GA1UECAwGVmllbm5hMQ8w
DQYDVQQHDAZWaWVubmExGDAWBgNVBAoMD0JpdG1lc3NhZ2UgR21iSDCCAiIwDQYJ
KoZIhvcNAQEBBQADggIPADCCAgoCggIBAN297lK71o4mpXBzmVnjW76m/r4lp+1H
F6T5IPbNZ3DpvibJRk2TXjEblpen2hdcJffvlRVCG3yQ8tgWLTVe6y7C/dDx7UEZ
BsvUvKoJqdTbsg9OF888Qq1ghsHmgjgCJ4bNDg/FLZryj0JxWcmvfR0cnoDZscFS
GuqDlcA/5Zc456Wkf5yjjYlVEz/qclfand2xkYCJM+OuCLnzsgWyW3NyAapC7PYG
mfmNE/vKboHOMVkScSxyf4RfNQlt/ZurNqQW8SgVE2ppicF6N+HDxDIsXIRo3zaW
DjF3qC3QctJwbc70xxtTiSTYj0L4j5hSypkJ7oO+U6mk+wdt4abTMS/on8MGVDZi
WwtnW7PYrvmuhrAaWEsETCySdVLWvTAEjhEqSoHWd0bIh0Qnhtso/VVjKsm/8zZI
NKNjV+BIyr2L9ZuE34nnbkEV6Dfj58IW4nwykFEljdt1im6hgpyVsTg+WJ1gMCxH
VBFqfFBI/BKLnqiBjQAwUBQOzWE+emV7GeXC1e4rO/7Ptj6nVAyG7MHbqjYZOKRv
DsAWx6eeRWNdfZrGTcmB6x1zVDI6gxp/GLRU2sfqs1g7p40I1qUS2YPng9BAw5TL
1WjJJwpMB5XiWy3qcg20FbiH2Q0olitOTEnHPL3Ecco0vQMgr5VQgDdM/O0TuWYB
LvFdU/H8CfOBAgMBAAGjUzBRMB0GA1UdDgQWBBRkgqkhWz8EvvFsOaxbg/MjoWuq
cTAfBgNVHSMEGDAWgBRkgqkhWz8EvvFsOaxbg/MjoWuqcTAPBgNVHRMBAf8EBTAD
AQH/MA0GCSqGSIb3DQEBCwUAA4ICAQCFLT/RMzFl4SRmGyfTbRBXDUORSqq1mmdz
LNl1+UqdfMYupLgsJXZWf3wEt+RRjZzXBVX1ZwjgBiQS1MTMrcJ+5x4JCOtknK17
3N0KKK58McAIn6BL44uAHAAmJsfptB1AnFf6adlRuVLeBs3Dgj2aU5nJ79ZKkClN
xJX8roGkkfn21lJ/lpEQwdz3klSIvE2Jr0HOsFBbeXre8q/zUQCRuEtRSjW45Ycx
Yv3aI1Mo0Ak5P7XDDmux6tD6+nn9lAzOGucTfmZbsH85WGV5eSRezb4xqYUfZq1+
1uF+aLRNdiV9+M0/PJJ82GjxWfL+n+hh3LTxecym3Gj1IEgfW8AMN/VJPRPWYOrg
zIwG7F6NDh3szaSbCgZ0htezHTWH0NW/+PO90dMdTwdJi2/WVAaqqCk6sTfGfKdx
sb6MAdLxmoYOZVj0eSaoAq0dJEOo/5yWwzJEHZYjXkc/xJVs6O60x2GLk5dXz6U9
dQIfVIPBeNYwA1ICbo5TJ3xjYUe3nQAT2TXlD2tFCj2VgGFfQRBRrkci/m+oJr6v
DbBw7WaVw843qm5/dfNH7Nx8FhFpCJ6T/ILqaoU1v5DBS71I6rkQCuBp4Aqp+Waw
2NqJB0/fVkvcoP66JMLImdbtRq8N9JMcofWDjq29HTpPQAnXl72J53H6NTFtFTpZ
MXyumh2iRw==
-----END CERTIFICATE-----

View File

@ -1,9 +1,10 @@
#!/bin/bash
[ -d /var/lib/buildbot/workers ] || mkdir /var/lib/buildbot/workers
buildbot-worker create-worker /var/lib/buildbot/workers/default "$1" "$2" "$3"
buildbot-worker create-worker --use-tls /var/lib/buildbot/workers/default "$1" "$2" "$3"
unset BUILDMASTER BUILDMASTER_PORT WORKERNAME WORKERPASS
unset BUILDMASTER WORKERNAME WORKERPASS
cd /var/lib/buildbot/workers/default
/usr/bin/dumb-init buildbot-worker start --nodaemon

9
docker/xenial/entrypoint.sh Executable file
View File

@ -0,0 +1,9 @@
#!/bin/bash
buildslave create-slave /var/lib/buildbot/slaves/default "$1" "$2" "$3"
unset BUILDMASTER BUILDMASTER_PORT WORKERNAME WORKERPASS
cd /var/lib/buildbot/slaves/default
/usr/bin/dumb-init buildslave start --nodaemon

278
pybitmessage.wsgi Normal file
View File

@ -0,0 +1,278 @@
#!/usr/bin/python2.7
#import datetime
from base64 import b64encode
from cStringIO import StringIO
import fcntl
from hashlib import sha1
import hmac
import httplib
import json
import os
import pprint
import requests
import shutil
from subprocess import call, Popen
import sys
import tempfile
import time
import traceback
from urlparse import parse_qs
gitHubSecret = ""
gitHubUsername = ""
gitHubToken = ""
transifexSecret = ""
transifexUsername = ""
transifexPassword = ""
branch = "v0.6"
lock = None
lockFile = ".webhook.lock"
os.chdir("/usr/src/PyBitmessage")
if os.environ.get("HOME") is None:
os.environ["HOME"] = "/var/www"
def debug(obj):
sys.stderr.write(pprint.pformat(obj) + "\n")
def verifyGitHubSignature (environ, payload_body):
signature = 'sha1=' + hmac.new(gitHubSecret, payload_body, sha1).hexdigest()
try:
if signature != environ.get('HTTP_X_HUB_SIGNATURE'):
return False
return True
except:
return False
def verifyTransifexSignature (environ, payload_body):
signature = b64encode(hmac.new(transifexSecret, payload_body, sha1).digest())
try:
debug(signature)
if signature != environ.get('HTTP_X_TX_SIGNATURE'):
return False
return True
except:
return False
def returnMessage(status = False, message = "Unimplemented"):
output = json.dumps({"status": "OK" if status else "FAIL", "message": message})
return [output, [('Content-type', 'text/plain'),
('Content-Length', str(len(output)))
]]
def updateLocalTranslationSource():
call(["git", "stash", "-q"])
call(["git", "checkout", "-q", branch])
call(["git", "pull", "-q"])
call(["pylupdate4", "src/translations/bitmessage.pro"])
def uploadTranslationSource():
headers = {"Authorization": "Basic " + b64encode(transifexUsername + ":" + transifexPassword)}
response = requests.put("https://www.transifex.com/api/2/project/pybitmessage/resource/pybitmessage/content/",
headers=headers, files={'bitmessage_en.ts': open("src/translations/bitmessage_en.ts", "rb")})
return response
def updateLocalTranslationDestination(ts, lang):
call(["git", "pull", "--all", "-q"])
call(["git", "stash", "-q"])
call(["git", "checkout", "-q", branch])
call(["git", "checkout", "-q", "-b", "translate_" + lang + "_" + str(ts)])
call(["git", "branch", "-q", "--set-upstream-to=origin/v0.6"])
def downloadTranslatedLanguage(ts, lang):
headers = {"Authorization": "Basic " + b64encode(transifexUsername + ":" + transifexPassword)}
resname = "pybitmessage_" + lang + ".ts"
fname = "bitmessage_" + lang.lower() + ".ts"
with open("src/translations/" + fname, "wt") as handle:
response = requests.get("https://www.transifex.com/api/2/project/pybitmessage/resource/pybitmessage/translation/" + lang + "/",
headers=headers)
if response.ok:
content = json.loads(response.content)["content"]
handle.write(content.encode("utf-8"))
# print "Response from github for pull request: %i, %s" % (response.status_code, response.content)
return response
def commitTranslatedLanguage(ts, lang):
call(["lrelease-qt4", "src/translations/bitmessage.pro"])
call(["git", "add", "src/translations/bitmessage_" + lang + ".ts", "src/translations/bitmessage_" + lang + ".qm"])
call(["git", "commit", "-q", "-S", "-m", "Auto-updated language %s from transifex" % (lang)])
newbranch = "translate_" + lang + "_" + str(ts)
call(["git", "push", "-q", "translations", newbranch + ":" + newbranch])
request = {
"title": "Translation update " + lang,
"body": "Auto-updated from transifex",
"head": "PyBitmessageTranslations:" + newbranch,
"base": branch
}
headers = {"Authorization": "token " + gitHubToken}
response = requests.post("https://api.github.com/repos/Bitmessage/PyBitmessage/pulls",
headers=headers, data=json.dumps(request))
# TODO: save pull request number
return response
# print "JSON dumps request: %s" % (json.dumps(request))
# print "Response from github for pull request: %i, %s" % (response.status_code, response.content)
def listPullRequests():
headers = {"Authorization": "token " + gitHubToken}
response = requests.get("https://api.github.com/repos/Bitmessage/PyBitmessage/pulls?state=open&base=%s&sort=created&direction=desc" % branch,
headers=headers)
pulls = []
if response.ok:
try:
data = json.loads(response.content)
for i in data:
if i['user']['login'] != gitHubUsername:
continue
if not i['head']['label'].startswith(gitHubUsername):
continue
# print i['number'], i['title'], i['user']['login'], i['head']['label'], i['head']['ref']
pulls.append({'number': i['number'], 'branch': i['head']['ref']})
except:
print "Exception"
traceback.print_exc()
pass
else:
print "Not ok"
return pulls
# print "JSON dumps request: %s" % (json.dumps(request))
# print "Response from github for pull request: %i, %s" % (response.status_code, response.content)
def rebasePullRequest(newbranch):
# newbranch = "translate_" + lang + "_" + str(ts)
call(["git", "pull", "--all", "-q"])
call(["git", "stash", "-q"])
call(["git", "checkout", "-q", newbranch])
call(["git", "branch", "-q", "--set-upstream-to=origin/v0.6"])
call(["git", "rebase", "-q"])
call(["git", "commit", "-q", "--no-edit", "--amend", "-S"])
call(["git", "push", "-q", "-f", "translations", newbranch + ":" + newbranch])
call(["git", "checkout", "-q", branch])
def checkIfPullRequestMerged(ts, lang):
# Get if a pull request has been merged
# GET /repos/:owner/:repo/pulls/:number/merge
# Response if pull request has been merged
# Status: 204 No Content
# Response if pull request has not been merged
# Status: 404 Not Found
return
def deleteBranch(ts, lang):
newbranch = "translate_" + lang + "_" + str(ts)
call(["git", "branch", "-q", "-D", newbranch])
# TODO: delete remote branch
def lockWait():
global lockFile, lock
lock = open(lockFile, "wb")
fcntl.lockf(lock, fcntl.LOCK_EX)
def unlock():
global lockFile, lock
fcntl.lockf(lock, fcntl.LOCK_UN)
if os.path.isfile(lockFile):
os.unlink(lockFile)
def application(environ, start_response):
status = '200 OK'
output = ''
lockWait()
length = int(environ.get('CONTENT_LENGTH', '0'))
body = environ['wsgi.input'].read(length)
# h environ['wsgi.input'] = body
if environ.get("HTTP_X_GITHUB_EVENT") == "ping":
if not verifyGitHubSignature(environ, body):
output, responseHeaders = returnMessage(False, "Checksum bad")
start_response(status, responseHeaders)
unlock()
return [output]
output, responseHeaders = returnMessage(True, "Test OK")
elif environ.get("HTTP_X_GITHUB_EVENT") == "push":
if not verifyGitHubSignature(environ, body):
output, responseHeaders = returnMessage(False, "Checksum bad")
start_response(status, responseHeaders)
unlock()
return [output]
try:
payload = json.loads(body)
if payload['ref'] != "refs/heads/" + branch:
unlock()
raise Exception
updateLocalTranslationSource()
response = uploadTranslationSource()
output, responseHeaders = returnMessage(True, "Processed: %i, %s:" % (response.status_code, response.content))
except:
output, responseHeaders = returnMessage(True, "Not processing")
elif "Transifex" in environ.get("HTTP_USER_AGENT"):
# debug(environ)
# debug(body)
if not verifyTransifexSignature(environ, body):
debug ("Verify Transifex Signature fail, but fuck them")
else:
debug ("Verify Transifex Signature ok")
# output, responseHeaders = returnMessage(False, "Checksum bad")
# start_response(status, responseHeaders)
# unlock()
# return [output]
try:
# debug(body)
payload = parse_qs(body)
# debug(payload)
if 'pybitmessage' in payload['project'] and 'pybitmessage' in payload['resource']:
if 'translated' in payload and '100' in payload['translated']:
ts = int(time.time())
updateLocalTranslationDestination(ts, payload['language'][0].lower())
downloadTranslatedLanguage(ts, payload['language'][0])
response = commitTranslatedLanguage(ts, payload['language'][0].lower())
if response.ok:
output, responseHeaders = returnMessage(True, "Processed.")
else:
output, responseHeaders = returnMessage(False, "Error: %i." % (response.status_code))
else:
output, responseHeaders = returnMessage(False, "Nothing to do")
else:
output, responseHeaders = returnMessage(False, "Nothing to do")
except:
output, responseHeaders = returnMessage(True, "Not processing")
else:
debug("Unknown command %s" % (environ.get("HTTP_X_GITHUB_EVENT")))
output, responseHeaders = returnMessage(True, "Unknown command, ignoring")
# output = ''
# for k, v in environ.items():
# output += '%.40s %s\n' % (k, v)
#responseHeaders = sendFile("ffb8c8eb-3d3b-4306-b65e-e0d1fa4f7ea0")
start_response(status, responseHeaders)
unlock()
return [output]
if __name__ == "__main__":
lockWait()
if len(sys.argv) < 2:
unlock()
sys.exit()
if sys.argv[1] == "commit":
updateLocalTranslationSource(tempdir)
response = uploadTranslationSource(tempdir)
print "Uploaded to transifex: %i, %s" % (response.status_code, response.content)
elif sys.argv[1] == "translated":
if len(sys.argv) < 3:
unlock()
sys.exit()
lang = sys.argv[2]
print "Cloning repo"
ts = int(time.time())
updateLocalTranslationDestination(ts, lang.lower())
print "Downloading translated file"
downloadTranslatedLanguage(ts, lang)
print "Creating pull request"
response = commitTranslatedLanguage(ts, lang.lower())
print "Pull request sent"
elif sys.argv[1] == "rebase":
for pull in listPullRequests():
rebasePullRequest(pull['branch'])
print "Rebased %s" % (pull['number'])
break
unlock()

View File

@ -63,7 +63,7 @@ fi
if [ ! -e "$config_file" ]; then
echo "No $config_file found, exiting"
exit 1
exit 0
fi
function aptinstall() {
@ -189,9 +189,9 @@ if [[ "$(declare -p travis_python)" =~ "declare -a" ]]; then
echo "skipping python-$pv due to TRAVIS_PYTHON=$TRAVIS_PYTHON"
continue
fi
#if [ ! -e "$ppath" ]; then
sudo apt -y install "python$pv" "python${pv}-dev"
#fi
if [ ! -e "$ppath" ]; then
sudo apt -y install "python$pv" "python${pv}-dev"
fi
if [ -n "$options" ]; then
virtualenv_init "$ppath" "$pv" "$options"

View File

@ -14,6 +14,7 @@ trusty()
xml=$(mktemp)
virt-install -r "$mempercore" --vcpus=2,maxvcpus=2,sockets=1,cores=1,threads=2 \
--cpu host \
-n trusty_libvirt_"${hostname}_${id}" -w network=default --nographics \
--disk path=/var/lib/libvirt/ephemeral/trusty_libvirt_"${hostname}_${id}".qcow2 \
--disk path=/var/lib/libvirt/ephemeral/trusty_libvirt_"${hostname}_${id}".iso,device=cdrom \
@ -40,6 +41,7 @@ xenial()
xml=$(mktemp)
virt-install -r "$mempercore" --vcpus=2,maxvcpus=2,sockets=1,cores=1,threads=2 \
--cpu host \
-n xenial_libvirt_"${hostname}_${id}" -w network=default --nographics \
--disk path=/var/lib/libvirt/ephemeral/xenial_libvirt_"${hostname}_${id}".qcow2 \
--disk path=/var/lib/libvirt/ephemeral/xenial_libvirt_"${hostname}_${id}".iso,device=cdrom \
@ -66,6 +68,7 @@ bionic()
xml=$(mktemp)
virt-install -r "$mempercore" --vcpus=2,maxvcpus=2,sockets=1,cores=1,threads=2 \
--cpu host \
-n bionic_libvirt_"${hostname}_${id}" -w network=default --nographics \
--disk path=/var/lib/libvirt/ephemeral/bionic_libvirt_"${hostname}_${id}".qcow2 \
--disk path=/var/lib/libvirt/ephemeral/bionic_libvirt_"${hostname}_${id}".iso,device=cdrom \
@ -92,6 +95,7 @@ focal()
xml=$(mktemp)
virt-install -r "$mempercore" --vcpus=2,maxvcpus=2,sockets=1,cores=1,threads=2 \
--cpu host \
-n focal_libvirt_"${hostname}_${id}" -w network=default --nographics \
--disk path=/var/lib/libvirt/ephemeral/focal_libvirt_"${hostname}_${id}".qcow2 \
--disk path=/var/lib/libvirt/ephemeral/focal_libvirt_"${hostname}_${id}".iso,device=cdrom \
@ -104,6 +108,32 @@ focal()
rm -f "$xml" /var/lib/libvirt/ephemeral/focal_libvirt_"${hostname}_${id}".{iso,qcow2}
}
jammy()
{
id="$1"
qemu-img create -b /var/lib/libvirt/backingstore/jammy.qcow2 \
-f qcow2 -F qcow2 \
/var/lib/libvirt/ephemeral/jammy_libvirt_"${hostname}_${id}".qcow2
cloud-localds /var/lib/libvirt/ephemeral/jammy_libvirt_"${hostname}_${id}".iso \
/var/lib/libvirt/backingstore/jammy.user
virsh undefine jammy_libvirt_"${hostname}_${id}"
xml=$(mktemp)
virt-install -r "$mempercore" --vcpus=2,maxvcpus=2,sockets=1,cores=1,threads=2 \
-n jammy_libvirt_"${hostname}_${id}" -w network=default --nographics \
--disk path=/var/lib/libvirt/ephemeral/jammy_libvirt_"${hostname}_${id}".qcow2 \
--disk path=/var/lib/libvirt/ephemeral/jammy_libvirt_"${hostname}_${id}".iso,device=cdrom \
--import --noautoconsole \
--print-xml --dry-run --check disk_size=off,path_in_use=off \
--boot=hd --os-type=Linux --os-variant ubuntu18.04 > "$xml"
virsh define "$xml"
rm -f "$xml" /var/lib/libvirt/ephemeral/jammy_libvirt_"${hostname}_${id}".{iso,qcow2}
}
elcapitan()
{
local id
@ -144,7 +174,9 @@ cores=$(grep -E '^cpu cores' /proc/cpuinfo |head -1|cut -d: -f2|tr -d '[:space:]
mem=$(sed -E 's/^MemTotal: +([0-9]+) kB/\1/p;d' < /proc/meminfo)
mempercore=$(((mem/1024-4096)/cores))
elcapitan 1
cp -f /usr/src/buildbot-workers/*.user /var/lib/libvirt/backingstore
# elcapitan 1
for i in $(seq "$cores"); do
trusty "$i" &
@ -161,4 +193,7 @@ wait
for i in $(seq "$cores"); do
focal "$i" &
done
for i in $(seq "$cores"); do
jammy "$i" &
done
wait