Improve kvm performance

- seems to better fit into the workflow

docker-install-all.sh

@@ -26,24 +26,6 @@ cores=$(grep -E '^cpu cores' /proc/cpuinfo |head -1|cut -d: -f2|tr -d '[:space:]
 mem=$(sed -E 's/^MemTotal: +([0-9]+) kB/\1/p;d' < /proc/meminfo)
 mempercore=$(((mem/1024-4096)/cores))
 
-# install kata containers
-if ! docker info 2>/dev/null|grep Runtimes:|grep -qs kata-qemu-virtiofs; then
-    echo "Installing kata...."
-    cid=$(docker run -d --runtime=runc -v /opt/kata:/opt/kata -v /var/run/dbus:/var/run/dbus -v /run/systemd:/run/systemd -v /etc/docker:/etc/docker -it katadocker/kata-deploy kata-deploy-docker install)
-
-    # wait for finish
-    while [ "$(docker container inspect -f '{{.State.Running}}' "$cid")" == "true" ]; do
-        sleep 1
-    done
-
-    # delete container
-    docker rm $cid
-
-    # update default container CPU and memory
-    sed -i "s/default_vcpus = .*/default_vcpus = 2/g;s/default_memory = .*/default-memory = $mempercore/g" /opt/kata/share/defaults/kata-containers/configuration-qemu-virtiofs.toml
-    echo "Kata installed"
-fi
-
 trusty
 xenial
 bionic

docker/bionic/Dockerfile

@@ -30,7 +30,7 @@ RUN apt-get install -yq --no-install-suggests --no-install-recommends \
     python-minimal python-setuptools python-all python openssl libssl-dev \
     dh-apparmor debhelper dh-python python-msgpack python-qt4 python-stdeb \
     python-all-dev python-crypto python-pycryptopp python-psutil \
-    fakeroot python-pytest
+    fakeroot python-pytest libglib2.0-bin
 
 # Code quality
 RUN apt-get install -yq --no-install-suggests --no-install-recommends \

docker/bionic/buildbot-ca.crt

@@ -0,0 +1,32 @@
+-----BEGIN CERTIFICATE-----
+MIIFczCCA1ugAwIBAgIUDGtpHV1pcNVBXVt69fRXvCwBdc0wDQYJKoZIhvcNAQEL
+BQAwSTELMAkGA1UEBhMCQVQxDzANBgNVBAgMBlZpZW5uYTEPMA0GA1UEBwwGVmll
+bm5hMRgwFgYDVQQKDA9CaXRtZXNzYWdlIEdtYkgwHhcNMjQwMzAzMTgyNzI5WhcN
+MjYxMTI4MTgyNzI5WjBJMQswCQYDVQQGEwJBVDEPMA0GA1UECAwGVmllbm5hMQ8w
+DQYDVQQHDAZWaWVubmExGDAWBgNVBAoMD0JpdG1lc3NhZ2UgR21iSDCCAiIwDQYJ
+KoZIhvcNAQEBBQADggIPADCCAgoCggIBAN297lK71o4mpXBzmVnjW76m/r4lp+1H
+F6T5IPbNZ3DpvibJRk2TXjEblpen2hdcJffvlRVCG3yQ8tgWLTVe6y7C/dDx7UEZ
+BsvUvKoJqdTbsg9OF888Qq1ghsHmgjgCJ4bNDg/FLZryj0JxWcmvfR0cnoDZscFS
+GuqDlcA/5Zc456Wkf5yjjYlVEz/qclfand2xkYCJM+OuCLnzsgWyW3NyAapC7PYG
+mfmNE/vKboHOMVkScSxyf4RfNQlt/ZurNqQW8SgVE2ppicF6N+HDxDIsXIRo3zaW
+DjF3qC3QctJwbc70xxtTiSTYj0L4j5hSypkJ7oO+U6mk+wdt4abTMS/on8MGVDZi
+WwtnW7PYrvmuhrAaWEsETCySdVLWvTAEjhEqSoHWd0bIh0Qnhtso/VVjKsm/8zZI
+NKNjV+BIyr2L9ZuE34nnbkEV6Dfj58IW4nwykFEljdt1im6hgpyVsTg+WJ1gMCxH
+VBFqfFBI/BKLnqiBjQAwUBQOzWE+emV7GeXC1e4rO/7Ptj6nVAyG7MHbqjYZOKRv
+DsAWx6eeRWNdfZrGTcmB6x1zVDI6gxp/GLRU2sfqs1g7p40I1qUS2YPng9BAw5TL
+1WjJJwpMB5XiWy3qcg20FbiH2Q0olitOTEnHPL3Ecco0vQMgr5VQgDdM/O0TuWYB
+LvFdU/H8CfOBAgMBAAGjUzBRMB0GA1UdDgQWBBRkgqkhWz8EvvFsOaxbg/MjoWuq
+cTAfBgNVHSMEGDAWgBRkgqkhWz8EvvFsOaxbg/MjoWuqcTAPBgNVHRMBAf8EBTAD
+AQH/MA0GCSqGSIb3DQEBCwUAA4ICAQCFLT/RMzFl4SRmGyfTbRBXDUORSqq1mmdz
+LNl1+UqdfMYupLgsJXZWf3wEt+RRjZzXBVX1ZwjgBiQS1MTMrcJ+5x4JCOtknK17
+3N0KKK58McAIn6BL44uAHAAmJsfptB1AnFf6adlRuVLeBs3Dgj2aU5nJ79ZKkClN
+xJX8roGkkfn21lJ/lpEQwdz3klSIvE2Jr0HOsFBbeXre8q/zUQCRuEtRSjW45Ycx
+Yv3aI1Mo0Ak5P7XDDmux6tD6+nn9lAzOGucTfmZbsH85WGV5eSRezb4xqYUfZq1+
+1uF+aLRNdiV9+M0/PJJ82GjxWfL+n+hh3LTxecym3Gj1IEgfW8AMN/VJPRPWYOrg
+zIwG7F6NDh3szaSbCgZ0htezHTWH0NW/+PO90dMdTwdJi2/WVAaqqCk6sTfGfKdx
+sb6MAdLxmoYOZVj0eSaoAq0dJEOo/5yWwzJEHZYjXkc/xJVs6O60x2GLk5dXz6U9
+dQIfVIPBeNYwA1ICbo5TJ3xjYUe3nQAT2TXlD2tFCj2VgGFfQRBRrkci/m+oJr6v
+DbBw7WaVw843qm5/dfNH7Nx8FhFpCJ6T/ILqaoU1v5DBS71I6rkQCuBp4Aqp+Waw
+2NqJB0/fVkvcoP66JMLImdbtRq8N9JMcofWDjq29HTpPQAnXl72J53H6NTFtFTpZ
+MXyumh2iRw==
+-----END CERTIFICATE-----

docker/bionic/entrypoint.sh

@@ -1,9 +1,10 @@
 #!/bin/bash
 
+[ -d /var/lib/buildbot/workers ] || mkdir /var/lib/buildbot/workers
 
-buildbot-worker create-worker /var/lib/buildbot/workers/default "$1" "$2" "$3"
+buildbot-worker create-worker --use-tls /var/lib/buildbot/workers/default "$1" "$2" "$3"
 
-unset BUILDMASTER BUILDMASTER_PORT WORKERNAME WORKERPASS
+unset BUILDMASTER WORKERNAME WORKERPASS
 
 cd /var/lib/buildbot/workers/default
 /usr/bin/dumb-init buildbot-worker start --nodaemon

docker/xenial/entrypoint.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+
+buildslave create-slave /var/lib/buildbot/slaves/default "$1" "$2" "$3"
+
+unset BUILDMASTER BUILDMASTER_PORT WORKERNAME WORKERPASS
+
+cd /var/lib/buildbot/slaves/default
+/usr/bin/dumb-init buildslave start --nodaemon

pybitmessage.wsgi

@@ -0,0 +1,278 @@
+#!/usr/bin/python2.7
+
+#import datetime
+from base64 import b64encode
+from cStringIO import StringIO
+import fcntl
+from hashlib import sha1
+import hmac
+import httplib
+import json
+import os
+import pprint
+import requests
+import shutil
+from subprocess import call, Popen
+import sys
+import tempfile
+import time
+import traceback
+from urlparse import parse_qs
+
+gitHubSecret = ""
+gitHubUsername = ""
+gitHubToken = ""
+transifexSecret = ""
+transifexUsername = ""
+transifexPassword = ""
+branch = "v0.6"
+lock = None
+lockFile = ".webhook.lock"
+
+os.chdir("/usr/src/PyBitmessage")
+if os.environ.get("HOME") is None:
+    os.environ["HOME"] = "/var/www"
+
+def debug(obj):
+    sys.stderr.write(pprint.pformat(obj) + "\n")
+
+def verifyGitHubSignature (environ, payload_body):
+    signature = 'sha1=' + hmac.new(gitHubSecret, payload_body, sha1).hexdigest()
+    try:
+        if signature != environ.get('HTTP_X_HUB_SIGNATURE'):
+            return False
+        return True
+    except:
+        return False
+
+def verifyTransifexSignature (environ, payload_body):
+    signature = b64encode(hmac.new(transifexSecret, payload_body, sha1).digest())
+    try:
+        debug(signature)
+        if signature != environ.get('HTTP_X_TX_SIGNATURE'):
+            return False
+        return True
+    except:
+        return False
+
+def returnMessage(status = False, message = "Unimplemented"):
+    output = json.dumps({"status": "OK" if status else "FAIL", "message": message})
+    return [output, [('Content-type', 'text/plain'),
+        ('Content-Length', str(len(output)))
+    ]]
+
+def updateLocalTranslationSource():
+    call(["git", "stash", "-q"])
+    call(["git", "checkout", "-q", branch])
+    call(["git", "pull", "-q"])
+    call(["pylupdate4", "src/translations/bitmessage.pro"])
+
+def uploadTranslationSource():
+    headers = {"Authorization": "Basic " + b64encode(transifexUsername + ":" + transifexPassword)}
+    response = requests.put("https://www.transifex.com/api/2/project/pybitmessage/resource/pybitmessage/content/",
+        headers=headers, files={'bitmessage_en.ts': open("src/translations/bitmessage_en.ts", "rb")})
+    return response
+
+def updateLocalTranslationDestination(ts, lang):
+    call(["git", "pull", "--all", "-q"])
+    call(["git", "stash", "-q"])
+    call(["git", "checkout", "-q", branch])
+    call(["git", "checkout", "-q", "-b", "translate_" + lang + "_" + str(ts)])
+    call(["git", "branch", "-q", "--set-upstream-to=origin/v0.6"])
+
+def downloadTranslatedLanguage(ts, lang):
+    headers = {"Authorization": "Basic " + b64encode(transifexUsername + ":" + transifexPassword)}
+    resname = "pybitmessage_" + lang + ".ts"
+    fname = "bitmessage_" + lang.lower() + ".ts"
+    with open("src/translations/" + fname, "wt") as handle:
+        response = requests.get("https://www.transifex.com/api/2/project/pybitmessage/resource/pybitmessage/translation/" + lang + "/",
+            headers=headers)
+        if response.ok:
+            content = json.loads(response.content)["content"]
+            handle.write(content.encode("utf-8"))
+#        print "Response from github for pull request: %i, %s" % (response.status_code, response.content)
+    return response
+
+def commitTranslatedLanguage(ts, lang):
+    call(["lrelease-qt4", "src/translations/bitmessage.pro"])
+    call(["git", "add", "src/translations/bitmessage_" + lang + ".ts", "src/translations/bitmessage_" + lang + ".qm"])
+    call(["git", "commit", "-q", "-S", "-m", "Auto-updated language %s from transifex" % (lang)])
+    newbranch = "translate_" + lang + "_" + str(ts)
+    call(["git", "push", "-q", "translations", newbranch + ":" + newbranch])
+    request = {
+        "title": "Translation update " + lang,
+        "body": "Auto-updated from transifex",
+        "head": "PyBitmessageTranslations:" + newbranch,
+        "base": branch
+    }
+    headers = {"Authorization": "token " + gitHubToken}
+    response = requests.post("https://api.github.com/repos/Bitmessage/PyBitmessage/pulls",
+        headers=headers, data=json.dumps(request))
+    # TODO: save pull request number
+    return response
+#    print "JSON dumps request: %s" % (json.dumps(request))
+#    print "Response from github for pull request: %i, %s" % (response.status_code, response.content)
+
+def listPullRequests():
+    headers = {"Authorization": "token " + gitHubToken}
+    response = requests.get("https://api.github.com/repos/Bitmessage/PyBitmessage/pulls?state=open&base=%s&sort=created&direction=desc" % branch,
+        headers=headers)
+    pulls = []
+    if response.ok:
+        try:
+            data = json.loads(response.content)
+            for i in data:
+                if i['user']['login'] != gitHubUsername:
+                    continue
+                if not i['head']['label'].startswith(gitHubUsername):
+                    continue
+#                print i['number'], i['title'], i['user']['login'], i['head']['label'], i['head']['ref']
+                pulls.append({'number': i['number'], 'branch': i['head']['ref']})
+        except:
+            print "Exception"
+            traceback.print_exc()
+            pass
+    else:
+        print "Not ok"
+    return pulls
+#    print "JSON dumps request: %s" % (json.dumps(request))
+#    print "Response from github for pull request: %i, %s" % (response.status_code, response.content)
+
+def rebasePullRequest(newbranch):
+#    newbranch = "translate_" + lang + "_" + str(ts)
+    call(["git", "pull", "--all", "-q"])
+    call(["git", "stash", "-q"])
+    call(["git", "checkout", "-q", newbranch])
+    call(["git", "branch", "-q", "--set-upstream-to=origin/v0.6"])
+    call(["git", "rebase", "-q"])
+    call(["git", "commit", "-q", "--no-edit", "--amend", "-S"])
+    call(["git", "push", "-q", "-f", "translations", newbranch + ":" + newbranch])
+    call(["git", "checkout", "-q", branch])
+
+def checkIfPullRequestMerged(ts, lang):
+#    Get if a pull request has been merged
+#    GET /repos/:owner/:repo/pulls/:number/merge
+#    Response if pull request has been merged
+#    Status: 204 No Content
+#    Response if pull request has not been merged
+#    Status: 404 Not Found
+    return
+
+def deleteBranch(ts, lang):
+    newbranch = "translate_" + lang + "_" + str(ts)
+    call(["git", "branch", "-q", "-D", newbranch])
+    # TODO: delete remote branch
+
+def lockWait():
+    global lockFile, lock
+    lock = open(lockFile, "wb")
+    fcntl.lockf(lock, fcntl.LOCK_EX)
+
+def unlock():
+    global lockFile, lock
+    fcntl.lockf(lock, fcntl.LOCK_UN)
+    if os.path.isfile(lockFile):
+        os.unlink(lockFile)
+
+def application(environ, start_response):
+    status = '200 OK'
+    output = ''
+
+    lockWait()
+    length = int(environ.get('CONTENT_LENGTH', '0'))
+    body = environ['wsgi.input'].read(length)
+#   h environ['wsgi.input'] = body
+
+    if environ.get("HTTP_X_GITHUB_EVENT") == "ping":
+        if not verifyGitHubSignature(environ, body):
+            output, responseHeaders = returnMessage(False, "Checksum bad")
+            start_response(status, responseHeaders)
+            unlock()
+            return [output]
+        output, responseHeaders = returnMessage(True, "Test OK")
+    elif environ.get("HTTP_X_GITHUB_EVENT") == "push":
+        if not verifyGitHubSignature(environ, body):
+            output, responseHeaders = returnMessage(False, "Checksum bad")
+            start_response(status, responseHeaders)
+            unlock()
+            return [output]
+        try:
+            payload = json.loads(body)
+            if payload['ref'] != "refs/heads/" + branch:
+                unlock()
+                raise Exception
+            updateLocalTranslationSource()
+            response = uploadTranslationSource()
+            output, responseHeaders = returnMessage(True, "Processed: %i, %s:" % (response.status_code, response.content))
+        except:
+            output, responseHeaders = returnMessage(True, "Not processing")
+    elif "Transifex" in environ.get("HTTP_USER_AGENT"):
+#        debug(environ)
+#        debug(body)
+        if not verifyTransifexSignature(environ, body):
+            debug ("Verify Transifex Signature fail, but fuck them")
+        else:
+            debug ("Verify Transifex Signature ok")
+#            output, responseHeaders = returnMessage(False, "Checksum bad")
+#            start_response(status, responseHeaders)
+#            unlock()
+#            return [output]
+        try:
+#            debug(body)
+            payload = parse_qs(body)
+#            debug(payload)
+            if 'pybitmessage' in payload['project'] and 'pybitmessage' in payload['resource']:
+                if 'translated' in payload and '100' in payload['translated']:
+                    ts = int(time.time())
+                    updateLocalTranslationDestination(ts, payload['language'][0].lower())
+                    downloadTranslatedLanguage(ts, payload['language'][0])
+                    response = commitTranslatedLanguage(ts, payload['language'][0].lower())
+                    if response.ok:
+                        output, responseHeaders = returnMessage(True, "Processed.")
+                    else:
+                        output, responseHeaders = returnMessage(False, "Error: %i." % (response.status_code))
+                else:
+                    output, responseHeaders = returnMessage(False, "Nothing to do")
+            else:
+                output, responseHeaders = returnMessage(False, "Nothing to do")
+        except:
+            output, responseHeaders = returnMessage(True, "Not processing")
+    else:
+        debug("Unknown command %s" % (environ.get("HTTP_X_GITHUB_EVENT")))
+        output, responseHeaders = returnMessage(True, "Unknown command, ignoring")
+#    output = ''
+#    for k, v in environ.items():
+#        output += '%.40s %s\n' % (k, v)
+    #responseHeaders = sendFile("ffb8c8eb-3d3b-4306-b65e-e0d1fa4f7ea0")
+    start_response(status, responseHeaders)
+    unlock()
+    return [output]
+
+if __name__ == "__main__":
+    lockWait()
+    if len(sys.argv) < 2:
+        unlock()
+        sys.exit()
+    if sys.argv[1] == "commit":
+        updateLocalTranslationSource(tempdir)
+        response = uploadTranslationSource(tempdir)
+        print "Uploaded to transifex: %i, %s" % (response.status_code, response.content)
+    elif sys.argv[1] == "translated":
+        if len(sys.argv) < 3:
+            unlock()
+            sys.exit()
+        lang = sys.argv[2]
+        print "Cloning repo"
+        ts = int(time.time())
+        updateLocalTranslationDestination(ts, lang.lower())
+        print "Downloading translated file"
+        downloadTranslatedLanguage(ts, lang)
+        print "Creating pull request"
+        response = commitTranslatedLanguage(ts, lang.lower())
+        print "Pull request sent"
+    elif sys.argv[1] == "rebase":
+        for pull in listPullRequests():
+            rebasePullRequest(pull['branch'])
+            print "Rebased %s" % (pull['number'])
+            break
+    unlock()

qemu.hook

@@ -70,7 +70,7 @@ def prepare_files(domain, env_):
 
         # resize
         cmd = ['/usr/bin/qemu-img',
-               'resize', disk, "+10G"
+               'resize', disk, "+20G"
                ]
         subprocess.call(cmd)
 

travis2bash.sh

@@ -63,7 +63,7 @@ fi
 
 if [ ! -e "$config_file" ]; then
     echo "No $config_file found, exiting"
-    exit 1
+    exit 0
 fi
 
 function aptinstall() {
@@ -189,9 +189,9 @@ if [[ "$(declare -p travis_python)" =~ "declare -a" ]]; then
             echo "skipping python-$pv due to TRAVIS_PYTHON=$TRAVIS_PYTHON"
             continue
         fi
-        #if [ ! -e "$ppath" ]; then
+        if [ ! -e "$ppath" ]; then
             sudo apt -y install "python$pv" "python${pv}-dev"
-        #fi
+        fi
 
         if [ -n "$options" ]; then
             virtualenv_init "$ppath" "$pv" "$options"

virt-install-all.sh

@@ -14,6 +14,7 @@ trusty()
     xml=$(mktemp)
 
     virt-install -r "$mempercore" --vcpus=2,maxvcpus=2,sockets=1,cores=1,threads=2 \
+        --cpu host \
         -n trusty_libvirt_"${hostname}_${id}" -w network=default --nographics \
         --disk path=/var/lib/libvirt/ephemeral/trusty_libvirt_"${hostname}_${id}".qcow2 \
         --disk path=/var/lib/libvirt/ephemeral/trusty_libvirt_"${hostname}_${id}".iso,device=cdrom \
@@ -40,6 +41,7 @@ xenial()
     xml=$(mktemp)
 
     virt-install -r "$mempercore" --vcpus=2,maxvcpus=2,sockets=1,cores=1,threads=2 \
+        --cpu host \
         -n xenial_libvirt_"${hostname}_${id}" -w network=default --nographics \
         --disk path=/var/lib/libvirt/ephemeral/xenial_libvirt_"${hostname}_${id}".qcow2 \
         --disk path=/var/lib/libvirt/ephemeral/xenial_libvirt_"${hostname}_${id}".iso,device=cdrom \
@@ -66,6 +68,7 @@ bionic()
     xml=$(mktemp)
     
     virt-install -r "$mempercore" --vcpus=2,maxvcpus=2,sockets=1,cores=1,threads=2 \
+        --cpu host \
         -n bionic_libvirt_"${hostname}_${id}" -w network=default --nographics \
         --disk path=/var/lib/libvirt/ephemeral/bionic_libvirt_"${hostname}_${id}".qcow2 \
         --disk path=/var/lib/libvirt/ephemeral/bionic_libvirt_"${hostname}_${id}".iso,device=cdrom \
@@ -92,6 +95,7 @@ focal()
     xml=$(mktemp)
     
     virt-install -r "$mempercore" --vcpus=2,maxvcpus=2,sockets=1,cores=1,threads=2 \
+        --cpu host \
         -n focal_libvirt_"${hostname}_${id}" -w network=default --nographics \
         --disk path=/var/lib/libvirt/ephemeral/focal_libvirt_"${hostname}_${id}".qcow2 \
         --disk path=/var/lib/libvirt/ephemeral/focal_libvirt_"${hostname}_${id}".iso,device=cdrom \
@@ -104,6 +108,32 @@ focal()
     rm -f "$xml" /var/lib/libvirt/ephemeral/focal_libvirt_"${hostname}_${id}".{iso,qcow2}
 }
 
+jammy()
+{
+    id="$1"
+    qemu-img create -b /var/lib/libvirt/backingstore/jammy.qcow2 \
+        -f qcow2 -F qcow2 \
+        /var/lib/libvirt/ephemeral/jammy_libvirt_"${hostname}_${id}".qcow2
+    cloud-localds /var/lib/libvirt/ephemeral/jammy_libvirt_"${hostname}_${id}".iso \
+        /var/lib/libvirt/backingstore/jammy.user
+
+    virsh undefine jammy_libvirt_"${hostname}_${id}"
+
+    xml=$(mktemp)
+    
+    virt-install -r "$mempercore" --vcpus=2,maxvcpus=2,sockets=1,cores=1,threads=2 \
+        -n jammy_libvirt_"${hostname}_${id}" -w network=default --nographics \
+        --disk path=/var/lib/libvirt/ephemeral/jammy_libvirt_"${hostname}_${id}".qcow2 \
+        --disk path=/var/lib/libvirt/ephemeral/jammy_libvirt_"${hostname}_${id}".iso,device=cdrom \
+        --import --noautoconsole \
+        --print-xml --dry-run --check disk_size=off,path_in_use=off \
+        --boot=hd --os-type=Linux --os-variant ubuntu18.04 > "$xml"
+
+    virsh define "$xml"
+
+    rm -f "$xml" /var/lib/libvirt/ephemeral/jammy_libvirt_"${hostname}_${id}".{iso,qcow2}
+}
+
 elcapitan()
 {
     local id
@@ -144,7 +174,9 @@ cores=$(grep -E '^cpu cores' /proc/cpuinfo |head -1|cut -d: -f2|tr -d '[:space:]
 mem=$(sed -E 's/^MemTotal: +([0-9]+) kB/\1/p;d' < /proc/meminfo)
 mempercore=$(((mem/1024-4096)/cores))
 
-elcapitan 1
+cp -f /usr/src/buildbot-workers/*.user /var/lib/libvirt/backingstore
+
+# elcapitan 1
 
 for i in $(seq "$cores"); do
     trusty "$i" &
@@ -161,4 +193,7 @@ wait
 for i in $(seq "$cores"); do
     focal "$i" &
 done
+for i in $(seq "$cores"); do
+    jammy "$i" &
+done
 wait