Initial commit

qemu.hook

@@ -0,0 +1,178 @@
+#!/usr/bin/python3
+"""
+Creates an empemeral image and patches it
+"""
+
+# Script /etc/libvirt/hooks/qemu
+# Don't forget to execute service libvirt-bin restart
+# Also see https://www.libvirt.org/hooks.html
+
+# This script make clean VM for each start using base image
+
+import os
+import subprocess
+import sys
+import tempfile
+import xml.etree.ElementTree as et
+
+from jinja2 import Environment, exceptions, FileSystemLoader
+
+
+BACKINGSTORE_PATH = '/var/lib/libvirt/backingstore/'
+IMAGES_PATH = '/var/lib/libvirt/images/'
+EPHEMERAL_PATH = '/var/lib/libvirt/ephemeral/'
+CLOUDINIT_PATH = '/mnt/nfsimages'
+VENDOR_EXT = '.vendor'
+USER_EXT = '.user'
+CLOUDINIT_EXT = '.iso'
+
+xml = None
+
+
+def parse_xml():
+    """
+    Parse XML provided in stdin
+    """
+    global xml
+    if xml is None:
+        data = sys.stdin.read()
+        xml = et.fromstring(data)
+    return xml
+
+
+def find_base(domain):
+    """
+    Convert VM name into hostname (part before first _)
+    """
+    return domain.split("_", 2)[0]
+
+
+def prepare_files(domain, env_):
+    """
+    Create the ephemeral images
+    """
+    xml = parse_xml()
+    base = find_base(domain)
+    for disk in xml.iterfind("./devices/disk[@device='disk']/source"):
+        disk = disk.get('file')
+        if not os.path.exists(os.path.join(BACKINGSTORE_PATH,
+                                           base + '.qcow2')):
+            continue
+        if not disk.startswith(EPHEMERAL_PATH):
+            continue
+        cmd = ['/usr/bin/qemu-img',
+               'create', '-b',
+               os.path.join(BACKINGSTORE_PATH, base + '.qcow2'),
+               '-f', 'qcow2',
+               '-F', 'qcow2',
+               disk]
+        subprocess.call(cmd)
+
+        # resize
+        cmd = ['/usr/bin/qemu-img',
+               'resize', disk, "+10G"
+               ]
+        subprocess.call(cmd)
+
+    try:
+        x = xml.find("./metadata/{http://buildbot.net/}auth")
+        username, password, master = (x.attrib["username"],
+                                        x.attrib["password"],
+                                        x.attrib["master"])
+    except Exception:
+        username = domain
+        password = domain
+        master = None
+
+    for cdrom in xml.iterfind("./devices/disk[@device='cdrom']/source"):
+        cdrom = cdrom.get('file')
+        if not cdrom.startswith(EPHEMERAL_PATH):
+            continue
+        # cloud init
+        if not os.path.exists(os.path.join(BACKINGSTORE_PATH,
+                                           base + USER_EXT)):
+            continue
+
+        hostname = os.uname()[1].replace('.', '_')
+
+        userdata = {
+            'guest': hostname,
+            'hostname': domain,
+            'buildbot_username': username,
+            'buildbot_password': password,
+            'buildbot_master': master
+        }
+
+        with tempfile.NamedTemporaryFile(mode='w+t') as temp_file:
+            # jinja user data
+            try:
+                temp_file.write(env_.get_template(domain + USER_EXT).
+                                render(userdata))
+            except exceptions.TemplateNotFound:
+                temp_file.write(env_.get_template(base + USER_EXT).
+                                render(userdata))
+            temp_file.flush()
+
+            # make ISO
+            cmd = ['/usr/bin/cloud-localds',
+                   # '-V', os.path.join(CLOUDINIT_PATH,
+                   #                    domain + VENDOR_EXT),
+                   # '-H', HOSTNAME,
+                   os.path.join(cdrom),
+                   temp_file.name
+                   ]
+            subprocess.call(cmd)
+
+
+def cleanup_files():
+    """
+    Delete the ephemeral images
+    """
+    xml = parse_xml()
+    for disk in xml.iterfind("./devices/disk[@device='disk']/source"):
+        disk = disk.get('file')
+        if disk.startswith(EPHEMERAL_PATH):
+            try:
+                os.remove(disk)
+            except FileNotFoundError:
+                pass
+
+    for cdrom in xml.iterfind("./devices/disk[@device='cdrom']/source"):
+        cdrom = cdrom.get('file')
+        if cdrom.startswith(EPHEMERAL_PATH):
+            try:
+                os.remove(cdrom)
+            except FileNotFoundError:
+                pass
+
+
+def get_buildbot_password():
+    """
+    Retrieve buildbot password for guest
+    """
+    fname = '/etc/libvirt/buildbot.pwd'
+    with open(fname) as file_handle:
+        content = [line.rstrip('\n') for line in file_handle]
+    try:
+        return content[0]
+    except (TypeError, IndexError):
+        return "password"
+
+
+if __name__ == "__main__":
+    try:
+        # pylint: disable=unbalanced-tuple-unpacking
+        VIR_DOMAIN, ACTION = sys.argv[1:3]
+    except IndexError:
+        sys.exit(1)
+
+    ENV = Environment(autoescape=False,
+                      loader=FileSystemLoader(BACKINGSTORE_PATH),
+                      trim_blocks=False)
+
+    if ACTION in ["prepare"]:
+        cleanup_files()
+        prepare_files(VIR_DOMAIN, ENV)
+
+    if ACTION in ["release"]:
+        cleanup_files()

test-checkdeps.sh

@@ -0,0 +1,102 @@
+#!/bin/bash
+
+# tests checkdeps.py with all possible subsets of dependency packages installed
+
+distro=$(grep ^VERSION_CODENAME= /etc/os-release | cut -d= -f2)
+
+case $distro in
+    trusty)
+        method="apt-get"
+        packages=(python-qt4 python-msgpack python-pyopencl python-setuptools build-essential libssl-dev libcap-dev python-prctl)
+        ;;
+    xenial)
+        method="apt-get"
+        packages=(python-qt4 python-msgpack python-pyopencl python-setuptools build-essential libssl-dev libcap-dev python-prctl)
+        ;;
+    bionic)
+        method="apt-get"
+        packages=(python-qt4 python-msgpack python-pyopencl python-setuptools build-essential libssl-dev libcap-dev python-prctl)
+        ;;
+    focal)
+        method="apt-get"
+        packages=(python-setuptools build-essential libssl-dev libcap-dev)
+        ;;
+    *)
+        echo "Unknown distro"
+        exit 0
+esac
+
+echo "Using $method on $distro"
+
+output=$(mktemp)
+succeeded=0
+failed=0
+
+# 0 - 2^n -1
+for i in $(seq 0 $((2**${#packages[@]}-1))); do
+    declare -a missing
+    missing=()
+    case $method in
+        apt-get)
+            cmd="sudo apt-get remove --autoremove -y -qq"
+            ;;
+        *);;
+     esac
+
+    # iterate packages
+    for j in $(seq 0 $((${#packages[@]}-1))); do
+        # should j-th item be included in i-th iteration?
+        if [ $(((i>>j)%2)) -eq 1 ]; then
+            case $method in
+                apt-get)
+                    cmd="$cmd ${packages[j]}+"
+                    ;;
+            esac
+        else
+            case $method in
+                apt-get)
+                    missing+=("${packages[j]}")
+                    cmd="$cmd ${packages[j]}"
+                    ;;
+            esac
+        fi
+    done
+    echo -n "${missing[*]}: "
+    $cmd &> /dev/null
+    #echo "python checkdeps.py"
+    python checkdeps.py > "$output" 2> /dev/null
+    toinstall=$(sed -E 's/.*apt-get install (.*)(`.*|$)/\1/p;d' < "$output" | xargs -n 1 echo)
+    toinstalla=($toinstall)
+    fail=0
+    for a in ${missing[*]}; do
+        for b in ${toinstalla[*]}; do
+            if [ "$a" == "$b" ]; then
+                break
+            fi
+        done
+        if ! [ "$a" == "$b" ]; then
+            fail=1
+        fi
+    done
+    for a in ${toinstalla[*]}; do
+        for b in ${missing[*]}; do
+            if [ "$a" == "$b" ]; then
+                break
+            fi
+        done
+        if ! [ "$a" == "$b" ]; then
+            fail=1
+        fi
+    done
+    if [ $fail -eq 0 ]; then
+        echo "OK"
+        succeeded=$((succeeded+1))
+    else
+        echo "FAIL"
+        echo "     " $toinstall
+        failed=$((failed+1))
+    fi
+done
+
+echo "Succeeded: $succeeded"
+echo "   Failed: $failed"

travis2bash.sh

@@ -0,0 +1,177 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC1003
+# pulled from https://raw.githubusercontent.com/jasperes/bash-yaml/master/script/yaml.sh
+# Licensed under MIT license
+
+# Based on https://gist.github.com/pkuczynski/8665367
+
+failure=0
+
+parse_yaml() {
+    local yaml_file=$1
+    local prefix=$2
+    local s
+    local w
+    local fs
+
+    s='[[:space:]]*'
+    w='[a-zA-Z0-9_.-]*'
+    fs="$(echo @|tr @ '\034')"
+
+    (
+        sed -e '/- [^\“]'"[^\']"'.*: /s|\([ ]*\)- \([[:space:]]*\)|\1-\'$'\n''  \1\2|g' |
+
+        sed -ne '/^--/s|--||g; s|\"|\\\"|g; s/[[:space:]]*$//g;' \
+            -e "/#.*[\"\']/!s| #.*||g; /^#/s|#.*||g;" \
+            -e "s|^\($s\)\($w\)$s:$s\"\(.*\)\"$s\$|\1$fs\2$fs\3|p" \
+            -e "s|^\($s\)\($w\)${s}[:-]$s\(.*\)$s\$|\1$fs\2$fs\3|p" |
+
+        awk -F"$fs" '{
+            indent = length($1)/2;
+            if (length($2) == 0) { conj[indent]="+";} else {conj[indent]="";}
+            vname[indent] = $2;
+            for (i in vname) {if (i > indent) {delete vname[i]}}
+                if (length($3) > 0) {
+                    vn=""; for (i=0; i<indent; i++) {vn=(vn)(vname[i])("_")}
+                    printf("%s%s%s%s=(\"%s\")\n", "'"$prefix"'",vn, $2, conj[indent-1],$3);
+                }
+            }' |
+
+        sed -e 's/_=/+=/g' |
+
+        awk 'BEGIN {
+                FS="=";
+                OFS="="
+            }
+            /(-|\.).*=/ {
+                gsub("-|\\.", "_", $1)
+            }
+            { print }'
+    ) < "$yaml_file"
+}
+
+create_variables() {
+    local yaml_file="$1"
+    local prefix="$2"
+    eval "$(parse_yaml "$yaml_file" "$prefix")"
+}
+
+if [ ! -e .travis.yml ]; then
+    echo "No .travis.yml found, exiting"
+    exit 1
+fi
+
+function aptinstall() {
+    # apt install
+    # shellcheck disable=SC2154
+    if [[ "$(declare -p travis_addons_apt_packages)" =~ "declare -a" ]]; then
+        echo -e "\nsudo apt -y update\n"
+        sudo apt -y update
+        for ((i = 0; i < ${#travis_addons_apt_packages[@]}; i++))
+        do
+            echo -e "\napt-get -yq --no-install-suggests --no-install-recommends install ${travis_addons_apt_packages[$i]}\n"
+            # shellcheck disable=SC2086
+            sudo apt-get -yq --no-install-suggests --no-install-recommends \
+            install \
+            ${travis_addons_apt_packages[$i]}
+        done
+    fi
+}
+
+function virtualenv_init() {
+    # init virtualenv directory
+    if [ -n "$1" -a -n "$2" ]; then
+        echo -e "\nvirtualenv -p \"$1\" ~/.venv-$2\n"
+        virtualenv -p "$1" ~/.venv-"$2"
+    else
+        echo -e "\nvirtualenv ~/.venv\n"
+        virtualenv ~/.venv
+    fi
+}
+
+function virtualenv_activate() {
+    # activate virtualenv
+    # shellcheck disable=SC1090
+    if [ -n "$1" ]; then
+        echo -e "\n. ~/.venv-$1/bin/activate\n"
+        . ~/.venv-$1/bin/activate
+    else
+        echo -e "\n. ~/.venv/bin/activate\n"
+        . ~/.venv/bin/activate
+    fi
+}
+
+function python_run() {
+    # shellcheck disable=SC2154
+    if [[ "$(declare -p travis_env)" =~ "declare -a" ]]; then
+        echo "Setting up environment variables"
+        for ((i = 0; i < ${#travis_env[@]}; i++))
+        do
+            echo -e "\n${travis_env[$i]}\n"
+            # shellcheck disable=SC2086
+            export ${travis_env[$i]}
+        done
+    fi
+
+    # shellcheck disable=SC2154
+    if [[ "$(declare -p travis_install)" =~ "declare -a" ]]; then
+        echo "Running \"install\""
+        if [ -n "$2" ]; then
+            if [ -e "requirements-${2}.txt" ]; then
+                echo "Running \"pip install -r requirements-${2}.txt\""
+                pip install -r "requirements-${2}.txt"
+            fi
+        fi
+        for ((i = 0; i < ${#travis_install[@]}; i++))
+        do
+            cd "$rundir" || continue
+            echo -e "\n${travis_install[$i]}\n"
+            if ! bash -c "${travis_install[$i]}"; then
+                failure=1
+            fi
+        done
+    fi
+    # shellcheck disable=SC2154
+    if [[ "$(declare -p travis_script)" =~ "declare -a" ]]; then
+        echo "Running \"script\""
+        for ((i = 0; i < ${#travis_script[@]}; i++))
+        do
+            cd "$rundir" || continue
+            echo -e "\n${travis_script[$i]}\n"
+            if ! bash -c "${travis_script[$i]}"; then
+                failure=1
+            fi
+        done
+    fi
+}
+
+create_variables .travis.yml travis_
+
+rundir=$(pwd)
+
+aptinstall
+
+# shellcheck disable=SC2154
+if [[ "$(declare -p travis_python)" =~ "declare -a" ]]; then
+    # shellcheck disable=SC2068
+    for pv in ${travis_python[@]}; do
+        # strip quotes
+        temp="${pv%\"}"
+        pv="${temp#\"}"
+        ppath=/usr/bin/python$pv
+        #if [ ! -e "$ppath" ]; then
+        sudo apt -y install "python$pv" "python${pv}-dev"
+        #fi
+        virtualenv_init "$ppath" "$pv"
+        virtualenv_activate "$pv"
+        python_run "$ppath" "$pv"
+        deactivate
+    done
+else
+    virtualenv_init
+    virtualenv_activate
+    python_run 
+    deactivate
+fi
+
+exit $failure

virt-install-all.sh

@@ -0,0 +1,160 @@
+#!/bin/bash
+
+trusty()
+{
+    id="$1"
+    qemu-img create -b /var/lib/libvirt/backingstore/trusty.qcow2 -f qcow2 \
+        /var/lib/libvirt/ephemeral/trusty_"${hostname}_${id}".qcow2
+    cloud-localds /var/lib/libvirt/ephemeral/trusty_"${hostname}_${id}".iso \
+        /var/lib/libvirt/backingstore/trusty.user
+
+    virsh undefine trusty_"${hostname}_${id}"
+
+    xml=$(mktemp)
+
+    virt-install -r $mempercore --vcpus=2,maxvcpus=2,sockets=1,cores=1,threads=2 \
+        -n trusty_"${hostname}_${id}" -w network=default --nographics \
+        --disk path=/var/lib/libvirt/ephemeral/trusty_"${hostname}_${id}".qcow2 \
+        --disk path=/var/lib/libvirt/ephemeral/trusty_"${hostname}_${id}".iso,device=cdrom \
+        --import --noautoconsole \
+        --print-xml --dry-run --check disk_size=off,path_in_use=off \
+        --boot=hd --os-type=Linux --os-variant ubuntu14.04 > "$xml"
+        
+    virsh define "$xml"
+
+    rm -f "$xml" /var/lib/libvirt/ephemeral/trusty_"${hostname}_${id}".{iso,qcow2}
+}
+
+xenial()
+{
+    id="$1"
+    qemu-img create -b /var/lib/libvirt/backingstore/xenial.qcow2 -f qcow2 \
+        /var/lib/libvirt/ephemeral/xenial_"${hostname}_${id}".qcow2
+    cloud-localds /var/lib/libvirt/ephemeral/xenial_"${hostname}_${id}".iso \
+        /var/lib/libvirt/backingstore/xenial.user
+
+    virsh undefine xenial_"${hostname}_${id}"
+
+    xml=$(mktemp)
+
+    virt-install -r $mempercore --vcpus=2,maxvcpus=2,sockets=1,cores=1,threads=2 \
+        -n xenial_"${hostname}_${id}" -w network=default --nographics \
+        --disk path=/var/lib/libvirt/ephemeral/xenial_"${hostname}_${id}".qcow2 \
+        --disk path=/var/lib/libvirt/ephemeral/xenial_"${hostname}_${id}".iso,device=cdrom \
+        --import --noautoconsole \
+        --print-xml --dry-run --check disk_size=off,path_in_use=off \
+        --boot=hd --os-type=Linux --os-variant ubuntu16.04 > "$xml"
+        
+    virsh define "$xml"
+
+    rm -f "$xml" /var/lib/libvirt/ephemeral/xenial_"${hostname}_${id}".{iso,qcow2}
+}
+
+bionic()
+{
+    id="$1"
+    qemu-img create -b /var/lib/libvirt/backingstore/bionic.qcow2 -f qcow2 \
+        /var/lib/libvirt/ephemeral/bionic_"${hostname}_${id}".qcow2
+    cloud-localds /var/lib/libvirt/ephemeral/bionic_"${hostname}_${id}".iso \
+        /var/lib/libvirt/backingstore/bionic.user
+
+    virsh undefine bionic_"${hostname}_${id}"
+
+    xml=$(mktemp)
+    
+    virt-install -r $mempercore --vcpus=2,maxvcpus=2,sockets=1,cores=1,threads=2 \
+        -n bionic_"${hostname}_${id}" -w network=default --nographics \
+        --disk path=/var/lib/libvirt/ephemeral/bionic_"${hostname}_${id}".qcow2 \
+        --disk path=/var/lib/libvirt/ephemeral/bionic_"${hostname}_${id}".iso,device=cdrom \
+        --import --noautoconsole \
+        --print-xml --dry-run --check disk_size=off,path_in_use=off \
+        --boot=hd --os-type=Linux --os-variant ubuntu18.04 > "$xml"
+
+    virsh define "$xml"
+
+    rm -f "$xml" /var/lib/libvirt/ephemeral/bionic_"${hostname}_${id}".{iso,qcow2}
+}
+
+focal()
+{
+    id="$1"
+    qemu-img create -b /var/lib/libvirt/backingstore/focal.qcow2 -f qcow2 \
+        /var/lib/libvirt/ephemeral/focal_"${hostname}_${id}".qcow2
+    cloud-localds /var/lib/libvirt/ephemeral/focal_"${hostname}_${id}".iso \
+        /var/lib/libvirt/backingstore/focal.user
+
+    virsh undefine focal_"${hostname}_${id}"
+
+    xml=$(mktemp)
+    
+    virt-install -r $mempercore --vcpus=2,maxvcpus=2,sockets=1,cores=1,threads=2 \
+        -n focal_"${hostname}_${id}" -w network=default --nographics \
+        --disk path=/var/lib/libvirt/ephemeral/focal_"${hostname}_${id}".qcow2 \
+        --disk path=/var/lib/libvirt/ephemeral/focal_"${hostname}_${id}".iso,device=cdrom \
+        --import --noautoconsole \
+        --print-xml --dry-run --check disk_size=off,path_in_use=off \
+        --boot=hd --os-type=Linux --os-variant ubuntu18.04 > "$xml"
+
+    virsh define "$xml"
+
+    rm -f "$xml" /var/lib/libvirt/ephemeral/focal_"${hostname}_${id}".{iso,qcow2}
+}
+
+elcapitan()
+{
+    local id
+    local threads
+    local cores
+    id="$1"
+
+    virsh undefine elcapitan_"${hostname}_${id}"
+
+    xml=$(mktemp)
+
+    cat /var/lib/libvirt/backingstore/elcapitan.xml > "$xml"
+
+    threads="$(grep -c processor /proc/cpuinfo)"
+    if [ "$threads" -gt 16 ]; then
+        threads=16
+    fi
+    cores="$((threads/2))"
+
+    cp /var/lib/libvirt/backingstore/enoch_rev2839_boot \
+        /var/lib/libvirt/ephemeral/enoch_rev2839_boot
+
+    sed -Ei "
+        s#<name>.*<#<name>elcapitan_${hostname}_${id}<#;
+        s#<vcpu (.*)>[0-9]+#<vcpu \\1>$threads#;
+        s#<topology (.*)cores='[0-9]+'#<topology \\1cores='$cores'#;
+        s#<source file='(.*\\.qcow2)'#<source file='/var/lib/libvirt/ephemeral/elcapitan_${hostname}_${id}.qcow2'#;
+        " "$xml"
+
+    virsh define "$xml"
+
+    rm -f "$xml" 
+}
+
+hostname=$(hostname|cut -d. -f1)
+
+cores=$(grep -E '^cpu cores' /proc/cpuinfo |head -1|cut -d: -f2|tr -d '[:space:]')
+mem=$(sed -E 's/^MemTotal: +([0-9]+) kB/\1/p;d' < /proc/meminfo)
+mempercore=$(((mem/1024-4096)/cores))
+
+elcapitan 1
+
+for i in $(seq "$cores"); do
+    trusty "$i" &
+done
+wait
+for i in $(seq "$cores"); do
+    xenial "$i" &
+done
+wait
+for i in $(seq "$cores"); do
+    bionic "$i" &
+done
+wait
+for i in $(seq "$cores"); do
+    focal "$i" &
+done
+wait