Initial commit

This commit is contained in:
Peter Šurda 2021-02-10 13:26:46 +01:00
commit c6e0e37381
Signed by: PeterSurda
GPG Key ID: 0C5F50C0B5F37D87
4 changed files with 617 additions and 0 deletions

178
qemu.hook Normal file
View File

@ -0,0 +1,178 @@
#!/usr/bin/python3
"""
Creates an empemeral image and patches it
"""
# Script /etc/libvirt/hooks/qemu
# Don't forget to execute service libvirt-bin restart
# Also see https://www.libvirt.org/hooks.html
# This script make clean VM for each start using base image
import os
import subprocess
import sys
import tempfile
import xml.etree.ElementTree as et
from jinja2 import Environment, exceptions, FileSystemLoader
BACKINGSTORE_PATH = '/var/lib/libvirt/backingstore/'
IMAGES_PATH = '/var/lib/libvirt/images/'
EPHEMERAL_PATH = '/var/lib/libvirt/ephemeral/'
CLOUDINIT_PATH = '/mnt/nfsimages'
VENDOR_EXT = '.vendor'
USER_EXT = '.user'
CLOUDINIT_EXT = '.iso'
xml = None
def parse_xml():
"""
Parse XML provided in stdin
"""
global xml
if xml is None:
data = sys.stdin.read()
xml = et.fromstring(data)
return xml
def find_base(domain):
"""
Convert VM name into hostname (part before first _)
"""
return domain.split("_", 2)[0]
def prepare_files(domain, env_):
"""
Create the ephemeral images
"""
xml = parse_xml()
base = find_base(domain)
for disk in xml.iterfind("./devices/disk[@device='disk']/source"):
disk = disk.get('file')
if not os.path.exists(os.path.join(BACKINGSTORE_PATH,
base + '.qcow2')):
continue
if not disk.startswith(EPHEMERAL_PATH):
continue
cmd = ['/usr/bin/qemu-img',
'create', '-b',
os.path.join(BACKINGSTORE_PATH, base + '.qcow2'),
'-f', 'qcow2',
'-F', 'qcow2',
disk]
subprocess.call(cmd)
# resize
cmd = ['/usr/bin/qemu-img',
'resize', disk, "+10G"
]
subprocess.call(cmd)
try:
x = xml.find("./metadata/{http://buildbot.net/}auth")
username, password, master = (x.attrib["username"],
x.attrib["password"],
x.attrib["master"])
except Exception:
username = domain
password = domain
master = None
for cdrom in xml.iterfind("./devices/disk[@device='cdrom']/source"):
cdrom = cdrom.get('file')
if not cdrom.startswith(EPHEMERAL_PATH):
continue
# cloud init
if not os.path.exists(os.path.join(BACKINGSTORE_PATH,
base + USER_EXT)):
continue
hostname = os.uname()[1].replace('.', '_')
userdata = {
'guest': hostname,
'hostname': domain,
'buildbot_username': username,
'buildbot_password': password,
'buildbot_master': master
}
with tempfile.NamedTemporaryFile(mode='w+t') as temp_file:
# jinja user data
try:
temp_file.write(env_.get_template(domain + USER_EXT).
render(userdata))
except exceptions.TemplateNotFound:
temp_file.write(env_.get_template(base + USER_EXT).
render(userdata))
temp_file.flush()
# make ISO
cmd = ['/usr/bin/cloud-localds',
# '-V', os.path.join(CLOUDINIT_PATH,
# domain + VENDOR_EXT),
# '-H', HOSTNAME,
os.path.join(cdrom),
temp_file.name
]
subprocess.call(cmd)
def cleanup_files():
"""
Delete the ephemeral images
"""
xml = parse_xml()
for disk in xml.iterfind("./devices/disk[@device='disk']/source"):
disk = disk.get('file')
if disk.startswith(EPHEMERAL_PATH):
try:
os.remove(disk)
except FileNotFoundError:
pass
for cdrom in xml.iterfind("./devices/disk[@device='cdrom']/source"):
cdrom = cdrom.get('file')
if cdrom.startswith(EPHEMERAL_PATH):
try:
os.remove(cdrom)
except FileNotFoundError:
pass
def get_buildbot_password():
"""
Retrieve buildbot password for guest
"""
fname = '/etc/libvirt/buildbot.pwd'
with open(fname) as file_handle:
content = [line.rstrip('\n') for line in file_handle]
try:
return content[0]
except (TypeError, IndexError):
return "password"
if __name__ == "__main__":
try:
# pylint: disable=unbalanced-tuple-unpacking
VIR_DOMAIN, ACTION = sys.argv[1:3]
except IndexError:
sys.exit(1)
ENV = Environment(autoescape=False,
loader=FileSystemLoader(BACKINGSTORE_PATH),
trim_blocks=False)
if ACTION in ["prepare"]:
cleanup_files()
prepare_files(VIR_DOMAIN, ENV)
if ACTION in ["release"]:
cleanup_files()

102
test-checkdeps.sh Normal file
View File

@ -0,0 +1,102 @@
#!/bin/bash
# tests checkdeps.py with all possible subsets of dependency packages installed
distro=$(grep ^VERSION_CODENAME= /etc/os-release | cut -d= -f2)
case $distro in
trusty)
method="apt-get"
packages=(python-qt4 python-msgpack python-pyopencl python-setuptools build-essential libssl-dev libcap-dev python-prctl)
;;
xenial)
method="apt-get"
packages=(python-qt4 python-msgpack python-pyopencl python-setuptools build-essential libssl-dev libcap-dev python-prctl)
;;
bionic)
method="apt-get"
packages=(python-qt4 python-msgpack python-pyopencl python-setuptools build-essential libssl-dev libcap-dev python-prctl)
;;
focal)
method="apt-get"
packages=(python-setuptools build-essential libssl-dev libcap-dev)
;;
*)
echo "Unknown distro"
exit 0
esac
echo "Using $method on $distro"
output=$(mktemp)
succeeded=0
failed=0
# 0 - 2^n -1
for i in $(seq 0 $((2**${#packages[@]}-1))); do
declare -a missing
missing=()
case $method in
apt-get)
cmd="sudo apt-get remove --autoremove -y -qq"
;;
*);;
esac
# iterate packages
for j in $(seq 0 $((${#packages[@]}-1))); do
# should j-th item be included in i-th iteration?
if [ $(((i>>j)%2)) -eq 1 ]; then
case $method in
apt-get)
cmd="$cmd ${packages[j]}+"
;;
esac
else
case $method in
apt-get)
missing+=("${packages[j]}")
cmd="$cmd ${packages[j]}"
;;
esac
fi
done
echo -n "${missing[*]}: "
$cmd &> /dev/null
#echo "python checkdeps.py"
python checkdeps.py > "$output" 2> /dev/null
toinstall=$(sed -E 's/.*apt-get install (.*)(`.*|$)/\1/p;d' < "$output" | xargs -n 1 echo)
toinstalla=($toinstall)
fail=0
for a in ${missing[*]}; do
for b in ${toinstalla[*]}; do
if [ "$a" == "$b" ]; then
break
fi
done
if ! [ "$a" == "$b" ]; then
fail=1
fi
done
for a in ${toinstalla[*]}; do
for b in ${missing[*]}; do
if [ "$a" == "$b" ]; then
break
fi
done
if ! [ "$a" == "$b" ]; then
fail=1
fi
done
if [ $fail -eq 0 ]; then
echo "OK"
succeeded=$((succeeded+1))
else
echo "FAIL"
echo " " $toinstall
failed=$((failed+1))
fi
done
echo "Succeeded: $succeeded"
echo " Failed: $failed"

177
travis2bash.sh Normal file
View File

@ -0,0 +1,177 @@
#!/usr/bin/env bash
# shellcheck disable=SC1003
# pulled from https://raw.githubusercontent.com/jasperes/bash-yaml/master/script/yaml.sh
# Licensed under MIT license
# Based on https://gist.github.com/pkuczynski/8665367
failure=0
parse_yaml() {
local yaml_file=$1
local prefix=$2
local s
local w
local fs
s='[[:space:]]*'
w='[a-zA-Z0-9_.-]*'
fs="$(echo @|tr @ '\034')"
(
sed -e '/- [^\“]'"[^\']"'.*: /s|\([ ]*\)- \([[:space:]]*\)|\1-\'$'\n'' \1\2|g' |
sed -ne '/^--/s|--||g; s|\"|\\\"|g; s/[[:space:]]*$//g;' \
-e "/#.*[\"\']/!s| #.*||g; /^#/s|#.*||g;" \
-e "s|^\($s\)\($w\)$s:$s\"\(.*\)\"$s\$|\1$fs\2$fs\3|p" \
-e "s|^\($s\)\($w\)${s}[:-]$s\(.*\)$s\$|\1$fs\2$fs\3|p" |
awk -F"$fs" '{
indent = length($1)/2;
if (length($2) == 0) { conj[indent]="+";} else {conj[indent]="";}
vname[indent] = $2;
for (i in vname) {if (i > indent) {delete vname[i]}}
if (length($3) > 0) {
vn=""; for (i=0; i<indent; i++) {vn=(vn)(vname[i])("_")}
printf("%s%s%s%s=(\"%s\")\n", "'"$prefix"'",vn, $2, conj[indent-1],$3);
}
}' |
sed -e 's/_=/+=/g' |
awk 'BEGIN {
FS="=";
OFS="="
}
/(-|\.).*=/ {
gsub("-|\\.", "_", $1)
}
{ print }'
) < "$yaml_file"
}
create_variables() {
local yaml_file="$1"
local prefix="$2"
eval "$(parse_yaml "$yaml_file" "$prefix")"
}
if [ ! -e .travis.yml ]; then
echo "No .travis.yml found, exiting"
exit 1
fi
function aptinstall() {
# apt install
# shellcheck disable=SC2154
if [[ "$(declare -p travis_addons_apt_packages)" =~ "declare -a" ]]; then
echo -e "\nsudo apt -y update\n"
sudo apt -y update
for ((i = 0; i < ${#travis_addons_apt_packages[@]}; i++))
do
echo -e "\napt-get -yq --no-install-suggests --no-install-recommends install ${travis_addons_apt_packages[$i]}\n"
# shellcheck disable=SC2086
sudo apt-get -yq --no-install-suggests --no-install-recommends \
install \
${travis_addons_apt_packages[$i]}
done
fi
}
function virtualenv_init() {
# init virtualenv directory
if [ -n "$1" -a -n "$2" ]; then
echo -e "\nvirtualenv -p \"$1\" ~/.venv-$2\n"
virtualenv -p "$1" ~/.venv-"$2"
else
echo -e "\nvirtualenv ~/.venv\n"
virtualenv ~/.venv
fi
}
function virtualenv_activate() {
# activate virtualenv
# shellcheck disable=SC1090
if [ -n "$1" ]; then
echo -e "\n. ~/.venv-$1/bin/activate\n"
. ~/.venv-$1/bin/activate
else
echo -e "\n. ~/.venv/bin/activate\n"
. ~/.venv/bin/activate
fi
}
function python_run() {
# shellcheck disable=SC2154
if [[ "$(declare -p travis_env)" =~ "declare -a" ]]; then
echo "Setting up environment variables"
for ((i = 0; i < ${#travis_env[@]}; i++))
do
echo -e "\n${travis_env[$i]}\n"
# shellcheck disable=SC2086
export ${travis_env[$i]}
done
fi
# shellcheck disable=SC2154
if [[ "$(declare -p travis_install)" =~ "declare -a" ]]; then
echo "Running \"install\""
if [ -n "$2" ]; then
if [ -e "requirements-${2}.txt" ]; then
echo "Running \"pip install -r requirements-${2}.txt\""
pip install -r "requirements-${2}.txt"
fi
fi
for ((i = 0; i < ${#travis_install[@]}; i++))
do
cd "$rundir" || continue
echo -e "\n${travis_install[$i]}\n"
if ! bash -c "${travis_install[$i]}"; then
failure=1
fi
done
fi
# shellcheck disable=SC2154
if [[ "$(declare -p travis_script)" =~ "declare -a" ]]; then
echo "Running \"script\""
for ((i = 0; i < ${#travis_script[@]}; i++))
do
cd "$rundir" || continue
echo -e "\n${travis_script[$i]}\n"
if ! bash -c "${travis_script[$i]}"; then
failure=1
fi
done
fi
}
create_variables .travis.yml travis_
rundir=$(pwd)
aptinstall
# shellcheck disable=SC2154
if [[ "$(declare -p travis_python)" =~ "declare -a" ]]; then
# shellcheck disable=SC2068
for pv in ${travis_python[@]}; do
# strip quotes
temp="${pv%\"}"
pv="${temp#\"}"
ppath=/usr/bin/python$pv
#if [ ! -e "$ppath" ]; then
sudo apt -y install "python$pv" "python${pv}-dev"
#fi
virtualenv_init "$ppath" "$pv"
virtualenv_activate "$pv"
python_run "$ppath" "$pv"
deactivate
done
else
virtualenv_init
virtualenv_activate
python_run
deactivate
fi
exit $failure

160
virt-install-all.sh Normal file
View File

@ -0,0 +1,160 @@
#!/bin/bash
trusty()
{
id="$1"
qemu-img create -b /var/lib/libvirt/backingstore/trusty.qcow2 -f qcow2 \
/var/lib/libvirt/ephemeral/trusty_"${hostname}_${id}".qcow2
cloud-localds /var/lib/libvirt/ephemeral/trusty_"${hostname}_${id}".iso \
/var/lib/libvirt/backingstore/trusty.user
virsh undefine trusty_"${hostname}_${id}"
xml=$(mktemp)
virt-install -r $mempercore --vcpus=2,maxvcpus=2,sockets=1,cores=1,threads=2 \
-n trusty_"${hostname}_${id}" -w network=default --nographics \
--disk path=/var/lib/libvirt/ephemeral/trusty_"${hostname}_${id}".qcow2 \
--disk path=/var/lib/libvirt/ephemeral/trusty_"${hostname}_${id}".iso,device=cdrom \
--import --noautoconsole \
--print-xml --dry-run --check disk_size=off,path_in_use=off \
--boot=hd --os-type=Linux --os-variant ubuntu14.04 > "$xml"
virsh define "$xml"
rm -f "$xml" /var/lib/libvirt/ephemeral/trusty_"${hostname}_${id}".{iso,qcow2}
}
xenial()
{
id="$1"
qemu-img create -b /var/lib/libvirt/backingstore/xenial.qcow2 -f qcow2 \
/var/lib/libvirt/ephemeral/xenial_"${hostname}_${id}".qcow2
cloud-localds /var/lib/libvirt/ephemeral/xenial_"${hostname}_${id}".iso \
/var/lib/libvirt/backingstore/xenial.user
virsh undefine xenial_"${hostname}_${id}"
xml=$(mktemp)
virt-install -r $mempercore --vcpus=2,maxvcpus=2,sockets=1,cores=1,threads=2 \
-n xenial_"${hostname}_${id}" -w network=default --nographics \
--disk path=/var/lib/libvirt/ephemeral/xenial_"${hostname}_${id}".qcow2 \
--disk path=/var/lib/libvirt/ephemeral/xenial_"${hostname}_${id}".iso,device=cdrom \
--import --noautoconsole \
--print-xml --dry-run --check disk_size=off,path_in_use=off \
--boot=hd --os-type=Linux --os-variant ubuntu16.04 > "$xml"
virsh define "$xml"
rm -f "$xml" /var/lib/libvirt/ephemeral/xenial_"${hostname}_${id}".{iso,qcow2}
}
bionic()
{
id="$1"
qemu-img create -b /var/lib/libvirt/backingstore/bionic.qcow2 -f qcow2 \
/var/lib/libvirt/ephemeral/bionic_"${hostname}_${id}".qcow2
cloud-localds /var/lib/libvirt/ephemeral/bionic_"${hostname}_${id}".iso \
/var/lib/libvirt/backingstore/bionic.user
virsh undefine bionic_"${hostname}_${id}"
xml=$(mktemp)
virt-install -r $mempercore --vcpus=2,maxvcpus=2,sockets=1,cores=1,threads=2 \
-n bionic_"${hostname}_${id}" -w network=default --nographics \
--disk path=/var/lib/libvirt/ephemeral/bionic_"${hostname}_${id}".qcow2 \
--disk path=/var/lib/libvirt/ephemeral/bionic_"${hostname}_${id}".iso,device=cdrom \
--import --noautoconsole \
--print-xml --dry-run --check disk_size=off,path_in_use=off \
--boot=hd --os-type=Linux --os-variant ubuntu18.04 > "$xml"
virsh define "$xml"
rm -f "$xml" /var/lib/libvirt/ephemeral/bionic_"${hostname}_${id}".{iso,qcow2}
}
focal()
{
id="$1"
qemu-img create -b /var/lib/libvirt/backingstore/focal.qcow2 -f qcow2 \
/var/lib/libvirt/ephemeral/focal_"${hostname}_${id}".qcow2
cloud-localds /var/lib/libvirt/ephemeral/focal_"${hostname}_${id}".iso \
/var/lib/libvirt/backingstore/focal.user
virsh undefine focal_"${hostname}_${id}"
xml=$(mktemp)
virt-install -r $mempercore --vcpus=2,maxvcpus=2,sockets=1,cores=1,threads=2 \
-n focal_"${hostname}_${id}" -w network=default --nographics \
--disk path=/var/lib/libvirt/ephemeral/focal_"${hostname}_${id}".qcow2 \
--disk path=/var/lib/libvirt/ephemeral/focal_"${hostname}_${id}".iso,device=cdrom \
--import --noautoconsole \
--print-xml --dry-run --check disk_size=off,path_in_use=off \
--boot=hd --os-type=Linux --os-variant ubuntu18.04 > "$xml"
virsh define "$xml"
rm -f "$xml" /var/lib/libvirt/ephemeral/focal_"${hostname}_${id}".{iso,qcow2}
}
elcapitan()
{
local id
local threads
local cores
id="$1"
virsh undefine elcapitan_"${hostname}_${id}"
xml=$(mktemp)
cat /var/lib/libvirt/backingstore/elcapitan.xml > "$xml"
threads="$(grep -c processor /proc/cpuinfo)"
if [ "$threads" -gt 16 ]; then
threads=16
fi
cores="$((threads/2))"
cp /var/lib/libvirt/backingstore/enoch_rev2839_boot \
/var/lib/libvirt/ephemeral/enoch_rev2839_boot
sed -Ei "
s#<name>.*<#<name>elcapitan_${hostname}_${id}<#;
s#<vcpu (.*)>[0-9]+#<vcpu \\1>$threads#;
s#<topology (.*)cores='[0-9]+'#<topology \\1cores='$cores'#;
s#<source file='(.*\\.qcow2)'#<source file='/var/lib/libvirt/ephemeral/elcapitan_${hostname}_${id}.qcow2'#;
" "$xml"
virsh define "$xml"
rm -f "$xml"
}
hostname=$(hostname|cut -d. -f1)
cores=$(grep -E '^cpu cores' /proc/cpuinfo |head -1|cut -d: -f2|tr -d '[:space:]')
mem=$(sed -E 's/^MemTotal: +([0-9]+) kB/\1/p;d' < /proc/meminfo)
mempercore=$(((mem/1024-4096)/cores))
elcapitan 1
for i in $(seq "$cores"); do
trusty "$i" &
done
wait
for i in $(seq "$cores"); do
xenial "$i" &
done
wait
for i in $(seq "$cores"); do
bionic "$i" &
done
wait
for i in $(seq "$cores"); do
focal "$i" &
done
wait