From ee7a93c2fd461609caf858e4b8d2e1923e3cda45 Mon Sep 17 00:00:00 2001 From: Peter Surda Date: Thu, 11 Feb 2021 10:26:44 +0100 Subject: [PATCH] Docker and permissions - add dockerfile for bionic builder - chmod scripts sexecutable - add docker-install-all.sh --- androiddev.sh | 0 buildbot-osx.sh | 0 docker-install-all.sh | 37 ++++++++++++++++++ docker/bionic/Dockerfile | 76 +++++++++++++++++++++++++++++++++++++ docker/bionic/entrypoint.sh | 5 +++ qemu.hook | 0 test-checkdeps.sh | 0 travis2bash.sh | 0 virt-install-all.sh | 68 +++++++++++++++++---------------- 9 files changed, 154 insertions(+), 32 deletions(-) mode change 100644 => 100755 androiddev.sh mode change 100644 => 100755 buildbot-osx.sh create mode 100755 docker-install-all.sh create mode 100644 docker/bionic/Dockerfile create mode 100755 docker/bionic/entrypoint.sh mode change 100644 => 100755 qemu.hook mode change 100644 => 100755 test-checkdeps.sh mode change 100644 => 100755 travis2bash.sh mode change 100644 => 100755 virt-install-all.sh diff --git a/androiddev.sh b/androiddev.sh old mode 100644 new mode 100755 diff --git a/buildbot-osx.sh b/buildbot-osx.sh old mode 100644 new mode 100755 diff --git a/docker-install-all.sh b/docker-install-all.sh new file mode 100755 index 0000000..5664dc1 --- /dev/null +++ b/docker-install-all.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +trusty() +{ + return +} + +xenial() +{ + return +} + +bionic() +{ + cd /usr/src/buildbot-scripts/docker/bionic || return + docker build -t pybm-build-bionic . +} + +focal() +{ + return +} + +cores=$(grep -E '^cpu cores' /proc/cpuinfo |head -1|cut -d: -f2|tr -d '[:space:]') +mem=$(sed -E 's/^MemTotal: +([0-9]+) kB/\1/p;d' < /proc/meminfo) +mempercore=$(((mem/1024-4096)/cores)) + +# install kata containers +docker run --runtime=runc -v /opt/kata:/opt/kata -v /var/run/dbus:/var/run/dbus -v /run/systemd:/run/systemd -v /etc/docker:/etc/docker -it katadocker/kata-deploy kata-deploy-docker install + +# update default container CPU and memory +sed -i "s/default_vcpus = .*/default_vcpus = 2/g;s/default_memory = .*/default-memory = $mempercore/g" /opt/kata/share/defaults/kata-containers/configuration-qemu-virtiofs.toml + +trusty +xenial +bionic +focal diff --git a/docker/bionic/Dockerfile b/docker/bionic/Dockerfile new file mode 100644 index 0000000..1ed1f88 --- /dev/null +++ b/docker/bionic/Dockerfile @@ -0,0 +1,76 @@ +FROM ubuntu:bionic AS pybm-build-bionic + +ENV DEBIAN_FRONTEND noninteractive + +RUN apt-get update + +RUN apt-get install -yq --no-install-suggests --no-install-recommends \ + software-properties-common + +RUN dpkg --add-architecture i386 + +RUN add-apt-repository ppa:deadsnakes/ppa + +RUN apt-get update + +RUN apt-get -y install sudo + +# travis xenial bionic +RUN apt-get install -yq --no-install-suggests --no-install-recommends \ + python-setuptools libssl-dev libpq-dev python-prctl python-dev \ + python-dev python-virtualenv python-pip virtualenv + +# travis focal +RUN apt-get install -yq --no-install-suggests --no-install-recommends \ + python-setuptools libpcap-dev libpq-dev python-prctl python-dev \ + libpcap-dev python3-virtualenv python3-pip virtualenv + +# dpkg +RUN apt-get install -yq --no-install-suggests --no-install-recommends \ + python-minimal python-setuptools python-all python openssl libssl-dev \ + dh-apparmor debhelper dh-python python-msgpack python-qt4 python-stdeb \ + python-all-dev + +# Code quality +RUN apt-get install -yq --no-install-suggests --no-install-recommends \ + pylint python-pycodestyle python3-pycodestyle pycodestyle python-flake8 \ + python3-flake8 flake8 python-pyflakes python3-pyflakes pyflakes pyflakes3 \ + curl + +# Wine32 +RUN apt-get install -yq --no-install-suggests --no-install-recommends \ + python python-pip wget wine-stable winetricks mingw-w64 wine32 wine64 xvfb + +# Wine64 +RUN apt-get install -yq --no-install-suggests --no-install-recommends \ + python python-pip wget wine-stable winetricks mingw-w64 wine32 wine64 xvfb + +# Buildbot +RUN apt-get install -yq --no-install-suggests --no-install-recommends \ + buildbot-slave git subversion python3-dev libffi-dev python3-setuptools \ + python3-pip dumb-init curl + +# python 3.7 +RUN apt-get install -yq --no-install-suggests --no-install-recommends \ + python3.7 python3.7-dev + +# .travis.yml +RUN apt-get install -yq --no-install-suggests --no-install-recommends \ + build-essential libcap-dev tor + + +# cleanup +RUN rm -rf /var/lib/apt/lists/* + +# travis2bash +RUN cp /usr/src/buildbot-scripts/travis2bash.sh /usr/local/bin + +#RUN useradd -ms /bin/bash buildbot + +RUN echo 'buildbot ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers + +COPY entrypoint.sh /entrypoint.sh + +USER buildbot + +ENTRYPOINT /entrypoint.sh "$BUILDMASTER" "$WORKERNAME" "$WORKERPASS" diff --git a/docker/bionic/entrypoint.sh b/docker/bionic/entrypoint.sh new file mode 100755 index 0000000..6d92402 --- /dev/null +++ b/docker/bionic/entrypoint.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +buildbot-worker create-worker /var/lib/buildbot/workers/default "$1" "$2" "$3" +cd /var/lib/buildbot/workers/default +/usr/bin/dumb-init buildbot-worker start --nodaemon diff --git a/qemu.hook b/qemu.hook old mode 100644 new mode 100755 diff --git a/test-checkdeps.sh b/test-checkdeps.sh old mode 100644 new mode 100755 diff --git a/travis2bash.sh b/travis2bash.sh old mode 100644 new mode 100755 diff --git a/virt-install-all.sh b/virt-install-all.sh old mode 100644 new mode 100755 index 396a749..ef160d3 --- a/virt-install-all.sh +++ b/virt-install-all.sh @@ -3,101 +3,105 @@ trusty() { id="$1" - qemu-img create -b /var/lib/libvirt/backingstore/trusty.qcow2 -f qcow2 \ - /var/lib/libvirt/ephemeral/trusty_"${hostname}_${id}".qcow2 - cloud-localds /var/lib/libvirt/ephemeral/trusty_"${hostname}_${id}".iso \ + qemu-img create -b /var/lib/libvirt/backingstore/trusty.qcow2 \ + -f qcow2 -F qcow2 \ + /var/lib/libvirt/ephemeral/trusty_libvirt_"${hostname}_${id}".qcow2 + cloud-localds /var/lib/libvirt/ephemeral/trusty_libvirt_"${hostname}_${id}".iso \ /var/lib/libvirt/backingstore/trusty.user - virsh undefine trusty_"${hostname}_${id}" + virsh undefine trusty_libvirt_"${hostname}_${id}" xml=$(mktemp) virt-install -r $mempercore --vcpus=2,maxvcpus=2,sockets=1,cores=1,threads=2 \ - -n trusty_"${hostname}_${id}" -w network=default --nographics \ - --disk path=/var/lib/libvirt/ephemeral/trusty_"${hostname}_${id}".qcow2 \ - --disk path=/var/lib/libvirt/ephemeral/trusty_"${hostname}_${id}".iso,device=cdrom \ + -n trusty_libvirt_"${hostname}_${id}" -w network=default --nographics \ + --disk path=/var/lib/libvirt/ephemeral/trusty_libvirt_"${hostname}_${id}".qcow2 \ + --disk path=/var/lib/libvirt/ephemeral/trusty_libvirt_"${hostname}_${id}".iso,device=cdrom \ --import --noautoconsole \ --print-xml --dry-run --check disk_size=off,path_in_use=off \ --boot=hd --os-type=Linux --os-variant ubuntu14.04 > "$xml" virsh define "$xml" - rm -f "$xml" /var/lib/libvirt/ephemeral/trusty_"${hostname}_${id}".{iso,qcow2} + rm -f "$xml" /var/lib/libvirt/ephemeral/trusty_libvirt_"${hostname}_${id}".{iso,qcow2} } xenial() { id="$1" - qemu-img create -b /var/lib/libvirt/backingstore/xenial.qcow2 -f qcow2 \ - /var/lib/libvirt/ephemeral/xenial_"${hostname}_${id}".qcow2 - cloud-localds /var/lib/libvirt/ephemeral/xenial_"${hostname}_${id}".iso \ + qemu-img create -b /var/lib/libvirt/backingstore/xenial.qcow2 \ + -f qcow2 -F qcow2 \ + /var/lib/libvirt/ephemeral/xenial_libvirt_"${hostname}_${id}".qcow2 + cloud-localds /var/lib/libvirt/ephemeral/xenial_libvirt_"${hostname}_${id}".iso \ /var/lib/libvirt/backingstore/xenial.user - virsh undefine xenial_"${hostname}_${id}" + virsh undefine xenial_libvirt_"${hostname}_${id}" xml=$(mktemp) virt-install -r $mempercore --vcpus=2,maxvcpus=2,sockets=1,cores=1,threads=2 \ - -n xenial_"${hostname}_${id}" -w network=default --nographics \ - --disk path=/var/lib/libvirt/ephemeral/xenial_"${hostname}_${id}".qcow2 \ - --disk path=/var/lib/libvirt/ephemeral/xenial_"${hostname}_${id}".iso,device=cdrom \ + -n xenial_libvirt_"${hostname}_${id}" -w network=default --nographics \ + --disk path=/var/lib/libvirt/ephemeral/xenial_libvirt_"${hostname}_${id}".qcow2 \ + --disk path=/var/lib/libvirt/ephemeral/xenial_libvirt_"${hostname}_${id}".iso,device=cdrom \ --import --noautoconsole \ --print-xml --dry-run --check disk_size=off,path_in_use=off \ --boot=hd --os-type=Linux --os-variant ubuntu16.04 > "$xml" virsh define "$xml" - rm -f "$xml" /var/lib/libvirt/ephemeral/xenial_"${hostname}_${id}".{iso,qcow2} + rm -f "$xml" /var/lib/libvirt/ephemeral/xenial_libvirt_"${hostname}_${id}".{iso,qcow2} } bionic() { id="$1" - qemu-img create -b /var/lib/libvirt/backingstore/bionic.qcow2 -f qcow2 \ - /var/lib/libvirt/ephemeral/bionic_"${hostname}_${id}".qcow2 - cloud-localds /var/lib/libvirt/ephemeral/bionic_"${hostname}_${id}".iso \ + qemu-img create -b /var/lib/libvirt/backingstore/bionic.qcow2 \ + -f qcow2 -F qcow2 \ + /var/lib/libvirt/ephemeral/bionic_libvirt_"${hostname}_${id}".qcow2 + cloud-localds /var/lib/libvirt/ephemeral/bionic_libvirt_"${hostname}_${id}".iso \ /var/lib/libvirt/backingstore/bionic.user - virsh undefine bionic_"${hostname}_${id}" + virsh undefine bionic_libvirt_"${hostname}_${id}" xml=$(mktemp) virt-install -r $mempercore --vcpus=2,maxvcpus=2,sockets=1,cores=1,threads=2 \ - -n bionic_"${hostname}_${id}" -w network=default --nographics \ - --disk path=/var/lib/libvirt/ephemeral/bionic_"${hostname}_${id}".qcow2 \ - --disk path=/var/lib/libvirt/ephemeral/bionic_"${hostname}_${id}".iso,device=cdrom \ + -n bionic_libvirt_"${hostname}_${id}" -w network=default --nographics \ + --disk path=/var/lib/libvirt/ephemeral/bionic_libvirt_"${hostname}_${id}".qcow2 \ + --disk path=/var/lib/libvirt/ephemeral/bionic_libvirt_"${hostname}_${id}".iso,device=cdrom \ --import --noautoconsole \ --print-xml --dry-run --check disk_size=off,path_in_use=off \ --boot=hd --os-type=Linux --os-variant ubuntu18.04 > "$xml" virsh define "$xml" - rm -f "$xml" /var/lib/libvirt/ephemeral/bionic_"${hostname}_${id}".{iso,qcow2} + rm -f "$xml" /var/lib/libvirt/ephemeral/bionic_libvirt_"${hostname}_${id}".{iso,qcow2} } focal() { id="$1" - qemu-img create -b /var/lib/libvirt/backingstore/focal.qcow2 -f qcow2 \ - /var/lib/libvirt/ephemeral/focal_"${hostname}_${id}".qcow2 - cloud-localds /var/lib/libvirt/ephemeral/focal_"${hostname}_${id}".iso \ + qemu-img create -b /var/lib/libvirt/backingstore/focal.qcow2 \ + -f qcow2 -F qcow2 \ + /var/lib/libvirt/ephemeral/focal_libvirt_"${hostname}_${id}".qcow2 + cloud-localds /var/lib/libvirt/ephemeral/focal_libvirt_"${hostname}_${id}".iso \ /var/lib/libvirt/backingstore/focal.user - virsh undefine focal_"${hostname}_${id}" + virsh undefine focal_libvirt_"${hostname}_${id}" xml=$(mktemp) virt-install -r $mempercore --vcpus=2,maxvcpus=2,sockets=1,cores=1,threads=2 \ - -n focal_"${hostname}_${id}" -w network=default --nographics \ - --disk path=/var/lib/libvirt/ephemeral/focal_"${hostname}_${id}".qcow2 \ - --disk path=/var/lib/libvirt/ephemeral/focal_"${hostname}_${id}".iso,device=cdrom \ + -n focal_libvirt_"${hostname}_${id}" -w network=default --nographics \ + --disk path=/var/lib/libvirt/ephemeral/focal_libvirt_"${hostname}_${id}".qcow2 \ + --disk path=/var/lib/libvirt/ephemeral/focal_libvirt_"${hostname}_${id}".iso,device=cdrom \ --import --noautoconsole \ --print-xml --dry-run --check disk_size=off,path_in_use=off \ --boot=hd --os-type=Linux --os-variant ubuntu18.04 > "$xml" virsh define "$xml" - rm -f "$xml" /var/lib/libvirt/ephemeral/focal_"${hostname}_${id}".{iso,qcow2} + rm -f "$xml" /var/lib/libvirt/ephemeral/focal_libvirt_"${hostname}_${id}".{iso,qcow2} } elcapitan()