[qemu] Use ssh to run tests inside of qemu machine (#1292)

Summary: QEMU tests now start a qemu instance in the background and then
SSH into the machine to run tests. This makes the logs much cleaner and
also enables running interactively in a normal terminal instead of a
serial console.

Type of change: /kind test-infra

Test Plan: Tested that `bazel test --config=qemu-bpf //...` works (apart
from a flake in stirling error bpf test that I'll address separately).

Signed-off-by: James Bartlett <jamesbartlett@pixielabs.ai>

Author: JamesMBartlett

bazel/cc_toolchains/sysroots/sysroots.bzl

@@ -19,44 +19,44 @@ load("//bazel/cc_toolchains:utils.bzl", "abi")
 
 SYSROOT_LOCATIONS = dict(
     sysroot_x86_64_glibc2_36_runtime = dict(
-        sha256 = "7dcdec4bd693329a1ed670a87e95814a87f0906bd0f0c14058f47003b9fcf52a",
+        sha256 = "4fe3499357bca3e6ac8f6b6fb0266849eb434e0bbc95f2cfeeaa2dce7fe3913a",
         strip_prefix = "",
-        urls = ["https://storage.googleapis.com/pixie-dev-public/sysroots/pl6/sysroot-amd64-runtime.tar.gz"],
+        urls = ["https://storage.googleapis.com/pixie-dev-public/sysroots/pl7/sysroot-amd64-runtime.tar.gz"],
     ),
     sysroot_x86_64_glibc2_36_build = dict(
-        sha256 = "b3c7cd7044ba3de0f3550bece6e21f69eb198e9a165eeab690257a749fe442d0",
+        sha256 = "0aace63714ccf25a894c5ce02cbe19fd77ac04c804aeb1edbf14e3acf803d552",
         strip_prefix = "",
-        urls = ["https://storage.googleapis.com/pixie-dev-public/sysroots/pl6/sysroot-amd64-build.tar.gz"],
+        urls = ["https://storage.googleapis.com/pixie-dev-public/sysroots/pl7/sysroot-amd64-build.tar.gz"],
     ),
     sysroot_x86_64_glibc2_36_test = dict(
-        sha256 = "bee652932886a34bbd5bd9ea3a3c52c08f462c005faa0e2353297dd5b9070989",
+        sha256 = "d324655373e8f4f4eb234bec60061b034c1efa476dc4321dda54b403013eb183",
         strip_prefix = "",
-        urls = ["https://storage.googleapis.com/pixie-dev-public/sysroots/pl6/sysroot-amd64-test.tar.gz"],
+        urls = ["https://storage.googleapis.com/pixie-dev-public/sysroots/pl7/sysroot-amd64-test.tar.gz"],
     ),
     sysroot_x86_64_glibc2_36_debug = dict(
-        sha256 = "5639c73b5afd7c16bbdcbfcbd54262ea232283a5f5b4852e51c77062dea62f2e",
+        sha256 = "1430be9aa4290060c45f2afddeef59460746ccea91435605ce3d6fe907f6a558",
         strip_prefix = "",
-        urls = ["https://storage.googleapis.com/pixie-dev-public/sysroots/pl6/sysroot-amd64-debug.tar.gz"],
+        urls = ["https://storage.googleapis.com/pixie-dev-public/sysroots/pl7/sysroot-amd64-debug.tar.gz"],
     ),
     sysroot_aarch64_glibc2_36_runtime = dict(
-        sha256 = "48ea0856326974dca562db5a4e7ef09befef50fdb7a13698966e06393a3b090c",
+        sha256 = "a7584932a57b9cc5cce8ddd81437d53af6ab9407499c12a3a75231ed8d63195d",
         strip_prefix = "",
-        urls = ["https://storage.googleapis.com/pixie-dev-public/sysroots/pl6/sysroot-arm64-runtime.tar.gz"],
+        urls = ["https://storage.googleapis.com/pixie-dev-public/sysroots/pl7/sysroot-arm64-runtime.tar.gz"],
     ),
     sysroot_aarch64_glibc2_36_build = dict(
-        sha256 = "304e86332754e473938d5614a12e8676fb36481096ab6915b7d667faa0373a74",
+        sha256 = "86fdedb79870366363916507700dffb84eb1ebf1e94decb12fad17e2d568455f",
         strip_prefix = "",
-        urls = ["https://storage.googleapis.com/pixie-dev-public/sysroots/pl6/sysroot-arm64-build.tar.gz"],
+        urls = ["https://storage.googleapis.com/pixie-dev-public/sysroots/pl7/sysroot-arm64-build.tar.gz"],
     ),
     sysroot_aarch64_glibc2_36_test = dict(
-        sha256 = "12be586d2562e1ed49e4ea5c178c8df73c596c522b75b9b1de1700e081c6fd1e",
+        sha256 = "8f51888a333544737086960fa49e212e65aa8c10624a0e92bd2c521f82c49c81",
         strip_prefix = "",
-        urls = ["https://storage.googleapis.com/pixie-dev-public/sysroots/pl6/sysroot-arm64-test.tar.gz"],
+        urls = ["https://storage.googleapis.com/pixie-dev-public/sysroots/pl7/sysroot-arm64-test.tar.gz"],
     ),
     sysroot_aarch64_glibc2_36_debug = dict(
-        sha256 = "4cb30d6f9e73f66a03ba34a74fb51171ac7d45365102c41db2add0b852c7db33",
+        sha256 = "96df14385ad58a61cc3dd921c8b5830451da2fedbd421b0ef5ebb6c3fb82554f",
         strip_prefix = "",
-        urls = ["https://storage.googleapis.com/pixie-dev-public/sysroots/pl6/sysroot-arm64-debug.tar.gz"],
+        urls = ["https://storage.googleapis.com/pixie-dev-public/sysroots/pl7/sysroot-arm64-debug.tar.gz"],
     ),
 )
 

bazel/test_runners/qemu_with_kernel/init

@@ -75,6 +75,9 @@ function mount_system_dirs() {
 
   mkdir -p /dev/shm
   mount -t tmpfs tmpfs /dev/shm
+
+  mkdir -p /dev/pts
+  mount -t devpts devpts /dev/pts
 }
 
 function mount_test_fs() {
@@ -83,6 +86,12 @@ function mount_test_fs() {
   mount -t 9p -o trans=virtio test_fs /test_fs || true
 }
 
+function setup_eth0() {
+  ip link set dev eth0 up
+  ifconfig eth0 10.0.2.15 netmask 255.255.255.0
+  route add default gw 10.0.2.2
+}
+
 function setup_networking() {
   ip link set dev lo up
 
@@ -103,7 +112,8 @@ ff02::2 ip6-allrouters
 ff02::3 ip6-allhosts
 
 EOF
-    fi
+  fi
+  setup_eth0
 }
 
 function setup_containers() {
@@ -166,9 +176,9 @@ Welcome to PX BPF Runner
 
 EOF
 
-test_runner="/test_fs/test_runner_inside_qemu.sh"
-if [[ -x "${test_runner}" ]]; then
-  exec /bin/bash -c "${test_runner}"
+run_sshd="/test_fs/run_sshd.sh"
+if [[ -x "${run_sshd}" ]]; then
+  exec /bin/bash -c "${run_sshd}"
 else
   exec /bin/exit_qemu_with_status 3
 fi

bazel/test_runners/qemu_with_kernel/launcher.sh

@@ -19,15 +19,22 @@ set -e
 
 KERNEL_IMAGE=%kernelimage%
 DISK_IMAGE=%diskimage%
-BAZEL_QEMU_TEST_RUNNER_PATH=%testrunnerinsideqemu%
+BAZEL_RUN_SSHD_PATH=%runsshd%
 RUN_QEMU_SCRIPT=%runqemuscript%
 
 # Create a tmp directory that serves as the /test_fs sanbox dir inside qemu.
 tmpdir_for_sandbox=$(mktemp -d)
 
+# shellcheck disable=SC2317
 function cleanup {
   retval=$?
   rm -rf "${tmpdir_for_sandbox:?}" || true
+
+  if [[ -n "${qemu_pid}" ]]; then
+    kill "${qemu_pid}" &> /dev/null || true
+    wait "${qemu_pid}" || true
+    qemu_pid=""
+  fi
   exit "${retval}"
 }
 
@@ -56,6 +63,10 @@ function path_qemu_sandbox() {
   echo "$v"
 }
 
+function qemu_is_running() {
+  kill -0 "${qemu_pid}" &> /dev/null
+}
+
 
 # We need to write and transform the environment variables so that they have the "correct"
 # paths when run inside of qemu.
@@ -104,34 +115,89 @@ qemu_warnings_file=$(strip_pwd_from_path "${TEST_WARNINGS_OUTPUT_FILE}")
 qemu_testlogs_dir="${qemu_warnings_file/${qemu_warnings_file##*/testlogs/}}"
 cp -afL "${testlogs_dir}/" "${tmpdir_for_sandbox}/${qemu_testlogs_dir}/"
 
-# Copy the test runner and test cmd into the sandbox.
-test_runner_file="${tmpdir_for_sandbox}/test_runner_inside_qemu.sh"
+# Create test tmp dir.
+if [[ -n "${TEST_TMPDIR}" ]]; then
+  mkdir -p "$(path_qemu_sandbox "${TEST_TMPDIR}")"
+fi
+
+# Copy the test cmd into the sandbox.
 test_base=${PWD//"${OLDPWD}"/\/test_fs}
 test_cmd_path="${tmpdir_for_sandbox}/test_cmd.sh"
 test_cmd_path_in_qemu="/test_fs/test_cmd.sh"
 
-echo "#!/bin/bash -e" > "${test_cmd_path}"
-echo "${@:1}" >> "${test_cmd_path}"
+cat <<EOF > "${test_cmd_path}"
+#!/bin/bash -e
+source /test_fs/test_env.sh
+cd ${test_base}
+export TESTING_UNDER_QEMU=true
+${@:1}
+EOF
 chmod +x "${test_cmd_path}"
 
 printf "export test_base=%s\n" "${test_base}" >> "${test_env_file}"
 printf "export test_exec_path=%s\n" "${test_cmd_path_in_qemu}" >> "${test_env_file}"
 
-cp -afL "${BAZEL_QEMU_TEST_RUNNER_PATH}" "${test_runner_file}"
+# Setup ssh.
+ssh_priv_key="${tmpdir_for_sandbox}/ssh_key"
+ssh_pub_key_inside_qemu="/test_fs/ssh_key.pub"
+ssh-keygen -f "${ssh_priv_key}" -N '' -b 1024 > /dev/null
 
-# Launch the qemu test.
-retval=0
-(exec env - \
+ssh_env_file="${tmpdir_for_sandbox}/ssh_env.sh"
+echo "#!/bin/bash" > "${ssh_env_file}"
+echo "export SSH_PUB_KEY=${ssh_pub_key_inside_qemu}" >> "${ssh_env_file}"
+
+run_sshd_file="${tmpdir_for_sandbox}/run_sshd.sh"
+cp -afL "${BAZEL_RUN_SSHD_PATH}" "${run_sshd_file}"
+
+monitor_sock="mon.sock"
+# Launch qemu.
+env - \
   QEMU_TEST_FS_PATH="${tmpdir_for_sandbox}" \
   QEMU_KERNEL_IMAGE="${KERNEL_IMAGE}" \
   QEMU_DISK_BASE_RO="${PWD}/${DISK_IMAGE}" \
-  "${RUN_QEMU_SCRIPT}" ) || retval=$?
+  MONITOR_SOCK="${monitor_sock}" \
+  "${RUN_QEMU_SCRIPT}" &> "qemu.log"  &
+qemu_pid="$!"
+
+echo "QEMU logs available at: $(pwd)/qemu.log"
+echo "QEMU monitor available at unix socket: $(pwd)/${monitor_sock}"
+
+
+echo 'Waiting for QEMU to boot'
+while qemu_is_running && ! echo "info usernet" | netcat -NU "${monitor_sock}" &> /dev/null; do
+  sleep 1
+done
+
+host_ssh_port="$(echo "info usernet" | netcat -NU "${monitor_sock}" | grep "HOST_FORWARD" | awk '{print $4}')"
+ssh_opts=(
+  -q
+  -i "${ssh_priv_key}"
+  -o "StrictHostKeyChecking=no"
+  -o "UserKnownHostsFile=/dev/null"
+  -p "${host_ssh_port}"
+  root@localhost
+)
+
+# Wait for qemu to boot and ssh to be ready
+echo 'Waiting for SSH to come online'
+while qemu_is_running && ! ssh "${ssh_opts[@]}" 'echo test' &> /dev/null; do
+  sleep 1
+done
+
+if ! qemu_is_running; then
+  echo 'QEMU failed to boot'
+  cat -v qemu.log
+  echo "Log available in sandbox at: $(pwd)/qemu.log"
+  exit 3
+fi
+
+retval=0
+ssh "${ssh_opts[@]}" '/bin/bash -c '"${test_cmd_path_in_qemu}" || retval=$?
 
 # We use a known path to find the testlogs directory so that we can copy the results back from qemu.
 testlogs_dir="${TEST_WARNINGS_OUTPUT_FILE%%/testlogs/*}/testlogs/"
 qemu_warnings_file=$(strip_pwd_from_path "${TEST_WARNINGS_OUTPUT_FILE}")
 qemu_testlogs_dir="${qemu_warnings_file/${qemu_warnings_file##*/testlogs/}}"
-
 cp -afL "${tmpdir_for_sandbox}/${qemu_testlogs_dir}/" "${testlogs_dir}/"
 
-exit $retval
+exit "${retval}"

bazel/test_runners/qemu_with_kernel/run_qemu.sh

@@ -27,6 +27,7 @@ function check_env_set() {
 check_env_set QEMU_TEST_FS_PATH
 check_env_set QEMU_KERNEL_IMAGE
 check_env_set QEMU_DISK_BASE_RO
+check_env_set MONITOR_SOCK
 
 QEMU_MEMORY=${QEMU_MEMORY:-4096M}
 QEMU_CPU_COUNT=${QEMU_CPU_COUNT:-1}
@@ -76,16 +77,25 @@ flags+=(-append "console=ttyS0 root=/dev/sda")
 # Disable graphics mode.
 flags+=(-nographic)
 
+# Enable ssh port forwarding.
+flags+=(-device "virtio-net-pci,netdev=net0")
+flags+=(-netdev "user,id=net0,hostfwd=tcp::0-:22")
+
+flags+=(-monitor "unix:${MONITOR_SOCK},server,nowait")
+
 retval=0
 qemu-system-x86_64 "${flags[@]}" || retval=$?
 
 if [[ "${retval}" -gt 0 ]]; then
     if [[ "${retval}" -lt 128 ]]; then
-	echo "QEMU failed to launch with status code: ${retval}"
+	    echo "QEMU failed to launch with status code: ${retval}"
     else
-	retval="$(echo "($retval-128-1)/2" | bc)"
-	echo "Test failed with status: ${retval}"
+	    retval="$(echo "($retval-128-1)/2" | bc)"
     fi
 fi
 
+if [[ "${retval}" -ne 0 ]]; then
+  echo "Running sshd inside qemu failed with status: ${retval}"
+fi
+
 exit "${retval}"

…u_with_kernel/test_runner_inside_qemu.sh …est_runners/qemu_with_kernel/run_sshd.shbazel/test_runners/qemu_with_kernel/test_runner_inside_qemu.sh renamed to bazel/test_runners/qemu_with_kernel/run_sshd.sh bazel/test_runners/qemu_with_kernel/test_runner_inside_qemu.sh renamed to bazel/test_runners/qemu_with_kernel/run_sshd.sh

@@ -25,19 +25,27 @@ function finish {
 trap finish EXIT
 trap finish ERR
 
+function check_env_set() {
+  if [[ -z "${!1}" ]]; then
+    echo "The environment variable \"$1\" needs to be set"
+    exit 1
+  fi
+}
+
 # This file is generated by the test launcher.
 # shellcheck disable=SC1091
-source /test_fs/test_env.sh
-
-cd "${test_base:?}"
+source /test_fs/ssh_env.sh
 
-if [[ -n "${GTEST_TMP_DIR}" ]]; then
-    mkdir -p "${GTEST_TMP_DIR}"
-fi
+check_env_set SSH_PUB_KEY
 
-export TESTING_UNDER_QEMU=true
+mkdir -p "/etc/ssh"
+echo "PermitRootLogin without-password" >> /etc/ssh/sshd_config
+mkdir -p "/root/.ssh"
+cat "${SSH_PUB_KEY}" >> /root/.ssh/authorized_keys
+ssh-keygen -A -N '' -b 1024 &> /dev/null
+useradd sshd
+mkdir -p "/run/sshd"
+mkdir -p "/var/log"
 
-# Actually run the test and capture the return value.
-retval=0
-"${test_exec_path:?}" || retval=$?
-exit $retval
+echo "Starting SSH Daemon"
+exec /usr/sbin/sshd -p 22 -e -D

bazel/test_runners/qemu_with_kernel/runner.bzl

@@ -60,15 +60,15 @@ def _test_runner_impl(ctx):
             "%diskimage%": "bazel/test_runners/qemu_with_kernel/disk_image.qcow2",
             "%kernelimage%": "bazel/test_runners/qemu_with_kernel/bzImage",
             "%runqemuscript%": "bazel/test_runners/qemu_with_kernel/run_qemu.sh",
-            "%testrunnerinsideqemu%": "bazel/test_runners/qemu_with_kernel/test_runner_inside_qemu.sh",
+            "%runsshd%": "bazel/test_runners/qemu_with_kernel/run_sshd.sh",
         },
         is_executable = True,
     )
 
     runfiles = ctx.runfiles(files = [
         disk_image,
         kernel_bzimage,
-        ctx.files._test_runner_inside_qemu[0],
+        ctx.files._run_sshd[0],
         ctx.files._run_qemu_script[0],
     ])
 
@@ -108,12 +108,12 @@ qemu_with_kernel_test_runner = rule(
             default = Label("//bazel/test_runners/qemu_with_kernel:run_qemu.sh"),
             allow_single_file = True,
         ),
-        "_test_launcher_tpl": attr.label(
-            default = Label("//bazel/test_runners/qemu_with_kernel:launcher.sh"),
+        "_run_sshd": attr.label(
+            default = Label("//bazel/test_runners/qemu_with_kernel:run_sshd.sh"),
             allow_single_file = True,
         ),
-        "_test_runner_inside_qemu": attr.label(
-            default = Label("//bazel/test_runners/qemu_with_kernel:test_runner_inside_qemu.sh"),
+        "_test_launcher_tpl": attr.label(
+            default = Label("//bazel/test_runners/qemu_with_kernel:launcher.sh"),
             allow_single_file = True,
         ),
     },

tools/docker/Makefile

@@ -50,7 +50,7 @@ GRAALVM_ARCHIVE_FNAME := graalvm-native-image-22.3.0.tar.gz
 GRAALVM_ARCHIVE_GS_PATH := gs://pixie-dev-public/graalvm-native-image-22.3.0-$(GRAALVM_IMAGE_VERSION).tar.gz
 
 ## Sysroot parameters
-SYSROOT_REV := pl6
+SYSROOT_REV := pl7
 SYSROOT_BUILD_DIR := $(BUILD_DIR)/sysroots
 SYSROOT_ARCHITECTURES := amd64 arm64
 SYSROOT_VARIANTS := runtime build test

tools/docker/sysroot_creator/package_groups/test.yaml

@@ -23,3 +23,5 @@ include:
 - aardvark-dns
 - netavark
 - podman
+# ssh is used to run qemu tests
+- openssh-server