[qemu] Add interactive runner using ssh (#1302)

Summary: Adds a way to get a shell inside qemu using SSH. To run the
interactive qemu session without any test runfiles, run: `bazel run
//bazel/test_runners/qemu_with_kernel:run_interactive`.
To run with a test's runfiles, run `bazel run
//<path/to/test>_qemu_interactive`, eg. `bazel run
//src/stirling/obj_tools:elf_reader_symbolizer_bpf_test_qemu_interactive`.
Note it won't work if you add `--config=qemu-bpf` because of the way
bazel's `run_under` flag works, so just leave out the `--config`.

Type of change: /kind test-infra.

Test Plan: Tested that the interactive runner opens a shell with the
expected runfiles. Also tested that closing the shell at any point
doesn't leave zombie qemu processes.

Signed-off-by: James Bartlett <jamesbartlett@pixielabs.ai>

Author: JamesMBartlett

.bazelrc

@@ -131,8 +131,8 @@ test:qemu-bpf --test_tag_filters=requires_bpf,-disabled,-no_qemu
 build:gcc --copt -Wno-error=sign-compare
 build:gcc --copt -Wno-error=stringop-truncation
 build:gcc --copt -Wno-error=maybe-uninitialized
-build:gcc --build_tag_filters=-no_gcc
-build:gcc --test_tag_filters=-no_gcc,-requires_root,-requires_bpf,-disabled
+build:gcc --build_tag_filters=-no_gcc,-qemu_interactive
+build:gcc --test_tag_filters=-no_gcc,-requires_root,-requires_bpf,-disabled,-qemu_interactive
 build:gcc --//bazel/cc_toolchains:compiler=gcc
 test:gcc --config=tmp-sandbox
 

bazel/pl_build_system.bzl

@@ -21,6 +21,7 @@ load("@io_bazel_rules_docker//go:image.bzl", "go_image")
 load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_context", "go_library", "go_test")
 load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test")
 load("@rules_python//python:defs.bzl", "py_test")
+load("//bazel:toolchain_transitions.bzl", "qemu_interactive_runner")
 
 pl_supported_go_sdk_versions = ["1.16", "1.17", "1.18", "1.19", "1.20"]
 
@@ -476,3 +477,27 @@ def pl_py_test(**kwargs):
 def pl_sh_test(**kwargs):
     _add_test_runner(kwargs)
     native.sh_test(**kwargs)
+
+def pl_cc_bpf_test(**kwargs):
+    pl_cc_test(**kwargs)
+    qemu_interactive_runner(
+        name = kwargs["name"] + "_qemu_interactive",
+        test = ":" + kwargs["name"],
+        tags = [
+            "manual",
+            "qemu_interactive",
+        ],
+        testonly = True,
+    )
+
+def pl_sh_bpf_test(**kwargs):
+    pl_sh_test(**kwargs)
+    qemu_interactive_runner(
+        name = kwargs["name"] + "_qemu_interactive",
+        test = ":" + kwargs["name"],
+        tags = [
+            "manual",
+            "qemu_interactive",
+        ],
+        testonly = True,
+    )

bazel/test_runners/qemu_with_kernel/BUILD.bazel

@@ -16,8 +16,11 @@
 
 load("@bazel_skylib//rules:common_settings.bzl", "string_flag")
 load("//bazel:pl_qemu_kernels.bzl", "kernel_flag_name", "qemu_image_to_deps")
+load("//bazel:toolchain_transitions.bzl", "qemu_interactive_runner")
 load("//bazel/test_runners/qemu_with_kernel:runner.bzl", "qemu_with_kernel_test_runner")
 
+exports_files(["interactive_runner.sh"])
+
 kernel_image_deps = qemu_image_to_deps()
 
 string_flag(
@@ -41,3 +44,12 @@ qemu_with_kernel_test_runner(
     kernel_image = select(kernel_select_list),
     visibility = ["//visibility:public"],
 )
+
+qemu_interactive_runner(
+    name = "run_interactive",
+    tags = [
+        "manual",
+        "qemu_interactive",
+    ],
+    visibility = ["//visibility:public"],
+)

bazel/test_runners/qemu_with_kernel/interactive_runner.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env -S -i /bin/bash
+
+# Copyright 2018- The Pixie Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# shellcheck shell=bash
+
+execroot="$(realpath "${PWD%%/execroot/px/*}/execroot")"
+# Make OLDPWD look like what bazel's test setup does.
+export OLDPWD="${execroot}/px"
+export RUNFILES_DIR="$PWD"
+export INTERACTIVE_MODE="true"
+
+# shellcheck disable=SC2288
+%qemu_runner_path% %test_path%

bazel/test_runners/qemu_with_kernel/launcher.sh

@@ -109,11 +109,13 @@ if [ -f "${RUNFILES_DIR}_manifest" ]; then
   rewrite_manifest "${RUNFILES_DIR}_manifest" > "${runfiles_path}_manifest"
 fi
 
-# Copy over testlog file.
-testlogs_dir="${TEST_WARNINGS_OUTPUT_FILE%%/testlogs/*}/testlogs/"
-qemu_warnings_file=$(strip_pwd_from_path "${TEST_WARNINGS_OUTPUT_FILE}")
-qemu_testlogs_dir="${qemu_warnings_file/${qemu_warnings_file##*/testlogs/}}"
-cp -afL "${testlogs_dir}/" "${tmpdir_for_sandbox}/${qemu_testlogs_dir}/"
+if [[ "${INTERACTIVE_MODE}" != "true" ]]; then
+  # Copy over testlog file.
+  testlogs_dir="${TEST_WARNINGS_OUTPUT_FILE%%/testlogs/*}/testlogs/"
+  qemu_warnings_file=$(strip_pwd_from_path "${TEST_WARNINGS_OUTPUT_FILE}")
+  qemu_testlogs_dir="${qemu_warnings_file/${qemu_warnings_file##*/testlogs/}}"
+  cp -afL "${testlogs_dir}/" "${tmpdir_for_sandbox}/${qemu_testlogs_dir}/"
+fi
 
 # Create test tmp dir.
 if [[ -n "${TEST_TMPDIR}" ]]; then
@@ -125,13 +127,27 @@ test_base=${PWD//"${OLDPWD}"/\/test_fs}
 test_cmd_path="${tmpdir_for_sandbox}/test_cmd.sh"
 test_cmd_path_in_qemu="/test_fs/test_cmd.sh"
 
-cat <<EOF > "${test_cmd_path}"
+if [[ "${INTERACTIVE_MODE}" != "true" ]]; then
+  cat <<EOF > "${test_cmd_path}"
 #!/bin/bash -e
 source /test_fs/test_env.sh
 cd ${test_base}
 export TESTING_UNDER_QEMU=true
 ${@:1}
 EOF
+else
+  cat <<EOF > "${test_cmd_path}"
+#!/bin/bash
+cd ${test_base}
+if [[ -n "${@:1}" ]]; then
+  echo "-------------------------------"
+  echo "Command to run test: "
+  echo "  ${@:1}"
+  echo "-------------------------------"
+fi
+/bin/bash -l
+EOF
+fi
 chmod +x "${test_cmd_path}"
 
 printf "export test_base=%s\n" "${test_base}" >> "${test_env_file}"
@@ -177,6 +193,9 @@ ssh_opts=(
   -p "${host_ssh_port}"
   root@localhost
 )
+if [[ "${INTERACTIVE_MODE}" == "true" ]]; then
+  ssh_opts+=(-t)
+fi
 
 # Wait for qemu to boot and ssh to be ready
 echo 'Waiting for SSH to come online'
@@ -194,10 +213,12 @@ fi
 retval=0
 ssh "${ssh_opts[@]}" '/bin/bash -c '"${test_cmd_path_in_qemu}" || retval=$?
 
-# We use a known path to find the testlogs directory so that we can copy the results back from qemu.
-testlogs_dir="${TEST_WARNINGS_OUTPUT_FILE%%/testlogs/*}/testlogs/"
-qemu_warnings_file=$(strip_pwd_from_path "${TEST_WARNINGS_OUTPUT_FILE}")
-qemu_testlogs_dir="${qemu_warnings_file/${qemu_warnings_file##*/testlogs/}}"
-cp -afL "${tmpdir_for_sandbox}/${qemu_testlogs_dir}/" "${testlogs_dir}/"
+if [[ "${INTERACTIVE_MODE}" != "true" ]]; then
+  # We use a known path to find the testlogs directory so that we can copy the results back from qemu.
+  testlogs_dir="${TEST_WARNINGS_OUTPUT_FILE%%/testlogs/*}/testlogs/"
+  qemu_warnings_file=$(strip_pwd_from_path "${TEST_WARNINGS_OUTPUT_FILE}")
+  qemu_testlogs_dir="${qemu_warnings_file/${qemu_warnings_file##*/testlogs/}}"
+  cp -afL "${tmpdir_for_sandbox}/${qemu_testlogs_dir}/" "${testlogs_dir}/"
+fi
 
 exit "${retval}"

bazel/test_runners/qemu_with_kernel/run_qemu.sh

@@ -84,7 +84,7 @@ flags+=(-netdev "user,id=net0,hostfwd=tcp::0-:22")
 flags+=(-monitor "unix:${MONITOR_SOCK},server,nowait")
 
 retval=0
-qemu-system-x86_64 "${flags[@]}" || retval=$?
+exec qemu-system-x86_64 "${flags[@]}" || retval=$?
 
 if [[ "${retval}" -gt 0 ]]; then
     if [[ "${retval}" -lt 128 ]]; then

bazel/test_runners/qemu_with_kernel/runner.bzl

@@ -122,3 +122,49 @@ qemu_with_kernel_test_runner = rule(
     ],
     executable = True,
 )
+
+def _interactive_impl(ctx):
+    transitive_runfiles = []
+    transitive_runfiles.append(ctx.attr._qemu_runner[DefaultInfo].default_runfiles)
+    if ctx.attr.test:
+        transitive_runfiles.append(ctx.attr.test[DefaultInfo].default_runfiles)
+    runfiles = ctx.runfiles()
+    runfiles = runfiles.merge_all(transitive_runfiles)
+
+    test_path = ""
+    if ctx.attr.test:
+        test_path = ctx.attr.test[DefaultInfo].files_to_run.executable.short_path
+        # TODO(james): Once bazel supports an ArgsProvider we can add test args here.
+
+    output_script = ctx.actions.declare_file(ctx.attr.name + ".sh")
+    ctx.actions.expand_template(
+        template = ctx.files._interactive_runner_tpl[0],
+        output = output_script,
+        substitutions = {
+            "%qemu_runner_path%": "bazel/test_runners/qemu_with_kernel/test_runner.sh",
+            "%test_path%": test_path,
+        },
+        is_executable = True,
+    )
+    return DefaultInfo(
+        files = depset(
+            [output_script],
+        ),
+        runfiles = runfiles,
+        executable = output_script,
+    )
+
+qemu_with_kernel_interactive_runner = rule(
+    implementation = _interactive_impl,
+    attrs = {
+        "test": attr.label(),
+        "_interactive_runner_tpl": attr.label(
+            default = Label("//bazel/test_runners/qemu_with_kernel:interactive_runner.sh"),
+            allow_single_file = True,
+        ),
+        "_qemu_runner": attr.label(
+            default = Label("//bazel/test_runners/qemu_with_kernel:runner"),
+        ),
+    },
+    executable = True,
+)

bazel/toolchain_transitions.bzl

@@ -15,6 +15,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 load("@com_github_fmeum_rules_meta//meta:defs.bzl", "meta")
+load("//bazel/test_runners/qemu_with_kernel:runner.bzl", "qemu_with_kernel_interactive_runner")
 
 cc_static_musl_binary = meta.wrap_with_transition(
     native.cc_binary,
@@ -40,3 +41,11 @@ cc_clang_binary = meta.wrap_with_transition(
     },
     executable = True,
 )
+
+qemu_interactive_runner = meta.wrap_with_transition(
+    qemu_with_kernel_interactive_runner,
+    {
+        "@//bazel/cc_toolchains:libc_version": meta.replace_with("glibc2_36"),
+    },
+    executable = True,
+)

src/stirling/bpf_tools/BUILD.bazel

@@ -14,7 +14,7 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-load("//bazel:pl_build_system.bzl", "pl_cc_binary", "pl_cc_library", "pl_cc_test")
+load("//bazel:pl_build_system.bzl", "pl_cc_binary", "pl_cc_bpf_test", "pl_cc_library", "pl_cc_test")
 
 package(default_visibility = ["//src/stirling:__subpackages__"])
 
@@ -60,7 +60,7 @@ pl_cc_test(
     ],
 )
 
-pl_cc_test(
+pl_cc_bpf_test(
     name = "bcc_wrapper_bpf_test",
     srcs = ["bcc_wrapper_bpf_test.cc"],
     tags = [
@@ -74,7 +74,7 @@ pl_cc_test(
     ],
 )
 
-pl_cc_test(
+pl_cc_bpf_test(
     name = "bcc_symbolizer_bpf_test",
     srcs = ["bcc_symbolizer_bpf_test.cc"],
     tags = [
@@ -87,7 +87,7 @@ pl_cc_test(
     ],
 )
 
-pl_cc_test(
+pl_cc_bpf_test(
     name = "bpftrace_wrapper_bpf_test",
     srcs = ["bpftrace_wrapper_bpf_test.cc"],
     tags = [
@@ -100,7 +100,7 @@ pl_cc_test(
     ],
 )
 
-pl_cc_test(
+pl_cc_bpf_test(
     name = "task_struct_resolver_bpf_test",
     srcs = ["task_struct_resolver_bpf_test.cc"],
     tags = [
@@ -116,7 +116,7 @@ pl_cc_test(
     ],
 )
 
-pl_cc_test(
+pl_cc_bpf_test(
     name = "uprobe_extra_trigger_bpf_test",
     srcs = ["uprobe_extra_trigger_bpf_test.cc"],
     tags = [

src/stirling/e2e_tests/BUILD.bazel

@@ -14,12 +14,12 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-load("//bazel:pl_build_system.bzl", "pl_cc_test", "pl_sh_test")
+load("//bazel:pl_build_system.bzl", "pl_cc_bpf_test", "pl_cc_test", "pl_sh_bpf_test", "pl_sh_test")
 load("//src/stirling/source_connectors/perf_profiler/testing:testing.bzl", "agent_libs", "agent_libs_arg")
 
 package(default_visibility = ["//src/stirling:__subpackages__"])
 
-pl_cc_test(
+pl_cc_bpf_test(
     name = "stirling_bpf_test",
     timeout = "moderate",
     srcs = ["stirling_bpf_test.cc"],
@@ -33,7 +33,7 @@ pl_cc_test(
     ],
 )
 
-pl_cc_test(
+pl_cc_bpf_test(
     name = "bpf_map_leak_bpf_test",
     timeout = "moderate",
     srcs = ["bpf_map_leak_bpf_test.cc"],
@@ -64,7 +64,7 @@ pl_cc_test(
     ],
 )
 
-pl_sh_test(
+pl_sh_bpf_test(
     name = "stirling_wrapper_kprobe_leak_bpf_test",
     srcs = ["stirling_wrapper_kprobe_leak_bpf_test.sh"],
     args = [
@@ -84,7 +84,7 @@ pl_sh_test(
     deps = ["//src/stirling/scripts:sh_library"],
 )
 
-pl_sh_test(
+pl_sh_bpf_test(
     name = "stirling_wrapper_bpf_test",
     srcs = ["stirling_wrapper_bpf_test.sh"],
     args = [
@@ -110,7 +110,7 @@ pl_sh_test(
     deps = ["//src/stirling/scripts:sh_library"],
 )
 
-pl_sh_test(
+pl_sh_bpf_test(
     name = "stirling_wrapper_bpftrace_bpf_test",
     srcs = ["stirling_wrapper_bpftrace_bpf_test.sh"],
     args = [
@@ -134,7 +134,7 @@ pl_sh_test(
     ],
 )
 
-pl_sh_test(
+pl_sh_bpf_test(
     name = "stirling_wrapper_container_bpf_test",
     srcs = ["stirling_wrapper_container_bpf_test.sh"],
     args = [
@@ -163,7 +163,7 @@ pl_sh_test(
     deps = ["//src/stirling/scripts:sh_library"],
 )
 
-pl_sh_test(
+pl_sh_bpf_test(
     name = "stirling_perf_bpf_test",
     srcs = ["stirling_perf_bpf_test.sh"],
     args = [
@@ -188,7 +188,7 @@ pl_sh_test(
     deps = ["//src/stirling/scripts:sh_library"],
 )
 
-pl_sh_test(
+pl_sh_bpf_test(
     name = "stirling_wrapper_jvm_stats_bpf_test",
     srcs = ["stirling_wrapper_jvm_stats_bpf_test.sh"],
     args = [
@@ -230,7 +230,7 @@ pl_sh_test(
     deps = ["//src/stirling/scripts:sh_library"],
 )
 
-pl_sh_test(
+pl_sh_bpf_test(
     name = "probe_cleaner_bpf_test",
     srcs = ["probe_cleaner_bpf_test.sh"],
     args = [