update system test execution

Author: ohmayr

.kokoro/system-single.sh

@@ -32,4 +32,17 @@ NOX_FILE_ARG=""
 
 [[ -z "${NOX_FILE}" ]] || NOX_FILE_ARG="-f ${NOX_FILE}"
 
-python3 -m nox ${NOX_SESSION_ARG} $NOX_FILE_ARG
+python3 -m pip install uv
+for attempt in 1 2 3; do
+  echo "Execution attempt $attempt of 3..."
+  if uvx --with 'nox[uv]' nox ${NOX_SESSION_ARG} $NOX_FILE_ARG; then
+    echo "Tests passed successfully!"
+    exit 0
+  fi
+  
+  echo "Tests failed. Backing off for 15 seconds..."
+  sleep 15
+done
+
+echo "Tests failed after 3 attempts. Hard failure."
+exit 1

.kokoro/system.sh

@@ -115,6 +115,8 @@ system_test_script="${PROJECT_ROOT}/.kokoro/system-single.sh"
 packages_with_system_tests_pattern=$(printf "|*%s*" "${packages_with_system_tests[@]}")
 packages_with_system_tests_pattern="${packages_with_system_tests_pattern:1}" # Remove the leading pipe
 
+declare -A pids
+
 # Run system tests for each package with directory packages/*/tests/system
 for path in `find 'packages' \
   \( -type d -wholename 'packages/*/tests/system' \) -o \
@@ -140,10 +142,23 @@ for path in `find 'packages' \
   set -e
 
   if [[ "${package_modified}" -gt 0 ]]; then
-      # Call the function - its internal exports won't affect the next loop
-      run_package_test "$package_name" || RETVAL=$?
+      echo "Dispatching ${package_name} in the background..."
+      run_package_test "$package_name" &
+      pids["$package_name"]=$!
   else
       echo "No changes in ${package_name}, skipping."
   fi
 done
-exit ${RETVAL}
+
+echo "============================================================"
+echo "Waiting for all concurrent system tests to complete..."
+echo "============================================================"
+
+for package in "${!pids[@]}"; do
+  wait ${pids[$package]} || {
+    echo "ERROR: System tests failed for $package"
+    RETVAL=1
+  }
+done
+
+exit ${RETVAL}