From 98bad33ab3a211e4f126b707eeab3ed1b6e02d27 Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Thu, 1 Oct 2026 00:52:09 -0400 Subject: guest-diag: reusable supervised physical-claim cycle harness (D0 pre-lock + abort checks) --- tools/guest-diag/phys_claim_cycle.sh | 426 +++++++++++++++++++++++++++++++++++ 1 file changed, 426 insertions(+) create mode 100755 tools/guest-diag/phys_claim_cycle.sh (limited to 'tools') diff --git a/tools/guest-diag/phys_claim_cycle.sh b/tools/guest-diag/phys_claim_cycle.sh new file mode 100755 index 0000000..7103fec --- /dev/null +++ b/tools/guest-diag/phys_claim_cycle.sh @@ -0,0 +1,426 @@ +#!/bin/bash +# phys_claim_cycle.sh - supervised physical-GPU claim/release cycle harness. +# +# Drives one or more full "D0 pre-lock" cycles of the physical-ownership +# backend (open-iov-vfio + gsp_phys_gpu=1 + gsp_phys_hazard_ack=1): +# +# start persistenced -> wait GPU+root port active -> power/control=on on +# GPU+port -> dmesg -C -> (stop persistenced && insmod) as ONE command -> +# verify lock/claim patterns -> hold -> verify while claimed -> +# rmmod -> verify release patterns -> restore auto + persistenced -> +# health check. +# +# !!! HOST-FREEZE HAZARD !!! +# Claiming the GPU unbinds the distribution NVIDIA driver. On this host the +# dGPU board then power-gate and fail to come back, hard-freezing the system +# with no crash record (observed twice on 2026-09-30). Run only in a +# supervised session, with other host work stopped and the PART 87 recovery +# plan ready. The harness itself never retries an aborted cycle. +# +# Usage: phys_claim_cycle.sh --yes [--hold SECONDS] [--cycles N] [--layout] +# [--ko PATH] [--no-restore] +# +# --yes required: acknowledge the host-freeze hazard +# --hold S seconds to keep the GPU claimed (default 30) +# --cycles N cycles to run, each only if the previous was green (default 1) +# --layout also pass gsp_phys_layout=1 (rung 3b); "stage 3b complete" +# then becomes an expected (but non-fatal) pattern +# --ko PATH module to insmod (default kernel/open-iov-vfio.ko) +# --no-restore do not restore power/control=auto or restart persistenced +# at the end of each cycle (leaves the D0 hold in place) +# +# Exit codes: 0 = all cycles green; 1 = --yes missing / usage error; +# 2 = abort (hard abort signature, failed verification, red +# in-window rail event); 3 = precondition failure. +# +# Per-cycle full dmesg is saved to +# /tmp/opencode/phys_claim_cycle__.log +# +# Env overrides: GPU_BDF (default 0000:01:00.0), PORT_BDF (default: the GPU's +# upstream bridge, normally 0000:00:01.0), PHYS_CLAIM_LOGDIR. + +set -u + +HERE="$(dirname "$(readlink -f "$0")")" +REPO_ROOT="$(readlink -f "$HERE/../..")" + +USE_YES=0 +HOLD=30 +CYCLES=1 +LAYOUT=0 +NO_RESTORE=0 +KO="$REPO_ROOT/kernel/open-iov-vfio.ko" + +GPU_BDF="${GPU_BDF:-0000:01:00.0}" +if [ -z "${PORT_BDF:-}" ] && [ -e "/sys/bus/pci/devices/$GPU_BDF" ]; then + PORT_BDF="$(basename "$(dirname "$(readlink -f "/sys/bus/pci/devices/$GPU_BDF")")")" +fi +PORT_BDF="${PORT_BDF:-0000:00:01.0}" + +LOGDIR="${PHYS_CLAIM_LOGDIR:-/tmp/opencode}" +TS="$(date +%Y%m%d_%H%M%S)" +WATCHERS='pdwatch|watch_ctx|uvmt_poll|invd\.py|bpwait|sempages|track_waits|guest_pin|guest_pmap|vmrestart|push_guest_tools|nvidia-smi -l|dmesg -w' + +usage() { + sed -n '1,50p' "$0" | grep -E '^#( |$)' | sed 's/^# \{0,1\}//' +} + +while [ $# -gt 0 ]; do + case "$1" in + --yes) USE_YES=1 ;; + --hold) HOLD="${2:-}"; shift ;; + --cycles) CYCLES="${2:-}"; shift ;; + --layout) LAYOUT=1 ;; + --ko) KO="${2:-}"; shift ;; + --no-restore) NO_RESTORE=1 ;; + -h|--help) usage; exit 0 ;; + *) echo "unknown argument: $1" >&2; usage >&2; exit 1 ;; + esac + shift +done + +case "$HOLD" in ''|*[!0-9]*) echo "--hold must be a non-negative integer" >&2; exit 1 ;; esac +case "$CYCLES" in ''|*[!0-9]*|0) echo "--cycles must be a positive integer" >&2; exit 1 ;; esac +if [ "$HOLD" -lt 30 ]; then + echo "WARN: --hold $HOLD is below the verified 30 s minimum" >&2 +fi + +if [ "$USE_YES" -ne 1 ]; then + cat >&2 <<'EOF' +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +HOST-FREEZE HAZARD - REFUSING TO RUN WITHOUT --yes + +The physical claim unbinds the distribution NVIDIA driver from the dGPU. +On this host the board can then power-gate and fail to come back, hard- +freezing the system with no crash record (observed twice on 2026-09-30). +Run only when supervised, with other host work stopped and the PART 87 +recovery plan ready. Re-run with --yes to acknowledge. +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +EOF + exit 1 +fi + +log() { echo "[$(date +%H:%M:%S)] $*"; } +warn() { echo "[$(date +%H:%M:%S)] WARN: $*" >&2; } +pre_fail() { echo "PRECONDITION FAIL: $*" >&2; exit 3; } + +INTERRUPTED=0 +on_signal() { + INTERRUPTED=1 + echo + echo "ABORT: interrupted; leaving module/lock state as-is (no cleanup)." >&2 + exit 2 +} +trap on_signal INT TERM + +# ---------------------------------------------------------------- preflight + +[ -c /dev/null ] || pre_fail "no /dev/null?" +if ! sudo -n true 2>/dev/null; then + sudo -v 2>/dev/null || pre_fail "current user cannot sudo" +fi + +[ -f "$KO" ] || pre_fail "module not found: $KO (build it or pass --ko)" +KO_HASH="$(sha256sum "$KO" | awk '{print $1}')" +KO_SRCVER="$(modinfo -F srcversion "$KO" 2>/dev/null || true)" +if [ -d "$REPO_ROOT/.git" ]; then + HEAD_COMMIT="$(git -C "$REPO_ROOT" rev-parse --short HEAD 2>/dev/null || true)" + log "preflight: repo HEAD ${HEAD_COMMIT:-unknown}" + if git -C "$REPO_ROOT" status --porcelain -- kernel 2>/dev/null | grep -q .; then + warn "kernel/ has uncommitted changes; HEAD does not describe $KO" + fi +else + HEAD_COMMIT="" + warn "not a git checkout; cannot record HEAD" +fi +log "preflight: ko $KO sha256 $KO_HASH srcversion ${KO_SRCVER:-unknown}" +if [ "$KO" -ot "$REPO_ROOT/kernel/gsp_phys_gpu.c" ]; then + warn "$KO is older than kernel/gsp_phys_gpu.c; rebuild before cycling" +fi + +[ -e "/sys/bus/pci/devices/$GPU_BDF" ] || pre_fail "GPU $GPU_BDF not present" +[ -e "/sys/bus/pci/devices/$PORT_BDF" ] || pre_fail "root port $PORT_BDF not present" + +if pgrep -f qemu-system >/dev/null 2>&1; then + pgrep -af qemu-system >&2 + pre_fail "QEMU is running; stop the VM first" +fi + +if compgen -G "/sys/bus/mdev/devices/*" >/dev/null 2>&1; then + ls -d /sys/bus/mdev/devices/* >&2 + pre_fail "mdev devices exist; remove them first" +fi + +WATCH_PIDS="$(pgrep -af "$WATCHERS" 2>/dev/null || true)" +if [ -n "$WATCH_PIDS" ]; then + echo "$WATCH_PIDS" >&2 + pre_fail "stale watcher process(es) running; stop them first" +fi + +if lsmod 2>/dev/null | awk '{print $1}' | grep -qx open_iov_vfio; then + pre_fail "open_iov_vfio is already loaded; rmmod it first" +fi + +command -v lspci >/dev/null 2>&1 || pre_fail "lspci not found" +command -v nvidia-smi >/dev/null 2>&1 || pre_fail "nvidia-smi not found" +command -v timeout >/dev/null 2>&1 || pre_fail "timeout not found" + +sudo systemctl start nvidia-persistenced 2>/dev/null || true +for _ in $(seq 1 10); do + systemctl is-active --quiet nvidia-persistenced 2>/dev/null && break + sleep 1 +done +systemctl is-active --quiet nvidia-persistenced 2>/dev/null \ + || pre_fail "nvidia-persistenced did not start" +if ! nvidia-smi -L >/dev/null 2>&1; then + pre_fail "nvidia-smi -L failed with persistenced running" +fi +log "preflight: persistenced active, nvidia-smi OK, GPU=$GPU_BDF port=$PORT_BDF" + +mkdir -p "$LOGDIR" || pre_fail "cannot create $LOGDIR" + +# ---------------------------------------------------------------- helpers + +save_dmesg() { + sudo dmesg > "$1" 2>/dev/null || true +} + +# Count rail-cycle lines strictly inside the locked window: after the first +# "power domain locked" line and before the first "power domain released" +# line (or the end of the log if the locks never released). Outside the +# window is normal RTD3. +in_window_rail_count() { + local f="$1" first rel + first="$(grep -n "power domain locked" "$f" 2>/dev/null | head -1 | cut -d: -f1)" + [ -n "$first" ] || { echo 0; return; } + rel="$(grep -n "power domain released" "$f" 2>/dev/null | head -1 | cut -d: -f1)" + if [ -n "$rel" ]; then + awk -v a="$first" -v b="$rel" \ + '/xHC error in resume|root hub lost power/ && NR > a && NR < b { n++ } END { print n+0 }' "$f" + else + awk -v a="$first" \ + '/xHC error in resume|root hub lost power/ && NR > a { n++ } END { print n+0 }' "$f" + fi +} + +rail_count() { + grep -c -E "xHC error in resume|root hub lost power" "$1" 2>/dev/null || true +} + +abort_signature() { + grep -q -E "not ready .*after resume" "$1" 2>/dev/null && return 0 + grep -q -E "board unresponsive|gsp-phys.*WARN|WARNING:.*gsp-phys" "$1" 2>/dev/null && return 0 + return 1 +} + +wait_active() { + local i + for i in $(seq 1 10); do + if [ "$(cat "/sys/bus/pci/devices/$GPU_BDF/power/runtime_status" 2>/dev/null)" = active ] && + [ "$(cat "/sys/bus/pci/devices/$PORT_BDF/power/runtime_status" 2>/dev/null)" = active ]; then + return 0 + fi + nvidia-smi -L >/dev/null 2>&1 || true + sleep 1 + done + return 1 +} + +set_power_control() { + local bdf="$1" val="$2" + sudo sh -c "echo '$val' > '/sys/bus/pci/devices/$bdf/power/control'" 2>/dev/null || return 1 + [ "$(cat "/sys/bus/pci/devices/$bdf/power/control" 2>/dev/null)" = "$val" ] || return 1 + return 0 +} + +pci_field() { + lspci -k -s "$1" 2>/dev/null | sed -n "s/.*Kernel driver in use: //p" | head -1 +} + +abort_now() { + local n="$1" dlog="$2" msg="$3" + save_dmesg "$dlog" + echo "ABORT: cycle $n: $msg" >&2 + echo "ABORT: state left as-is (no rmmod, no restore). Log: $dlog" >&2 + exit 2 +} + +# ---------------------------------------------------------------- cycles + +ALL_GREEN=1 +for n in $(seq 1 "$CYCLES"); do + [ "$INTERRUPTED" -eq 0 ] || exit 2 + DLOG="$LOGDIR/phys_claim_cycle_${TS}_${n}.log" + log "=== cycle $n/$CYCLES (hold ${HOLD}s, layout=$LAYOUT, ko=$(basename "$KO")) ===" + + sudo systemctl start nvidia-persistenced 2>/dev/null || true + if ! wait_active; then + pre_fail "cycle $n: GPU/port did not reach runtime_status=active" + fi + + set_power_control "$GPU_BDF" on || pre_fail "cycle $n: cannot set GPU power/control=on" + set_power_control "$PORT_BDF" on || pre_fail "cycle $n: cannot set port power/control=on" + + sudo dmesg -C 2>/dev/null || pre_fail "cycle $n: dmesg -C failed" + + LAYOUT_ARG="" + [ "$LAYOUT" -eq 1 ] && LAYOUT_ARG="gsp_phys_layout=1" + # One command: stop persistenced and insmod with the D0 pre-lock in place. + if ! { sudo systemctl stop nvidia-persistenced && sudo insmod "$KO" \ + gsp_phys_gpu=1 gsp_phys_hazard_ack=1 $LAYOUT_ARG; }; then + save_dmesg "$DLOG" + echo "ABORT: cycle $n: stop-persistenced && insmod failed" >&2 + exit 2 + fi + sleep 1 + save_dmesg "$DLOG" + + if grep -q "gsp-phys: refusing claim" "$DLOG" 2>/dev/null; then + echo "PRECONDITION FAIL: cycle $n: module refused the claim (GPU/port were already runtime-suspended)" >&2 + warn "no claim was taken; unloading the module is safe" + timeout 60 sudo rmmod open_iov_vfio 2>/dev/null || true + save_dmesg "$DLOG" + exit 3 + fi + + if abort_signature "$DLOG"; then + abort_now "$n" "$DLOG" "hard abort signature after insmod" + fi + + NR_LOCKS="$(grep -c "power domain locked" "$DLOG" 2>/dev/null || true)" + OK_GPU_LOCK=0; grep -q "power domain locked: $GPU_BDF" "$DLOG" && OK_GPU_LOCK=1 + OK_PORT_LOCK=0; grep -q "power domain locked: $PORT_BDF" "$DLOG" && OK_PORT_LOCK=1 + OK_UNBIND=0; grep -q "unbinding nvidia from" "$DLOG" && OK_UNBIND=1 + OK_CLAIM=0; grep -q "physical GPU claimed" "$DLOG" && OK_CLAIM=1 + OK_BOARD=0; grep -q "board check after claim.*id=10de:1ed3" "$DLOG" && OK_BOARD=1 + OK_STAGE3B=1 + if [ "$LAYOUT" -eq 1 ]; then + OK_STAGE3B=0; grep -q "stage 3b complete" "$DLOG" && OK_STAGE3B=1 + fi + + if [ "$NR_LOCKS" -eq 0 ] && [ "$OK_CLAIM" -eq 0 ]; then + echo "ABORT: cycle $n: no locks and no claim took effect" >&2 + warn "no D0 locks were taken; unloading the module is safe" + timeout 60 sudo rmmod open_iov_vfio 2>/dev/null || true + save_dmesg "$DLOG" + exit 2 + fi + if [ "$OK_GPU_LOCK" -ne 1 ] || [ "$OK_PORT_LOCK" -ne 1 ] || \ + [ "$OK_UNBIND" -ne 1 ] || [ "$OK_CLAIM" -ne 1 ] || [ "$OK_BOARD" -ne 1 ]; then + abort_now "$n" "$DLOG" \ + "claim patterns incomplete (locks=$NR_LOCKS gpu=$OK_GPU_LOCK port=$OK_PORT_LOCK unbind=$OK_UNBIND claim=$OK_CLAIM board=$OK_BOARD)" + fi + if [ "$NR_LOCKS" -ne 5 ]; then + warn "cycle $n: expected 5 power domain locked lines, saw $NR_LOCKS" + ALL_GREEN=0 + fi + if [ "$LAYOUT" -eq 1 ] && [ "$OK_STAGE3B" -ne 1 ]; then + warn "cycle $n: gsp_phys_layout=1 but no 'stage 3b complete' line" + fi + + IN_RAIL_BEFORE="$(in_window_rail_count "$DLOG")" + OUT_RAIL_BEFORE="$(( $(rail_count "$DLOG") - IN_RAIL_BEFORE ))" + log "cycle $n: claimed and verified (locks=$NR_LOCKS, rail outside=$OUT_RAIL_BEFORE). Holding ${HOLD}s" + sleep "$HOLD" + + save_dmesg "$DLOG" + if abort_signature "$DLOG"; then + abort_now "$n" "$DLOG" "hard abort signature during hold" + fi + + LSPCI_NN="$(lspci -nn -s "$GPU_BDF" 2>/dev/null || true)" + DRV="$(pci_field "$GPU_BDF")" + RSTAT_GPU="$(cat "/sys/bus/pci/devices/$GPU_BDF/power/runtime_status" 2>/dev/null || echo '?')" + RSTAT_PORT="$(cat "/sys/bus/pci/devices/$PORT_BDF/power/runtime_status" 2>/dev/null || echo '?')" + HOLD_RED=0 + echo "$LSPCI_NN" | grep -q "10de:1ed3" || HOLD_RED=1 + [ "$DRV" = "open-iov-gsp-phys" ] || HOLD_RED=1 + [ "$RSTAT_GPU" = active ] && [ "$RSTAT_PORT" = active ] || HOLD_RED=1 + if [ "$HOLD_RED" -ne 0 ]; then + abort_now "$n" "$DLOG" \ + "hold verification failed (lspci='$LSPCI_NN' driver='$DRV' gpu=$RSTAT_GPU port=$RSTAT_PORT)" + fi + + timeout 60 sudo rmmod open_iov_vfio >/dev/null 2>&1 + RMMOD_RC=$? + sleep 1 + save_dmesg "$DLOG" + if [ "$RMMOD_RC" -eq 124 ]; then + abort_now "$n" "$DLOG" "rmmod timed out after 60s" + fi + if [ "$RMMOD_RC" -ne 0 ]; then + abort_now "$n" "$DLOG" "rmmod failed (rc=$RMMOD_RC)" + fi + if abort_signature "$DLOG"; then + abort_now "$n" "$DLOG" "hard abort signature after rmmod" + fi + + OK_REBIND=0; grep -q "rebound to nvidia (rc=1)" "$DLOG" && OK_REBIND=1 + OK_BEFORE=0; grep -q "board check before rebind.*id=10de:1ed3" "$DLOG" && OK_BEFORE=1 + NR_RELEASED="$(grep -c "power domain released" "$DLOG" 2>/dev/null || true)" + if [ "$OK_REBIND" -ne 1 ] || [ "$NR_RELEASED" -ne 5 ]; then + ALL_GREEN=0 + warn "cycle $n: release verification failed (rebind=$OK_REBIND released=$NR_RELEASED); locks may still be held - do not retry" + save_dmesg "$DLOG" + break + fi + if [ "$OK_BEFORE" -ne 1 ]; then + warn "cycle $n: missing 'board check before rebind id=10de:1ed3' line" + ALL_GREEN=0 + fi + + if [ "$NO_RESTORE" -eq 0 ]; then + set_power_control "$GPU_BDF" auto || warn "cycle $n: failed to restore GPU power/control=auto" + set_power_control "$PORT_BDF" auto || warn "cycle $n: failed to restore port power/control=auto" + sudo systemctl start nvidia-persistenced 2>/dev/null || true + wait_active || warn "cycle $n: GPU/port not active after persistenced restart" + else + log "cycle $n: --no-restore: leaving power/control=on and persistenced stopped" + fi + + HEALTH_RED=0 + nvidia-smi -L >/dev/null 2>&1 || HEALTH_RED=1 + [ "$(pci_field "$GPU_BDF")" = "nvidia" ] || HEALTH_RED=1 + if [ "$NO_RESTORE" -eq 0 ]; then + [ "$(cat "/sys/bus/pci/devices/$GPU_BDF/power/control" 2>/dev/null)" = auto ] || HEALTH_RED=1 + [ "$(cat "/sys/bus/pci/devices/$PORT_BDF/power/control" 2>/dev/null)" = auto ] || HEALTH_RED=1 + fi + [ "$(cat "/sys/bus/pci/devices/$GPU_BDF/d3cold_allowed" 2>/dev/null)" = 1 ] || HEALTH_RED=1 + + IN_RAIL="$(in_window_rail_count "$DLOG")" + OUT_RAIL="$(( $(rail_count "$DLOG") - IN_RAIL ))" + if [ "$IN_RAIL" -gt 0 ]; then + warn "cycle $n: RED - $IN_RAIL rail-cycle line(s) INSIDE the locked window (lock failed)" + ALL_GREEN=0 + fi + if [ "$OUT_RAIL" -gt 0 ]; then + warn "cycle $n: normal RTD3 rail cycle outside the locked window ($OUT_RAIL line(s))" + fi + + if [ "$HEALTH_RED" -ne 0 ]; then + ALL_GREEN=0 + warn "cycle $n: health check failed (nvidia-smi/driver/control/d3cold)" + fi + if [ "$OK_STAGE3B" -ne 1 ]; then + ALL_GREEN=0 + fi + + if [ "$ALL_GREEN" -eq 1 ] && [ "$IN_RAIL" -eq 0 ] && [ "$HEALTH_RED" -eq 0 ]; then + log "cycle $n: GREEN (locks=$NR_LOCKS, released=$NR_RELEASED, in-window rail=0, outside rail=$OUT_RAIL, hold=${HOLD}s)" + else + log "cycle $n: RED (locks=$NR_LOCKS, released=$NR_RELEASED, in-window rail=$IN_RAIL, outside rail=$OUT_RAIL, health_ok=$((1-HEALTH_RED)))" + if [ "$IN_RAIL" -gt 0 ]; then + echo "ABORT: cycle $n: rail cycle inside the locked window; not retrying" >&2 + save_dmesg "$DLOG" + exit 2 + fi + break + fi +done + +if [ "$ALL_GREEN" -eq 1 ]; then + log "VERDICT: all $CYCLES cycle(s) green (exit 0)" + exit 0 +fi +echo "VERDICT: not green; do not retry in this boot (exit 2)" >&2 +exit 2 -- cgit v1.2.3