guest-diag: reusable supervised physical-claim cycle harness (D0 pre-lock + abort checks)

1 files changed, 426 insertions, 0 deletions

diff --git a/tools/guest-diag/phys_claim_cycle.sh b/tools/guest-diag/phys_claim_cycle.sh
new file mode 100755
index 0000000..7103fec
--- /dev/null
+++ b/tools/guest-diag/phys_claim_cycle.sh
@@ -0,0 +1,426 @@
+#!/bin/bash
+# phys_claim_cycle.sh - supervised physical-GPU claim/release cycle harness.
+#
+# Drives one or more full "D0 pre-lock" cycles of the physical-ownership
+# backend (open-iov-vfio + gsp_phys_gpu=1 + gsp_phys_hazard_ack=1):
+#
+#   start persistenced -> wait GPU+root port active -> power/control=on on
+#   GPU+port -> dmesg -C -> (stop persistenced && insmod) as ONE command ->
+#   verify lock/claim patterns -> hold -> verify while claimed ->
+#   rmmod -> verify release patterns -> restore auto + persistenced ->
+#   health check.
+#
+# !!! HOST-FREEZE HAZARD !!!
+# Claiming the GPU unbinds the distribution NVIDIA driver. On this host the
+# dGPU board then power-gate and fail to come back, hard-freezing the system
+# with no crash record (observed twice on 2026-09-30). Run only in a
+# supervised session, with other host work stopped and the PART 87 recovery
+# plan ready. The harness itself never retries an aborted cycle.
+#
+# Usage: phys_claim_cycle.sh --yes [--hold SECONDS] [--cycles N] [--layout]
+#                             [--ko PATH] [--no-restore]
+#
+#   --yes        required: acknowledge the host-freeze hazard
+#   --hold S     seconds to keep the GPU claimed (default 30)
+#   --cycles N   cycles to run, each only if the previous was green (default 1)
+#   --layout     also pass gsp_phys_layout=1 (rung 3b); "stage 3b complete"
+#                then becomes an expected (but non-fatal) pattern
+#   --ko PATH    module to insmod (default kernel/open-iov-vfio.ko)
+#   --no-restore do not restore power/control=auto or restart persistenced
+#                at the end of each cycle (leaves the D0 hold in place)
+#
+# Exit codes: 0 = all cycles green; 1 = --yes missing / usage error;
+#             2 = abort (hard abort signature, failed verification, red
+#             in-window rail event); 3 = precondition failure.
+#
+# Per-cycle full dmesg is saved to
+#   /tmp/opencode/phys_claim_cycle_<timestamp>_<n>.log
+#
+# Env overrides: GPU_BDF (default 0000:01:00.0), PORT_BDF (default: the GPU's
+# upstream bridge, normally 0000:00:01.0), PHYS_CLAIM_LOGDIR.
+
+set -u
+
+HERE="$(dirname "$(readlink -f "$0")")"
+REPO_ROOT="$(readlink -f "$HERE/../..")"
+
+USE_YES=0
+HOLD=30
+CYCLES=1
+LAYOUT=0
+NO_RESTORE=0
+KO="$REPO_ROOT/kernel/open-iov-vfio.ko"
+
+GPU_BDF="${GPU_BDF:-0000:01:00.0}"
+if [ -z "${PORT_BDF:-}" ] && [ -e "/sys/bus/pci/devices/$GPU_BDF" ]; then
+    PORT_BDF="$(basename "$(dirname "$(readlink -f "/sys/bus/pci/devices/$GPU_BDF")")")"
+fi
+PORT_BDF="${PORT_BDF:-0000:00:01.0}"
+
+LOGDIR="${PHYS_CLAIM_LOGDIR:-/tmp/opencode}"
+TS="$(date +%Y%m%d_%H%M%S)"
+WATCHERS='pdwatch|watch_ctx|uvmt_poll|invd\.py|bpwait|sempages|track_waits|guest_pin|guest_pmap|vmrestart|push_guest_tools|nvidia-smi -l|dmesg -w'
+
+usage() {
+    sed -n '1,50p' "$0" | grep -E '^#( |$)' | sed 's/^# \{0,1\}//'
+}
+
+while [ $# -gt 0 ]; do
+    case "$1" in
+        --yes)        USE_YES=1 ;;
+        --hold)       HOLD="${2:-}"; shift ;;
+        --cycles)     CYCLES="${2:-}"; shift ;;
+        --layout)     LAYOUT=1 ;;
+        --ko)         KO="${2:-}"; shift ;;
+        --no-restore) NO_RESTORE=1 ;;
+        -h|--help)    usage; exit 0 ;;
+        *) echo "unknown argument: $1" >&2; usage >&2; exit 1 ;;
+    esac
+    shift
+done
+
+case "$HOLD" in ''|*[!0-9]*) echo "--hold must be a non-negative integer" >&2; exit 1 ;; esac
+case "$CYCLES" in ''|*[!0-9]*|0) echo "--cycles must be a positive integer" >&2; exit 1 ;; esac
+if [ "$HOLD" -lt 30 ]; then
+    echo "WARN: --hold $HOLD is below the verified 30 s minimum" >&2
+fi
+
+if [ "$USE_YES" -ne 1 ]; then
+    cat >&2 <<'EOF'
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+HOST-FREEZE HAZARD - REFUSING TO RUN WITHOUT --yes
+
+The physical claim unbinds the distribution NVIDIA driver from the dGPU.
+On this host the board can then power-gate and fail to come back, hard-
+freezing the system with no crash record (observed twice on 2026-09-30).
+Run only when supervised, with other host work stopped and the PART 87
+recovery plan ready. Re-run with --yes to acknowledge.
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+EOF
+    exit 1
+fi
+
+log()  { echo "[$(date +%H:%M:%S)] $*"; }
+warn() { echo "[$(date +%H:%M:%S)] WARN: $*" >&2; }
+pre_fail() { echo "PRECONDITION FAIL: $*" >&2; exit 3; }
+
+INTERRUPTED=0
+on_signal() {
+    INTERRUPTED=1
+    echo
+    echo "ABORT: interrupted; leaving module/lock state as-is (no cleanup)." >&2
+    exit 2
+}
+trap on_signal INT TERM
+
+# ---------------------------------------------------------------- preflight
+
+[ -c /dev/null ] || pre_fail "no /dev/null?"
+if ! sudo -n true 2>/dev/null; then
+    sudo -v 2>/dev/null || pre_fail "current user cannot sudo"
+fi
+
+[ -f "$KO" ] || pre_fail "module not found: $KO (build it or pass --ko)"
+KO_HASH="$(sha256sum "$KO" | awk '{print $1}')"
+KO_SRCVER="$(modinfo -F srcversion "$KO" 2>/dev/null || true)"
+if [ -d "$REPO_ROOT/.git" ]; then
+    HEAD_COMMIT="$(git -C "$REPO_ROOT" rev-parse --short HEAD 2>/dev/null || true)"
+    log "preflight: repo HEAD ${HEAD_COMMIT:-unknown}"
+    if git -C "$REPO_ROOT" status --porcelain -- kernel 2>/dev/null | grep -q .; then
+        warn "kernel/ has uncommitted changes; HEAD does not describe $KO"
+    fi
+else
+    HEAD_COMMIT=""
+    warn "not a git checkout; cannot record HEAD"
+fi
+log "preflight: ko $KO sha256 $KO_HASH srcversion ${KO_SRCVER:-unknown}"
+if [ "$KO" -ot "$REPO_ROOT/kernel/gsp_phys_gpu.c" ]; then
+    warn "$KO is older than kernel/gsp_phys_gpu.c; rebuild before cycling"
+fi
+
+[ -e "/sys/bus/pci/devices/$GPU_BDF" ] || pre_fail "GPU $GPU_BDF not present"
+[ -e "/sys/bus/pci/devices/$PORT_BDF" ] || pre_fail "root port $PORT_BDF not present"
+
+if pgrep -f qemu-system >/dev/null 2>&1; then
+    pgrep -af qemu-system >&2
+    pre_fail "QEMU is running; stop the VM first"
+fi
+
+if compgen -G "/sys/bus/mdev/devices/*" >/dev/null 2>&1; then
+    ls -d /sys/bus/mdev/devices/* >&2
+    pre_fail "mdev devices exist; remove them first"
+fi
+
+WATCH_PIDS="$(pgrep -af "$WATCHERS" 2>/dev/null || true)"
+if [ -n "$WATCH_PIDS" ]; then
+    echo "$WATCH_PIDS" >&2
+    pre_fail "stale watcher process(es) running; stop them first"
+fi
+
+if lsmod 2>/dev/null | awk '{print $1}' | grep -qx open_iov_vfio; then
+    pre_fail "open_iov_vfio is already loaded; rmmod it first"
+fi
+
+command -v lspci >/dev/null 2>&1 || pre_fail "lspci not found"
+command -v nvidia-smi >/dev/null 2>&1 || pre_fail "nvidia-smi not found"
+command -v timeout >/dev/null 2>&1 || pre_fail "timeout not found"
+
+sudo systemctl start nvidia-persistenced 2>/dev/null || true
+for _ in $(seq 1 10); do
+    systemctl is-active --quiet nvidia-persistenced 2>/dev/null && break
+    sleep 1
+done
+systemctl is-active --quiet nvidia-persistenced 2>/dev/null \
+    || pre_fail "nvidia-persistenced did not start"
+if ! nvidia-smi -L >/dev/null 2>&1; then
+    pre_fail "nvidia-smi -L failed with persistenced running"
+fi
+log "preflight: persistenced active, nvidia-smi OK, GPU=$GPU_BDF port=$PORT_BDF"
+
+mkdir -p "$LOGDIR" || pre_fail "cannot create $LOGDIR"
+
+# ---------------------------------------------------------------- helpers
+
+save_dmesg() {
+    sudo dmesg > "$1" 2>/dev/null || true
+}
+
+# Count rail-cycle lines strictly inside the locked window: after the first
+# "power domain locked" line and before the first "power domain released"
+# line (or the end of the log if the locks never released). Outside the
+# window is normal RTD3.
+in_window_rail_count() {
+    local f="$1" first rel
+    first="$(grep -n "power domain locked" "$f" 2>/dev/null | head -1 | cut -d: -f1)"
+    [ -n "$first" ] || { echo 0; return; }
+    rel="$(grep -n "power domain released" "$f" 2>/dev/null | head -1 | cut -d: -f1)"
+    if [ -n "$rel" ]; then
+        awk -v a="$first" -v b="$rel" \
+            '/xHC error in resume|root hub lost power/ && NR > a && NR < b { n++ } END { print n+0 }' "$f"
+    else
+        awk -v a="$first" \
+            '/xHC error in resume|root hub lost power/ && NR > a { n++ } END { print n+0 }' "$f"
+    fi
+}
+
+rail_count() {
+    grep -c -E "xHC error in resume|root hub lost power" "$1" 2>/dev/null || true
+}
+
+abort_signature() {
+    grep -q -E "not ready .*after resume" "$1" 2>/dev/null && return 0
+    grep -q -E "board unresponsive|gsp-phys.*WARN|WARNING:.*gsp-phys" "$1" 2>/dev/null && return 0
+    return 1
+}
+
+wait_active() {
+    local i
+    for i in $(seq 1 10); do
+        if [ "$(cat "/sys/bus/pci/devices/$GPU_BDF/power/runtime_status" 2>/dev/null)" = active ] &&
+           [ "$(cat "/sys/bus/pci/devices/$PORT_BDF/power/runtime_status" 2>/dev/null)" = active ]; then
+            return 0
+        fi
+        nvidia-smi -L >/dev/null 2>&1 || true
+        sleep 1
+    done
+    return 1
+}
+
+set_power_control() {
+    local bdf="$1" val="$2"
+    sudo sh -c "echo '$val' > '/sys/bus/pci/devices/$bdf/power/control'" 2>/dev/null || return 1
+    [ "$(cat "/sys/bus/pci/devices/$bdf/power/control" 2>/dev/null)" = "$val" ] || return 1
+    return 0
+}
+
+pci_field() {
+    lspci -k -s "$1" 2>/dev/null | sed -n "s/.*Kernel driver in use: //p" | head -1
+}
+
+abort_now() {
+    local n="$1" dlog="$2" msg="$3"
+    save_dmesg "$dlog"
+    echo "ABORT: cycle $n: $msg" >&2
+    echo "ABORT: state left as-is (no rmmod, no restore). Log: $dlog" >&2
+    exit 2
+}
+
+# ---------------------------------------------------------------- cycles
+
+ALL_GREEN=1
+for n in $(seq 1 "$CYCLES"); do
+    [ "$INTERRUPTED" -eq 0 ] || exit 2
+    DLOG="$LOGDIR/phys_claim_cycle_${TS}_${n}.log"
+    log "=== cycle $n/$CYCLES (hold ${HOLD}s, layout=$LAYOUT, ko=$(basename "$KO")) ==="
+
+    sudo systemctl start nvidia-persistenced 2>/dev/null || true
+    if ! wait_active; then
+        pre_fail "cycle $n: GPU/port did not reach runtime_status=active"
+    fi
+
+    set_power_control "$GPU_BDF" on || pre_fail "cycle $n: cannot set GPU power/control=on"
+    set_power_control "$PORT_BDF" on || pre_fail "cycle $n: cannot set port power/control=on"
+
+    sudo dmesg -C 2>/dev/null || pre_fail "cycle $n: dmesg -C failed"
+
+    LAYOUT_ARG=""
+    [ "$LAYOUT" -eq 1 ] && LAYOUT_ARG="gsp_phys_layout=1"
+    # One command: stop persistenced and insmod with the D0 pre-lock in place.
+    if ! { sudo systemctl stop nvidia-persistenced && sudo insmod "$KO" \
+            gsp_phys_gpu=1 gsp_phys_hazard_ack=1 $LAYOUT_ARG; }; then
+        save_dmesg "$DLOG"
+        echo "ABORT: cycle $n: stop-persistenced && insmod failed" >&2
+        exit 2
+    fi
+    sleep 1
+    save_dmesg "$DLOG"
+
+    if grep -q "gsp-phys: refusing claim" "$DLOG" 2>/dev/null; then
+        echo "PRECONDITION FAIL: cycle $n: module refused the claim (GPU/port were already runtime-suspended)" >&2
+        warn "no claim was taken; unloading the module is safe"
+        timeout 60 sudo rmmod open_iov_vfio 2>/dev/null || true
+        save_dmesg "$DLOG"
+        exit 3
+    fi
+
+    if abort_signature "$DLOG"; then
+        abort_now "$n" "$DLOG" "hard abort signature after insmod"
+    fi
+
+    NR_LOCKS="$(grep -c "power domain locked" "$DLOG" 2>/dev/null || true)"
+    OK_GPU_LOCK=0; grep -q "power domain locked: $GPU_BDF" "$DLOG" && OK_GPU_LOCK=1
+    OK_PORT_LOCK=0; grep -q "power domain locked: $PORT_BDF" "$DLOG" && OK_PORT_LOCK=1
+    OK_UNBIND=0;  grep -q "unbinding nvidia from" "$DLOG" && OK_UNBIND=1
+    OK_CLAIM=0;   grep -q "physical GPU claimed" "$DLOG" && OK_CLAIM=1
+    OK_BOARD=0;   grep -q "board check after claim.*id=10de:1ed3" "$DLOG" && OK_BOARD=1
+    OK_STAGE3B=1
+    if [ "$LAYOUT" -eq 1 ]; then
+        OK_STAGE3B=0; grep -q "stage 3b complete" "$DLOG" && OK_STAGE3B=1
+    fi
+
+    if [ "$NR_LOCKS" -eq 0 ] && [ "$OK_CLAIM" -eq 0 ]; then
+        echo "ABORT: cycle $n: no locks and no claim took effect" >&2
+        warn "no D0 locks were taken; unloading the module is safe"
+        timeout 60 sudo rmmod open_iov_vfio 2>/dev/null || true
+        save_dmesg "$DLOG"
+        exit 2
+    fi
+    if [ "$OK_GPU_LOCK" -ne 1 ] || [ "$OK_PORT_LOCK" -ne 1 ] || \
+       [ "$OK_UNBIND" -ne 1 ] || [ "$OK_CLAIM" -ne 1 ] || [ "$OK_BOARD" -ne 1 ]; then
+        abort_now "$n" "$DLOG" \
+            "claim patterns incomplete (locks=$NR_LOCKS gpu=$OK_GPU_LOCK port=$OK_PORT_LOCK unbind=$OK_UNBIND claim=$OK_CLAIM board=$OK_BOARD)"
+    fi
+    if [ "$NR_LOCKS" -ne 5 ]; then
+        warn "cycle $n: expected 5 power domain locked lines, saw $NR_LOCKS"
+        ALL_GREEN=0
+    fi
+    if [ "$LAYOUT" -eq 1 ] && [ "$OK_STAGE3B" -ne 1 ]; then
+        warn "cycle $n: gsp_phys_layout=1 but no 'stage 3b complete' line"
+    fi
+
+    IN_RAIL_BEFORE="$(in_window_rail_count "$DLOG")"
+    OUT_RAIL_BEFORE="$(( $(rail_count "$DLOG") - IN_RAIL_BEFORE ))"
+    log "cycle $n: claimed and verified (locks=$NR_LOCKS, rail outside=$OUT_RAIL_BEFORE). Holding ${HOLD}s"
+    sleep "$HOLD"
+
+    save_dmesg "$DLOG"
+    if abort_signature "$DLOG"; then
+        abort_now "$n" "$DLOG" "hard abort signature during hold"
+    fi
+
+    LSPCI_NN="$(lspci -nn -s "$GPU_BDF" 2>/dev/null || true)"
+    DRV="$(pci_field "$GPU_BDF")"
+    RSTAT_GPU="$(cat "/sys/bus/pci/devices/$GPU_BDF/power/runtime_status" 2>/dev/null || echo '?')"
+    RSTAT_PORT="$(cat "/sys/bus/pci/devices/$PORT_BDF/power/runtime_status" 2>/dev/null || echo '?')"
+    HOLD_RED=0
+    echo "$LSPCI_NN" | grep -q "10de:1ed3" || HOLD_RED=1
+    [ "$DRV" = "open-iov-gsp-phys" ] || HOLD_RED=1
+    [ "$RSTAT_GPU" = active ] && [ "$RSTAT_PORT" = active ] || HOLD_RED=1
+    if [ "$HOLD_RED" -ne 0 ]; then
+        abort_now "$n" "$DLOG" \
+            "hold verification failed (lspci='$LSPCI_NN' driver='$DRV' gpu=$RSTAT_GPU port=$RSTAT_PORT)"
+    fi
+
+    timeout 60 sudo rmmod open_iov_vfio >/dev/null 2>&1
+    RMMOD_RC=$?
+    sleep 1
+    save_dmesg "$DLOG"
+    if [ "$RMMOD_RC" -eq 124 ]; then
+        abort_now "$n" "$DLOG" "rmmod timed out after 60s"
+    fi
+    if [ "$RMMOD_RC" -ne 0 ]; then
+        abort_now "$n" "$DLOG" "rmmod failed (rc=$RMMOD_RC)"
+    fi
+    if abort_signature "$DLOG"; then
+        abort_now "$n" "$DLOG" "hard abort signature after rmmod"
+    fi
+
+    OK_REBIND=0;  grep -q "rebound to nvidia (rc=1)" "$DLOG" && OK_REBIND=1
+    OK_BEFORE=0;  grep -q "board check before rebind.*id=10de:1ed3" "$DLOG" && OK_BEFORE=1
+    NR_RELEASED="$(grep -c "power domain released" "$DLOG" 2>/dev/null || true)"
+    if [ "$OK_REBIND" -ne 1 ] || [ "$NR_RELEASED" -ne 5 ]; then
+        ALL_GREEN=0
+        warn "cycle $n: release verification failed (rebind=$OK_REBIND released=$NR_RELEASED); locks may still be held - do not retry"
+        save_dmesg "$DLOG"
+        break
+    fi
+    if [ "$OK_BEFORE" -ne 1 ]; then
+        warn "cycle $n: missing 'board check before rebind id=10de:1ed3' line"
+        ALL_GREEN=0
+    fi
+
+    if [ "$NO_RESTORE" -eq 0 ]; then
+        set_power_control "$GPU_BDF" auto || warn "cycle $n: failed to restore GPU power/control=auto"
+        set_power_control "$PORT_BDF" auto || warn "cycle $n: failed to restore port power/control=auto"
+        sudo systemctl start nvidia-persistenced 2>/dev/null || true
+        wait_active || warn "cycle $n: GPU/port not active after persistenced restart"
+    else
+        log "cycle $n: --no-restore: leaving power/control=on and persistenced stopped"
+    fi
+
+    HEALTH_RED=0
+    nvidia-smi -L >/dev/null 2>&1 || HEALTH_RED=1
+    [ "$(pci_field "$GPU_BDF")" = "nvidia" ] || HEALTH_RED=1
+    if [ "$NO_RESTORE" -eq 0 ]; then
+        [ "$(cat "/sys/bus/pci/devices/$GPU_BDF/power/control" 2>/dev/null)" = auto ] || HEALTH_RED=1
+        [ "$(cat "/sys/bus/pci/devices/$PORT_BDF/power/control" 2>/dev/null)" = auto ] || HEALTH_RED=1
+    fi
+    [ "$(cat "/sys/bus/pci/devices/$GPU_BDF/d3cold_allowed" 2>/dev/null)" = 1 ] || HEALTH_RED=1
+
+    IN_RAIL="$(in_window_rail_count "$DLOG")"
+    OUT_RAIL="$(( $(rail_count "$DLOG") - IN_RAIL ))"
+    if [ "$IN_RAIL" -gt 0 ]; then
+        warn "cycle $n: RED - $IN_RAIL rail-cycle line(s) INSIDE the locked window (lock failed)"
+        ALL_GREEN=0
+    fi
+    if [ "$OUT_RAIL" -gt 0 ]; then
+        warn "cycle $n: normal RTD3 rail cycle outside the locked window ($OUT_RAIL line(s))"
+    fi
+
+    if [ "$HEALTH_RED" -ne 0 ]; then
+        ALL_GREEN=0
+        warn "cycle $n: health check failed (nvidia-smi/driver/control/d3cold)"
+    fi
+    if [ "$OK_STAGE3B" -ne 1 ]; then
+        ALL_GREEN=0
+    fi
+
+    if [ "$ALL_GREEN" -eq 1 ] && [ "$IN_RAIL" -eq 0 ] && [ "$HEALTH_RED" -eq 0 ]; then
+        log "cycle $n: GREEN (locks=$NR_LOCKS, released=$NR_RELEASED, in-window rail=0, outside rail=$OUT_RAIL, hold=${HOLD}s)"
+    else
+        log "cycle $n: RED (locks=$NR_LOCKS, released=$NR_RELEASED, in-window rail=$IN_RAIL, outside rail=$OUT_RAIL, health_ok=$((1-HEALTH_RED)))"
+        if [ "$IN_RAIL" -gt 0 ]; then
+            echo "ABORT: cycle $n: rail cycle inside the locked window; not retrying" >&2
+            save_dmesg "$DLOG"
+            exit 2
+        fi
+        break
+    fi
+done
+
+if [ "$ALL_GREEN" -eq 1 ]; then
+    log "VERDICT: all $CYCLES cycle(s) green (exit 0)"
+    exit 0
+fi
+echo "VERDICT: not green; do not retry in this boot (exit 2)" >&2
+exit 2