diff options
Diffstat (limited to 'tools')
| -rwxr-xr-x | tools/guest-diag/phys_claim_cycle.sh | 160 |
1 files changed, 141 insertions, 19 deletions
diff --git a/tools/guest-diag/phys_claim_cycle.sh b/tools/guest-diag/phys_claim_cycle.sh index 898aa15..3b146c6 100755 --- a/tools/guest-diag/phys_claim_cycle.sh +++ b/tools/guest-diag/phys_claim_cycle.sh @@ -17,8 +17,12 @@ # supervised session, with other host work stopped and the PART 87 recovery # plan ready. The harness itself never retries an aborted cycle. # +# RULE: never stop after FWSEC alone on this host. --boot (the full boot +# lifecycle) is the only write mode; --fwsec is retired and refused. +# # Usage: phys_claim_cycle.sh --yes [--hold SECONDS] [--cycles N] [--layout] -# [--dryrun] [--fwsec] [--ko PATH] [--no-restore] +# [--dryrun] [--boot-dryrun] [--boot] [--keep] +# [--ko PATH] [--no-restore] # # --yes required: acknowledge the host-freeze hazard # --hold S seconds to keep the GPU claimed (default 30) @@ -28,17 +32,41 @@ # --dryrun also pass gsp_phys_fwsec_dryrun=1 (implies --layout): run the # READ-ONLY stage-3c validation (VBIOS/BIT/FWSEC parse and the # staging/placement plan, no device writes); expects the -# "gsp-phys-fwsec-dryrun:" and "stage 3c dry-run complete" lines -# --fwsec also pass gsp_phys_fwsec=1 (implies --layout): run the REAL -# stage-3c write sequence (GSP falcon reset, WPR2 carve, FWSEC); -# expects "stage 3c complete" and, on release, the WPR2-down -# teardown line; a failed teardown keeps the locks and refuses -# the nvidia rebind (the cycle goes red; do not retry) +# "gsp-phys-fwsec-dryrun:" and "stage 3c dry-run complete" +# lines. It is also implied by --boot-dryrun (alias) +# --boot-dryrun implies --dryrun and also passes gsp_phys_boot_dryrun=1 +# (implies --layout): host-memory-only stage-3d staging +# rehearsal (radix3 build, queue/PTE/args fill, all placement +# math; ZERO BAR0 writes); additionally expects the +# "stage 3d prime complete" line +# --boot pass gsp_phys_boot=1 (implies --layout): the ONLY write mode. +# Runs the full boot lifecycle (FWSEC, Booter Load, RISC-V +# start, FALCON_OS, RM-init poll). Requires "stage 3d +# complete"; "stage 3e complete" = full green, "stage 3e +# incomplete" = non-fatal warning (RM-init not observed, boot +# still valid); on release requires "teardown complete: WPR2 +# down" (missing = red; do not retry) +# --keep after a GREEN cycle do NOT rmmod: leave the module loaded and +# the GPU claimed by open-iov-gsp-phys (power/control=on on +# GPU+port, nvidia-persistenced stopped) and skip the release +# verification (no rebind/released lines); the hold checks +# (lspci id, driver, runtime active) and abort signatures still +# run. Prints a notice with the exact recovery steps. Compatible +# with --boot/--boot-dryrun/--dryrun/--layout (it only changes +# the ending); single cycle only (--cycles 1). Exit 0 = the +# claimed session is intentionally left running. +# --fwsec RETIRED (2026-10-01). A FWSEC-only carve is a one-way door on +# this fused part: it left WPR2 up and was not host-recoverable. +# It is refused immediately and never reaches insmod. Use +# --boot; never stop after FWSEC alone on this host. # --ko PATH module to insmod (default kernel/open-iov-vfio.ko) # --no-restore do not restore power/control=auto or restart persistenced # at the end of each cycle (leaves the D0 hold in place) # -# Exit codes: 0 = all cycles green; 1 = --yes missing / usage error; +# --boot is mutually exclusive with --dryrun/--boot-dryrun. +# +# Exit codes: 0 = all cycles green (with --keep: the claimed session was left +# running intentionally); 1 = --yes missing / usage error; # 2 = abort (hard abort signature, failed verification, red # in-window rail event); 3 = precondition failure. # @@ -59,6 +87,9 @@ CYCLES=1 LAYOUT=0 DRYRUN=0 FWSEC=0 +BOOT=0 +BOOT_DRYRUN=0 +KEEP=0 NO_RESTORE=0 KO="$REPO_ROOT/kernel/open-iov-vfio.ko" @@ -73,7 +104,7 @@ TS="$(date +%Y%m%d_%H%M%S)" WATCHERS='pdwatch|watch_ctx|uvmt_poll|invd\.py|bpwait|sempages|track_waits|guest_pin|guest_pmap|vmrestart|push_guest_tools|nvidia-smi -l|dmesg -w' usage() { - sed -n '1,50p' "$0" | grep -E '^#( |$)' | sed 's/^# \{0,1\}//' + awk '/^[^#]/{exit} {print}' "$0" | grep -E '^#( |$)' | sed 's/^# \{0,1\}//' } while [ $# -gt 0 ]; do @@ -83,7 +114,16 @@ while [ $# -gt 0 ]; do --cycles) CYCLES="${2:-}"; shift ;; --layout) LAYOUT=1 ;; --dryrun) DRYRUN=1; LAYOUT=1 ;; - --fwsec) FWSEC=1; LAYOUT=1 ;; + --boot-dryrun) BOOT_DRYRUN=1; LAYOUT=1; DRYRUN=1 ;; + --boot) BOOT=1; LAYOUT=1 ;; + --keep) KEEP=1 ;; + --fwsec) + cat >&2 <<'EOF' +--fwsec is RETIRED: FWSEC-only is a one-way door on this fused part. +A FWSEC-only carve left WPR2 up and was not host-recoverable (2026-10-01). +Never stop after FWSEC alone on this host. Use --boot (full boot lifecycle). +EOF + exit 1 ;; --ko) KO="${2:-}"; shift ;; --no-restore) NO_RESTORE=1 ;; -h|--help) usage; exit 0 ;; @@ -94,14 +134,28 @@ done case "$HOLD" in ''|*[!0-9]*) echo "--hold must be a non-negative integer" >&2; exit 1 ;; esac case "$CYCLES" in ''|*[!0-9]*|0) echo "--cycles must be a positive integer" >&2; exit 1 ;; esac +if [ "$KEEP" -eq 1 ] && [ "$CYCLES" -ne 1 ]; then + echo "--keep leaves the claimed session running: single cycle only (--cycles 1)" >&2 + exit 1 +fi if [ "$DRYRUN" -eq 1 ] && [ "$FWSEC" -eq 1 ]; then echo "--dryrun and --fwsec are mutually exclusive" >&2 exit 1 fi +if [ "$BOOT" -eq 1 ] && { [ "$DRYRUN" -eq 1 ] || [ "$BOOT_DRYRUN" -eq 1 ]; }; then + echo "--boot is mutually exclusive with --dryrun/--boot-dryrun" >&2 + exit 1 +fi if [ "$HOLD" -lt 30 ]; then echo "WARN: --hold $HOLD is below the verified 30 s minimum" >&2 fi +MODE="none" +[ "$LAYOUT" -eq 1 ] && MODE="layout" +[ "$DRYRUN" -eq 1 ] && MODE="dryrun" +[ "$BOOT_DRYRUN" -eq 1 ] && MODE="boot-dryrun" +[ "$BOOT" -eq 1 ] && MODE="boot" + if [ "$USE_YES" -ne 1 ]; then cat >&2 <<'EOF' !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! @@ -268,7 +322,7 @@ ALL_GREEN=1 for n in $(seq 1 "$CYCLES"); do [ "$INTERRUPTED" -eq 0 ] || exit 2 DLOG="$LOGDIR/phys_claim_cycle_${TS}_${n}.log" - log "=== cycle $n/$CYCLES (hold ${HOLD}s, layout=$LAYOUT, ko=$(basename "$KO")) ===" + log "=== cycle $n/$CYCLES (mode=$MODE, hold ${HOLD}s, layout=$LAYOUT, ko=$(basename "$KO")) ===" sudo systemctl start nvidia-persistenced 2>/dev/null || true if ! wait_active; then @@ -284,11 +338,14 @@ for n in $(seq 1 "$CYCLES"); do [ "$LAYOUT" -eq 1 ] && LAYOUT_ARG="gsp_phys_layout=1" DRYRUN_ARG="" [ "$DRYRUN" -eq 1 ] && DRYRUN_ARG="gsp_phys_fwsec_dryrun=1" - FWSEC_ARG="" - [ "$FWSEC" -eq 1 ] && FWSEC_ARG="gsp_phys_fwsec=1" + BOOTDRY_ARG="" + [ "$BOOT_DRYRUN" -eq 1 ] && BOOTDRY_ARG="gsp_phys_boot_dryrun=1" + BOOT_ARG="" + [ "$BOOT" -eq 1 ] && BOOT_ARG="gsp_phys_boot=1" # One command: stop persistenced and insmod with the D0 pre-lock in place. + # gsp_phys_fwsec=1 is never passed: --fwsec refuses before this point. if ! { sudo systemctl stop nvidia-persistenced && sudo insmod "$KO" \ - gsp_phys_gpu=1 gsp_phys_hazard_ack=1 $LAYOUT_ARG $DRYRUN_ARG $FWSEC_ARG; }; then + gsp_phys_gpu=1 gsp_phys_hazard_ack=1 $LAYOUT_ARG $DRYRUN_ARG $BOOTDRY_ARG $BOOT_ARG; }; then save_dmesg "$DLOG" echo "ABORT: cycle $n: stop-persistenced && insmod failed" >&2 exit 2 @@ -326,6 +383,24 @@ for n in $(seq 1 "$CYCLES"); do OK_DRYRUN=1 fi fi + OK_BOOTPRIME=1 + if [ "$BOOT_DRYRUN" -eq 1 ]; then + OK_BOOTPRIME=0 + grep -q "stage 3d prime complete" "$DLOG" && OK_BOOTPRIME=1 + fi + OK_BOOT=1 + OK_BOOT3E=0 + if [ "$BOOT" -eq 1 ]; then + OK_BOOT=0 + if grep -q "stage 3d complete" "$DLOG"; then + OK_BOOT=1 + if grep -q "stage 3e complete" "$DLOG"; then + OK_BOOT3E=2 + elif grep -q "stage 3e incomplete" "$DLOG"; then + OK_BOOT3E=1 + fi + fi + fi OK_FWSEC=1 if [ "$FWSEC" -eq 1 ]; then OK_FWSEC=0 @@ -354,6 +429,18 @@ for n in $(seq 1 "$CYCLES"); do if [ "$DRYRUN" -eq 1 ] && [ "$OK_DRYRUN" -ne 1 ]; then warn "cycle $n: gsp_phys_fwsec_dryrun=1 but the read-only dry-run did not complete (missing 'gsp-phys-fwsec-dryrun:'/'stage 3c dry-run complete')" fi + if [ "$BOOT_DRYRUN" -eq 1 ] && [ "$OK_BOOTPRIME" -ne 1 ]; then + warn "cycle $n: gsp_phys_boot_dryrun=1 but 'stage 3d prime complete' is missing (staging rehearsal did not complete)" + fi + if [ "$BOOT" -eq 1 ]; then + if [ "$OK_BOOT" -ne 1 ]; then + warn "cycle $n: gsp_phys_boot=1 but 'stage 3d complete' is missing (boot lifecycle did not report success)" + elif [ "$OK_BOOT3E" -eq 0 ]; then + warn "cycle $n: gsp_phys_boot=1: neither 'stage 3e complete' nor 'stage 3e incomplete' present (RM-init status not reported)" + elif [ "$OK_BOOT3E" -eq 1 ]; then + warn "cycle $n: gsp_phys_boot=1: 'stage 3e incomplete' (RM-init not observed; stage 3d boot remains valid)" + fi + fi if [ "$FWSEC" -eq 1 ] && [ "$OK_FWSEC" -ne 1 ]; then warn "cycle $n: gsp_phys_fwsec=1 but 'stage 3c complete' is missing (FWSEC did not report success)" fi @@ -381,6 +468,28 @@ for n in $(seq 1 "$CYCLES"); do "hold verification failed (lspci='$LSPCI_NN' driver='$DRV' gpu=$RSTAT_GPU port=$RSTAT_PORT)" fi + if [ "$KEEP" -eq 1 ]; then + save_dmesg "$DLOG" + log "cycle $n: --keep: no rmmod, no release verification; the claimed session stays up" + cat <<EOF +=============================================================================== +--keep: the claimed session is INTENTIONALLY LEFT RUNNING. + lspci: $LSPCI_NN + driver=$DRV gpu=$RSTAT_GPU port=$RSTAT_PORT + power/control=on (GPU $GPU_BDF and port $PORT_BDF) + nvidia-persistenced stopped; open_iov_vfio loaded and holding the claim. +To end the claimed session: + sudo rmmod open_iov_vfio + echo auto | sudo tee /sys/bus/pci/devices/$GPU_BDF/power/control + echo auto | sudo tee /sys/bus/pci/devices/$PORT_BDF/power/control + sudo systemctl start nvidia-persistenced +Cycle log: $DLOG +=============================================================================== +EOF + log "VERDICT: cycle $n claimed, verified and left running (--keep, mode=$MODE, exit 0)" + exit 0 + fi + timeout 60 sudo rmmod open_iov_vfio >/dev/null 2>&1 RMMOD_RC=$? sleep 1 @@ -398,7 +507,7 @@ for n in $(seq 1 "$CYCLES"); do OK_REBIND=0; grep -q "rebound to nvidia (rc=1)" "$DLOG" && OK_REBIND=1 OK_BEFORE=0; grep -q "board check before rebind.*id=10de:1ed3" "$DLOG" && OK_BEFORE=1 OK_TEARDOWN=1 - if [ "$FWSEC" -eq 1 ]; then + if [ "$FWSEC" -eq 1 ] || [ "$BOOT" -eq 1 ]; then OK_TEARDOWN=0 grep -q "teardown complete: WPR2 down" "$DLOG" && OK_TEARDOWN=1 fi @@ -413,8 +522,12 @@ for n in $(seq 1 "$CYCLES"); do warn "cycle $n: missing 'board check before rebind id=10de:1ed3' line" ALL_GREEN=0 fi - if [ "$FWSEC" -eq 1 ] && [ "$OK_TEARDOWN" -ne 1 ]; then - warn "cycle $n: gsp_phys_fwsec=1 but no 'teardown complete: WPR2 down' on release (WPR2 may still be up; do not retry)" + if { [ "$FWSEC" -eq 1 ] || [ "$BOOT" -eq 1 ]; } && [ "$OK_TEARDOWN" -ne 1 ]; then + if [ "$BOOT" -eq 1 ]; then + warn "cycle $n: gsp_phys_boot=1 but no 'teardown complete: WPR2 down' on release (WPR2 may still be up; do not retry)" + else + warn "cycle $n: gsp_phys_fwsec=1 but no 'teardown complete: WPR2 down' on release (WPR2 may still be up; do not retry)" + fi ALL_GREEN=0 fi @@ -456,6 +569,15 @@ for n in $(seq 1 "$CYCLES"); do if [ "$OK_DRYRUN" -ne 1 ]; then ALL_GREEN=0 fi + if [ "$OK_BOOTPRIME" -ne 1 ]; then + ALL_GREEN=0 + fi + if [ "$OK_BOOT" -ne 1 ]; then + ALL_GREEN=0 + fi + if [ "$BOOT" -eq 1 ] && [ "$OK_BOOT3E" -eq 0 ]; then + ALL_GREEN=0 + fi if [ "$OK_FWSEC" -ne 1 ]; then ALL_GREEN=0 fi @@ -474,8 +596,8 @@ for n in $(seq 1 "$CYCLES"); do done if [ "$ALL_GREEN" -eq 1 ]; then - log "VERDICT: all $CYCLES cycle(s) green (exit 0)" + log "VERDICT: all $CYCLES cycle(s) green (mode=$MODE, exit 0)" exit 0 fi -echo "VERDICT: not green; do not retry in this boot (exit 2)" >&2 +echo "VERDICT: not green (mode=$MODE); do not retry in this boot (exit 2)" >&2 exit 2 |