diff options
Diffstat (limited to 'tools/guest-diag/phys_claim_cycle.sh')
| -rwxr-xr-x | tools/guest-diag/phys_claim_cycle.sh | 52 |
1 files changed, 38 insertions, 14 deletions
diff --git a/tools/guest-diag/phys_claim_cycle.sh b/tools/guest-diag/phys_claim_cycle.sh index 40577f9..0e00ae8 100755 --- a/tools/guest-diag/phys_claim_cycle.sh +++ b/tools/guest-diag/phys_claim_cycle.sh @@ -7,7 +7,8 @@ # start persistenced -> wait GPU+root port active -> power/control=on on # GPU+port -> dmesg -C -> (stop persistenced && insmod) as ONE command -> # verify lock/claim patterns -> hold -> verify while claimed -> -# rmmod -> verify release patterns -> restore auto + persistenced -> +# rmmod -> verify release patterns (power domain left LOCKED by default; +# with --restore-power: power/control=auto + persistenced restored) -> # health check. # # !!! HOST-FREEZE HAZARD !!! @@ -22,7 +23,7 @@ # # Usage: phys_claim_cycle.sh --yes [--hold SECONDS] [--cycles N] [--layout] # [--dryrun] [--boot-dryrun] [--boot] [--keep] -# [--ko PATH] [--fw NAME] [--no-restore] +# [--ko PATH] [--fw NAME] [--restore-power] # # --yes required: acknowledge the host-freeze hazard # --hold S seconds to keep the GPU claimed (default 30) @@ -68,8 +69,13 @@ # the "gsp-nocat:" prefix. The unique-record summary is # printed at the end of stage 3e (success or failure) with or # without this flag; per-record logging is verbose. -# --no-restore do not restore power/control=auto or restart persistenced -# at the end of each cycle (leaves the D0 hold in place) +# --restore-power restore power/control=auto and restart persistenced at +# the end of each cycle; expects 5 "power domain released" +# lines and d3cold_allowed=1 on the GPU. Default off: the +# module latches the D0 hold, power/control stays "on", +# persistenced stays stopped; expects 5 "power domain left +# LOCKED" lines and d3cold_allowed=0 on the GPU. +# --no-restore accepted as a no-op: the safe locked default is now built in # # --boot is mutually exclusive with --dryrun/--boot-dryrun. # @@ -99,7 +105,7 @@ BOOT=0 BOOT_DRYRUN=0 NOCAT=0 KEEP=0 -NO_RESTORE=0 +RESTORE_POWER=0 KO="$REPO_ROOT/kernel/open-iov-vfio.ko" FW="" @@ -137,7 +143,8 @@ EOF --ko) KO="${2:-}"; shift ;; --fw) FW="${2:-}"; shift ;; --nocat) NOCAT=1 ;; - --no-restore) NO_RESTORE=1 ;; + --restore-power) RESTORE_POWER=1 ;; + --no-restore) : ;; -h|--help) usage; exit 0 ;; *) echo "unknown argument: $1" >&2; usage >&2; exit 1 ;; esac @@ -531,10 +538,16 @@ EOF OK_TEARDOWN=0 grep -q "teardown complete: WPR2 down" "$DLOG" && OK_TEARDOWN=1 fi - NR_RELEASED="$(grep -c "power domain released" "$DLOG" 2>/dev/null || true)" + if [ "$RESTORE_POWER" -eq 1 ]; then + NR_RELEASED="$(grep -c "power domain released" "$DLOG" 2>/dev/null || true)" + RELEASE_KIND="released" + else + NR_RELEASED="$(grep -c "power domain left LOCKED" "$DLOG" 2>/dev/null || true)" + RELEASE_KIND="locked" + fi if [ "$OK_REBIND" -ne 1 ] || [ "$NR_RELEASED" -ne 5 ]; then ALL_GREEN=0 - warn "cycle $n: release verification failed (rebind=$OK_REBIND released=$NR_RELEASED); locks may still be held - do not retry" + warn "cycle $n: release verification failed (rebind=$OK_REBIND $RELEASE_KIND=$NR_RELEASED); locks may still be held - do not retry" save_dmesg "$DLOG" break fi @@ -551,23 +564,34 @@ EOF ALL_GREEN=0 fi - if [ "$NO_RESTORE" -eq 0 ]; then + if [ "$RESTORE_POWER" -eq 1 ]; then set_power_control "$GPU_BDF" auto || warn "cycle $n: failed to restore GPU power/control=auto" set_power_control "$PORT_BDF" auto || warn "cycle $n: failed to restore port power/control=auto" sudo systemctl start nvidia-persistenced 2>/dev/null || true wait_active || warn "cycle $n: GPU/port not active after persistenced restart" else - log "cycle $n: --no-restore: leaving power/control=on and persistenced stopped" + log "cycle $n: power domain left LOCKED (default): power/control=on retained, persistenced stopped" fi HEALTH_RED=0 nvidia-smi -L >/dev/null 2>&1 || HEALTH_RED=1 [ "$(pci_field "$GPU_BDF")" = "nvidia" ] || HEALTH_RED=1 - if [ "$NO_RESTORE" -eq 0 ]; then + if [ "$RESTORE_POWER" -eq 1 ]; then [ "$(cat "/sys/bus/pci/devices/$GPU_BDF/power/control" 2>/dev/null)" = auto ] || HEALTH_RED=1 [ "$(cat "/sys/bus/pci/devices/$PORT_BDF/power/control" 2>/dev/null)" = auto ] || HEALTH_RED=1 + [ "$(cat "/sys/bus/pci/devices/$GPU_BDF/d3cold_allowed" 2>/dev/null)" = 1 ] || HEALTH_RED=1 + else + # Latched mode: the module keeps dev->no_d3cold set, which is not + # visible in sysfs (d3cold_allowed is the user policy and stays 1), + # and the root port stays pinned in D0 even if the udev nvidia bind + # rule rewrites the GPU's power/control to auto (60-nvidia.rules). + # The latch is verified by the "power domain left LOCKED" count and + # the absence of an in-window rail cycle, so do not require + # control=on/d3cold_allowed=0 here. + [ "$(cat "/sys/bus/pci/devices/$PORT_BDF/power/control" 2>/dev/null)" = on ] || HEALTH_RED=1 + [ "$(cat "/sys/bus/pci/devices/$PORT_BDF/power/runtime_status" 2>/dev/null)" = active ] || HEALTH_RED=1 + log "cycle $n: power policy (diagnostic): GPU control=$(cat "/sys/bus/pci/devices/$GPU_BDF/power/control" 2>/dev/null) d3cold_allowed=$(cat "/sys/bus/pci/devices/$GPU_BDF/d3cold_allowed" 2>/dev/null) (dev->no_d3cold is not visible in sysfs)" fi - [ "$(cat "/sys/bus/pci/devices/$GPU_BDF/d3cold_allowed" 2>/dev/null)" = 1 ] || HEALTH_RED=1 IN_RAIL="$(in_window_rail_count "$DLOG")" OUT_RAIL="$(( $(rail_count "$DLOG") - IN_RAIL ))" @@ -603,9 +627,9 @@ EOF fi if [ "$ALL_GREEN" -eq 1 ] && [ "$IN_RAIL" -eq 0 ] && [ "$HEALTH_RED" -eq 0 ]; then - log "cycle $n: GREEN (locks=$NR_LOCKS, released=$NR_RELEASED, in-window rail=0, outside rail=$OUT_RAIL, hold=${HOLD}s)" + log "cycle $n: GREEN (locks=$NR_LOCKS, $RELEASE_KIND=$NR_RELEASED, in-window rail=0, outside rail=$OUT_RAIL, hold=${HOLD}s)" else - log "cycle $n: RED (locks=$NR_LOCKS, released=$NR_RELEASED, in-window rail=$IN_RAIL, outside rail=$OUT_RAIL, health_ok=$((1-HEALTH_RED)))" + log "cycle $n: RED (locks=$NR_LOCKS, $RELEASE_KIND=$NR_RELEASED, in-window rail=$IN_RAIL, outside rail=$OUT_RAIL, health_ok=$((1-HEALTH_RED)))" if [ "$IN_RAIL" -gt 0 ]; then echo "ABORT: cycle $n: rail cycle inside the locked window; not retrying" >&2 save_dmesg "$DLOG" |