path: root/tools/guest-diag/phys_claim_cycle.sh

#!/bin/bash
# phys_claim_cycle.sh - supervised physical-GPU claim/release cycle harness.
#
# Drives one or more full "D0 pre-lock" cycles of the physical-ownership
# backend (open-iov-vfio + gsp_phys_gpu=1 + gsp_phys_hazard_ack=1):
#
#   start persistenced -> wait GPU+root port active -> power/control=on on
#   GPU+port -> dmesg -C -> (stop persistenced && insmod) as ONE command ->
#   verify lock/claim patterns -> hold -> verify while claimed ->
#   rmmod -> verify release patterns -> restore auto + persistenced ->
#   health check.
#
# !!! HOST-FREEZE HAZARD !!!
# Claiming the GPU unbinds the distribution NVIDIA driver. On this host the
# dGPU board then power-gate and fail to come back, hard-freezing the system
# with no crash record (observed twice on 2026-09-30). Run only in a
# supervised session, with other host work stopped and the PART 87 recovery
# plan ready. The harness itself never retries an aborted cycle.
#
# RULE: never stop after FWSEC alone on this host. --boot (the full boot
# lifecycle) is the only write mode; --fwsec is retired and refused.
#
# Usage: phys_claim_cycle.sh --yes [--hold SECONDS] [--cycles N] [--layout]
#                             [--dryrun] [--boot-dryrun] [--boot] [--keep]
#                             [--ko PATH] [--fw NAME] [--no-restore]
#
#   --yes        required: acknowledge the host-freeze hazard
#   --hold S     seconds to keep the GPU claimed (default 30)
#   --cycles N   cycles to run, each only if the previous was green (default 1)
#   --layout     also pass gsp_phys_layout=1 (rung 3b); "stage 3b complete"
#                then becomes an expected (but non-fatal) pattern
#   --dryrun     also pass gsp_phys_fwsec_dryrun=1 (implies --layout): run the
#                READ-ONLY stage-3c validation (VBIOS/BIT/FWSEC parse and the
#                staging/placement plan, no device writes); expects the
#                "gsp-phys-fwsec-dryrun:" and "stage 3c dry-run complete"
#                lines. It is also implied by --boot-dryrun (alias)
#   --boot-dryrun implies --dryrun and also passes gsp_phys_boot_dryrun=1
#                (implies --layout): host-memory-only stage-3d staging
#                rehearsal (radix3 build, queue/PTE/args fill, all placement
#                math; ZERO BAR0 writes); additionally expects the
#                "stage 3d prime complete" line
#   --boot       pass gsp_phys_boot=1 (implies --layout): the ONLY write mode.
#                Runs the full boot lifecycle (FWSEC, Booter Load, RISC-V
#                start, FALCON_OS, RM-init poll). Requires "stage 3d
#                complete"; "stage 3e complete" = full green, "stage 3e
#                incomplete" = non-fatal warning (RM-init not observed, boot
#                still valid); on release requires "teardown complete: WPR2
#                down" (missing = red; do not retry)
#   --keep       after a GREEN cycle do NOT rmmod: leave the module loaded and
#                the GPU claimed by open-iov-gsp-phys (power/control=on on
#                GPU+port, nvidia-persistenced stopped) and skip the release
#                verification (no rebind/released lines); the hold checks
#                (lspci id, driver, runtime active) and abort signatures still
#                run. Prints a notice with the exact recovery steps. Compatible
#                with --boot/--boot-dryrun/--dryrun/--layout (it only changes
#                the ending); single cycle only (--cycles 1). Exit 0 = the
#                claimed session is intentionally left running.
#   --fwsec      RETIRED (2026-10-01). A FWSEC-only carve is a one-way door on
#                this fused part: it left WPR2 up and was not host-recoverable.
#                It is refused immediately and never reaches insmod. Use
#                --boot; never stop after FWSEC alone on this host.
#   --ko PATH    module to insmod (default kernel/open-iov-vfio.ko)
#   --fw NAME    GSP firmware to stage (request_firmware name under
#                /lib/firmware; default: the module's own default, currently
#                nvidia/570.211.01/gsp_tu10x.bin)
#   --nocat      pass gsp_phys_nocat=1: during --boot, log every GSP-RM NOCAT
#                journal record (status-queue function 0x1020) to dmesg with
#                the "gsp-nocat:" prefix. The unique-record summary is
#                printed at the end of stage 3e (success or failure) with or
#                without this flag; per-record logging is verbose.
#   --no-restore do not restore power/control=auto or restart persistenced
#                at the end of each cycle (leaves the D0 hold in place)
#
# --boot is mutually exclusive with --dryrun/--boot-dryrun.
#
# Exit codes: 0 = all cycles green (with --keep: the claimed session was left
#             running intentionally); 1 = --yes missing / usage error;
#             2 = abort (hard abort signature, failed verification, red
#             in-window rail event); 3 = precondition failure.
#
# Per-cycle full dmesg is saved to
#   /tmp/opencode/phys_claim_cycle_<timestamp>_<n>.log
#
# Env overrides: GPU_BDF (default 0000:01:00.0), PORT_BDF (default: the GPU's
# upstream bridge, normally 0000:00:01.0), PHYS_CLAIM_LOGDIR.

set -u

HERE="$(dirname "$(readlink -f "$0")")"
REPO_ROOT="$(readlink -f "$HERE/../..")"

USE_YES=0
HOLD=30
CYCLES=1
LAYOUT=0
DRYRUN=0
FWSEC=0
BOOT=0
BOOT_DRYRUN=0
NOCAT=0
KEEP=0
NO_RESTORE=0
KO="$REPO_ROOT/kernel/open-iov-vfio.ko"
FW=""

GPU_BDF="${GPU_BDF:-0000:01:00.0}"
if [ -z "${PORT_BDF:-}" ] && [ -e "/sys/bus/pci/devices/$GPU_BDF" ]; then
    PORT_BDF="$(basename "$(dirname "$(readlink -f "/sys/bus/pci/devices/$GPU_BDF")")")"
fi
PORT_BDF="${PORT_BDF:-0000:00:01.0}"

LOGDIR="${PHYS_CLAIM_LOGDIR:-/tmp/opencode}"
TS="$(date +%Y%m%d_%H%M%S)"
WATCHERS='pdwatch|watch_ctx|uvmt_poll|invd\.py|bpwait|sempages|track_waits|guest_pin|guest_pmap|vmrestart|push_guest_tools|nvidia-smi -l|dmesg -w'

usage() {
    awk '/^[^#]/{exit} {print}' "$0" | grep -E '^#( |$)' | sed 's/^# \{0,1\}//'
}

while [ $# -gt 0 ]; do
    case "$1" in
        --yes)        USE_YES=1 ;;
        --hold)       HOLD="${2:-}"; shift ;;
        --cycles)     CYCLES="${2:-}"; shift ;;
        --layout)     LAYOUT=1 ;;
        --dryrun)     DRYRUN=1; LAYOUT=1 ;;
        --boot-dryrun) BOOT_DRYRUN=1; LAYOUT=1; DRYRUN=1 ;;
        --boot)       BOOT=1; LAYOUT=1 ;;
        --keep)       KEEP=1 ;;
        --fwsec)
            cat >&2 <<'EOF'
--fwsec is RETIRED: FWSEC-only is a one-way door on this fused part.
A FWSEC-only carve left WPR2 up and was not host-recoverable (2026-10-01).
Never stop after FWSEC alone on this host. Use --boot (full boot lifecycle).
EOF
            exit 1 ;;
        --ko)         KO="${2:-}"; shift ;;
        --fw)         FW="${2:-}"; shift ;;
        --nocat)      NOCAT=1 ;;
        --no-restore) NO_RESTORE=1 ;;
        -h|--help)    usage; exit 0 ;;
        *) echo "unknown argument: $1" >&2; usage >&2; exit 1 ;;
    esac
    shift
done

case "$HOLD" in ''|*[!0-9]*) echo "--hold must be a non-negative integer" >&2; exit 1 ;; esac
case "$CYCLES" in ''|*[!0-9]*|0) echo "--cycles must be a positive integer" >&2; exit 1 ;; esac
if [ "$KEEP" -eq 1 ] && [ "$CYCLES" -ne 1 ]; then
    echo "--keep leaves the claimed session running: single cycle only (--cycles 1)" >&2
    exit 1
fi
if [ "$DRYRUN" -eq 1 ] && [ "$FWSEC" -eq 1 ]; then
    echo "--dryrun and --fwsec are mutually exclusive" >&2
    exit 1
fi
if [ "$BOOT" -eq 1 ] && { [ "$DRYRUN" -eq 1 ] || [ "$BOOT_DRYRUN" -eq 1 ]; }; then
    echo "--boot is mutually exclusive with --dryrun/--boot-dryrun" >&2
    exit 1
fi
if [ "$HOLD" -lt 30 ]; then
    echo "WARN: --hold $HOLD is below the verified 30 s minimum" >&2
fi

MODE="none"
[ "$LAYOUT" -eq 1 ] && MODE="layout"
[ "$DRYRUN" -eq 1 ] && MODE="dryrun"
[ "$BOOT_DRYRUN" -eq 1 ] && MODE="boot-dryrun"
[ "$BOOT" -eq 1 ] && MODE="boot"

if [ "$USE_YES" -ne 1 ]; then
    cat >&2 <<'EOF'
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
HOST-FREEZE HAZARD - REFUSING TO RUN WITHOUT --yes

The physical claim unbinds the distribution NVIDIA driver from the dGPU.
On this host the board can then power-gate and fail to come back, hard-
freezing the system with no crash record (observed twice on 2026-09-30).
Run only when supervised, with other host work stopped and the PART 87
recovery plan ready. Re-run with --yes to acknowledge.
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
EOF
    exit 1
fi

log()  { echo "[$(date +%H:%M:%S)] $*"; }
warn() { echo "[$(date +%H:%M:%S)] WARN: $*" >&2; }
pre_fail() { echo "PRECONDITION FAIL: $*" >&2; exit 3; }

INTERRUPTED=0
on_signal() {
    INTERRUPTED=1
    echo
    echo "ABORT: interrupted; leaving module/lock state as-is (no cleanup)." >&2
    exit 2
}
trap on_signal INT TERM

# ---------------------------------------------------------------- preflight

[ -c /dev/null ] || pre_fail "no /dev/null?"
if ! sudo -n true 2>/dev/null; then
    sudo -v 2>/dev/null || pre_fail "current user cannot sudo"
fi

[ -f "$KO" ] || pre_fail "module not found: $KO (build it or pass --ko)"
KO_HASH="$(sha256sum "$KO" | awk '{print $1}')"
KO_SRCVER="$(modinfo -F srcversion "$KO" 2>/dev/null || true)"
if [ -d "$REPO_ROOT/.git" ]; then
    HEAD_COMMIT="$(git -C "$REPO_ROOT" rev-parse --short HEAD 2>/dev/null || true)"
    log "preflight: repo HEAD ${HEAD_COMMIT:-unknown}"
    if git -C "$REPO_ROOT" status --porcelain -- kernel 2>/dev/null | grep -q .; then
        warn "kernel/ has uncommitted changes; HEAD does not describe $KO"
    fi
else
    HEAD_COMMIT=""
    warn "not a git checkout; cannot record HEAD"
fi
log "preflight: ko $KO sha256 $KO_HASH srcversion ${KO_SRCVER:-unknown}"
if [ "$KO" -ot "$REPO_ROOT/kernel/gsp_phys_gpu.c" ]; then
    warn "$KO is older than kernel/gsp_phys_gpu.c; rebuild before cycling"
fi

[ -e "/sys/bus/pci/devices/$GPU_BDF" ] || pre_fail "GPU $GPU_BDF not present"
[ -e "/sys/bus/pci/devices/$PORT_BDF" ] || pre_fail "root port $PORT_BDF not present"

if pgrep -f qemu-system >/dev/null 2>&1; then
    pgrep -af qemu-system >&2
    pre_fail "QEMU is running; stop the VM first"
fi

if compgen -G "/sys/bus/mdev/devices/*" >/dev/null 2>&1; then
    ls -d /sys/bus/mdev/devices/* >&2
    pre_fail "mdev devices exist; remove them first"
fi

WATCH_PIDS="$(pgrep -af "$WATCHERS" 2>/dev/null || true)"
if [ -n "$WATCH_PIDS" ]; then
    echo "$WATCH_PIDS" >&2
    pre_fail "stale watcher process(es) running; stop them first"
fi

if lsmod 2>/dev/null | awk '{print $1}' | grep -qx open_iov_vfio; then
    pre_fail "open_iov_vfio is already loaded; rmmod it first"
fi

command -v lspci >/dev/null 2>&1 || pre_fail "lspci not found"
command -v nvidia-smi >/dev/null 2>&1 || pre_fail "nvidia-smi not found"
command -v timeout >/dev/null 2>&1 || pre_fail "timeout not found"

sudo systemctl start nvidia-persistenced 2>/dev/null || true
for _ in $(seq 1 10); do
    systemctl is-active --quiet nvidia-persistenced 2>/dev/null && break
    sleep 1
done
systemctl is-active --quiet nvidia-persistenced 2>/dev/null \
    || pre_fail "nvidia-persistenced did not start"
if ! nvidia-smi -L >/dev/null 2>&1; then
    pre_fail "nvidia-smi -L failed with persistenced running"
fi
log "preflight: persistenced active, nvidia-smi OK, GPU=$GPU_BDF port=$PORT_BDF"

mkdir -p "$LOGDIR" || pre_fail "cannot create $LOGDIR"

# ---------------------------------------------------------------- helpers

save_dmesg() {
    sudo dmesg > "$1" 2>/dev/null || true
}

# Count rail-cycle lines strictly inside the locked window: after the first
# "power domain locked" line and before the first "power domain released"
# line (or the end of the log if the locks never released). Outside the
# window is normal RTD3.
in_window_rail_count() {
    local f="$1" first rel
    first="$(grep -n "power domain locked" "$f" 2>/dev/null | head -1 | cut -d: -f1)"
    [ -n "$first" ] || { echo 0; return; }
    rel="$(grep -n "power domain released" "$f" 2>/dev/null | head -1 | cut -d: -f1)"
    if [ -n "$rel" ]; then
        awk -v a="$first" -v b="$rel" \
            '/xHC error in resume|root hub lost power/ && NR > a && NR < b { n++ } END { print n+0 }' "$f"
    else
        awk -v a="$first" \
            '/xHC error in resume|root hub lost power/ && NR > a { n++ } END { print n+0 }' "$f"
    fi
}

rail_count() {
    grep -c -E "xHC error in resume|root hub lost power" "$1" 2>/dev/null || true
}

abort_signature() {
    grep -q -E "not ready .*after resume" "$1" 2>/dev/null && return 0
    grep -q -E "board unresponsive|gsp-phys.*WARN|WARNING:.*gsp-phys" "$1" 2>/dev/null && return 0
    return 1
}

wait_active() {
    local i
    for i in $(seq 1 10); do
        if [ "$(cat "/sys/bus/pci/devices/$GPU_BDF/power/runtime_status" 2>/dev/null)" = active ] &&
           [ "$(cat "/sys/bus/pci/devices/$PORT_BDF/power/runtime_status" 2>/dev/null)" = active ]; then
            return 0
        fi
        nvidia-smi -L >/dev/null 2>&1 || true
        sleep 1
    done
    return 1
}

set_power_control() {
    local bdf="$1" val="$2"
    sudo sh -c "echo '$val' > '/sys/bus/pci/devices/$bdf/power/control'" 2>/dev/null || return 1
    [ "$(cat "/sys/bus/pci/devices/$bdf/power/control" 2>/dev/null)" = "$val" ] || return 1
    return 0
}

pci_field() {
    lspci -k -s "$1" 2>/dev/null | sed -n "s/.*Kernel driver in use: //p" | head -1
}

abort_now() {
    local n="$1" dlog="$2" msg="$3"
    save_dmesg "$dlog"
    echo "ABORT: cycle $n: $msg" >&2
    echo "ABORT: state left as-is (no rmmod, no restore). Log: $dlog" >&2
    exit 2
}

# ---------------------------------------------------------------- cycles

ALL_GREEN=1
for n in $(seq 1 "$CYCLES"); do
    [ "$INTERRUPTED" -eq 0 ] || exit 2
    DLOG="$LOGDIR/phys_claim_cycle_${TS}_${n}.log"
    log "=== cycle $n/$CYCLES (mode=$MODE, hold ${HOLD}s, layout=$LAYOUT, ko=$(basename "$KO")) ==="

    sudo systemctl start nvidia-persistenced 2>/dev/null || true
    if ! wait_active; then
        pre_fail "cycle $n: GPU/port did not reach runtime_status=active"
    fi

    set_power_control "$GPU_BDF" on || pre_fail "cycle $n: cannot set GPU power/control=on"
    set_power_control "$PORT_BDF" on || pre_fail "cycle $n: cannot set port power/control=on"

    sudo dmesg -C 2>/dev/null || pre_fail "cycle $n: dmesg -C failed"

    LAYOUT_ARG=""
    [ "$LAYOUT" -eq 1 ] && LAYOUT_ARG="gsp_phys_layout=1"
    DRYRUN_ARG=""
    [ "$DRYRUN" -eq 1 ] && DRYRUN_ARG="gsp_phys_fwsec_dryrun=1"
    BOOTDRY_ARG=""
    [ "$BOOT_DRYRUN" -eq 1 ] && BOOTDRY_ARG="gsp_phys_boot_dryrun=1"
    BOOT_ARG=""
    [ "$BOOT" -eq 1 ] && BOOT_ARG="gsp_phys_boot=1"
    FW_ARG=""
    [ -n "$FW" ] && FW_ARG="gsp_phys_fw=$FW"
    NOCAT_ARG=""
    [ "$NOCAT" -eq 1 ] && NOCAT_ARG="gsp_phys_nocat=1"
    # One command: stop persistenced and insmod with the D0 pre-lock in place.
    # gsp_phys_fwsec=1 is never passed: --fwsec refuses before this point.
    if ! { sudo systemctl stop nvidia-persistenced && sudo insmod "$KO" \
            gsp_phys_gpu=1 gsp_phys_hazard_ack=1 $LAYOUT_ARG $DRYRUN_ARG $BOOTDRY_ARG $BOOT_ARG $FW_ARG $NOCAT_ARG; }; then
        save_dmesg "$DLOG"
        echo "ABORT: cycle $n: stop-persistenced && insmod failed" >&2
        exit 2
    fi
    sleep 1
    save_dmesg "$DLOG"

    if grep -q "gsp-phys: refusing claim" "$DLOG" 2>/dev/null; then
        echo "PRECONDITION FAIL: cycle $n: module refused the claim (GPU/port were already runtime-suspended)" >&2
        warn "no claim was taken; unloading the module is safe"
        timeout 60 sudo rmmod open_iov_vfio 2>/dev/null || true
        save_dmesg "$DLOG"
        exit 3
    fi

    if abort_signature "$DLOG"; then
        abort_now "$n" "$DLOG" "hard abort signature after insmod"
    fi

    NR_LOCKS="$(grep -c "power domain locked" "$DLOG" 2>/dev/null || true)"
    OK_GPU_LOCK=0; grep -q "power domain locked: $GPU_BDF" "$DLOG" && OK_GPU_LOCK=1
    OK_PORT_LOCK=0; grep -q "power domain locked: $PORT_BDF" "$DLOG" && OK_PORT_LOCK=1
    OK_UNBIND=0;  grep -q "unbinding nvidia from" "$DLOG" && OK_UNBIND=1
    OK_CLAIM=0;   grep -q "physical GPU claimed" "$DLOG" && OK_CLAIM=1
    OK_BOARD=0;   grep -q "board check after claim.*id=10de:1ed3" "$DLOG" && OK_BOARD=1
    OK_STAGE3B=1
    if [ "$LAYOUT" -eq 1 ]; then
        OK_STAGE3B=0; grep -q "stage 3b complete" "$DLOG" && OK_STAGE3B=1
    fi
    OK_DRYRUN=1
    if [ "$DRYRUN" -eq 1 ]; then
        OK_DRYRUN=0
        if grep -q "gsp-phys-fwsec-dryrun:" "$DLOG" && \
           grep -q "stage 3c dry-run complete" "$DLOG"; then
            OK_DRYRUN=1
        fi
    fi
    OK_BOOTPRIME=1
    if [ "$BOOT_DRYRUN" -eq 1 ]; then
        OK_BOOTPRIME=0
        grep -q "stage 3d prime complete" "$DLOG" && OK_BOOTPRIME=1
    fi
    OK_BOOT=1
    OK_BOOT3E=0
    if [ "$BOOT" -eq 1 ]; then
        OK_BOOT=0
        if grep -q "stage 3d complete" "$DLOG"; then
            OK_BOOT=1
            if grep -q "stage 3e complete" "$DLOG"; then
                OK_BOOT3E=2
            elif grep -q "stage 3e incomplete" "$DLOG"; then
                OK_BOOT3E=1
            fi
        fi
    fi
    OK_FWSEC=1
    if [ "$FWSEC" -eq 1 ]; then
        OK_FWSEC=0
        grep -q "stage 3c complete" "$DLOG" && OK_FWSEC=1
    fi

    if [ "$NR_LOCKS" -eq 0 ] && [ "$OK_CLAIM" -eq 0 ]; then
        echo "ABORT: cycle $n: no locks and no claim took effect" >&2
        warn "no D0 locks were taken; unloading the module is safe"
        timeout 60 sudo rmmod open_iov_vfio 2>/dev/null || true
        save_dmesg "$DLOG"
        exit 2
    fi
    if [ "$OK_GPU_LOCK" -ne 1 ] || [ "$OK_PORT_LOCK" -ne 1 ] || \
       [ "$OK_UNBIND" -ne 1 ] || [ "$OK_CLAIM" -ne 1 ] || [ "$OK_BOARD" -ne 1 ]; then
        abort_now "$n" "$DLOG" \
            "claim patterns incomplete (locks=$NR_LOCKS gpu=$OK_GPU_LOCK port=$OK_PORT_LOCK unbind=$OK_UNBIND claim=$OK_CLAIM board=$OK_BOARD)"
    fi
    if [ "$NR_LOCKS" -ne 5 ]; then
        warn "cycle $n: expected 5 power domain locked lines, saw $NR_LOCKS"
        ALL_GREEN=0
    fi
    if [ "$LAYOUT" -eq 1 ] && [ "$OK_STAGE3B" -ne 1 ]; then
        warn "cycle $n: gsp_phys_layout=1 but no 'stage 3b complete' line"
    fi
    if [ "$DRYRUN" -eq 1 ] && [ "$OK_DRYRUN" -ne 1 ]; then
        warn "cycle $n: gsp_phys_fwsec_dryrun=1 but the read-only dry-run did not complete (missing 'gsp-phys-fwsec-dryrun:'/'stage 3c dry-run complete')"
    fi
    if [ "$BOOT_DRYRUN" -eq 1 ] && [ "$OK_BOOTPRIME" -ne 1 ]; then
        warn "cycle $n: gsp_phys_boot_dryrun=1 but 'stage 3d prime complete' is missing (staging rehearsal did not complete)"
    fi
    if [ "$BOOT" -eq 1 ]; then
        if [ "$OK_BOOT" -ne 1 ]; then
            warn "cycle $n: gsp_phys_boot=1 but 'stage 3d complete' is missing (boot lifecycle did not report success)"
        elif [ "$OK_BOOT3E" -eq 0 ]; then
            warn "cycle $n: gsp_phys_boot=1: neither 'stage 3e complete' nor 'stage 3e incomplete' present (RM-init status not reported)"
        elif [ "$OK_BOOT3E" -eq 1 ]; then
            warn "cycle $n: gsp_phys_boot=1: 'stage 3e incomplete' (RM-init not observed; stage 3d boot remains valid)"
        fi
    fi
    if [ "$FWSEC" -eq 1 ] && [ "$OK_FWSEC" -ne 1 ]; then
        warn "cycle $n: gsp_phys_fwsec=1 but 'stage 3c complete' is missing (FWSEC did not report success)"
    fi

    IN_RAIL_BEFORE="$(in_window_rail_count "$DLOG")"
    OUT_RAIL_BEFORE="$(( $(rail_count "$DLOG") - IN_RAIL_BEFORE ))"
    log "cycle $n: claimed and verified (locks=$NR_LOCKS, rail outside=$OUT_RAIL_BEFORE). Holding ${HOLD}s"
    sleep "$HOLD"

    save_dmesg "$DLOG"
    if abort_signature "$DLOG"; then
        abort_now "$n" "$DLOG" "hard abort signature during hold"
    fi

    LSPCI_NN="$(lspci -nn -s "$GPU_BDF" 2>/dev/null || true)"
    DRV="$(pci_field "$GPU_BDF")"
    RSTAT_GPU="$(cat "/sys/bus/pci/devices/$GPU_BDF/power/runtime_status" 2>/dev/null || echo '?')"
    RSTAT_PORT="$(cat "/sys/bus/pci/devices/$PORT_BDF/power/runtime_status" 2>/dev/null || echo '?')"
    HOLD_RED=0
    echo "$LSPCI_NN" | grep -q "10de:1ed3" || HOLD_RED=1
    [ "$DRV" = "open-iov-gsp-phys" ] || HOLD_RED=1
    [ "$RSTAT_GPU" = active ] && [ "$RSTAT_PORT" = active ] || HOLD_RED=1
    if [ "$HOLD_RED" -ne 0 ]; then
        abort_now "$n" "$DLOG" \
            "hold verification failed (lspci='$LSPCI_NN' driver='$DRV' gpu=$RSTAT_GPU port=$RSTAT_PORT)"
    fi

    if [ "$KEEP" -eq 1 ]; then
        save_dmesg "$DLOG"
        log "cycle $n: --keep: no rmmod, no release verification; the claimed session stays up"
        cat <<EOF
===============================================================================
--keep: the claimed session is INTENTIONALLY LEFT RUNNING.
  lspci: $LSPCI_NN
  driver=$DRV  gpu=$RSTAT_GPU  port=$RSTAT_PORT
  power/control=on (GPU $GPU_BDF and port $PORT_BDF)
  nvidia-persistenced stopped; open_iov_vfio loaded and holding the claim.
To end the claimed session:
  sudo rmmod open_iov_vfio
  echo auto | sudo tee /sys/bus/pci/devices/$GPU_BDF/power/control
  echo auto | sudo tee /sys/bus/pci/devices/$PORT_BDF/power/control
  sudo systemctl start nvidia-persistenced
Cycle log: $DLOG
===============================================================================
EOF
        log "VERDICT: cycle $n claimed, verified and left running (--keep, mode=$MODE, exit 0)"
        exit 0
    fi

    timeout 60 sudo rmmod open_iov_vfio >/dev/null 2>&1
    RMMOD_RC=$?
    sleep 1
    save_dmesg "$DLOG"
    if [ "$RMMOD_RC" -eq 124 ]; then
        abort_now "$n" "$DLOG" "rmmod timed out after 60s"
    fi
    if [ "$RMMOD_RC" -ne 0 ]; then
        abort_now "$n" "$DLOG" "rmmod failed (rc=$RMMOD_RC)"
    fi
    if abort_signature "$DLOG"; then
        abort_now "$n" "$DLOG" "hard abort signature after rmmod"
    fi

    OK_REBIND=0;  grep -q "rebound to nvidia (rc=1)" "$DLOG" && OK_REBIND=1
    OK_BEFORE=0;  grep -q "board check before rebind.*id=10de:1ed3" "$DLOG" && OK_BEFORE=1
    OK_TEARDOWN=1
    if [ "$FWSEC" -eq 1 ] || [ "$BOOT" -eq 1 ]; then
        OK_TEARDOWN=0
        grep -q "teardown complete: WPR2 down" "$DLOG" && OK_TEARDOWN=1
    fi
    NR_RELEASED="$(grep -c "power domain released" "$DLOG" 2>/dev/null || true)"
    if [ "$OK_REBIND" -ne 1 ] || [ "$NR_RELEASED" -ne 5 ]; then
        ALL_GREEN=0
        warn "cycle $n: release verification failed (rebind=$OK_REBIND released=$NR_RELEASED); locks may still be held - do not retry"
        save_dmesg "$DLOG"
        break
    fi
    if [ "$OK_BEFORE" -ne 1 ]; then
        warn "cycle $n: missing 'board check before rebind id=10de:1ed3' line"
        ALL_GREEN=0
    fi
    if { [ "$FWSEC" -eq 1 ] || [ "$BOOT" -eq 1 ]; } && [ "$OK_TEARDOWN" -ne 1 ]; then
        if [ "$BOOT" -eq 1 ]; then
            warn "cycle $n: gsp_phys_boot=1 but no 'teardown complete: WPR2 down' on release (WPR2 may still be up; do not retry)"
        else
            warn "cycle $n: gsp_phys_fwsec=1 but no 'teardown complete: WPR2 down' on release (WPR2 may still be up; do not retry)"
        fi
        ALL_GREEN=0
    fi

    if [ "$NO_RESTORE" -eq 0 ]; then
        set_power_control "$GPU_BDF" auto || warn "cycle $n: failed to restore GPU power/control=auto"
        set_power_control "$PORT_BDF" auto || warn "cycle $n: failed to restore port power/control=auto"
        sudo systemctl start nvidia-persistenced 2>/dev/null || true
        wait_active || warn "cycle $n: GPU/port not active after persistenced restart"
    else
        log "cycle $n: --no-restore: leaving power/control=on and persistenced stopped"
    fi

    HEALTH_RED=0
    nvidia-smi -L >/dev/null 2>&1 || HEALTH_RED=1
    [ "$(pci_field "$GPU_BDF")" = "nvidia" ] || HEALTH_RED=1
    if [ "$NO_RESTORE" -eq 0 ]; then
        [ "$(cat "/sys/bus/pci/devices/$GPU_BDF/power/control" 2>/dev/null)" = auto ] || HEALTH_RED=1
        [ "$(cat "/sys/bus/pci/devices/$PORT_BDF/power/control" 2>/dev/null)" = auto ] || HEALTH_RED=1
    fi
    [ "$(cat "/sys/bus/pci/devices/$GPU_BDF/d3cold_allowed" 2>/dev/null)" = 1 ] || HEALTH_RED=1

    IN_RAIL="$(in_window_rail_count "$DLOG")"
    OUT_RAIL="$(( $(rail_count "$DLOG") - IN_RAIL ))"
    if [ "$IN_RAIL" -gt 0 ]; then
        warn "cycle $n: RED - $IN_RAIL rail-cycle line(s) INSIDE the locked window (lock failed)"
        ALL_GREEN=0
    fi
    if [ "$OUT_RAIL" -gt 0 ]; then
        warn "cycle $n: normal RTD3 rail cycle outside the locked window ($OUT_RAIL line(s))"
    fi

    if [ "$HEALTH_RED" -ne 0 ]; then
        ALL_GREEN=0
        warn "cycle $n: health check failed (nvidia-smi/driver/control/d3cold)"
    fi
    if [ "$OK_STAGE3B" -ne 1 ]; then
        ALL_GREEN=0
    fi
    if [ "$OK_DRYRUN" -ne 1 ]; then
        ALL_GREEN=0
    fi
    if [ "$OK_BOOTPRIME" -ne 1 ]; then
        ALL_GREEN=0
    fi
    if [ "$OK_BOOT" -ne 1 ]; then
        ALL_GREEN=0
    fi
    if [ "$BOOT" -eq 1 ] && [ "$OK_BOOT3E" -eq 0 ]; then
        ALL_GREEN=0
    fi
    if [ "$OK_FWSEC" -ne 1 ]; then
        ALL_GREEN=0
    fi

    if [ "$ALL_GREEN" -eq 1 ] && [ "$IN_RAIL" -eq 0 ] && [ "$HEALTH_RED" -eq 0 ]; then
        log "cycle $n: GREEN (locks=$NR_LOCKS, released=$NR_RELEASED, in-window rail=0, outside rail=$OUT_RAIL, hold=${HOLD}s)"
    else
        log "cycle $n: RED (locks=$NR_LOCKS, released=$NR_RELEASED, in-window rail=$IN_RAIL, outside rail=$OUT_RAIL, health_ok=$((1-HEALTH_RED)))"
        if [ "$IN_RAIL" -gt 0 ]; then
            echo "ABORT: cycle $n: rail cycle inside the locked window; not retrying" >&2
            save_dmesg "$DLOG"
            exit 2
        fi
        break
    fi
done

if [ "$ALL_GREEN" -eq 1 ]; then
    log "VERDICT: all $CYCLES cycle(s) green (mode=$MODE, exit 0)"
    exit 0
fi
echo "VERDICT: not green (mode=$MODE); do not retry in this boot (exit 2)" >&2
exit 2