vgsp: m178 GR launch reliability + diagnostics; tools: pin latching robustness

Kernel: authorship instrumentation + swrel gate for in-flight GR release VAs;
first-poll shortening + fast extended re-arm (params gr_retry_fast=1,
gr_first_poll_ms=2500); per-attempt win0 bank recompute + shadow recopy in the
outcome-retry loop; poll-break/deposit-readback diagnostics; C1-C10 sema
diagnostics (crel-entry, relx, opdef-clear, walk, pub-adv, hold-fallback,
busy-defer, CAND-OVERWRITE, inline recs); busy-defer hold instead of
software-release consume; counter fast path; QMD word 48/49 skip.
Tools: heartbeat thread, scanner-alive probe on reinstall, full periodic
re-assert, #nopid handling, per-client reset fix (debounce), dead-pid drop,
supervisor grace+threshold.

1 files changed, 32 insertions, 12 deletions

diff --git a/tools/guest-diag/pin_supervisor.sh b/tools/guest-diag/pin_supervisor.sh
index 5bbab0c..f1f4eae 100755
--- a/tools/guest-diag/pin_supervisor.sh
+++ b/tools/guest-diag/pin_supervisor.sh
@@ -6,10 +6,12 @@
 # first cudaMalloc.
 #
 # Both tools write a heartbeat file; this script checks, every 5 s, that each
-# process is alive (exact script path) and that its heartbeat mtime is < 30 s.
-# On failure it kills the tool (exact path match) and restarts it, appending
-# RESTART lines to the supervisor log. After 10 restarts in one hour for a
-# tool it logs GIVEUP and exits 1. Only one semfix_uvmt instance is ever
+# process is alive (exact script path) and that its heartbeat mtime is < 75 s.
+# A startup grace (120 s since the tool started, skipped early when the tool
+# log already shows first successful work) keeps a slow cold start from being
+# restarted. On failure it kills the tool (exact path match) and restarts it,
+# appending RESTART lines to the supervisor log. After 10 restarts in one hour
+# for a tool it logs GIVEUP and exits 1. Only one semfix_uvmt instance is ever
 # started (QEMU's QMP socket accepts a single client).
 #
 # Usage: pin_supervisor.sh LOGDIR TAG
@@ -30,6 +32,8 @@ PIN_LOG="$LOGDIR/12-pin_gr_stream-$TAG.log"
 SEMFIX_LOG="$LOGDIR/12-semfix-$TAG.log"
 WINDOW=3600
 CAP=10
+STALE=75
+GRACE=120
 mkdir -p "$LOGDIR" /tmp/opencode
 
 log() { echo "[$(date +%H:%M:%S)] $*" | tee -a "$SUPLOG"; }
@@ -57,6 +61,7 @@ start_pin() {
     touch "$PIN_HB" 2>/dev/null || true
     ( setsid nohup env SEMFIX_COUNTER_RETRY=10 \
         python3 "$PIN" >> "$PIN_LOG" 2>&1 & )
+    pin_started=$(date +%s)
     log "START pin_gr_stream"
 }
 
@@ -70,6 +75,7 @@ start_semfix() {
     touch "$SEMFIX_HB" 2>/dev/null || true
     ( setsid nohup env SEMFIX_SKIP_PB=0 SEMFIX_COUNTER_RETRY=10 \
         python3 "$SEMFIX" 0.5 >> "$SEMFIX_LOG" 2>&1 & )
+    semfix_started=$(date +%s)
     log "START semfix_uvmt"
 }
 
@@ -89,6 +95,8 @@ count_list() {  # $1 list -> element count
 
 restarts_pin=""
 restarts_semfix=""
+pin_started=$(date +%s)
+semfix_started=$(date +%s)
 
 restart() {  # $1 label, $2 script path, $3 start function
     local now list n
@@ -114,16 +122,26 @@ restart() {  # $1 label, $2 script path, $3 start function
     "$3"
 }
 
-check_tool() {  # $1 label, $2 path, $3 heartbeat file, $4 start function
-    local age
+check_tool() {  # $1 label, $2 path, $3 heartbeat file, $4 start function,
+                # $5 tool start TS, $6 tool log file, $7 first-work pattern
+    local age runtime
     age=$(hb_age "$3")
-    if tool_alive "$2" && [ "$age" -le 30 ]; then
+    if tool_alive "$2" && [ "$age" -le "$STALE" ]; then
         return 0
     fi
-    if ! tool_alive "$2"; then
-        log "$1 not running (heartbeat age ${age}s) - restarting"
-    else
+    if tool_alive "$2"; then
+        # Startup grace: a cold tool can spend its first ~2 min installing the
+        # guest scanner or connecting QMP with a lagging heartbeat; only
+        # enforce staleness once the grace has passed or the log shows the
+        # tool has already done successful work.
+        runtime=$(( $(date +%s) - $5 ))
+        if [ "$runtime" -lt "$GRACE" ] && \
+                ! grep -q -- "$7" "$6" 2>/dev/null; then
+            return 0
+        fi
         log "$1 heartbeat stale (${age}s) - restarting"
+    else
+        log "$1 not running (heartbeat age ${age}s) - restarting"
     fi
     restart "$1" "$2" "$4"
 }
@@ -133,6 +151,8 @@ start_pin
 start_semfix
 while true; do
     sleep 5
-    check_tool pin_gr_stream "$PIN" "$PIN_HB" start_pin
-    check_tool semfix_uvmt "$SEMFIX" "$SEMFIX_HB" start_semfix
+    check_tool pin_gr_stream "$PIN" "$PIN_HB" start_pin "$pin_started" \
+        "$PIN_LOG" "listening on vsock"
+    check_tool semfix_uvmt "$SEMFIX" "$SEMFIX_HB" start_semfix \
+        "$semfix_started" "$SEMFIX_LOG" "semfix_uvmt: watching"
 done