tools: kill-cascade purge + stall watchdogs + monotonic client identity (tools-fix c8fb5377)

Dead-pid purge via in-band #nopid (zombie-aware probe), monotonic
(pid,starttime) client identity so trace-ring replays can never reset an
active client (fixes the m178 mid-run reset regression avenue), pre-capture
and post-capture stall watchdogs that re-assert pins and escalate loudly,
heartbeat threads for both tools, supervisor STALE 75s/GRACE 120s.

1 files changed, 44 insertions, 13 deletions

diff --git a/tools/guest-diag/pin_supervisor.sh b/tools/guest-diag/pin_supervisor.sh
index 5bbab0c..b87d5c1 100755
--- a/tools/guest-diag/pin_supervisor.sh
+++ b/tools/guest-diag/pin_supervisor.sh
@@ -6,11 +6,24 @@
 # first cudaMalloc.
 #
 # Both tools write a heartbeat file; this script checks, every 5 s, that each
-# process is alive (exact script path) and that its heartbeat mtime is < 30 s.
-# On failure it kills the tool (exact path match) and restarts it, appending
-# RESTART lines to the supervisor log. After 10 restarts in one hour for a
-# tool it logs GIVEUP and exits 1. Only one semfix_uvmt instance is ever
-# started (QEMU's QMP socket accepts a single client).
+# process is alive (exact script path) and that its heartbeat mtime is < 75 s.
+# A startup grace (120 s, skipped early once the tool log shows it has done
+# first useful work) keeps a slow cold start from being restarted. On failure
+# it kills the tool (exact path match) and restarts it, appending RESTART
+# lines to the supervisor log. After 10 restarts in one hour for a tool it
+# logs GIVEUP and exits 1. Only one semfix_uvmt instance is ever started
+# (QEMU's QMP socket accepts a single client).
+#
+# Liveness policy (do not tighten without re-checking the soak evidence):
+#  * the tools refresh their heartbeats from dedicated loops/threads even
+#    while idle (pin_gr_stream: 5 s thread; semfix: per loop + around every
+#    blocking guest-agent/dmesg call), so 75 s never trips on a healthy
+#    client between phases;
+#  * a DEAD process is restarted on the next 5 s tick regardless of grace;
+#  * dead-pid/state recovery (the kill cascade) is handled INSIDE the tools
+#    (in-band #nopid + idle liveness probe + stall watchdog), NOT by this
+#    supervisor - a log-based restart here could fire mid-run of a healthy
+#    client (the m178 lesson).
 #
 # Usage: pin_supervisor.sh LOGDIR TAG
 # Logs: LOGDIR/12-pin-supervisor-TAG.log (supervisor)
@@ -30,6 +43,8 @@ PIN_LOG="$LOGDIR/12-pin_gr_stream-$TAG.log"
 SEMFIX_LOG="$LOGDIR/12-semfix-$TAG.log"
 WINDOW=3600
 CAP=10
+STALE=75
+GRACE=120
 mkdir -p "$LOGDIR" /tmp/opencode
 
 log() { echo "[$(date +%H:%M:%S)] $*" | tee -a "$SUPLOG"; }
@@ -57,6 +72,7 @@ start_pin() {
     touch "$PIN_HB" 2>/dev/null || true
     ( setsid nohup env SEMFIX_COUNTER_RETRY=10 \
         python3 "$PIN" >> "$PIN_LOG" 2>&1 & )
+    pin_started=$(date +%s)
     log "START pin_gr_stream"
 }
 
@@ -70,6 +86,7 @@ start_semfix() {
     touch "$SEMFIX_HB" 2>/dev/null || true
     ( setsid nohup env SEMFIX_SKIP_PB=0 SEMFIX_COUNTER_RETRY=10 \
         python3 "$SEMFIX" 0.5 >> "$SEMFIX_LOG" 2>&1 & )
+    semfix_started=$(date +%s)
     log "START semfix_uvmt"
 }
 
@@ -89,6 +106,8 @@ count_list() {  # $1 list -> element count
 
 restarts_pin=""
 restarts_semfix=""
+pin_started=$(date +%s)
+semfix_started=$(date +%s)
 
 restart() {  # $1 label, $2 script path, $3 start function
     local now list n
@@ -114,16 +133,26 @@ restart() {  # $1 label, $2 script path, $3 start function
     "$3"
 }
 
-check_tool() {  # $1 label, $2 path, $3 heartbeat file, $4 start function
-    local age
+check_tool() {  # $1 label, $2 path, $3 heartbeat file, $4 start function,
+                # $5 tool start TS, $6 tool log file, $7 first-work pattern
+    local age runtime
     age=$(hb_age "$3")
-    if tool_alive "$2" && [ "$age" -le 30 ]; then
+    if tool_alive "$2" && [ "$age" -le "$STALE" ]; then
         return 0
     fi
-    if ! tool_alive "$2"; then
-        log "$1 not running (heartbeat age ${age}s) - restarting"
-    else
+    if tool_alive "$2"; then
+        # Startup grace: a cold tool can spend its first ~2 min installing the
+        # guest scanner or connecting QMP with a lagging heartbeat; only
+        # enforce staleness once the grace has passed or the log shows the
+        # tool has already done successful work.
+        runtime=$(( $(date +%s) - $5 ))
+        if [ "$runtime" -lt "$GRACE" ] && \
+                ! grep -q -- "$7" "$6" 2>/dev/null; then
+            return 0
+        fi
         log "$1 heartbeat stale (${age}s) - restarting"
+    else
+        log "$1 not running (heartbeat age ${age}s) - restarting"
     fi
     restart "$1" "$2" "$4"
 }
@@ -133,6 +162,8 @@ start_pin
 start_semfix
 while true; do
     sleep 5
-    check_tool pin_gr_stream "$PIN" "$PIN_HB" start_pin
-    check_tool semfix_uvmt "$SEMFIX" "$SEMFIX_HB" start_semfix
+    check_tool pin_gr_stream "$PIN" "$PIN_HB" start_pin "$pin_started" \
+        "$PIN_LOG" "listening on vsock"
+    check_tool semfix_uvmt "$SEMFIX" "$SEMFIX_HB" start_semfix \
+        "$semfix_started" "$SEMFIX_LOG" "semfix_uvmt: watching"
 done