vgsp: m178 GR launch reliability + diagnostics; tools: pin latching robustness

Kernel: authorship instrumentation + swrel gate for in-flight GR release VAs;
first-poll shortening + fast extended re-arm (params gr_retry_fast=1,
gr_first_poll_ms=2500); per-attempt win0 bank recompute + shadow recopy in the
outcome-retry loop; poll-break/deposit-readback diagnostics; C1-C10 sema
diagnostics (crel-entry, relx, opdef-clear, walk, pub-adv, hold-fallback,
busy-defer, CAND-OVERWRITE, inline recs); busy-defer hold instead of
software-release consume; counter fast path; QMD word 48/49 skip.
Tools: heartbeat thread, scanner-alive probe on reinstall, full periodic
re-assert, #nopid handling, per-client reset fix (debounce), dead-pid drop,
supervisor grace+threshold.

1 files changed, 47 insertions, 23 deletions

diff --git a/tools/guest-diag/semfix_uvmt.py b/tools/guest-diag/semfix_uvmt.py
index fa3ae01..812076e 100755
--- a/tools/guest-diag/semfix_uvmt.py
+++ b/tools/guest-diag/semfix_uvmt.py
@@ -53,7 +53,8 @@ SKIP_PB = os.environ.get("SEMFIX_SKIP_PB", "0") == "1"
 # Seconds between counter-page pin attempts (the guest-agent exec blocks for
 # its full timeout when the client saturates the guest).
 COUNTER_RETRY_S = float(os.environ.get("SEMFIX_COUNTER_RETRY", "10"))
-# Liveness file checked by pin_supervisor.sh (mtime must stay < 30 s).
+# Liveness file checked by pin_supervisor.sh (mtime must stay < 75 s; the
+# supervisor grants a 120 s startup grace on top).
 SEMFIX_HB_FILE = os.environ.get("SEMFIX_HEARTBEAT_FILE",
                                 "/tmp/opencode/semfix_heartbeat")
 # Every client pid ever observed. Module-level on purpose: the per-client
@@ -239,6 +240,23 @@ def cdt_pid():
     return int(pids[-1]) if pids else None
 
 
+def pid_alive(pid):
+    """Probe /proc/<pid> in the guest before a blocking exec on it.
+
+    A killed client's pid stays latched in the watcher state; retrying its
+    counter pin every COUNTER_RETRY_S ran a full 25 s agent exec per attempt
+    and starved the scanner (the post-kill agent hammering). Assume alive on
+    a failed probe so a transient agent error cannot drop a live client.
+    """
+    try:
+        r = subprocess.run(
+            [VMEXEC, "exec", "test -d /proc/%d && echo ALIVE" % pid],
+            capture_output=True, text=True, timeout=15)
+    except Exception:
+        return True
+    return "ALIVE" in (r.stdout or "")
+
+
 def pin_counter_page(pid, pinned_pids):
     """Pin the client's completion-counter page from its own pagemap.
 
@@ -282,10 +300,9 @@ def main():
     last_seen_t = {}
     dseen = set()
     counter_pids = set()
+    counter_dead = set()
     counter_next_try = {}
     last_pid = None
-    pending_pid = None
-    pending_count = 0
     # Stale-sem tracking: consecutive unreadable reads, first-unreadable time,
     # and the per-sem read skip deadline once a sem has failed 3 times.
     unread_count = {}
@@ -332,7 +349,6 @@ def main():
             if last_pid is None:
                 seen_pids.add(p)
                 last_pid = p
-                pending_pid = None
                 continue
             if p == last_pid:
                 seen_pids.add(p)
@@ -343,20 +359,13 @@ def main():
                 log("semfix: pid %d seen before/backwards - no reset" % p)
                 seen_pids.add(p)
                 continue
-            if pending_pid != p:
-                pending_pid = p
-                pending_count = 1
-                log("semfix: pid change %d -> %d (debounce; reset if it "
-                    "repeats)" % (last_pid, p))
-                continue
-            pending_count += 1
-            if pending_count < 2:
-                continue
-            # Same unseen pid reported on two consecutive polls: real new
-            # client. seen_pids keeps old pids from ever triggering this.
+            # Any unseen pid above the current one is a real new client.
+            # Stream.poll dedupes pids against its own list, so each pid
+            # arrives in exactly one poll batch and never repeats - the old
+            # two-poll debounce therefore never fired after the first client
+            # (the post-kill counter latch bug). seen_pids still guards
+            # replays of old pids; reset immediately instead.
             seen_pids.add(p)
-            pending_pid = None
-            pending_count = 0
             stream.reset()
             known_sems.clear()
             seen_sem_pin.clear()
@@ -366,6 +375,7 @@ def main():
             seen_ch.clear()
             seen_pb.clear()
             counter_pids.clear()
+            counter_dead.clear()
             sem_pin_fail.clear()
             sem_pin_skip_until.clear()
             unread_count.clear()
@@ -382,6 +392,7 @@ def main():
         # as a fallback for channel/pushbuffer discovery and to recover sems
         # the scanner may have missed.
         if loop % TRACE_EVERY == 0:
+            _hb()
             try:
                 t_sems, t_pbs, t_chans = guest_trace()
             except Exception:
@@ -542,6 +553,7 @@ def main():
                 log("semfix: pb=0x%x pushbuf miss [%s]" % (pb, diag_str(d)))
 
         if DLOG:
+            _hb()
             dmesg_releases(dseen)
 
         # Retry incomplete push-buffer pins: gva2gpa of the kernel vmalloc
@@ -559,20 +571,32 @@ def main():
         # block for its whole timeout under client load, so retry on a slow
         # cadence: hammering the agent here starves it and wedges the run.
         if last_pid is None and loop % TRACE_EVERY == 0:
+            _hb()
             last_pid = cdt_pid()
             _hb()
             if last_pid:
                 seen_pids.add(last_pid)
                 log("counter: client pid %d (agent)" % last_pid)
-        if last_pid and last_pid not in counter_pids:
+        if last_pid and last_pid not in counter_pids \
+                and last_pid not in counter_dead:
             now = time.time()
             if now >= counter_next_try.get(last_pid, 0):
-                try:
-                    pin_counter_page(last_pid, counter_pids)
-                except Exception as e:
-                    log("counter: pin failed: %s" % e)
                 _hb()
-                counter_next_try[last_pid] = now + COUNTER_RETRY_S
+                if not pid_alive(last_pid):
+                    # Killed clients stay latched as last_pid forever; probe
+                    # first so the 10 s cadence does not block the agent for a
+                    # full 25 s exec on every retry (the post-kill hammering
+                    # that starved the scanner). A new client's #pid resets
+                    # counter_dead and takes over below.
+                    log("counter: pid %d is gone - dropping" % last_pid)
+                    counter_dead.add(last_pid)
+                else:
+                    try:
+                        pin_counter_page(last_pid, counter_pids)
+                    except Exception as e:
+                        log("counter: pin failed: %s" % e)
+                    _hb()
+                    counter_next_try[last_pid] = now + COUNTER_RETRY_S
 
         time.sleep(interval)