vgsp: m178 GR launch reliability + diagnostics; tools: pin latching robustness

Kernel: authorship instrumentation + swrel gate for in-flight GR release VAs;
first-poll shortening + fast extended re-arm (params gr_retry_fast=1,
gr_first_poll_ms=2500); per-attempt win0 bank recompute + shadow recopy in the
outcome-retry loop; poll-break/deposit-readback diagnostics; C1-C10 sema
diagnostics (crel-entry, relx, opdef-clear, walk, pub-adv, hold-fallback,
busy-defer, CAND-OVERWRITE, inline recs); busy-defer hold instead of
software-release consume; counter fast path; QMD word 48/49 skip.
Tools: heartbeat thread, scanner-alive probe on reinstall, full periodic
re-assert, #nopid handling, per-client reset fix (debounce), dead-pid drop,
supervisor grace+threshold.

1 files changed, 82 insertions, 9 deletions

diff --git a/tools/guest-diag/pin_gr_stream.py b/tools/guest-diag/pin_gr_stream.py
index 0072b20..1cd3eb1 100644
--- a/tools/guest-diag/pin_gr_stream.py
+++ b/tools/guest-diag/pin_gr_stream.py
@@ -32,7 +32,8 @@ LO = os.environ.get("PINWIN_LO", "0x200000000")
 HI = os.environ.get("PINWIN_HI", "0x206000000")
 PAMAX = os.environ.get("PINWIN_PA_MAX", "0x180000000")
 SEMSTREAM = os.environ.get("SEM_STREAM_FILE", "/tmp/opencode/sem_stream.txt")
-# Liveness file checked by pin_supervisor.sh (mtime must stay < 30 s).
+# Liveness file checked by pin_supervisor.sh (mtime must stay < 75 s; the
+# supervisor grants a 120 s startup grace on top).
 HEARTBEAT_FILE = os.environ.get("PIN_GR_HEARTBEAT", "/tmp/opencode/pin_gr_heartbeat")
 
 
@@ -48,6 +49,16 @@ def _heartbeat():
         pass
 
 
+def _hb_loop():
+    """Refresh the heartbeat on a fixed cadence, independent of data-path
+    activity: recv can be legally idle for minutes between client phases and
+    the supervisor used to read that silence as a hung tool (r198 restarts at
+    34 s staleness while injection bursts were running)."""
+    while True:
+        _heartbeat()
+        time.sleep(5)
+
+
 def _mirror(line):
     """Append guest control lines (#pid/#sem) to the shared host file so the
     single-QMP watcher can consume them without guest-agent round trips."""
@@ -67,6 +78,17 @@ def vexec(cmd, timeout=40):
         return ""
 
 
+def _scanner_alive():
+    """True when the in-guest scanner process still exists.
+
+    install_and_start() pkill's and restarts the scanner, which resets every
+    in-guest latch; probing first avoids killing a healthy scanner that is
+    merely silent because no client exists yet (r198 pre-client churn)."""
+    out = vexec("sudo sh -c 'kill -0 $(cat /tmp/guest_pin_stream.pid) "
+                "2>/dev/null && echo ALIVE'")
+    return "ALIVE" in (out or "")
+
+
 def _start_cmd():
     with open(GUEST_SCRIPT) as f:
         b64 = subprocess.run(["base64", "-w0"], stdin=f,
@@ -187,11 +209,18 @@ def _demand_loop():
 
 
 _DEMAND_STARTED = False
+_HB_STARTED = False
 
 
 def main():
-    global _DEMAND_STARTED
+    global _DEMAND_STARTED, _HB_STARTED
     _heartbeat()
+    if not _HB_STARTED:
+        _HB_STARTED = True
+        try:
+            threading.Thread(target=_hb_loop, daemon=True).start()
+        except Exception as e:
+            log("WARN: heartbeat thread failed to start: %s" % e)
     if not install_and_start():
         log("WARN: could not start guest scanner (will retry via vsock wait)")
     if not _DEMAND_STARTED:
@@ -218,9 +247,16 @@ def main():
             now = time.time()
             # No guest scanner connection for 30 s: it died or never started
             # (a boot without it streams no pins and every client then hangs).
-            # Re-run install_and_start() at most once per 15 s.
-            if now - last_conn > 30.0 and now - last_reinstall >= 15.0:
+            # Probe the scanner first instead of reinstalling blindly:
+            # install_and_start() pkill's a live scanner and resets all
+            # in-guest latches, and pre-client silence is normal (r198 churn).
+            if now - last_conn > 30.0 and now - last_reinstall >= 60.0:
                 last_reinstall = now
+                if _scanner_alive():
+                    _heartbeat()
+                    log("no guest connection for %.0fs but scanner alive; "
+                        "not reinstalling" % (now - last_conn))
+                    continue
                 log("reinstall attempt: no guest connection for %.0fs"
                     % (now - last_conn))
                 try:
@@ -243,8 +279,9 @@ def main():
                 try:
                     chunk = conn.recv(1 << 20)
                 except socket.timeout:
-                    if runs:
-                        _flush(runs, seen)
+                    # Run even with no pending runs: the periodic full
+                    # re-assert inside _flush needs a quiet-period tick too.
+                    if _flush(runs, seen):
                         runs = []
                     continue
                 except OSError:
@@ -265,6 +302,14 @@ def main():
                             % (m.group(1) if m else "?"))
                         _mirror(text)
                         continue
+                    if text.startswith("#nopid"):
+                        # Client exited: drop the per-pair dedupe map so the
+                        # next client's pairs are injected unconditionally
+                        # (the guest emits this on the pid -> None edge).
+                        seen = {}
+                        log("guest reports no client pid")
+                        _mirror(text)
+                        continue
                     if text.startswith("#sem"):
                         _mirror(text)
                         continue
@@ -303,6 +348,10 @@ def main():
             pass
 
 
+_FULL_REASSERT_S = 5.0
+_full_reassert_next = 0.0
+
+
 def _flush(runs, seen):
     """Inject run lines (one batched sudo pass), return number of pages pinned.
 
@@ -311,7 +360,18 @@ def _flush(runs, seen):
     deadline. The batch writes all runs in a single shell, retrying only the
     ones the kernel queue rejected (ENOSPC) individually.
     """
-    todo = []
+    global _full_reassert_next
+    now = time.time()
+    # Every _FULL_REASSERT_S re-assert every pair pinned so far, not only the
+    # ones in this batch. The guest streams deltas only and the kernel session
+    # flush wipes the pin table when a client's channels close, so without
+    # this a flush while the same pid continues (a second context in one
+    # process) strands the GR-window pages for good. Writes are idempotent;
+    # contiguous pages are coalesced into one run line.
+    full = now >= _full_reassert_next
+    if full:
+        _full_reassert_next = now + _FULL_REASSERT_S
+    todo = {}
     for va, pa, n in runs:
         # The CE window (0x120000000..0x121000000) and the UVM semaphore
         # pool page range (0x121000000..0x122000000) are re-asserted by the
@@ -324,9 +384,22 @@ def _flush(runs, seen):
         # pool/tracker page (pending release starved, every client's first
         # seconds).
         ce_win = 0x120000000 <= va < 0x122000000
-        if not ce_win and seen.get(va) == pa and n == 1:
+        if not full and not ce_win and seen.get(va) == pa and n == 1:
             continue
-        todo.append((va, pa, n))
+        todo[(va, pa, n)] = True
+    if full:
+        items = sorted(seen.items())
+        i = 0
+        while i < len(items):
+            va, pa = items[i]
+            n = 1
+            while (i + n < len(items)
+                   and items[i + n][0] == va + n * 0x1000
+                   and items[i + n][1] == pa + n * 0x1000):
+                n += 1
+            todo[(va, pa, n)] = True
+            i += n
+    todo = list(todo)
     if not todo:
         return 0