tools: kill-cascade purge + stall watchdogs + monotonic client identity (tools-fix c8fb5377)

Dead-pid purge via in-band #nopid (zombie-aware probe), monotonic
(pid,starttime) client identity so trace-ring replays can never reset an
active client (fixes the m178 mid-run reset regression avenue), pre-capture
and post-capture stall watchdogs that re-assert pins and escalate loudly,
heartbeat threads for both tools, supervisor STALE 75s/GRACE 120s.

1 files changed, 45 insertions, 2 deletions

diff --git a/tools/guest-diag/guest_pin_stream.py b/tools/guest-diag/guest_pin_stream.py
index 4bfaaf8..bb253a2 100644
--- a/tools/guest-diag/guest_pin_stream.py
+++ b/tools/guest-diag/guest_pin_stream.py
@@ -1398,6 +1398,34 @@ def nvidia_fds(pid):
     return kind
 
 
+def pid_starttime(pid):
+    """Starttime (clock ticks, field 22) of /proc/<pid>, or None.
+
+    Part of the client identity (pid, starttime): pids are recycled, so a
+    per-client reset keyed on the pid alone can mis-fire on a recycled pid.
+    Parsed after the last ')' so comm names with spaces/parens are safe."""
+    try:
+        with open('/proc/%d/stat' % pid) as f:
+            st = f.read()
+        rp = st.rfind(')')
+        if rp < 0:
+            return None
+        toks = st[rp + 2:].split()
+        if len(toks) < 20:      # starttime = field 22 overall = index 19
+            return None
+        return int(toks[19])
+    except (OSError, ValueError):
+        return None
+
+
+def pid_line(tag, pid, start):
+    """'#<tag> <pid> [starttime]' - the optional starttime keeps the line
+    backward compatible with older hosts (they parse the pid only)."""
+    if start is None:
+        return "#%s %d\n" % (tag, pid)
+    return "#%s %d %d\n" % (tag, pid, start)
+
+
 def cdt_pid():
     prefixes = tuple(p for p in os.environ.get("CLIENT_PREFIXES",
                                                "cdt,va,victory").split(",") if p)
@@ -1617,6 +1645,7 @@ def main():
     except OSError:
         pass
     last_pid = None
+    last_start = None
     last_log = 0.0
     scrub_next = 0.0
     demand_next = 0.0
@@ -1631,7 +1660,19 @@ def main():
         pid = cdt_pid()
         loop += 1
         if pid is None:
+            if last_pid is not None and sock is not None:
+                # In-band client-death signal: the host tools purge the dead
+                # client's per-client state on this line, so the NEXT client
+                # works without a VM cycle (the kill cascade). Best-effort -
+                # if the socket is gone the host still has its own probe.
+                try:
+                    sock.sendall(pid_line("nopid", last_pid,
+                                          last_start).encode())
+                except OSError:
+                    sock.close()
+                    sock = None
             last_pid = None
+            last_start = None
             seen = {}
             seen_sems = set()
             seen_pbs = set()
@@ -1640,6 +1681,7 @@ def main():
             continue
         if pid != last_pid:
             last_pid = pid
+            last_start = pid_starttime(pid)
             seen = {}
             seen_sems = set()
             seen_pbs = set()
@@ -1670,7 +1712,7 @@ def main():
                 dbg("switch seed: +%d event kptr(s)" % _seeded)
             if sock is not None:
                 try:
-                    sock.sendall(("#pid %d\n" % pid).encode())
+                    sock.sendall(pid_line("pid", pid, last_start).encode())
                 except OSError:
                     sock.close()
                     sock = None
@@ -2025,7 +2067,8 @@ def main():
                 sock.connect((CID, PORT))
                 sock.settimeout(None)
                 if last_pid is not None:
-                    sock.sendall(("#pid %d\n" % last_pid).encode())
+                    sock.sendall(pid_line("pid", last_pid,
+                                          last_start).encode())
             except OSError:
                 if sock:
                     sock.close()