tools: revert m178 latching changes (client-#3 hang regression); kernel m178 stays

The m178 tool latching changes caused a reproducible pre-capture hang of the
third CUDA client after two green clients (r199: smiley #3; r200: multikernel
#3; both recovered by reverting tools only, r201). Prime suspect: semfix
per-client reset re-firing mid-run (same pid reset twice 2 min apart while the
client was active). Tools restored to the m177-proven state until the reset
logic is fixed with a monotonic per-pid guard.

1 files changed, 23 insertions, 47 deletions

diff --git a/tools/guest-diag/semfix_uvmt.py b/tools/guest-diag/semfix_uvmt.py
index 812076e..fa3ae01 100755
--- a/tools/guest-diag/semfix_uvmt.py
+++ b/tools/guest-diag/semfix_uvmt.py
@@ -53,8 +53,7 @@ SKIP_PB = os.environ.get("SEMFIX_SKIP_PB", "0") == "1"
 # Seconds between counter-page pin attempts (the guest-agent exec blocks for
 # its full timeout when the client saturates the guest).
 COUNTER_RETRY_S = float(os.environ.get("SEMFIX_COUNTER_RETRY", "10"))
-# Liveness file checked by pin_supervisor.sh (mtime must stay < 75 s; the
-# supervisor grants a 120 s startup grace on top).
+# Liveness file checked by pin_supervisor.sh (mtime must stay < 30 s).
 SEMFIX_HB_FILE = os.environ.get("SEMFIX_HEARTBEAT_FILE",
                                 "/tmp/opencode/semfix_heartbeat")
 # Every client pid ever observed. Module-level on purpose: the per-client
@@ -240,23 +239,6 @@ def cdt_pid():
     return int(pids[-1]) if pids else None
 
 
-def pid_alive(pid):
-    """Probe /proc/<pid> in the guest before a blocking exec on it.
-
-    A killed client's pid stays latched in the watcher state; retrying its
-    counter pin every COUNTER_RETRY_S ran a full 25 s agent exec per attempt
-    and starved the scanner (the post-kill agent hammering). Assume alive on
-    a failed probe so a transient agent error cannot drop a live client.
-    """
-    try:
-        r = subprocess.run(
-            [VMEXEC, "exec", "test -d /proc/%d && echo ALIVE" % pid],
-            capture_output=True, text=True, timeout=15)
-    except Exception:
-        return True
-    return "ALIVE" in (r.stdout or "")
-
-
 def pin_counter_page(pid, pinned_pids):
     """Pin the client's completion-counter page from its own pagemap.
 
@@ -300,9 +282,10 @@ def main():
     last_seen_t = {}
     dseen = set()
     counter_pids = set()
-    counter_dead = set()
     counter_next_try = {}
     last_pid = None
+    pending_pid = None
+    pending_count = 0
     # Stale-sem tracking: consecutive unreadable reads, first-unreadable time,
     # and the per-sem read skip deadline once a sem has failed 3 times.
     unread_count = {}
@@ -349,6 +332,7 @@ def main():
             if last_pid is None:
                 seen_pids.add(p)
                 last_pid = p
+                pending_pid = None
                 continue
             if p == last_pid:
                 seen_pids.add(p)
@@ -359,13 +343,20 @@ def main():
                 log("semfix: pid %d seen before/backwards - no reset" % p)
                 seen_pids.add(p)
                 continue
-            # Any unseen pid above the current one is a real new client.
-            # Stream.poll dedupes pids against its own list, so each pid
-            # arrives in exactly one poll batch and never repeats - the old
-            # two-poll debounce therefore never fired after the first client
-            # (the post-kill counter latch bug). seen_pids still guards
-            # replays of old pids; reset immediately instead.
+            if pending_pid != p:
+                pending_pid = p
+                pending_count = 1
+                log("semfix: pid change %d -> %d (debounce; reset if it "
+                    "repeats)" % (last_pid, p))
+                continue
+            pending_count += 1
+            if pending_count < 2:
+                continue
+            # Same unseen pid reported on two consecutive polls: real new
+            # client. seen_pids keeps old pids from ever triggering this.
             seen_pids.add(p)
+            pending_pid = None
+            pending_count = 0
             stream.reset()
             known_sems.clear()
             seen_sem_pin.clear()
@@ -375,7 +366,6 @@ def main():
             seen_ch.clear()
             seen_pb.clear()
             counter_pids.clear()
-            counter_dead.clear()
             sem_pin_fail.clear()
             sem_pin_skip_until.clear()
             unread_count.clear()
@@ -392,7 +382,6 @@ def main():
         # as a fallback for channel/pushbuffer discovery and to recover sems
         # the scanner may have missed.
         if loop % TRACE_EVERY == 0:
-            _hb()
             try:
                 t_sems, t_pbs, t_chans = guest_trace()
             except Exception:
@@ -553,7 +542,6 @@ def main():
                 log("semfix: pb=0x%x pushbuf miss [%s]" % (pb, diag_str(d)))
 
         if DLOG:
-            _hb()
             dmesg_releases(dseen)
 
         # Retry incomplete push-buffer pins: gva2gpa of the kernel vmalloc
@@ -571,32 +559,20 @@ def main():
         # block for its whole timeout under client load, so retry on a slow
         # cadence: hammering the agent here starves it and wedges the run.
         if last_pid is None and loop % TRACE_EVERY == 0:
-            _hb()
             last_pid = cdt_pid()
             _hb()
             if last_pid:
                 seen_pids.add(last_pid)
                 log("counter: client pid %d (agent)" % last_pid)
-        if last_pid and last_pid not in counter_pids \
-                and last_pid not in counter_dead:
+        if last_pid and last_pid not in counter_pids:
             now = time.time()
             if now >= counter_next_try.get(last_pid, 0):
+                try:
+                    pin_counter_page(last_pid, counter_pids)
+                except Exception as e:
+                    log("counter: pin failed: %s" % e)
                 _hb()
-                if not pid_alive(last_pid):
-                    # Killed clients stay latched as last_pid forever; probe
-                    # first so the 10 s cadence does not block the agent for a
-                    # full 25 s exec on every retry (the post-kill hammering
-                    # that starved the scanner). A new client's #pid resets
-                    # counter_dead and takes over below.
-                    log("counter: pid %d is gone - dropping" % last_pid)
-                    counter_dead.add(last_pid)
-                else:
-                    try:
-                        pin_counter_page(last_pid, counter_pids)
-                    except Exception as e:
-                        log("counter: pin failed: %s" % e)
-                    _hb()
-                    counter_next_try[last_pid] = now + COUNTER_RETRY_S
+                counter_next_try[last_pid] = now + COUNTER_RETRY_S
 
         time.sleep(interval)