guest-diag: semfix trace reads are remotely timed and SSH-preferred (stuck QGA tails stopped pool/tracker pins and cascaded every client after ~5 to a DEVICE freeze)

1 files changed, 33 insertions, 5 deletions

diff --git a/tools/guest-diag/semfix_watch.py b/tools/guest-diag/semfix_watch.py
index 5124805..a4b2fc0 100644
--- a/tools/guest-diag/semfix_watch.py
+++ b/tools/guest-diag/semfix_watch.py
@@ -189,18 +189,46 @@ def guest_trace():
 
     Read *before* clearing: the create-time probes fire once per client and
     would be wiped by a clear that races them.
+
+    2026-10-16 robustness fix (the cascade root): this used to run a bare
+    `tail` over the guest agent. A tracefs read that blocks under a busy
+    guest left *uninterruptible* tail processes behind (observed dozens in
+    D state), the guest-agent exec then blocked for its full 90 s timeout
+    every iteration, and the pool/tracker pins this tool supplies stopped
+    flowing - after ~5 clients every new client froze at DEVICE waiting on
+    its UVM tracker with `pending release starved ... pool/waiter page`.
+    Now the read is bounded remotely (`timeout -s KILL 12`) so no tail can
+    outlive its call, and SSH (which stays healthy when the guest agent
+    wedges) is preferred with the agent as fallback.
     """
-    inner = ("tail -c 131072 /sys/kernel/tracing/trace "
+    inner = ("timeout -s KILL 12 tail -c 131072 /sys/kernel/tracing/trace "
              "| grep -E \"uvmt|uvch|uvbp|uvpbf\" "
              "| tail -400")
     if not os.environ.get("KEEP_TRACE"):
         inner += "; echo > /sys/kernel/tracing/trace"
-    cmd = "sudo sh -c '%s'" % inner
+    outer = "sudo sh -c '%s'" % inner
+    out = ""
+    ssh = os.environ.get("SEMFIX_SSH_PORT", "2222")
+    ssh_cmd = [
+        "sshpass", "-p", os.environ.get("VGPU_GUEST_PASS", "ubuntu"),
+        "ssh", "-o", "StrictHostKeyChecking=no",
+        "-o", "UserKnownHostsFile=/dev/null",
+        "-o", "PubkeyAuthentication=no", "-o", "ConnectTimeout=6",
+        "-p", ssh, "ubuntu@127.0.0.1", outer,
+    ]
     try:
-        out = subprocess.run([VMEXEC, "exec", cmd], capture_output=True,
-                             text=True, timeout=90).stdout
+        r = subprocess.run(ssh_cmd, capture_output=True, text=True,
+                           timeout=25)
+        out = r.stdout or ""
     except Exception:
-        return set(), set(), set()
+        out = ""
+    if not out:
+        try:
+            out = subprocess.run([VMEXEC, "exec", outer],
+                                 capture_output=True, text=True,
+                                 timeout=25).stdout or ""
+        except Exception:
+            return set(), set(), set()
     sems = {int(m, 16) for m in re.findall(r"sem=0x([0-9a-fA-F]+)", out)}
     pbs = {int(m, 16) for m in re.findall(r"pb=0x([0-9a-fA-F]+)", out)}
     chans = {int(m, 16) for m in re.findall(r"ch=0x([0-9a-fA-F]+)", out)}