From 159aa45c87efcc5634407d3ca766d88bd796cbfe Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Fri, 16 Oct 2026 17:38:21 -0400 Subject: guest-diag: semfix trace reads are remotely timed and SSH-preferred (stuck QGA tails stopped pool/tracker pins and cascaded every client after ~5 to a DEVICE freeze) --- tools/guest-diag/semfix_watch.py | 38 +++++++++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 5 deletions(-) (limited to 'tools/guest-diag/semfix_watch.py') diff --git a/tools/guest-diag/semfix_watch.py b/tools/guest-diag/semfix_watch.py index 5124805..a4b2fc0 100644 --- a/tools/guest-diag/semfix_watch.py +++ b/tools/guest-diag/semfix_watch.py @@ -189,18 +189,46 @@ def guest_trace(): Read *before* clearing: the create-time probes fire once per client and would be wiped by a clear that races them. + + 2026-10-16 robustness fix (the cascade root): this used to run a bare + `tail` over the guest agent. A tracefs read that blocks under a busy + guest left *uninterruptible* tail processes behind (observed dozens in + D state), the guest-agent exec then blocked for its full 90 s timeout + every iteration, and the pool/tracker pins this tool supplies stopped + flowing - after ~5 clients every new client froze at DEVICE waiting on + its UVM tracker with `pending release starved ... pool/waiter page`. + Now the read is bounded remotely (`timeout -s KILL 12`) so no tail can + outlive its call, and SSH (which stays healthy when the guest agent + wedges) is preferred with the agent as fallback. """ - inner = ("tail -c 131072 /sys/kernel/tracing/trace " + inner = ("timeout -s KILL 12 tail -c 131072 /sys/kernel/tracing/trace " "| grep -E \"uvmt|uvch|uvbp|uvpbf\" " "| tail -400") if not os.environ.get("KEEP_TRACE"): inner += "; echo > /sys/kernel/tracing/trace" - cmd = "sudo sh -c '%s'" % inner + outer = "sudo sh -c '%s'" % inner + out = "" + ssh = os.environ.get("SEMFIX_SSH_PORT", "2222") + ssh_cmd = [ + "sshpass", "-p", os.environ.get("VGPU_GUEST_PASS", "ubuntu"), + "ssh", "-o", "StrictHostKeyChecking=no", + "-o", "UserKnownHostsFile=/dev/null", + "-o", "PubkeyAuthentication=no", "-o", "ConnectTimeout=6", + "-p", ssh, "ubuntu@127.0.0.1", outer, + ] try: - out = subprocess.run([VMEXEC, "exec", cmd], capture_output=True, - text=True, timeout=90).stdout + r = subprocess.run(ssh_cmd, capture_output=True, text=True, + timeout=25) + out = r.stdout or "" except Exception: - return set(), set(), set() + out = "" + if not out: + try: + out = subprocess.run([VMEXEC, "exec", outer], + capture_output=True, text=True, + timeout=25).stdout or "" + except Exception: + return set(), set(), set() sems = {int(m, 16) for m in re.findall(r"sem=0x([0-9a-fA-F]+)", out)} pbs = {int(m, 16) for m in re.findall(r"pb=0x([0-9a-fA-F]+)", out)} chans = {int(m, 16) for m in re.findall(r"ch=0x([0-9a-fA-F]+)", out)} -- cgit v1.2.3