diff options
Diffstat (limited to 'tools/guest-diag/semfix_watch.py')
| -rw-r--r-- | tools/guest-diag/semfix_watch.py | 106 |
1 files changed, 95 insertions, 11 deletions
diff --git a/tools/guest-diag/semfix_watch.py b/tools/guest-diag/semfix_watch.py index 5124805..4b60fd6 100644 --- a/tools/guest-diag/semfix_watch.py +++ b/tools/guest-diag/semfix_watch.py @@ -58,12 +58,47 @@ CHAN_MAX_CANDS = int(os.environ.get("CHAN_MAX_CANDS", "256")) class Qmp: def __init__(self, path): + self.path = path + self._cache = {} + self._fails = 0 + self._last_reconnect = 0.0 + self._reconnecting = False self.s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) self.s.connect(path) self.s.settimeout(15) self._rd() self.cmd("qmp_capabilities") - self._cache = {} + + def reconnect(self): + """Re-establish the QMP connection after the socket died. + + QEMU's QMP socket accepts a single client and closes when the guest + reboots; cmd() used to swallow every exception and return {}, so all + reads stayed empty forever ("uvmt(hb) sem=... unreadable") and + pinning stopped. Rate-limited to one attempt per second so a failing + monitor does not turn every read into a connect storm.""" + if self._reconnecting or time.time() - self._last_reconnect < 1.0: + return + self._reconnecting = True + self._last_reconnect = time.time() + try: + try: + self.s.close() + except Exception: + pass + self.s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self.s.connect(self.path) + self.s.settimeout(15) + self._rd() + self.cmd("qmp_capabilities") + self._cache = {} + self._ncpu = 0 + self._cpu = 0 + self._fails = 0 + except Exception: + pass + finally: + self._reconnecting = False def _rd(self): """Read one QMP reply (JSON object ends with a newline). Poll with a @@ -91,8 +126,17 @@ class Qmp: def cmd(self, name, **args): try: self.s.sendall(json.dumps({"execute": name, "arguments": args}).encode() + b"\n") - return json.loads(self._rd() or "{}") + reply = self._rd() + if not reply: + raise OSError("empty QMP reply") + self._fails = 0 + return json.loads(reply) except Exception: + # Broken socket (QEMU restart, monitor hiccup): try exactly one + # reconnect, rate-limited inside reconnect(). + self._fails = getattr(self, "_fails", 0) + 1 + if not getattr(self, "_reconnecting", False): + self.reconnect() return {} def hmp(self, line): @@ -143,8 +187,14 @@ class Qmp: break m = re.search(r":(.*)$", out, re.S) if not m: + if not out.strip(): + # Empty monitor output: the socket is broken, not the address. + self.reconnect() return [] vals = [int(x, 16) for x in re.findall(r"0x([0-9a-fA-F]+)", m.group(1))][:n] + if not vals: + self.reconnect() + return [] if cache: self._cache[key] = vals return vals @@ -189,18 +239,46 @@ def guest_trace(): Read *before* clearing: the create-time probes fire once per client and would be wiped by a clear that races them. + + 2026-10-16 robustness fix (the cascade root): this used to run a bare + `tail` over the guest agent. A tracefs read that blocks under a busy + guest left *uninterruptible* tail processes behind (observed dozens in + D state), the guest-agent exec then blocked for its full 90 s timeout + every iteration, and the pool/tracker pins this tool supplies stopped + flowing - after ~5 clients every new client froze at DEVICE waiting on + its UVM tracker with `pending release starved ... pool/waiter page`. + Now the read is bounded remotely (`timeout -s KILL 12`) so no tail can + outlive its call, and SSH (which stays healthy when the guest agent + wedges) is preferred with the agent as fallback. """ - inner = ("tail -c 131072 /sys/kernel/tracing/trace " + inner = ("timeout -s KILL 12 tail -c 131072 /sys/kernel/tracing/trace " "| grep -E \"uvmt|uvch|uvbp|uvpbf\" " "| tail -400") if not os.environ.get("KEEP_TRACE"): inner += "; echo > /sys/kernel/tracing/trace" - cmd = "sudo sh -c '%s'" % inner + outer = "sudo sh -c '%s'" % inner + out = "" + ssh = os.environ.get("SEMFIX_SSH_PORT", "2222") + ssh_cmd = [ + "sshpass", "-p", os.environ.get("VGPU_GUEST_PASS", "ubuntu"), + "ssh", "-o", "StrictHostKeyChecking=no", + "-o", "UserKnownHostsFile=/dev/null", + "-o", "PubkeyAuthentication=no", "-o", "ConnectTimeout=6", + "-p", ssh, "ubuntu@127.0.0.1", outer, + ] try: - out = subprocess.run([VMEXEC, "exec", cmd], capture_output=True, - text=True, timeout=90).stdout + r = subprocess.run(ssh_cmd, capture_output=True, text=True, + timeout=25) + out = r.stdout or "" except Exception: - return set(), set(), set() + out = "" + if not out: + try: + out = subprocess.run([VMEXEC, "exec", outer], + capture_output=True, text=True, + timeout=25).stdout or "" + except Exception: + return set(), set(), set() sems = {int(m, 16) for m in re.findall(r"sem=0x([0-9a-fA-F]+)", out)} pbs = {int(m, 16) for m in re.findall(r"pb=0x([0-9a-fA-F]+)", out)} chans = {int(m, 16) for m in re.findall(r"ch=0x([0-9a-fA-F]+)", out)} @@ -369,8 +447,11 @@ def resolve_pushbuffer_wide(q, sem, d): def pin(va, pa): cmd = "printf '0x%x,0x%x' > %s/sema_map_add" % (va, pa, PARAM) for _ in range(40): - r = subprocess.run(["sudo", "-n", "sh", "-c", cmd], - capture_output=True, text=True) + try: + r = subprocess.run(["sudo", "-n", "sh", "-c", cmd], + capture_output=True, text=True, timeout=5) + except subprocess.TimeoutExpired: + continue if r.returncode == 0 and "timed out" not in (r.stderr or ""): return True time.sleep(0.05) # async queue full: drain and retry @@ -380,8 +461,11 @@ def pin(va, pa): def pin_bulk(va, pa, nbytes): cmd = "printf '0x%x,0x%x,0x%x' > %s/sema_map_add" % (va, pa, nbytes, PARAM) for _ in range(40): - r = subprocess.run(["sudo", "-n", "sh", "-c", cmd], - capture_output=True, text=True) + try: + r = subprocess.run(["sudo", "-n", "sh", "-c", cmd], + capture_output=True, text=True, timeout=5) + except subprocess.TimeoutExpired: + continue if r.returncode == 0 and "timed out" not in (r.stderr or ""): return True time.sleep(0.05) |