summaryrefslogtreecommitdiffstats
path: root/tools/guest-diag/pin_gr_stream.py
diff options
context:
space:
mode:
authorArthur Hanson Rasmusson <arthur@vgpu.io>2026-10-17 20:35:21 -0400
committerArthur Hanson Rasmusson <arthur@vgpu.io>2026-10-17 20:35:21 -0400
commitb16269daec0ca5838b8343e9b5f4cabd23015167 (patch)
treebca44ef8495beb29e4d68af3bf6e33d78e696221 /tools/guest-diag/pin_gr_stream.py
parent33de4c9bc524ac27582fde5911ad32d89d0a7de9 (diff)
downloadopen-iov-b16269daec0ca5838b8343e9b5f4cabd23015167.tar.gz
open-iov-b16269daec0ca5838b8343e9b5f4cabd23015167.zip
tools: revert m178 latching changes (client-#3 hang regression); kernel m178 stays
The m178 tool latching changes caused a reproducible pre-capture hang of the third CUDA client after two green clients (r199: smiley #3; r200: multikernel #3; both recovered by reverting tools only, r201). Prime suspect: semfix per-client reset re-firing mid-run (same pid reset twice 2 min apart while the client was active). Tools restored to the m177-proven state until the reset logic is fixed with a monotonic per-pid guard.
Diffstat (limited to 'tools/guest-diag/pin_gr_stream.py')
-rw-r--r--tools/guest-diag/pin_gr_stream.py91
1 files changed, 9 insertions, 82 deletions
diff --git a/tools/guest-diag/pin_gr_stream.py b/tools/guest-diag/pin_gr_stream.py
index 1cd3eb1..0072b20 100644
--- a/tools/guest-diag/pin_gr_stream.py
+++ b/tools/guest-diag/pin_gr_stream.py
@@ -32,8 +32,7 @@ LO = os.environ.get("PINWIN_LO", "0x200000000")
HI = os.environ.get("PINWIN_HI", "0x206000000")
PAMAX = os.environ.get("PINWIN_PA_MAX", "0x180000000")
SEMSTREAM = os.environ.get("SEM_STREAM_FILE", "/tmp/opencode/sem_stream.txt")
-# Liveness file checked by pin_supervisor.sh (mtime must stay < 75 s; the
-# supervisor grants a 120 s startup grace on top).
+# Liveness file checked by pin_supervisor.sh (mtime must stay < 30 s).
HEARTBEAT_FILE = os.environ.get("PIN_GR_HEARTBEAT", "/tmp/opencode/pin_gr_heartbeat")
@@ -49,16 +48,6 @@ def _heartbeat():
pass
-def _hb_loop():
- """Refresh the heartbeat on a fixed cadence, independent of data-path
- activity: recv can be legally idle for minutes between client phases and
- the supervisor used to read that silence as a hung tool (r198 restarts at
- 34 s staleness while injection bursts were running)."""
- while True:
- _heartbeat()
- time.sleep(5)
-
-
def _mirror(line):
"""Append guest control lines (#pid/#sem) to the shared host file so the
single-QMP watcher can consume them without guest-agent round trips."""
@@ -78,17 +67,6 @@ def vexec(cmd, timeout=40):
return ""
-def _scanner_alive():
- """True when the in-guest scanner process still exists.
-
- install_and_start() pkill's and restarts the scanner, which resets every
- in-guest latch; probing first avoids killing a healthy scanner that is
- merely silent because no client exists yet (r198 pre-client churn)."""
- out = vexec("sudo sh -c 'kill -0 $(cat /tmp/guest_pin_stream.pid) "
- "2>/dev/null && echo ALIVE'")
- return "ALIVE" in (out or "")
-
-
def _start_cmd():
with open(GUEST_SCRIPT) as f:
b64 = subprocess.run(["base64", "-w0"], stdin=f,
@@ -209,18 +187,11 @@ def _demand_loop():
_DEMAND_STARTED = False
-_HB_STARTED = False
def main():
- global _DEMAND_STARTED, _HB_STARTED
+ global _DEMAND_STARTED
_heartbeat()
- if not _HB_STARTED:
- _HB_STARTED = True
- try:
- threading.Thread(target=_hb_loop, daemon=True).start()
- except Exception as e:
- log("WARN: heartbeat thread failed to start: %s" % e)
if not install_and_start():
log("WARN: could not start guest scanner (will retry via vsock wait)")
if not _DEMAND_STARTED:
@@ -247,16 +218,9 @@ def main():
now = time.time()
# No guest scanner connection for 30 s: it died or never started
# (a boot without it streams no pins and every client then hangs).
- # Probe the scanner first instead of reinstalling blindly:
- # install_and_start() pkill's a live scanner and resets all
- # in-guest latches, and pre-client silence is normal (r198 churn).
- if now - last_conn > 30.0 and now - last_reinstall >= 60.0:
+ # Re-run install_and_start() at most once per 15 s.
+ if now - last_conn > 30.0 and now - last_reinstall >= 15.0:
last_reinstall = now
- if _scanner_alive():
- _heartbeat()
- log("no guest connection for %.0fs but scanner alive; "
- "not reinstalling" % (now - last_conn))
- continue
log("reinstall attempt: no guest connection for %.0fs"
% (now - last_conn))
try:
@@ -279,9 +243,8 @@ def main():
try:
chunk = conn.recv(1 << 20)
except socket.timeout:
- # Run even with no pending runs: the periodic full
- # re-assert inside _flush needs a quiet-period tick too.
- if _flush(runs, seen):
+ if runs:
+ _flush(runs, seen)
runs = []
continue
except OSError:
@@ -302,14 +265,6 @@ def main():
% (m.group(1) if m else "?"))
_mirror(text)
continue
- if text.startswith("#nopid"):
- # Client exited: drop the per-pair dedupe map so the
- # next client's pairs are injected unconditionally
- # (the guest emits this on the pid -> None edge).
- seen = {}
- log("guest reports no client pid")
- _mirror(text)
- continue
if text.startswith("#sem"):
_mirror(text)
continue
@@ -348,10 +303,6 @@ def main():
pass
-_FULL_REASSERT_S = 5.0
-_full_reassert_next = 0.0
-
-
def _flush(runs, seen):
"""Inject run lines (one batched sudo pass), return number of pages pinned.
@@ -360,18 +311,7 @@ def _flush(runs, seen):
deadline. The batch writes all runs in a single shell, retrying only the
ones the kernel queue rejected (ENOSPC) individually.
"""
- global _full_reassert_next
- now = time.time()
- # Every _FULL_REASSERT_S re-assert every pair pinned so far, not only the
- # ones in this batch. The guest streams deltas only and the kernel session
- # flush wipes the pin table when a client's channels close, so without
- # this a flush while the same pid continues (a second context in one
- # process) strands the GR-window pages for good. Writes are idempotent;
- # contiguous pages are coalesced into one run line.
- full = now >= _full_reassert_next
- if full:
- _full_reassert_next = now + _FULL_REASSERT_S
- todo = {}
+ todo = []
for va, pa, n in runs:
# The CE window (0x120000000..0x121000000) and the UVM semaphore
# pool page range (0x121000000..0x122000000) are re-asserted by the
@@ -384,22 +324,9 @@ def _flush(runs, seen):
# pool/tracker page (pending release starved, every client's first
# seconds).
ce_win = 0x120000000 <= va < 0x122000000
- if not full and not ce_win and seen.get(va) == pa and n == 1:
+ if not ce_win and seen.get(va) == pa and n == 1:
continue
- todo[(va, pa, n)] = True
- if full:
- items = sorted(seen.items())
- i = 0
- while i < len(items):
- va, pa = items[i]
- n = 1
- while (i + n < len(items)
- and items[i + n][0] == va + n * 0x1000
- and items[i + n][1] == pa + n * 0x1000):
- n += 1
- todo[(va, pa, n)] = True
- i += n
- todo = list(todo)
+ todo.append((va, pa, n))
if not todo:
return 0