diff options
Diffstat (limited to 'tools/guest-diag/pin_gr_stream.py')
| -rw-r--r-- | tools/guest-diag/pin_gr_stream.py | 190 |
1 files changed, 170 insertions, 20 deletions
diff --git a/tools/guest-diag/pin_gr_stream.py b/tools/guest-diag/pin_gr_stream.py index 6ba51ee..35c2ea3 100644 --- a/tools/guest-diag/pin_gr_stream.py +++ b/tools/guest-diag/pin_gr_stream.py @@ -100,25 +100,62 @@ def _scanner_alive(): return "ALIVE" in (out or "") +def _demand_write_cmd(vas): + """Build a transport-proof append command for the guest demand file. + + r582: the old `printf '%s' '<lines with real LF>'` form was mangled by + the guest-agent transport under load - the newlines arrived as LITERAL + backslash-n text and merged VAs onto one line, and the guest scanner's + parser then crashed on `int('0x20400f000\\\\n0x121000000...', 16)` + (ValueError in main; r581 boot scanner log). The scanner's death stopped + ALL new pins while the host only re-asserted known pairs - the observed + pool-pin-latency (0x121000000 never arrived while other injections + flowed). One `echo <va> >> file` per VA carries no newlines and no + backslashes at all, so no transport layer can escape them.""" + return "; ".join("echo 0x%x >> /tmp/guest_want_vas.txt" % va + for va in vas) + + def _demand_send(vas): """Append VA hints to the guest scanner's demand file (best-effort). Same channel pin_gr_stream._demand_loop uses for host-dmesg stalls; the stall watchdog uses it to ask the scanner to re-resolve the fixed pre-capture VAs (counter page / UVM pool) by content/pagemap.""" - lines = "".join("0x%x\n" % va for va in vas) + cmd = _demand_write_cmd(vas) + if not cmd: + return try: - vexec("printf '%%s' '%s' >> /tmp/guest_want_vas.txt" % lines, 30) + vexec(cmd, 30) except Exception: pass -def _start_cmd(): +def _push_guest_script(): + """Push the scanner into the guest in argv-safe chunks. + + SESSION 153-CONTINUED-64: the one-shot `echo <b64> | base64 -d >` form + put the whole ~135 KB base64 of the (grown) scanner on vgpu_vm.py's + argv, and the host's MAX_ARG_STRLEN (128 KB) rejected the exec before + it ever reached the guest ('OSError: Argument list too long'), so no + scanner started and the boot streamed zero pins. Push 60 KB chunks and + print the guest-side md5 for verification. + """ with open(GUEST_SCRIPT) as f: b64 = subprocess.run(["base64", "-w0"], stdin=f, capture_output=True, text=True, - timeout=10).stdout - prefixes = os.environ.get("CLIENT_PREFIXES", "cdt,va,victory") + timeout=10).stdout.strip() + vexec("rm -f /tmp/gps.b64", 30) + for off in range(0, len(b64), 60000): + vexec("printf %%s '%s' >> /tmp/gps.b64" % b64[off:off + 60000], 30) + return vexec("base64 -d /tmp/gps.b64 > /tmp/guest_pin_stream.py && " + "rm -f /tmp/gps.b64 && md5sum /tmp/guest_pin_stream.py " + "&& echo PUSH_OK", 60) or "" + + +def _start_cmd(): + prefixes = os.environ.get("CLIENT_PREFIXES", + "cdt,va,victory,Xorg,X,glxgears,gzdoom") vma_every = os.environ.get("STREAM_VMA_EVERY", "1") # Kill EVERY existing scanner, not just the pid-file one: a retry after # a partially-failed first attempt (pid file not written yet) used to @@ -132,14 +169,13 @@ def _start_cmd(): # failed silently (r149). return ("P=$(printf '^python3 /tmp/guest_pin_%%s.py' stream); " "sudo pkill -f \"$P\" 2>/dev/null; sleep 1; " - "echo '%s' | base64 -d > /tmp/guest_pin_stream.py; " "sudo rm -f /tmp/guest_pin_stream.pid; " "sudo setsid nohup env CLIENT_PREFIXES='%s' " "STREAM_VMA_EVERY='%s' " "python3 /tmp/guest_pin_stream.py %d %d %s %s %s " ">/tmp/guest_pin_stream.log 2>&1 & sleep 2; " "sudo cat /tmp/guest_pin_stream.pid 2>/dev/null" - % (b64, prefixes, vma_every, 2, PORT, LO, HI, PAMAX)) + % (prefixes, vma_every, 2, PORT, LO, HI, PAMAX)) def install_and_start(): @@ -148,6 +184,14 @@ def install_and_start(): # a boot without the scanner streams no pins and the client then hangs. for attempt in range(5): _heartbeat() + try: + out_push = _push_guest_script() + if "PUSH_OK" not in (out_push or ""): + log("WARN: scanner push attempt %d incomplete: %s" + % (attempt + 1, (out_push or "")[-120:])) + except Exception as e: + log("WARN: scanner push attempt %d failed: %s" + % (attempt + 1, e)) out = vexec(_start_cmd(), 60) if any(ch.isdigit() for ch in out): return True @@ -201,35 +245,141 @@ def _demand_loop(): that was not yet written when first asked is picked up on a retry. This closes the stall class where the one-shot #sem kprobe event was lost and the chain resolver never learns the client. + + r908: the GR ring VA itself ('GR ring h=0x... ... (va=0x...)', the + path-B "RAM content scan found no candidate" and "not at fb" rows) is + harvested too - the ring page is a kernel allocation the userspace + pagemap cannot cover, so the guest's content resolver is the only + source, and the pump needs the ring readable to decode its entries. + Same bounded/deduped tracking (distinct VAs, 20 s resend, 300 s + expiry) as the stalled-VA class. """ track = {} # va -> [first_seen, last_sent] + # SESSION 153-CONTINUED-64: strict demand anchors (entry va -> [first, + # last_sent]) from the anchored debugfs lines. The guest validates the + # exact [entry_va, +len) read window with the host's full stream parse + # before pinning, so the setup-push pin can never land on a page whose + # bytes the host's read will reject. Resent fast (2 s) while the hold + # is up; the loose cadence (20 s) is too slow for the client's churn. + strict_track = {} + it = 0 while True: - time.sleep(3) - try: - r = subprocess.run(["sudo", "-n", "dmesg"], - capture_output=True, text=True, timeout=20) - except Exception: - continue + # SESSION 153-CONTINUED-57: the 3 s pass was the losing half of the + # r1096 race - the beef setup-hold's demanded pages resolved in the + # guest ~2-4 s after the arm, while the pump re-read the entry + # every ~110 ms and moved on. Poll fast (0.75 s) so a newly armed + # demand goes to the guest within a second; the expensive dmesg + # harvest stays on a slower cadence (every 4th pass) because the + # debugfs setup-demand set is the time-critical source. + time.sleep(0.75) + it += 1 + r = None + if it % 4 == 1: + try: + r = subprocess.run(["sudo", "-n", "dmesg"], + capture_output=True, text=True, timeout=20) + except Exception: + continue now = time.time() - for m in re.finditer(r"(?:stalled VA #\d+|pending release starved va)=(0x[0-9a-f]+)", - (r.stdout or "")[-200000:]): - va = int(m.group(1), 16) + for m in re.finditer( + r"(?:stalled VA #\d+[:=]\s*" + r"|pending release starved va=" + r"|GR ring h=0x[0-9a-f]+ rank=\d+[^\n]*?\(va=)" + r"(0x[0-9a-f]+)" + # r914: an unreadable GR push names its pushbuffer VA - + # the GL client's push page is a kernel allocation the + # userspace pagemap cannot cover, so the guest's content + # resolver is the only source (same class as the ring VA + # demand above). Harvest it so the guest resolves and + # injects the page. + r"|sema-pushdec: h=0x[0-9a-f]+ va=(0x[0-9a-f]+)" + r"[^\n]*unreadable=1" + # SESSION 153-CONTINUED-46: a big GL push whose trusted + # chain answered an all-zero page is the beef draw that + # never captures ("sema-pushzero: ... rescued=0"). The + # promote rescue inside the read may heal it; when it + # cannot, the guest content resolver is the last source - + # harvest the VA so it resolves and injects the real page. + r"|sema-pushzero: h=0x[0-9a-f]+ va=(0x[0-9a-f]+)", + (r.stdout or "")[-2000000:] if r else ""): + va = int(m.group(1) or m.group(2) or m.group(3), 16) track.setdefault(va, [now, 0.0]) + # SESSION 153-CONTINUED-51: the pushzero line now carries the + # push length; a zero-head WINDOW push spans 1-2 pages, and the + # demand of a single page leaves the capture zero - expand the + # range and demand every page so the guest pagemap resolves the + # whole push at once (bulk demand instead of one page per cycle). + for m in re.finditer( + r"sema-pushzero: h=0x[0-9a-f]+ va=(0x[0-9a-f]+) len=(\d+)", + (r.stdout or "")[-2000000:] if r else ""): + zva = int(m.group(1), 16) + zlen = int(m.group(2)) + for zoff in range(0, min(zlen, 0x8000), 0x1000): + track.setdefault(zva + zoff, [now, 0.0]) + # SESSION 153-CONTINUED-56: the armed setup-hold's demand VA set is + # exposed by the service as STATE (debugfs), not as a dmesg row, so + # it cannot race the ring rotation under vgsp-sema spam (r1090 lost + # the setup-push pages that way). Read it every pass and feed the + # same track/resend machinery: new VAs are sent this pass (last=0), + # existing ones keep the 20 s resend / 300 s expiry cadence. + try: + rr = subprocess.run( + ["sudo", "-n", "cat", + "/sys/kernel/debug/vgsp/sema_setup_demand"], + capture_output=True, text=True, timeout=10) + if rr.returncode == 0: + for line in (rr.stdout or "").splitlines(): + ma = re.match( + r"\s*0x([0-9a-f]+)\s+0x([0-9a-f]+)\s+(\d+)\s*$", + line) + if ma: + ev = int(ma.group(1), 16) + off = int(ma.group(2), 16) + ln = int(ma.group(3)) + if ev and 0 < ln <= 0x40000: + strict_track.setdefault(ev, [now, 0.0, + off, ln]) + continue + for m in re.finditer(r"0x[0-9a-f]+", line): + sva = int(m.group(0), 16) & ~0xfff + if sva: + track.setdefault(sva, [now, 0.0]) + except Exception: + pass for va in list(track): first, last = track[va] if now - first > 300.0: del track[va] + for ev in list(strict_track): + first, last, off, ln = strict_track[ev] + if now - first > 300.0: + del strict_track[ev] + cmds = [] todo = [va for va, (f, l) in track.items() if now - l > 20.0] - if not todo: + if todo: + todo = todo[:48] + cmds.append(_demand_write_cmd(todo)) + stodo = [ev for ev, (f, l, o, ln) in strict_track.items() + if now - l > 2.0] + if stodo: + stodo = stodo[:16] + cmds.append("; ".join( + "echo S 0x%x 0x%x %d >> /tmp/guest_want_vas.txt" + % (ev, strict_track[ev][2], strict_track[ev][3]) + for ev in stodo)) + if not cmds: continue - lines = "".join("0x%x\n" % va for va in todo) try: - vexec("printf '%%s' '%s' >> /tmp/guest_want_vas.txt" % lines, 30) + vexec("; ".join(cmds), 30) except Exception: continue for va in todo: track[va][1] = now - log("demand: sent %d stalled VA(s) to guest" % len(todo)) + for ev in stodo: + strict_track[ev][1] = now + log("demand: sent %d stalled VA(s)%s to guest" % + (len(todo), + " + %d strict anchor(s)" % len(stodo) if stodo else "")) _DEMAND_STARTED = False |