gsprm: reply-latency logging + param dump cap; guest-diag: CDT_PAUSE, watcher reaping, wider CE stream window

11 files changed, 322 insertions, 22 deletions

diff --git a/STATE-CURRENT.md b/STATE-CURRENT.md
index 7a83f54..67b3d11 100644
--- a/STATE-CURRENT.md
+++ b/STATE-CURRENT.md
@@ -1,6 +1,6 @@
 # Open-IOV Current State
 
-> **Snapshot: 2026-09-29 17:59.** This is a dated public status page, not live
+> **Snapshot: 2026-09-29 23:57.** This is a dated public status page, not live
 > state; it is refreshed on every progress check. Live working notes are
 > maintained outside this repository; the public known-knowns/unknowns document
 > is authoritative here:
@@ -44,8 +44,9 @@ chain succeeds.
 ## Right now (frontier)
 
 - The stock guest driver initializes fully; the CUDA client reaches
-  `cuCtxCreate` within seconds and runs there for minutes before failing with
-  a forced `rc=999` (a timeout path, not an RM error record).
+  `cuCtxCreate` within seconds and now fails with a forced `rc=999` a few
+  seconds after the compute-channel phase (a timeout path, not an RM error
+  record).
 - The per-channel completion counters in guest memory are enumerated,
   registered and published; the client's per-channel waits clear, and the
   per-channel GPFIFO work on copy channels is consumed.
@@ -60,19 +61,25 @@ chain succeeds.
   page show all 20 compute-channel counter slots written at the correct
   16-byte stride (host releases now land exactly where the client polls), and
   auto-registration walks the full channel rank range.
-- **The registration wait is named and unblocked**: the ~16 s stall is a
-  single tracker wait (value 3) on the first copy-engine channel created
-  during UVM registration; pinning the copy-engine push buffer early (about a
-  second after it appears, instead of tens of seconds) is what lets
-  registration complete.
-- **The frontier has moved past registration** to the ~1.6 s that follow it
-  (a burst of external-allocation operations and rollback) — that is the
-  failure now being captured with the stack sampler and ioctl trace.
-- Active work: delivering the copy-engine push-buffer translation before the
-  client's first tracker wait, then diagnosing the post-registration failure.
-- One guest-side wedge occurred on the fastest-pin configuration (enlarged
-  trace buffer); it recovered by VM restart and is being retried with adjusted
-  probe settings. The host stayed stable throughout.
+- **The registration wall is root-caused and fixed**: the host consumed
+  channel submissions without delivering their tracking-semaphore release;
+  every consumed engine submission now delivers the incrementing release, and
+  the pool value advances within a second (previously it stalled until the
+  deadline). Registration no longer dominates the run.
+- **New frontier: the guest's own GSP health check**: a second after the
+  compute-channel phase the guest driver runs a GSP health check while one of
+  its resource-manager replies is pending, then rolls the context back. Two
+  suspects under test: reply-path latency now that releases are applied at a
+  high rate, and a reply whose parameter size is not canonical (the guest
+  logs `NV_ERR_INVALID_PARAMETER` on its reply-cache path).
+- Active work: an extended debug loop on the health-check failure - more
+  one-shot breakpoint tooling and fresh failure-instant guest/host snapshots
+  are being produced, with kernel and tool edits still in the tree awaiting
+  the next build/commit.
+- Guest health note: agent/SSH timeouts during a run are usually guest-agent
+  starvation and recover when the client stops; one hard wedge occurred only
+  when pushed memsets were re-enabled, so bulk pushed operations stay disabled
+  while stale translations can exist.
 - Execution backend (chosen, not started): the module will own the physical
   GPU and run its real GSP firmware as the resource-manager server, so real
   work executes with an unmodified host driver and no host daemon; see
diff --git a/kernel/gsp_emulation.c b/kernel/gsp_emulation.c
index 833ce42..4d8642a 100644
--- a/kernel/gsp_emulation.c
+++ b/kernel/gsp_emulation.c
@@ -302,6 +302,9 @@ static void gsp_emu_poll_boot(struct gsp_emu_context *ctx)
 			gsp_emu_shadow_write32(ctx, GSP_EMU_REG_FALCON_DEBUGINFO, 0);
 			nv_vgpu_info(ctx->vgpu, "GSP-BOOT: cleared CrashCat wayfinder (was 0x%x)",
 				     wfl0);
+		} else if (wfl0) {
+			pr_info_ratelimited("GSP-BOOT: DEBUGINFO nonzero (0x%x), not a wayfinder\n",
+					    wfl0);
 		}
 	}
 
diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c
index d1d87e9..777c63c 100644
--- a/kernel/gsp_rm_emulation.c
+++ b/kernel/gsp_rm_emulation.c
@@ -2545,6 +2545,7 @@ static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx,
 	u8 *reply;
 	u32 out_status = 0;
 	u32 out_params_size = 0;
+	u64 t0 = ktime_get_ns();
 
 	memcpy(&req, req_data, GSP_RM_CTRL_HDR_SIZE);
 
@@ -2730,7 +2731,7 @@ static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx,
 		/* Dump small request params so mapping/alloc controls that
 		 * carry PAs can be identified (rate-limited). */
 		if (req.paramsSize && req.paramsSize <= 96 &&
-		    rm->ctrl_param_dumps < 60) {
+		    rm->ctrl_param_dumps < 400) {
 			rm->ctrl_param_dumps++;
 			nv_vgpu_info(ctx->vgpu,
 				     "GSPRM:   ctrl 0x%x params[0..%u]=%*phN",
@@ -2765,6 +2766,15 @@ static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx,
 	memcpy(reply + 16, &out_params_size, 4);
 	memcpy(reply + 20, &req.flags, 4);
 
+	{
+		u64 dt_us = div_u64(ktime_get_ns() - t0, 1000);
+
+		nv_vgpu_info(ctx->vgpu,
+			     "GSPRM: RM_CONTROL reply cmd=0x%x hClient=0x%x hObject=0x%x in=%u out=%u status=0x%x us=%llu",
+			     req.cmd, req.hClient, req.hObject, req.paramsSize,
+			     out_params_size, out_status, dt_us);
+	}
+
 	if (gsp_rm_send(ctx, GSPRM_RPC_FUNC_GSP_RM_CONTROL, sequence, 0,
 			reply, GSP_RM_CTRL_HDR_SIZE + out_params_size))
 		nv_vgpu_err(ctx->vgpu, "GSPRM: RM_CONTROL reply send failed cmd=0x%x",
diff --git a/kernel/main.c b/kernel/main.c
index 590d956..f427828 100644
--- a/kernel/main.c
+++ b/kernel/main.c
@@ -772,7 +772,7 @@ static ssize_t nv_vgpu_vfio_dev_write(struct vfio_device *vdev, const char __use
 	struct nv_vgpu_device *vgpu = container_of(vdev, struct nv_vgpu_device, vfio_dev);
 	loff_t pos = *ppos;
 	
-	pr_info("gsp-resman-vfio: VFIO device write count=%zu pos=%lld (0x%llx)\n", count, pos, pos);
+	pr_info_ratelimited("gsp-resman-vfio: VFIO device write count=%zu pos=%lld (0x%llx)\n", count, pos, pos);
 
 	/* Config space region - route through the per-register filter so the
 	 * guest can only modify writable registers (not vendor/device IDs,
diff --git a/tools/guest-diag/bp999v2.py b/tools/guest-diag/bp999v2.py
new file mode 100644
index 0000000..766bc3b
--- /dev/null
+++ b/tools/guest-diag/bp999v2.py
@@ -0,0 +1,89 @@
+# gdb script: break at the 999/719 RETURN sites of the libcuda scanner
+# FUN_0054fa00 and dump the record values the scanner actually read (they are
+# still in its stack locals, so the host clearing the notification afterwards
+# cannot race this capture).
+#
+# Scanner layout (disassembly of libcuda+0x44fa00):
+#   rec  = *(*(*(ctx+0x9488)+0x10)+0x10)
+#   locals: rbp-0x40 rec[+0x00], rbp-0x3c rec[+0x04], rbp-0x38 rec[+0x08],
+#           rbp-0x34 rec[+0x0c] (s16), rbp-0x32 rec[+0x0e] (s16)
+#   0x44fb80 -> return 999 when force != 0 and (rec[+8]|rec[+0xc]|rec[+0xe])
+#   0x44fb6c -> return 0x2cf
+# Run: sudo gdb -p <cdt-pid> -batch -x /tmp/bp999v2.py
+import gdb
+
+out = gdb.execute("info proc mappings", to_string=True)
+base = None
+for line in out.splitlines():
+    if "libcuda" in line and "r-xp" in line:
+        f = line.split()
+        base = int(f[0], 16) - int(f[3], 16)
+        break
+print("BASE=0x%x" % (base or 0))
+if base is None:
+    raise SystemExit
+
+gdb.execute("set pagination off")
+gdb.execute("set confirm off")
+gdb.execute("break *0x%x" % (base + 0x44fb80))
+gdb.execute("break *0x%x" % (base + 0x44fb6c))
+try:
+    gdb.execute("continue")
+except gdb.error as e:
+    print("CONTINUE_ERR %s" % e)
+print("STOPPED")
+
+try:
+    rip = int(gdb.parse_and_eval("$rip")) & 0xffffffffffffffff
+    print("RIP=0x%x off=0x%x" % (rip, rip - base))
+    rdi = int(gdb.parse_and_eval("$rdi")) & 0xffffffffffffffff
+    rsi = int(gdb.parse_and_eval("$rsi")) & 0xffffffff
+    rbp = int(gdb.parse_and_eval("$rbp")) & 0xffffffffffffffff
+    print("rdi=0x%x rsi(force)=%d rbp=0x%x" % (rdi, rsi, rbp))
+    print(gdb.execute("x/5wx 0x%x" % (rbp - 0x40), to_string=True))
+    print(gdb.execute("x/2hx 0x%x" % (rbp - 0x34), to_string=True))
+except Exception as e:
+    print("locals_fail %s" % e)
+
+try:
+    print(gdb.execute("bt 14", to_string=True))
+except Exception as e:
+    print("bt_fail %s" % e)
+
+
+def rd64(a):
+    try:
+        return int(gdb.parse_and_eval("*(unsigned long *)0x%x" % a)) & 0xffffffffffffffff
+    except Exception:
+        return None
+
+
+def rd32(a):
+    try:
+        return int(gdb.parse_and_eval("*(unsigned int *)0x%x" % a)) & 0xffffffff
+    except Exception:
+        return None
+
+
+try:
+    pl = rd64(rdi)
+    n = 0
+    while pl and n < 64:
+        mgr = rd64(pl + 0x9488)
+        print("ctx[%d]=0x%x mgr=0x%x" % (n, pl, mgr or 0))
+        if mgr:
+            reg = rd64(mgr + 0x10)
+            if reg:
+                rec = rd64(reg + 0x10)
+                print("  reg=0x%x rec=0x%x" % (reg, rec or 0))
+                if rec:
+                    for i in range(0, 0x28, 4):
+                        print("  rec[+0x%02x]=0x%08x" % (i, rd32(rec + i) or 0))
+        pl = rd64(pl + 0x9470)
+        n += 1
+except Exception as e:
+    print("record_dump_fail %s" % e)
+try:
+    gdb.execute("detach")
+except Exception:
+    pass
\ No newline at end of file
diff --git a/tools/guest-diag/bpwait.py b/tools/guest-diag/bpwait.py
new file mode 100644
index 0000000..9b708c8
--- /dev/null
+++ b/tools/guest-diag/bpwait.py
@@ -0,0 +1,74 @@
+# gdb script: catch the tracker-gate exit with a NONZERO dispatcher status and
+# dump the wait-entry array the gate polls. This names the wait that ends the
+# client's context-init spin (the value that becomes the CUDA error).
+#
+# Site (libcuda+0x23f9e7): the gate loops
+#     call 0xd50e50            ; wait dispatcher
+#   0x23f9ec: mov %eax,%edx
+#   0x23f9ee: test %eax,%eax
+#   0x23f9f0: je 0x23f9c0      ; still pending -> poll again
+#   0x23f9f2: mov %edx,%eax    ; <- nonzero status: gate exits with it
+# Entry object: { *(void**)rsi; u32 count @+0x10; void *array @+0x18 }
+# Array entries are 0x28 bytes: type @+0x0, then type-specific fields.
+# Run: sudo gdb -p <cdt-pid> -batch -x /tmp/bpwait.py
+import gdb
+
+out = gdb.execute("info proc mappings", to_string=True)
+base = None
+for line in out.splitlines():
+    if "libcuda" in line and "r-xp" in line:
+        f = line.split()
+        base = int(f[0], 16) - int(f[3], 16)
+        break
+print("BASE=0x%x" % (base or 0))
+if base is None:
+    raise SystemExit
+
+gdb.execute("set pagination off")
+gdb.execute("set confirm off")
+gdb.execute("break *0x%x" % (base + 0x23f9f2))
+try:
+    gdb.execute("continue")
+except gdb.error as e:
+    print("CONTINUE_ERR %s" % e)
+print("STOPPED")
+
+
+def rd64(a):
+    try:
+        return int(gdb.parse_and_eval("*(unsigned long *)0x%x" % a)) & 0xffffffffffffffff
+    except Exception:
+        return None
+
+
+def rd32(a):
+    try:
+        return int(gdb.parse_and_eval("*(unsigned int *)0x%x" % a)) & 0xffffffff
+    except Exception:
+        return None
+
+
+try:
+    rip = int(gdb.parse_and_eval("$rip")) & 0xffffffffffffffff
+    edx = int(gdb.parse_and_eval("$edx")) & 0xffffffff
+    rsi = int(gdb.parse_and_eval("$rsi")) & 0xffffffffffffffff
+    print("RIP=0x%x off=0x%x status(edx)=0x%x" % (rip, rip - base, edx))
+    print(gdb.execute("bt 16", to_string=True))
+    ent = rd64(rsi)
+    cnt = rd32(rsi + 0x10)
+    arr = rd64(rsi + 0x18)
+    print("entry_obj=0x%x count=0x%x array=0x%x" % (ent or 0, cnt or 0, arr or 0))
+    if ent and arr:
+        print("entity hdr: " + " ".join("+0x%x=0x%x" % (o, rd32(ent + o) or 0)
+                                        for o in (0, 4, 8, 0xc, 0x10, 0x14, 0x18, 0x1c)))
+    if arr and cnt and cnt < 64:
+        for i in range(cnt):
+            e = arr + i * 0x28
+            words = [rd32(e + o) or 0 for o in (0, 4, 8, 0xc, 0x10, 0x14, 0x18, 0x1c, 0x20, 0x24)]
+            print("entry[%d] " % i + " ".join("0x%08x" % w for w in words))
+except Exception as e:
+    print("dump_fail %s" % e)
+try:
+    gdb.execute("detach")
+except Exception:
+    pass
\ No newline at end of file
diff --git a/tools/guest-diag/guest_pin_stream.py b/tools/guest-diag/guest_pin_stream.py
index 2f99bbd..94e4de9 100644
--- a/tools/guest-diag/guest_pin_stream.py
+++ b/tools/guest-diag/guest_pin_stream.py
@@ -32,9 +32,13 @@ REGION = 0x200000
 SPAN = int(os.environ.get("STREAM_SPAN", "0x80000"), 16)
 # CE push-buffer window: one 1 MB allocation per CE channel at
 # 0x120000000 + 0x100000*rank; SPAN2 covers the 1 MB boundary inside a
-# 2 MB region.
+# 2 MB region. Extend HI2 to 0x121400000: the UVM semaphore pool (the
+# tracker sems the client's context-init wait polls, pool word at
+# 0x121000000 + 4*idx) lives above the push buffers and must reach the host
+# through this fast stream - otherwise only the slow QMP chain resolution
+# pins it, and the pin lands after the client's RM thread-state deadline.
 LO2 = int(os.environ.get("STREAM_LO2", "0x120000000"), 16)
-HI2 = int(os.environ.get("STREAM_HI2", "0x120600000"), 16)
+HI2 = int(os.environ.get("STREAM_HI2", "0x121400000"), 16)
 SPAN2 = int(os.environ.get("STREAM_SPAN2", "0x180000"), 16)
 TRACE = ("/sys/kernel/tracing/trace"
          if os.path.exists("/sys/kernel/tracing/trace")
diff --git a/tools/guest-diag/push_guest_tools.sh b/tools/guest-diag/push_guest_tools.sh
index 58ebdb3..7114b03 100755
--- a/tools/guest-diag/push_guest_tools.sh
+++ b/tools/guest-diag/push_guest_tools.sh
@@ -27,6 +27,8 @@ push_b64 "$REPO/tools/test/cuda_drv_test.c" /tmp/cdt.c
 push_b64 "$HERE/dumpctx.py"     /tmp/dumpctx.py
 push_b64 "$HERE/trkinfo.py"     /tmp/trkinfo.py
 push_b64 "$HERE/bp999.py"       /tmp/bp999.py
+push_b64 "$HERE/bp999v2.py"     /tmp/bp999v2.py
+push_b64 "$HERE/bpwait.py"      /tmp/bpwait.py
 push_b64 "$HERE/ctx999.py"      /tmp/ctx999.py
 push_b64 "$HERE/scanerr.py"     /tmp/scanerr.py
 push_b64 "$HERE/sem_counters.py" /tmp/sem_counters.py
diff --git a/tools/guest-diag/run_cycle.sh b/tools/guest-diag/run_cycle.sh
new file mode 100755
index 0000000..a566469
--- /dev/null
+++ b/tools/guest-diag/run_cycle.sh
@@ -0,0 +1,102 @@
+#!/bin/bash
+# run_cycle.sh - one instrumented CUDA bring-up cycle with full capture.
+#
+# Recipe (PART 70 + PART 71): restart VM/module, push tools, install kprobes
+# (uvmt ON, uvsv/uvsvr OFF, 16 MB ring), start pin_gr_stream + semfix, launch
+# cdt (strace -f -e ioctl) holding on failure, snapshot host+guest dmesg.
+#
+# Usage: tools/guest-diag/run_cycle.sh [hold-seconds] [strace|nostrace]
+set -uo pipefail
+
+HERE="$(cd "$(dirname "$0")" && pwd)"
+REPO="$HERE/../.."
+VMEXEC="$REPO/scripts/vgpu_vm.py"
+OUT=/tmp/opencode
+HOLD="${1:-900}"
+TRACE="${2:-strace}"
+
+mkdir -p "$OUT"
+echo "[*] vmrestart"
+timeout 180 "$HERE/vmrestart.sh" || { echo "vmrestart failed"; exit 1; }
+
+echo "[*] waiting for guest"
+for i in $(seq 1 24); do
+    if timeout 10 "$VMEXEC" exec "echo UP" 2>/dev/null | grep -q UP; then
+        echo "    guest up"
+        break
+    fi
+    sleep 5
+done
+
+echo "[*] setting module params"
+sudo sh -c 'echo Y > /sys/module/open_iov_vfio/parameters/sema_fake_rel; echo Y > /sys/module/open_iov_vfio/parameters/sema_skip_ops'
+
+echo "[*] pushing guest tools"
+timeout 300 "$HERE/push_guest_tools.sh" >/dev/null 2>&1 || { echo "push failed"; exit 1; }
+
+echo "[*] loading nvidia modules in guest (kprobe symbols need them)"
+timeout 90 "$VMEXEC" exec "nvidia-smi -L >/dev/null 2>&1; lsmod | grep -q nvidia_uvm && echo UVM_LOADED" 2>/dev/null | grep -q UVM_LOADED || echo "    WARN: nvidia_uvm not loaded"
+
+echo "[*] installing guest kprobes (uvmt ON; uvsv/uvsvr OFF; 16 MB ring)"
+timeout 60 "$VMEXEC" exec "sh /tmp/kprobes.sh" 2>/dev/null | grep -E "ok|FAIL" | head -8
+timeout 30 "$VMEXEC" exec "TR=/sys/kernel/tracing; [ -d \$TR ] || TR=/sys/kernel/debug/tracing; echo 16384 > \$TR/buffer_size_kb; echo 0 > \$TR/events/kprobes/uvsv/enable; echo 0 > \$TR/events/kprobes/uvsvr/enable; echo 1 > \$TR/events/kprobes/uvmt/enable; echo 1 > \$TR/events/kprobes/uvtw/enable; echo 1 > \$TR/events/kprobes/uvbp/enable; echo 1 > \$TR/events/kprobes/uvch/enable" >/dev/null 2>&1
+
+echo "[*] starting host dmesg capture"
+sudo dmesg -w > "$OUT/hostdmesg.log" 2>&1 &
+DMESG_PID=$!
+
+echo "[*] starting pin_gr_stream + semfix"
+( setsid nohup env SEMFIX_SKIP_PB=1 SEMFIX_COUNTER_RETRY=10 \
+    python3 "$HERE/pin_gr_stream.py" > "$OUT/pin_gr_stream.log" 2>&1 & )
+sleep 1
+( setsid nohup env SEMFIX_SKIP_PB=1 SEMFIX_COUNTER_RETRY=10 \
+    python3 "$HERE/semfix_uvmt.py" 0.5 > "$OUT/semfix.log" 2>&1 & )
+sleep 2
+
+echo "[*] launching cdt (HOLD=$HOLD, $TRACE)"
+if [ "$TRACE" = "strace" ]; then
+    timeout 30 "$VMEXEC" exec \
+        "rm -f /tmp/cdt.out /tmp/strace.log; CDT_HOLD=$HOLD nohup strace -f -tt -e trace=ioctl -o /tmp/strace.log /tmp/cdt >/tmp/cdt.out 2>&1 & echo STARTED" \
+        >/dev/null 2>&1
+elif [ "$TRACE" = "gdb" ]; then
+    timeout 30 "$VMEXEC" exec "rm -f /tmp/cdt.out /tmp/bpwait.out; sync; echo CLEARED" >/dev/null 2>&1
+    timeout 30 "$VMEXEC" exec \
+        "rm -f /tmp/cdt.out /tmp/bpwait.out; CDT_PAUSE=60 CDT_HOLD=$HOLD nohup /tmp/cdt >/tmp/cdt.out 2>&1 & echo STARTED" \
+        >/dev/null 2>&1
+    PID=""
+    for i in $(seq 1 12); do
+        PID=$(timeout 20 "$VMEXEC" exec "pgrep -x cdt" 2>/dev/null | grep -E '^[0-9]+$' | head -1)
+        [ -n "$PID" ] && break
+        sleep 2
+    done
+    echo "    cdt pid=$PID; arming bpwait (tracker-gate exit, serial+file)"
+    timeout 40 "$VMEXEC" exec \
+        "nohup sh -c \"gdb -p '$PID' -batch -x /tmp/bpwait.py < /dev/null 2>&1 | tee /tmp/bpwait.out > /dev/ttyS0\" >/dev/null 2>&1 & echo ARMED" \
+        >/dev/null 2>&1
+else
+    timeout 30 "$VMEXEC" exec \
+        "rm -f /tmp/cdt.out; CDT_HOLD=$HOLD nohup /tmp/cdt >/tmp/cdt.out 2>&1 & echo STARTED" \
+        >/dev/null 2>&1
+fi
+
+echo "[*] waiting for the run to finish (watch cdt.out)"
+for i in $(seq 1 90); do
+    sleep 2
+    OUT_TXT=$(timeout 20 "$VMEXEC" exec "cat /tmp/cdt.out 2>/dev/null" 2>/dev/null | tail -6)
+    if echo "$OUT_TXT" | grep -q "FAILED rc="; then
+        echo "    failure observed after ~$((i*2))s"
+        break
+    fi
+done
+
+echo "[*] snapshotting guest dmesg"
+timeout 60 "$VMEXEC" exec "dmesg > /tmp/guestdmesg.log; cp /tmp/cdt.out /tmp/cdt.out.snap; [ -f /tmp/strace.log ] && cp /tmp/strace.log /tmp/strace.log.snap; [ -f /tmp/bpwait.out ] && cp /tmp/bpwait.out /tmp/bpwait.out.snap; echo DONE" >/dev/null 2>&1
+sleep 1
+sudo kill "$DMESG_PID" 2>/dev/null
+
+echo "[*] collecting artifacts to $OUT"
+for f in cdt.out.snap strace.log.snap bpwait.out.snap guestdmesg.log; do
+    timeout 60 "$VMEXEC" exec "cat /tmp/$f" > "$OUT/$f" 2>/dev/null
+done
+echo "[*] done: $OUT/cdt.out.snap $OUT/strace.log.snap $OUT/guestdmesg.log $OUT/hostdmesg.log"
+tail -5 "$OUT/cdt.out.snap" 2>/dev/null
\ No newline at end of file
diff --git a/tools/guest-diag/vmrestart.sh b/tools/guest-diag/vmrestart.sh
index 5d15852..8c15a89 100755
--- a/tools/guest-diag/vmrestart.sh
+++ b/tools/guest-diag/vmrestart.sh
@@ -9,8 +9,9 @@ sudo pkill -f qemu-system-x86_64 2>/dev/null
 # Reap host-side watchers too: a pin_gr_stream/semfix pair left over from the
 # previous boot holds the vsock port and the single QMP socket slot and will
 # silently corrupt the next run (two QMP clients).
-pkill -f "guest-diag/pin_gr_stream.py" 2>/dev/null
-pkill -f "guest-diag/semfix_uvmt.py" 2>/dev/null
+pkill -f "pin_gr_stream" 2>/dev/null
+pkill -f "semfix_uvmt" 2>/dev/null
+pkill -f "guest_pin_stream" 2>/dev/null
 sleep 4
 sudo rm -f /tmp/qemu-guest-agent.sock /tmp/qemu-qmp.sock
 
diff --git a/tools/test/cuda_drv_test.c b/tools/test/cuda_drv_test.c
index 8995a55..f3ee02f 100644
--- a/tools/test/cuda_drv_test.c
+++ b/tools/test/cuda_drv_test.c
@@ -99,6 +99,14 @@ int main(void)
     printf("device 0 = %d\n", dev);
 
     MARK("cuCtxCreate");
+    {
+        const char *pause = getenv("CDT_PAUSE");
+        if (pause && atoi(pause) > 0) {
+            printf("CDT_PAUSE: sleeping %d s before cuCtxCreate\n", atoi(pause));
+            fflush(stdout);
+            sleep((unsigned)atoi(pause));
+        }
+    }
     CHECK(cuCtxCreate(&ctx, 0, dev));
     printf("context = %p\n", ctx);