sema-svc: async VA->PA injection queue; guest-diag: fast counter+pin injector, wait sampler

1 files changed, 73 insertions, 0 deletions

diff --git a/tools/guest-diag/track_waits.sh b/tools/guest-diag/track_waits.sh
new file mode 100644
index 0000000..4c9cd1a
--- /dev/null
+++ b/tools/guest-diag/track_waits.sh
@@ -0,0 +1,73 @@
+#!/bin/bash
+# track_waits.sh - guest-side wait sampler: while a CUDA client (cdt) runs,
+# repeatedly dump the per-channel completion counters (values), the tracker
+# wait entries (trkinfo.py, every 2nd pass) and the push pages the host could
+# not resolve (read_va.py), so the last signal before a failure is recorded.
+#
+# Written for the guest (no host paths). Run as root in the guest:
+#   sudo /tmp/track_waits.sh <pid> [samples] [sleep]
+# Logs: /tmp/trk.log, /tmp/cnt.log, /tmp/push.log
+set -u
+
+PID="${1:?usage: track_waits.sh <pid> [samples] [sleep]}"
+N="${2:-600}"
+SLEEP="${3:-1}"
+PAGE_VA="${PAGE_VA:-0x20400f000}"
+COUNT="${COUNT:-20}"
+STRIDE="${STRIDE:-16}"
+OFFSET="${OFFSET:-0xec0}"
+
+: > /tmp/trk.log
+: > /tmp/cnt.log
+: > /tmp/push.log
+
+python3 - "$PID" "$PAGE_VA" "$COUNT" "$STRIDE" "$OFFSET" <<'PY' >> /tmp/cnt.log 2>&1 &
+import os, sys, time
+pid = int(sys.argv[1]); page = int(sys.argv[2], 16)
+count = int(sys.argv[3]); stride = int(sys.argv[4], 16); off = int(sys.argv[5], 16)
+pm = os.open('/proc/%d/pagemap' % pid, os.O_RDONLY)
+mem = os.open('/proc/%d/mem' % pid, os.O_RDONLY)
+while True:
+    ts = time.strftime('%T')
+    vals = []
+    for i in range(count):
+        va = page + off + i * stride
+        try:
+            os.lseek(pm, (va // 4096) * 8, 0)
+            e = int.from_bytes(os.read(pm, 8), 'little')
+            present = (e >> 63) & 1
+            if present:
+                v = int.from_bytes(os.pread(mem, 4, va), 'little')
+                vals.append('%d:0x%x' % (i, v))
+            else:
+                vals.append('%d:-' % i)
+        except OSError:
+            vals.append('%d:ERR' % i)
+    print('%s %s' % (ts, ' '.join(vals)), flush=True)
+    time.sleep(0.5)
+PY
+CNTPID=$!
+
+for i in $(seq 1 "$N"); do
+    TS=$(date +%T.%3N)
+    kill -0 "$PID" 2>/dev/null || { echo "PROCESS GONE at $TS (sample $i)" >> /tmp/trk.log; break; }
+
+    if [ $((i % 2)) -eq 1 ]; then
+        echo "=== $TS sample $i ===" >> /tmp/trk.log
+        timeout 8 gdb -p "$PID" -batch -x /tmp/trkinfo.py 2>&1 \
+            | grep -vE "^\[|Reading symbols|^0x[0-9a-f]+ in |warning:" >> /tmp/trk.log
+    fi
+
+    {
+        echo "=== $TS push pages ==="
+        python3 /tmp/guest_pmap.py "$PID" 0x201000000 0x203400000 0x203400020 2>&1
+        for va in 0x201000000 0x203400000 0x203400020; do
+            echo "--- $va"
+            timeout 5 python3 /tmp/read_va.py "$PID" "$va" 64 --u32 2>&1
+        done
+    } >> /tmp/push.log
+
+    sleep "$SLEEP"
+done
+kill "$CNTPID" 2>/dev/null
+echo "sampler done" >> /tmp/trk.log