guest-diag: adaptive miss-driven pinner; cdt holds on failure for inspection

pin_miss_pages.sh watches the host journal for release-miss VAs, resolves
each page for the live cdt pid through the guest pagemap and injects it (the
injection rewinds the pump so releases consumed under a stale pin are
re-read); it re-pins when a new cdt pid appears.  cuda_drv_test prints
cuGetErrorName/String on failure and, with CDT_HOLD=<seconds>, stays alive so
the RM error record and tracker state can be inspected after cuCtxCreate has
given up.

1 files changed, 59 insertions, 0 deletions

diff --git a/tools/guest-diag/pin_miss_pages.sh b/tools/guest-diag/pin_miss_pages.sh
new file mode 100755
index 0000000..80b8c77
--- /dev/null
+++ b/tools/guest-diag/pin_miss_pages.sh
@@ -0,0 +1,59 @@
+#!/bin/bash
+# pin_miss_pages.sh - adaptive tracker-sem pinner.
+#
+# Watches the host journal for "release miss va=0x..." lines, resolves those
+# pages for the *live* cdt pid via the guest's pagemap (pmapall.py) and injects
+# the translation with sema_map_add (which also rewinds the pump by 8 entries,
+# so releases already consumed under a stale pin are re-read). Re-pins when a
+# new cdt pid appears: pins are per-process pages and survive across cdt runs,
+# which is exactly the stale-pin poisoning the rewind fixes - but the *new*
+# process's pages must still be injected.
+#
+# Usage: tools/guest-diag/pin_miss_pages.sh [minutes]   (default 30)
+set -u
+cd "$(dirname "$(readlink -f "$0")")/../.."
+EXEC() { timeout 40 ./scripts/vgpu_vm.py exec --timeout 35 "$1" 2>/dev/null; }
+
+B64=$(base64 -w0 tools/guest-diag/guest_pmap_all.py)
+EXEC "echo '$B64' | base64 -d > /tmp/pmapall.py" >/dev/null
+
+MAXMIN=${1:-30}
+START=$(date '+%Y-%m-%d %H:%M:%S')
+seen=""
+last_pid=""
+
+for ((i = 1; i <= MAXMIN * 4; i++)); do
+    sleep 15
+    pid=$(EXEC 'pgrep -x cdt | head -1' | tail -1)
+    if [ -z "$pid" ]; then
+        echo "[$((i * 15))s] no cdt"
+        continue
+    fi
+    if [ "$pid" != "$last_pid" ]; then
+        seen=""
+        last_pid="$pid"
+        echo "[$((i * 15))s] new cdt pid $pid"
+    fi
+    vas=$(sudo journalctl -k --since "$START" --no-pager 2>/dev/null |
+        grep -aoE 'release miss va=0x[0-9a-f]+' | grep -oE '0x[0-9a-f]+$' | sort -u)
+    vas=$(printf '%s\n0x20400f000\n' "$vas" | sort -u)
+    for va in $vas; do
+        page=$(printf '0x%x' $((va & -4096)))
+        case " $seen " in
+        *" $page "*) continue ;;
+        esac
+        line=$(EXEC "sudo python3 /tmp/pmapall.py $pid $page 1" |
+            grep -aE '^0x[0-9a-f]+,0x[0-9a-f]+$' | head -1)
+        if [ -n "$line" ]; then
+            echo "$line,0x1000,1" | sudo tee /sys/module/open_iov_vfio/parameters/sema_map_add >/dev/null
+            echo "[$((i * 15))s] pinned $line (pid $pid)"
+            seen="$seen $page"
+        fi
+    done
+    st=$(EXEC 'sudo tail -1 /tmp/cdt.out' | tail -1)
+    case "$st" in
+    *FAILED* | *"ALL OK"*) echo "[$((i * 15))s] cdt finished: $st" ;;
+    esac
+done
+echo "=== final cdt.out ==="
+EXEC 'sudo cat /tmp/cdt.out'