guest-diag: adaptive miss-driven pinner; cdt holds on failure for inspection

pin_miss_pages.sh watches the host journal for release-miss VAs, resolves
each page for the live cdt pid through the guest pagemap and injects it (the
injection rewinds the pump so releases consumed under a stale pin are
re-read); it re-pins when a new cdt pid appears.  cuda_drv_test prints
cuGetErrorName/String on failure and, with CDT_HOLD=<seconds>, stays alive so
the RM error record and tracker state can be inspected after cuCtxCreate has
given up.

2 files changed, 83 insertions, 1 deletions

diff --git a/tools/guest-diag/pin_miss_pages.sh b/tools/guest-diag/pin_miss_pages.sh
new file mode 100755
index 0000000..80b8c77
--- /dev/null
+++ b/tools/guest-diag/pin_miss_pages.sh
@@ -0,0 +1,59 @@
+#!/bin/bash
+# pin_miss_pages.sh - adaptive tracker-sem pinner.
+#
+# Watches the host journal for "release miss va=0x..." lines, resolves those
+# pages for the *live* cdt pid via the guest's pagemap (pmapall.py) and injects
+# the translation with sema_map_add (which also rewinds the pump by 8 entries,
+# so releases already consumed under a stale pin are re-read). Re-pins when a
+# new cdt pid appears: pins are per-process pages and survive across cdt runs,
+# which is exactly the stale-pin poisoning the rewind fixes - but the *new*
+# process's pages must still be injected.
+#
+# Usage: tools/guest-diag/pin_miss_pages.sh [minutes]   (default 30)
+set -u
+cd "$(dirname "$(readlink -f "$0")")/../.."
+EXEC() { timeout 40 ./scripts/vgpu_vm.py exec --timeout 35 "$1" 2>/dev/null; }
+
+B64=$(base64 -w0 tools/guest-diag/guest_pmap_all.py)
+EXEC "echo '$B64' | base64 -d > /tmp/pmapall.py" >/dev/null
+
+MAXMIN=${1:-30}
+START=$(date '+%Y-%m-%d %H:%M:%S')
+seen=""
+last_pid=""
+
+for ((i = 1; i <= MAXMIN * 4; i++)); do
+    sleep 15
+    pid=$(EXEC 'pgrep -x cdt | head -1' | tail -1)
+    if [ -z "$pid" ]; then
+        echo "[$((i * 15))s] no cdt"
+        continue
+    fi
+    if [ "$pid" != "$last_pid" ]; then
+        seen=""
+        last_pid="$pid"
+        echo "[$((i * 15))s] new cdt pid $pid"
+    fi
+    vas=$(sudo journalctl -k --since "$START" --no-pager 2>/dev/null |
+        grep -aoE 'release miss va=0x[0-9a-f]+' | grep -oE '0x[0-9a-f]+$' | sort -u)
+    vas=$(printf '%s\n0x20400f000\n' "$vas" | sort -u)
+    for va in $vas; do
+        page=$(printf '0x%x' $((va & -4096)))
+        case " $seen " in
+        *" $page "*) continue ;;
+        esac
+        line=$(EXEC "sudo python3 /tmp/pmapall.py $pid $page 1" |
+            grep -aE '^0x[0-9a-f]+,0x[0-9a-f]+$' | head -1)
+        if [ -n "$line" ]; then
+            echo "$line,0x1000,1" | sudo tee /sys/module/open_iov_vfio/parameters/sema_map_add >/dev/null
+            echo "[$((i * 15))s] pinned $line (pid $pid)"
+            seen="$seen $page"
+        fi
+    done
+    st=$(EXEC 'sudo tail -1 /tmp/cdt.out' | tail -1)
+    case "$st" in
+    *FAILED* | *"ALL OK"*) echo "[$((i * 15))s] cdt finished: $st" ;;
+    esac
+done
+echo "=== final cdt.out ==="
+EXEC 'sudo cat /tmp/cdt.out'
diff --git a/tools/test/cuda_drv_test.c b/tools/test/cuda_drv_test.c
index 20ad88a..8995a55 100644
--- a/tools/test/cuda_drv_test.c
+++ b/tools/test/cuda_drv_test.c
@@ -15,7 +15,9 @@
  *   gcc -O1 -o /tmp/cdt cuda_drv_test.c -ldl
  */
 #include <stdio.h>
+#include <stdlib.h>
 #include <string.h>
+#include <unistd.h>
 #include <dlfcn.h>
 #include <stdint.h>
 
@@ -24,9 +26,30 @@ typedef int CUdevice;
 typedef void *CUcontext;
 typedef uint64_t CUdeviceptr;
 
+/* On failure: print cuGetErrorName/String and, when CDT_HOLD=<seconds> is
+ * set, keep the process alive so the live state (trackers, RM error record)
+ * can be inspected with gdb after cuCtxCreate has given up. */
+static void fail_info(CUresult r)
+{
+    const char *(*errname)(CUresult, const char **) = dlsym(RTLD_DEFAULT, "cuGetErrorName");
+    const char *(*errstr)(CUresult, const char **) = dlsym(RTLD_DEFAULT, "cuGetErrorString");
+    const char *s = NULL;
+    const char *hold = getenv("CDT_HOLD");
+
+    if (errname && errname(r, &s) == 0 && s)
+        printf("cuGetErrorName: %s\n", s);
+    if (errstr && errstr(r, &s) == 0 && s)
+        printf("cuGetErrorString: %s\n", s);
+    if (hold && atoi(hold) > 0) {
+        printf("CDT_HOLD: sleeping %d s for inspection\n", atoi(hold));
+        fflush(stdout);
+        sleep((unsigned)atoi(hold));
+    }
+}
+
 #define CHECK(expr) do { \
     CUresult r = (expr); \
-    if (r) { printf("%-24s FAILED rc=%d\n", #expr, r); return 1; } \
+    if (r) { printf("%-24s FAILED rc=%d\n", #expr, r); fail_info(r); return 1; } \
     printf("%-24s OK\n", #expr); \
 } while (0)