From 03d8df9ece5bd2d81703b3eeac07e4d889639674 Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Mon, 28 Sep 2026 13:14:16 -0400 Subject: guest-diag: adaptive miss-driven pinner; cdt holds on failure for inspection pin_miss_pages.sh watches the host journal for release-miss VAs, resolves each page for the live cdt pid through the guest pagemap and injects it (the injection rewinds the pump so releases consumed under a stale pin are re-read); it re-pins when a new cdt pid appears. cuda_drv_test prints cuGetErrorName/String on failure and, with CDT_HOLD=, stays alive so the RM error record and tracker state can be inspected after cuCtxCreate has given up. --- tools/guest-diag/pin_miss_pages.sh | 59 ++++++++++++++++++++++++++++++++++++++ tools/test/cuda_drv_test.c | 25 +++++++++++++++- 2 files changed, 83 insertions(+), 1 deletion(-) create mode 100755 tools/guest-diag/pin_miss_pages.sh diff --git a/tools/guest-diag/pin_miss_pages.sh b/tools/guest-diag/pin_miss_pages.sh new file mode 100755 index 0000000..80b8c77 --- /dev/null +++ b/tools/guest-diag/pin_miss_pages.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# pin_miss_pages.sh - adaptive tracker-sem pinner. +# +# Watches the host journal for "release miss va=0x..." lines, resolves those +# pages for the *live* cdt pid via the guest's pagemap (pmapall.py) and injects +# the translation with sema_map_add (which also rewinds the pump by 8 entries, +# so releases already consumed under a stale pin are re-read). Re-pins when a +# new cdt pid appears: pins are per-process pages and survive across cdt runs, +# which is exactly the stale-pin poisoning the rewind fixes - but the *new* +# process's pages must still be injected. +# +# Usage: tools/guest-diag/pin_miss_pages.sh [minutes] (default 30) +set -u +cd "$(dirname "$(readlink -f "$0")")/../.." +EXEC() { timeout 40 ./scripts/vgpu_vm.py exec --timeout 35 "$1" 2>/dev/null; } + +B64=$(base64 -w0 tools/guest-diag/guest_pmap_all.py) +EXEC "echo '$B64' | base64 -d > /tmp/pmapall.py" >/dev/null + +MAXMIN=${1:-30} +START=$(date '+%Y-%m-%d %H:%M:%S') +seen="" +last_pid="" + +for ((i = 1; i <= MAXMIN * 4; i++)); do + sleep 15 + pid=$(EXEC 'pgrep -x cdt | head -1' | tail -1) + if [ -z "$pid" ]; then + echo "[$((i * 15))s] no cdt" + continue + fi + if [ "$pid" != "$last_pid" ]; then + seen="" + last_pid="$pid" + echo "[$((i * 15))s] new cdt pid $pid" + fi + vas=$(sudo journalctl -k --since "$START" --no-pager 2>/dev/null | + grep -aoE 'release miss va=0x[0-9a-f]+' | grep -oE '0x[0-9a-f]+$' | sort -u) + vas=$(printf '%s\n0x20400f000\n' "$vas" | sort -u) + for va in $vas; do + page=$(printf '0x%x' $((va & -4096))) + case " $seen " in + *" $page "*) continue ;; + esac + line=$(EXEC "sudo python3 /tmp/pmapall.py $pid $page 1" | + grep -aE '^0x[0-9a-f]+,0x[0-9a-f]+$' | head -1) + if [ -n "$line" ]; then + echo "$line,0x1000,1" | sudo tee /sys/module/open_iov_vfio/parameters/sema_map_add >/dev/null + echo "[$((i * 15))s] pinned $line (pid $pid)" + seen="$seen $page" + fi + done + st=$(EXEC 'sudo tail -1 /tmp/cdt.out' | tail -1) + case "$st" in + *FAILED* | *"ALL OK"*) echo "[$((i * 15))s] cdt finished: $st" ;; + esac +done +echo "=== final cdt.out ===" +EXEC 'sudo cat /tmp/cdt.out' diff --git a/tools/test/cuda_drv_test.c b/tools/test/cuda_drv_test.c index 20ad88a..8995a55 100644 --- a/tools/test/cuda_drv_test.c +++ b/tools/test/cuda_drv_test.c @@ -15,7 +15,9 @@ * gcc -O1 -o /tmp/cdt cuda_drv_test.c -ldl */ #include +#include #include +#include #include #include @@ -24,9 +26,30 @@ typedef int CUdevice; typedef void *CUcontext; typedef uint64_t CUdeviceptr; +/* On failure: print cuGetErrorName/String and, when CDT_HOLD= is + * set, keep the process alive so the live state (trackers, RM error record) + * can be inspected with gdb after cuCtxCreate has given up. */ +static void fail_info(CUresult r) +{ + const char *(*errname)(CUresult, const char **) = dlsym(RTLD_DEFAULT, "cuGetErrorName"); + const char *(*errstr)(CUresult, const char **) = dlsym(RTLD_DEFAULT, "cuGetErrorString"); + const char *s = NULL; + const char *hold = getenv("CDT_HOLD"); + + if (errname && errname(r, &s) == 0 && s) + printf("cuGetErrorName: %s\n", s); + if (errstr && errstr(r, &s) == 0 && s) + printf("cuGetErrorString: %s\n", s); + if (hold && atoi(hold) > 0) { + printf("CDT_HOLD: sleeping %d s for inspection\n", atoi(hold)); + fflush(stdout); + sleep((unsigned)atoi(hold)); + } +} + #define CHECK(expr) do { \ CUresult r = (expr); \ - if (r) { printf("%-24s FAILED rc=%d\n", #expr, r); return 1; } \ + if (r) { printf("%-24s FAILED rc=%d\n", #expr, r); fail_info(r); return 1; } \ printf("%-24s OK\n", #expr); \ } while (0) -- cgit v1.2.3