diff options
Diffstat (limited to 'tools/guest-diag/cycle_cdt.sh')
| -rwxr-xr-x | tools/guest-diag/cycle_cdt.sh | 46 |
1 files changed, 46 insertions, 0 deletions
diff --git a/tools/guest-diag/cycle_cdt.sh b/tools/guest-diag/cycle_cdt.sh new file mode 100755 index 0000000..009bdfc --- /dev/null +++ b/tools/guest-diag/cycle_cdt.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# cycle_cdt.sh - full test cycle: push tools, start cdt, inject the libcuda +# tracker page mapping when the tracker wait is reached, poke UVM's tracker +# when it stalls, and report where cdt ends up. +set -u +cd "$(dirname "$(readlink -f "$0")")/../.." + +echo "[*] waiting for guest" +for i in $(seq 1 30); do + timeout 20 ./scripts/vgpu_vm.py exec --timeout 12 'uptime >/dev/null 2>&1 && echo READY' 2>/dev/null | grep -q READY && break + sleep 5 +done + +./tools/guest-diag/push_guest_tools.sh >/tmp/opencode/push.log 2>&1 || { echo "PUSH FAILED"; tail -3 /tmp/opencode/push.log; exit 1; } +echo "[*] tools pushed" + +timeout 40 ./scripts/vgpu_vm.py exec --timeout 35 'sudo nvidia-modprobe -c 0; sudo nvidia-modprobe -u; sudo modprobe nvidia-uvm; sudo sh -c "echo 1 > /sys/module/nvidia_uvm/parameters/uvm_debug_prints"; sudo rm -f /tmp/cdt.out; sudo setsid /tmp/cdt > /tmp/cdt.out 2>&1 & sleep 2; tail -1 /tmp/cdt.out' 2>/dev/null | tail -1 +echo "[*] cdt started" + +# wait for the libcuda tracker wait and inject the page mapping +SEEN_CTX=0 +for i in $(seq 1 24); do + sleep 10 + STATE=$(timeout 20 ./scripts/vgpu_vm.py exec --timeout 15 'tail -1 /tmp/cdt.out' 2>/dev/null | tail -1) + echo " [$((i*10))s] $STATE" + if echo "$STATE" | grep -q "cuCtxCreate"; then + SEEN_CTX=1 + if [ ! -f /tmp/opencode/injected_tracker ]; then + if /tmp/opencode/inject_tracker.sh >/tmp/opencode/inject.log 2>&1; then + touch /tmp/opencode/injected_tracker + echo " [*] tracker mapping injected: $(tail -1 /tmp/opencode/inject.log)" + fi + fi + fi + if echo "$STATE" | grep -qE "FAILED|ALL OK|cuMemAlloc"; then + echo "[*] cuCtxCreate finished: $STATE" + break + fi +done + +# if UVM's tracking semaphore lags, poke it forward +SEMAPA=$(sudo sh -c 'echo 0x121000010,1 > /sys/module/open_iov_vfio/parameters/sema_probe' 2>/dev/null; sleep 2; sudo dmesg -T | grep -A2 "sema-probe: va=0x121000010" | grep -oE "pa=0x[0-9a-f]+" | head -1 | cut -d= -f2) +if [ -n "$SEMAPA" ]; then + CUR=$(sudo dmesg -T | grep -A3 "sema-probe: va=0x121000010" | grep -oE "\[0\] 0x[0-9a-f]+" | tail -1 | awk '{print $2}') + echo "[*] UVM tracking sema pa=$SEMAPA value=$CUR" +fi |
