#!/bin/bash # cycle_cdt.sh - full test cycle: push tools, start cdt, inject the libcuda # tracker page mapping when the tracker wait is reached, poke UVM's tracker # when it stalls, and report where cdt ends up. set -u cd "$(dirname "$(readlink -f "$0")")/../.." echo "[*] waiting for guest" for i in $(seq 1 30); do timeout 20 ./scripts/vgpu_vm.py exec --timeout 12 'uptime >/dev/null 2>&1 && echo READY' 2>/dev/null | grep -q READY && break sleep 5 done ./tools/guest-diag/push_guest_tools.sh >/tmp/opencode/push.log 2>&1 || { echo "PUSH FAILED"; tail -3 /tmp/opencode/push.log; exit 1; } echo "[*] tools pushed" timeout 40 ./scripts/vgpu_vm.py exec --timeout 35 'sudo nvidia-modprobe -c 0; sudo nvidia-modprobe -u; sudo modprobe nvidia-uvm; sudo sh -c "echo 1 > /sys/module/nvidia_uvm/parameters/uvm_debug_prints"; sudo rm -f /tmp/cdt.out; sudo setsid /tmp/cdt > /tmp/cdt.out 2>&1 & sleep 2; tail -1 /tmp/cdt.out' 2>/dev/null | tail -1 echo "[*] cdt started" # wait for the libcuda tracker wait and inject the page mapping SEEN_CTX=0 for i in $(seq 1 24); do sleep 10 STATE=$(timeout 20 ./scripts/vgpu_vm.py exec --timeout 15 'tail -1 /tmp/cdt.out' 2>/dev/null | tail -1) echo " [$((i*10))s] $STATE" if echo "$STATE" | grep -q "cuCtxCreate"; then SEEN_CTX=1 if [ ! -f /tmp/opencode/injected_tracker ]; then if /tmp/opencode/inject_tracker.sh >/tmp/opencode/inject.log 2>&1; then touch /tmp/opencode/injected_tracker echo " [*] tracker mapping injected: $(tail -1 /tmp/opencode/inject.log)" fi fi fi if echo "$STATE" | grep -qE "FAILED|ALL OK|cuMemAlloc"; then echo "[*] cuCtxCreate finished: $STATE" break fi done # if UVM's tracking semaphore lags, poke it forward SEMAPA=$(sudo sh -c 'echo 0x121000010,1 > /sys/module/vgsp/parameters/sema_probe' 2>/dev/null; sleep 2; sudo dmesg -T | grep -A2 "sema-probe: va=0x121000010" | grep -oE "pa=0x[0-9a-f]+" | head -1 | cut -d= -f2) if [ -n "$SEMAPA" ]; then CUR=$(sudo dmesg -T | grep -A3 "sema-probe: va=0x121000010" | grep -oE "\[0\] 0x[0-9a-f]+" | tail -1 | awk '{print $2}') echo "[*] UVM tracking sema pa=$SEMAPA value=$CUR" fi