blob: 3aaa723fcf7849f2e4a7814bd7d861ab5f3bf915 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
|
#!/bin/bash
# cycle_cdt.sh - full test cycle: push tools, start cdt, inject the libcuda
# tracker page mapping when the tracker wait is reached, poke UVM's tracker
# when it stalls, and report where cdt ends up.
set -u
cd "$(dirname "$(readlink -f "$0")")/../.."
echo "[*] waiting for guest"
for i in $(seq 1 30); do
timeout 20 ./scripts/vgpu_vm.py exec --timeout 12 'uptime >/dev/null 2>&1 && echo READY' 2>/dev/null | grep -q READY && break
sleep 5
done
./tools/guest-diag/push_guest_tools.sh >/tmp/opencode/push.log 2>&1 || { echo "PUSH FAILED"; tail -3 /tmp/opencode/push.log; exit 1; }
echo "[*] tools pushed"
timeout 40 ./scripts/vgpu_vm.py exec --timeout 35 'sudo nvidia-modprobe -c 0; sudo nvidia-modprobe -u; sudo modprobe nvidia-uvm; sudo sh -c "echo 1 > /sys/module/nvidia_uvm/parameters/uvm_debug_prints"; sudo rm -f /tmp/cdt.out; sudo setsid /tmp/cdt > /tmp/cdt.out 2>&1 & sleep 2; tail -1 /tmp/cdt.out' 2>/dev/null | tail -1
echo "[*] cdt started"
# wait for the libcuda tracker wait and inject the page mapping
SEEN_CTX=0
for i in $(seq 1 24); do
sleep 10
STATE=$(timeout 20 ./scripts/vgpu_vm.py exec --timeout 15 'tail -1 /tmp/cdt.out' 2>/dev/null | tail -1)
echo " [$((i*10))s] $STATE"
if echo "$STATE" | grep -q "cuCtxCreate"; then
SEEN_CTX=1
if [ ! -f /tmp/opencode/injected_tracker ]; then
if /tmp/opencode/inject_tracker.sh >/tmp/opencode/inject.log 2>&1; then
touch /tmp/opencode/injected_tracker
echo " [*] tracker mapping injected: $(tail -1 /tmp/opencode/inject.log)"
fi
fi
fi
if echo "$STATE" | grep -qE "FAILED|ALL OK|cuMemAlloc"; then
echo "[*] cuCtxCreate finished: $STATE"
break
fi
done
# if UVM's tracking semaphore lags, poke it forward
SEMAPA=$(sudo sh -c 'echo 0x121000010,1 > /sys/module/vgsp/parameters/sema_probe' 2>/dev/null; sleep 2; sudo dmesg -T | grep -A2 "sema-probe: va=0x121000010" | grep -oE "pa=0x[0-9a-f]+" | head -1 | cut -d= -f2)
if [ -n "$SEMAPA" ]; then
CUR=$(sudo dmesg -T | grep -A3 "sema-probe: va=0x121000010" | grep -oE "\[0\] 0x[0-9a-f]+" | tail -1 | awk '{print $2}')
echo "[*] UVM tracking sema pa=$SEMAPA value=$CUR"
fi
|
