blob: cc333fbe16e4e5e85cca78af22f157b2bb940254 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
|
#!/bin/bash
# cuda_cycle.sh - one full CUDA bring-up attempt: restart the VM, push the
# guest tools, start cdt holding on failure, wait for the 20 GR channels,
# register/pin the completion counters, and arm the RM-error scanner so the
# next forced 999 names its record.
#
# Usage: tools/guest-diag/cuda_cycle.sh [hold-seconds]
# Output: guest /tmp/cdt.out, /tmp/scanerr.out; host sema-svc log in dmesg.
set -uo pipefail
HERE="$(cd "$(dirname "$0")" && pwd)"
REPO="$HERE/../.."
VMEXEC="$REPO/scripts/vgpu_vm.py"
HOLD="${1:-600}"
echo "[*] vmrestart"
timeout 150 "$HERE/vmrestart.sh" || { echo "vmrestart failed"; exit 1; }
echo "[*] waiting for guest"
for i in $(seq 1 20); do
if timeout 10 "$VMEXEC" exec "echo UP" 2>/dev/null | grep -q UP; then
break
fi
sleep 5
done
echo "[*] pushing guest tools"
timeout 240 "$HERE/push_guest_tools.sh" >/dev/null 2>&1 || { echo "push failed"; exit 1; }
echo "[*] starting cdt (CDT_HOLD=$HOLD)"
timeout 30 "$VMEXEC" exec \
"rm -f /tmp/cdt.out /tmp/scanerr.out; CDT_HOLD=$HOLD nohup /tmp/cdt >/tmp/cdt.out 2>&1 & echo STARTED" \
>/dev/null 2>&1
echo "[*] waiting for 20 GR channels"
# The client times out its context-init wait in ~30 s, so detection must be
# cheap: count the per-channel first-pump log lines straight from dmesg
# instead of round-tripping through the chan_dump parameter every second.
for i in $(seq 1 240); do
n=$(sudo dmesg -T | grep -c "sema-chan: h=0x5c" || true)
if [ "$n" -ge 20 ]; then
echo " GR channels up (poll $i)"
break
fi
sleep 0.5
done
echo "[*] injecting counters"
timeout 150 "$HERE/inject_counters.sh" 2>/dev/null | grep -c registered
PID=$(timeout 20 "$VMEXEC" exec "pgrep -x cdt" 2>/dev/null | grep -E '^[0-9]+$' | head -1)
echo "[*] cdt pid=$PID; arming scanerr"
timeout 30 "$VMEXEC" exec \
"nohup gdb -p $PID -batch -x /tmp/scanerr.py >/tmp/scanerr.out 2>&1 & echo ARMED" \
>/dev/null 2>&1
echo "[*] cycle armed: watch /tmp/cdt.out and /tmp/scanerr.out in the guest"
|
