blob: 82d278953d7610c03348ca154b81770ff33c2083 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
|
#!/bin/bash
# pin_client_sems.sh - bridge the CUDA client's device mappings into the
# emulation for the live cdt process.
#
# Two things are pinned, both per-process (a new cdt maps new physical pages
# at the same VAs, so re-run this for every cdt run):
# 1. the /dev/nvidia0 window: the driver's channel rings/USERDs (the GR
# channels' GPFIFO VAs live here), so the pump can run their pushes;
# 2. the semaphore pages libcuda's trackers poll (found by scanning the
# client's memory for sem references, then translated through the guest
# pagemap).
#
# Usage: tools/guest-diag/pin_client_sems.sh [iterations] [sleep-secs]
set -u
cd "$(dirname "$(readlink -f "$0")")/../.."
VMEXEC=./scripts/vgpu_vm.py
ITER=${1:-1}
SLEEP=${2:-10}
PID=$(timeout 20 $VMEXEC exec --timeout 15 'pgrep -x cdt | head -1' 2>/dev/null | tail -1)
[ -n "$PID" ] || { echo "no cdt pid"; exit 1; }
echo "cdt pid=$PID"
B64=$(gzip -9 < tools/guest-diag/sempages.py | base64 -w0)
timeout 30 $VMEXEC exec --timeout 25 "echo '$B64' | base64 -d | gunzip > /tmp/sempages.py" >/dev/null 2>&1
B64=$(gzip -9 < tools/guest-diag/memprobe.py | base64 -w0)
timeout 30 $VMEXEC exec --timeout 25 "echo '$B64' | base64 -d | gunzip > /tmp/memprobe.py" >/dev/null 2>&1
B64=$(base64 -w0 tools/guest-diag/guest_pmap_all.py)
timeout 30 $VMEXEC exec --timeout 25 "echo '$B64' | base64 -d > /tmp/pmapall.py" >/dev/null 2>&1
pin_range() { # pin_range <lo-dec> <hi-dec> <label>
local cur=$1 hi=$2 label=$3 total=0
while [ "$cur" -lt "$hi" ]; do
mapfile -t lines < <(timeout 60 $VMEXEC exec --timeout 50 "sudo python3 /tmp/pmapall.py $PID $(printf '0x%x' "$cur") 32" 2>/dev/null | grep -aE '^0x[0-9a-f]+,0x[0-9a-f]+$')
[ "${#lines[@]}" -gt 0 ] || break
for l in "${lines[@]}"; do
echo "$l,0x1000,1" | sudo tee /sys/module/open_iov_vfio/parameters/sema_map_add >/dev/null
done
total=$((total + ${#lines[@]}))
local last=${lines[-1]%%,*}
cur=$(( 16#${last#0x} + 0x1000 ))
done
echo " $label: pinned $total page(s)"
}
for it in $(seq 1 "$ITER"); do
# 1. /dev/nvidia0 windows (channel rings/USERDs).
mapfile -t n0 < <(timeout 30 $VMEXEC exec --timeout 25 "awk '/\\/dev\\/nvidia0/ {print \$1}' /proc/$PID/maps" 2>/dev/null | grep -E '^[0-9a-f]+-[0-9a-f]+$')
for r in "${n0[@]}"; do
lo=$(( 16#${r%-*} ))
hi=$(( 16#${r#*-} ))
pin_range "$lo" "$hi" "nvidia0 $r"
done
# 2. client semaphore pages.
lines=$(timeout 150 $VMEXEC exec --timeout 140 "sudo timeout 120 python3 /tmp/sempages.py $PID" 2>/dev/null | grep -E '^0x[0-9a-f]+$')
n=0
for va in $lines; do
line=$(timeout 30 $VMEXEC exec --timeout 25 "sudo python3 /tmp/pmapall.py $PID $va 1" 2>/dev/null | grep -E '^0x[0-9a-f]+,0x[0-9a-f]+$' | head -1)
[ -n "$line" ] || continue
echo "$line,0x1000,1" | sudo tee /sys/module/open_iov_vfio/parameters/sema_map_add >/dev/null || continue
n=$((n + 1))
done
echo "iter $it: pinned $n sem page(s) (pid $PID)"
[ "$it" -lt "$ITER" ] && sleep "$SLEEP"
done
|
