diff options
Diffstat (limited to 'tools')
| -rwxr-xr-x | tools/guest-diag/inject_counters.sh | 137 | ||||
| -rwxr-xr-x | tools/guest-diag/push_guest_tools.sh | 1 | ||||
| -rw-r--r-- | tools/guest-diag/track_waits.sh | 73 |
3 files changed, 173 insertions, 38 deletions
diff --git a/tools/guest-diag/inject_counters.sh b/tools/guest-diag/inject_counters.sh index b4e20bd..8a56558 100755 --- a/tools/guest-diag/inject_counters.sh +++ b/tools/guest-diag/inject_counters.sh @@ -1,27 +1,43 @@ #!/bin/bash # inject_counters.sh - register the running CUDA client's per-channel GPU # completion counters with the host sema service (the PART 45/46 manual -# procedure, made durable). The service then publishes each GR channel's -# consumed GPFIFO count into the client's counter, which is the completion -# signal its trackers poll. +# procedure, made durable). The host then tracks each GR channel's consumed +# GPFIFO progress in the counter the client polls. # # Usage: tools/guest-diag/inject_counters.sh [guest-pid] [page-va] [count] # guest-pid default: pgrep -x cdt in the guest # page-va default: 0x20400f000 (observed counter page VA) # count default: 20 (one per GR channel) # -# Handle order: GR channels (eng=0) from the host channel dump, ascending, are -# zipped with ascending counter index (the observed 1:1 layout). Override the -# list with GRS="0x...,0x...,...". -set -euo pipefail +# Env: +# WATCH=<seconds> keep watching for new GR channels and register them, +# instead of a single pass (default 0 = single pass). +# MAP=1 also enqueue the sema_map_add pin per channel (default +# on). The pin teaches the host the counter page's VA->PA, +# which is what lets the client's own release pushes into +# the counter page apply; without it the host logs +# "release miss ... (no translation)" and the counters +# never move. The sysfs write is async now (queue), so +# this is cheap. +# MAP=0 skip the pins (counters only advance from the host's +# fallback put publishing). +# GRS=0x..,0x.. override the channel handle list. +# +# Handle order: GR channels (eng=0) from the host channel dump, descending, +# are zipped with ascending counter index (the observed 1:1 layout; the +# last-created channel owns counter 0). +set -uo pipefail HERE="$(cd "$(dirname "$0")" && pwd)" VMEXEC="$HERE/../../scripts/vgpu_vm.py" PARAM=/sys/module/open_iov_vfio/parameters +DONE=/tmp/inject_counters.done PID="${1:-}" PAGE_VA="${2:-0x20400f000}" COUNT="${3:-20}" +WATCH="${WATCH:-0}" +MAP="${MAP:-1}" if [ -z "$PID" ]; then PID=$(timeout 20 "$VMEXEC" exec "pgrep -x cdt" 2>/dev/null \ @@ -36,41 +52,86 @@ fi b64=$(base64 -w0 "$HERE/sem_counters.py") timeout 30 "$VMEXEC" exec "echo '$b64' | base64 -d > /tmp/sem_counters.py" >/dev/null -# Channel handles: GR channels from the live channel dump, ascending. -if [ -z "${GRS:-}" ]; then +all_grs() { echo 1 | sudo tee "$PARAM/chan_dump" >/dev/null sleep 1 - GRS=$(sudo dmesg | grep -o 'sema-chan-dump: h=0x[0-9a-f]* eng=0' \ - | grep -o '0x[0-9a-f]*' | sort -u | paste -sd,) -fi -if [ -z "$GRS" ]; then - echo "ERROR: no GR channel handles (chan_dump empty; is a VM attached?)" >&2 - exit 1 -fi + sudo dmesg | grep -o 'sema-chan-dump: h=0x[0-9a-f]* eng=0' \ + | grep -o '0x[0-9a-f]*' | sort -u | paste -sd, || true +} + +register_handles() { + local grs="$1" + [ -z "$grs" ] && return 0 + + # Counter index order: observed DESCENDING by channel handle (the client's + # last-created channel owns counter 0). ORDER=asc restores creation order. + if [ "${ORDER:-desc}" = "desc" ]; then + grs=$(echo "$grs" | tr ',' '\n' | sort -r | paste -sd,) + fi -# Counter index order: observed DESCENDING by channel handle (the client's -# last-created channel owns counter 0; verified live: trackers waiting on -# counter idx2/idx3 belong to the channels 2nd/3rd from the top). ORDER=asc -# restores creation order. -if [ "${ORDER:-desc}" = "desc" ]; then - GRS=$(echo "$GRS" | tr ',' '\n' | sort -r | paste -sd,) + local pairs + pairs=$(timeout 30 "$VMEXEC" exec \ + "python3 /tmp/sem_counters.py $PID $PAGE_VA --count=$COUNT --pairs $grs" \ + 2>/dev/null | grep -E '^0x[0-9a-f]+,0x[0-9a-f]+,[0-9]+$' || true) + if [ -z "$pairs" ]; then + echo "WARN: enumeration returned no pairs for: $grs" >&2 + return 1 + fi + + local h va pa + while IFS=, read -r h va pa; do + # sema_counter is a direct state write (no service-thread round trip), + # so this loop completes in milliseconds. + if printf '%s,%s' "$h" "$pa" | sudo tee "$PARAM/sema_counter" >/dev/null; then + echo "$h" >> "$DONE" + echo "registered h=$h counter va=$va pa=$pa" + else + echo "WARN: sema_counter write failed for h=$h pa=$pa" >&2 + fi + if [ "$MAP" = "1" ]; then + printf '%s,%s' "$va" "$pa" | sudo tee "$PARAM/sema_map_add" >/dev/null \ + || echo "WARN: sema_map_add timed out for h=$h (pin skipped)" >&2 + fi + done <<< "$pairs" + return 0 +} + +: > "$DONE" + +if [ -n "${GRS:-}" ]; then + register_handles "$GRS" + exit 0 fi -PAIRS=$(timeout 30 "$VMEXEC" exec \ - "python3 /tmp/sem_counters.py $PID $PAGE_VA --count=$COUNT --pairs $GRS" \ - 2>/dev/null | grep -E '^0x[0-9a-f]+,0x[0-9a-f]+,[0-9]+$') -if [ -z "$PAIRS" ]; then - echo "ERROR: enumeration returned no pairs (pagemap access?)" >&2 - exit 1 +GRS=$(all_grs) + +if [ "$WATCH" = "0" ]; then + if [ -z "$GRS" ]; then + echo "ERROR: no GR channel handles (chan_dump empty; is a VM attached?)" >&2 + exit 1 + fi + register_handles "$GRS" + exit 0 fi -while IFS=, read -r h va pa; do - # printf: sysfs writes carry a trailing newline that kstrtoull tolerates - # for both fields; use it anyway (no newline is also fine). - printf '%s,%s' "$h" "$pa" | sudo tee "$PARAM/sema_counter" >/dev/null - # Pin the counter VA->PA too: a push that releases the counter then - # resolves and applies through the normal release path, and pins survive - # the periodic map rebuilds. - printf '%s,%s' "$va" "$pa" | sudo tee "$PARAM/sema_map_add" >/dev/null - echo "registered h=$h counter va=$va pa=$pa" -done <<< "$PAIRS" +# Watch mode: register newly appearing channels until the deadline passes and +# no new handles show up for a few consecutive polls. +deadline=$(( $(date +%s) + WATCH )) +quiet=0 +while [ "$(date +%s)" -lt "$deadline" ]; do + GRS=$(all_grs) + new="" + for h in $(echo "$GRS" | tr ',' ' '); do + grep -qx "$h" "$DONE" 2>/dev/null || new="$new,$h" + done + new="${new#,}" + if [ -n "$new" ]; then + register_handles "$new" || true + quiet=0 + else + quiet=$((quiet + 1)) + [ "$quiet" -ge 30 ] && break + fi + sleep 2 +done +echo "watch done: $(wc -l < "$DONE") channel(s) registered" diff --git a/tools/guest-diag/push_guest_tools.sh b/tools/guest-diag/push_guest_tools.sh index b0711d4..93f7e6a 100755 --- a/tools/guest-diag/push_guest_tools.sh +++ b/tools/guest-diag/push_guest_tools.sh @@ -33,6 +33,7 @@ push_b64 "$HERE/sem_counters.py" /tmp/sem_counters.py push_b64 "$HERE/sem_slots.py" /tmp/sem_slots.py push_b64 "$HERE/read_va.py" /tmp/read_va.py push_b64 "$HERE/guest_pmap.py" /tmp/guest_pmap.py +push_b64 "$HERE/track_waits.sh" /tmp/track_waits.sh echo "[*] compiling in guest..." "$VMEXEC" exec "gcc -O1 -w -o /tmp/ptio /tmp/ptio.c && \ diff --git a/tools/guest-diag/track_waits.sh b/tools/guest-diag/track_waits.sh new file mode 100644 index 0000000..4c9cd1a --- /dev/null +++ b/tools/guest-diag/track_waits.sh @@ -0,0 +1,73 @@ +#!/bin/bash +# track_waits.sh - guest-side wait sampler: while a CUDA client (cdt) runs, +# repeatedly dump the per-channel completion counters (values), the tracker +# wait entries (trkinfo.py, every 2nd pass) and the push pages the host could +# not resolve (read_va.py), so the last signal before a failure is recorded. +# +# Written for the guest (no host paths). Run as root in the guest: +# sudo /tmp/track_waits.sh <pid> [samples] [sleep] +# Logs: /tmp/trk.log, /tmp/cnt.log, /tmp/push.log +set -u + +PID="${1:?usage: track_waits.sh <pid> [samples] [sleep]}" +N="${2:-600}" +SLEEP="${3:-1}" +PAGE_VA="${PAGE_VA:-0x20400f000}" +COUNT="${COUNT:-20}" +STRIDE="${STRIDE:-16}" +OFFSET="${OFFSET:-0xec0}" + +: > /tmp/trk.log +: > /tmp/cnt.log +: > /tmp/push.log + +python3 - "$PID" "$PAGE_VA" "$COUNT" "$STRIDE" "$OFFSET" <<'PY' >> /tmp/cnt.log 2>&1 & +import os, sys, time +pid = int(sys.argv[1]); page = int(sys.argv[2], 16) +count = int(sys.argv[3]); stride = int(sys.argv[4], 16); off = int(sys.argv[5], 16) +pm = os.open('/proc/%d/pagemap' % pid, os.O_RDONLY) +mem = os.open('/proc/%d/mem' % pid, os.O_RDONLY) +while True: + ts = time.strftime('%T') + vals = [] + for i in range(count): + va = page + off + i * stride + try: + os.lseek(pm, (va // 4096) * 8, 0) + e = int.from_bytes(os.read(pm, 8), 'little') + present = (e >> 63) & 1 + if present: + v = int.from_bytes(os.pread(mem, 4, va), 'little') + vals.append('%d:0x%x' % (i, v)) + else: + vals.append('%d:-' % i) + except OSError: + vals.append('%d:ERR' % i) + print('%s %s' % (ts, ' '.join(vals)), flush=True) + time.sleep(0.5) +PY +CNTPID=$! + +for i in $(seq 1 "$N"); do + TS=$(date +%T.%3N) + kill -0 "$PID" 2>/dev/null || { echo "PROCESS GONE at $TS (sample $i)" >> /tmp/trk.log; break; } + + if [ $((i % 2)) -eq 1 ]; then + echo "=== $TS sample $i ===" >> /tmp/trk.log + timeout 8 gdb -p "$PID" -batch -x /tmp/trkinfo.py 2>&1 \ + | grep -vE "^\[|Reading symbols|^0x[0-9a-f]+ in |warning:" >> /tmp/trk.log + fi + + { + echo "=== $TS push pages ===" + python3 /tmp/guest_pmap.py "$PID" 0x201000000 0x203400000 0x203400020 2>&1 + for va in 0x201000000 0x203400000 0x203400020; do + echo "--- $va" + timeout 5 python3 /tmp/read_va.py "$PID" "$va" 64 --u32 2>&1 + done + } >> /tmp/push.log + + sleep "$SLEEP" +done +kill "$CNTPID" 2>/dev/null +echo "sampler done" >> /tmp/trk.log |