sema-svc: async VA->PA injection queue; guest-diag: fast counter+pin injector, wait sampler

3 files changed, 173 insertions, 38 deletions

diff --git a/tools/guest-diag/inject_counters.sh b/tools/guest-diag/inject_counters.sh
index b4e20bd..8a56558 100755
--- a/tools/guest-diag/inject_counters.sh
+++ b/tools/guest-diag/inject_counters.sh
@@ -1,27 +1,43 @@
 #!/bin/bash
 # inject_counters.sh - register the running CUDA client's per-channel GPU
 # completion counters with the host sema service (the PART 45/46 manual
-# procedure, made durable). The service then publishes each GR channel's
-# consumed GPFIFO count into the client's counter, which is the completion
-# signal its trackers poll.
+# procedure, made durable). The host then tracks each GR channel's consumed
+# GPFIFO progress in the counter the client polls.
 #
 # Usage: tools/guest-diag/inject_counters.sh [guest-pid] [page-va] [count]
 #   guest-pid  default: pgrep -x cdt in the guest
 #   page-va    default: 0x20400f000 (observed counter page VA)
 #   count      default: 20 (one per GR channel)
 #
-# Handle order: GR channels (eng=0) from the host channel dump, ascending, are
-# zipped with ascending counter index (the observed 1:1 layout). Override the
-# list with GRS="0x...,0x...,...".
-set -euo pipefail
+# Env:
+#   WATCH=<seconds>  keep watching for new GR channels and register them,
+#                    instead of a single pass (default 0 = single pass).
+#   MAP=1            also enqueue the sema_map_add pin per channel (default
+#                    on). The pin teaches the host the counter page's VA->PA,
+#                    which is what lets the client's own release pushes into
+#                    the counter page apply; without it the host logs
+#                    "release miss ... (no translation)" and the counters
+#                    never move. The sysfs write is async now (queue), so
+#                    this is cheap.
+#   MAP=0            skip the pins (counters only advance from the host's
+#                    fallback put publishing).
+#   GRS=0x..,0x..    override the channel handle list.
+#
+# Handle order: GR channels (eng=0) from the host channel dump, descending,
+# are zipped with ascending counter index (the observed 1:1 layout; the
+# last-created channel owns counter 0).
+set -uo pipefail
 
 HERE="$(cd "$(dirname "$0")" && pwd)"
 VMEXEC="$HERE/../../scripts/vgpu_vm.py"
 PARAM=/sys/module/open_iov_vfio/parameters
+DONE=/tmp/inject_counters.done
 
 PID="${1:-}"
 PAGE_VA="${2:-0x20400f000}"
 COUNT="${3:-20}"
+WATCH="${WATCH:-0}"
+MAP="${MAP:-1}"
 
 if [ -z "$PID" ]; then
     PID=$(timeout 20 "$VMEXEC" exec "pgrep -x cdt" 2>/dev/null \
@@ -36,41 +52,86 @@ fi
 b64=$(base64 -w0 "$HERE/sem_counters.py")
 timeout 30 "$VMEXEC" exec "echo '$b64' | base64 -d > /tmp/sem_counters.py" >/dev/null
 
-# Channel handles: GR channels from the live channel dump, ascending.
-if [ -z "${GRS:-}" ]; then
+all_grs() {
     echo 1 | sudo tee "$PARAM/chan_dump" >/dev/null
     sleep 1
-    GRS=$(sudo dmesg | grep -o 'sema-chan-dump: h=0x[0-9a-f]* eng=0' \
-        | grep -o '0x[0-9a-f]*' | sort -u | paste -sd,)
-fi
-if [ -z "$GRS" ]; then
-    echo "ERROR: no GR channel handles (chan_dump empty; is a VM attached?)" >&2
-    exit 1
-fi
+    sudo dmesg | grep -o 'sema-chan-dump: h=0x[0-9a-f]* eng=0' \
+        | grep -o '0x[0-9a-f]*' | sort -u | paste -sd, || true
+}
+
+register_handles() {
+    local grs="$1"
+    [ -z "$grs" ] && return 0
+
+    # Counter index order: observed DESCENDING by channel handle (the client's
+    # last-created channel owns counter 0). ORDER=asc restores creation order.
+    if [ "${ORDER:-desc}" = "desc" ]; then
+        grs=$(echo "$grs" | tr ',' '\n' | sort -r | paste -sd,)
+    fi
 
-# Counter index order: observed DESCENDING by channel handle (the client's
-# last-created channel owns counter 0; verified live: trackers waiting on
-# counter idx2/idx3 belong to the channels 2nd/3rd from the top). ORDER=asc
-# restores creation order.
-if [ "${ORDER:-desc}" = "desc" ]; then
-    GRS=$(echo "$GRS" | tr ',' '\n' | sort -r | paste -sd,)
+    local pairs
+    pairs=$(timeout 30 "$VMEXEC" exec \
+        "python3 /tmp/sem_counters.py $PID $PAGE_VA --count=$COUNT --pairs $grs" \
+        2>/dev/null | grep -E '^0x[0-9a-f]+,0x[0-9a-f]+,[0-9]+$' || true)
+    if [ -z "$pairs" ]; then
+        echo "WARN: enumeration returned no pairs for: $grs" >&2
+        return 1
+    fi
+
+    local h va pa
+    while IFS=, read -r h va pa; do
+        # sema_counter is a direct state write (no service-thread round trip),
+        # so this loop completes in milliseconds.
+        if printf '%s,%s' "$h" "$pa" | sudo tee "$PARAM/sema_counter" >/dev/null; then
+            echo "$h" >> "$DONE"
+            echo "registered h=$h counter va=$va pa=$pa"
+        else
+            echo "WARN: sema_counter write failed for h=$h pa=$pa" >&2
+        fi
+        if [ "$MAP" = "1" ]; then
+            printf '%s,%s' "$va" "$pa" | sudo tee "$PARAM/sema_map_add" >/dev/null \
+                || echo "WARN: sema_map_add timed out for h=$h (pin skipped)" >&2
+        fi
+    done <<< "$pairs"
+    return 0
+}
+
+: > "$DONE"
+
+if [ -n "${GRS:-}" ]; then
+    register_handles "$GRS"
+    exit 0
 fi
 
-PAIRS=$(timeout 30 "$VMEXEC" exec \
-    "python3 /tmp/sem_counters.py $PID $PAGE_VA --count=$COUNT --pairs $GRS" \
-    2>/dev/null | grep -E '^0x[0-9a-f]+,0x[0-9a-f]+,[0-9]+$')
-if [ -z "$PAIRS" ]; then
-    echo "ERROR: enumeration returned no pairs (pagemap access?)" >&2
-    exit 1
+GRS=$(all_grs)
+
+if [ "$WATCH" = "0" ]; then
+    if [ -z "$GRS" ]; then
+        echo "ERROR: no GR channel handles (chan_dump empty; is a VM attached?)" >&2
+        exit 1
+    fi
+    register_handles "$GRS"
+    exit 0
 fi
 
-while IFS=, read -r h va pa; do
-    # printf: sysfs writes carry a trailing newline that kstrtoull tolerates
-    # for both fields; use it anyway (no newline is also fine).
-    printf '%s,%s' "$h" "$pa" | sudo tee "$PARAM/sema_counter" >/dev/null
-    # Pin the counter VA->PA too: a push that releases the counter then
-    # resolves and applies through the normal release path, and pins survive
-    # the periodic map rebuilds.
-    printf '%s,%s' "$va" "$pa" | sudo tee "$PARAM/sema_map_add" >/dev/null
-    echo "registered h=$h counter va=$va pa=$pa"
-done <<< "$PAIRS"
+# Watch mode: register newly appearing channels until the deadline passes and
+# no new handles show up for a few consecutive polls.
+deadline=$(( $(date +%s) + WATCH ))
+quiet=0
+while [ "$(date +%s)" -lt "$deadline" ]; do
+    GRS=$(all_grs)
+    new=""
+    for h in $(echo "$GRS" | tr ',' ' '); do
+        grep -qx "$h" "$DONE" 2>/dev/null || new="$new,$h"
+    done
+    new="${new#,}"
+    if [ -n "$new" ]; then
+        register_handles "$new" || true
+        quiet=0
+    else
+        quiet=$((quiet + 1))
+        [ "$quiet" -ge 30 ] && break
+    fi
+    sleep 2
+done
+echo "watch done: $(wc -l < "$DONE") channel(s) registered"
diff --git a/tools/guest-diag/push_guest_tools.sh b/tools/guest-diag/push_guest_tools.sh
index b0711d4..93f7e6a 100755
--- a/tools/guest-diag/push_guest_tools.sh
+++ b/tools/guest-diag/push_guest_tools.sh
@@ -33,6 +33,7 @@ push_b64 "$HERE/sem_counters.py" /tmp/sem_counters.py
 push_b64 "$HERE/sem_slots.py"   /tmp/sem_slots.py
 push_b64 "$HERE/read_va.py"     /tmp/read_va.py
 push_b64 "$HERE/guest_pmap.py"  /tmp/guest_pmap.py
+push_b64 "$HERE/track_waits.sh" /tmp/track_waits.sh
 
 echo "[*] compiling in guest..."
 "$VMEXEC" exec "gcc -O1 -w -o /tmp/ptio /tmp/ptio.c && \
diff --git a/tools/guest-diag/track_waits.sh b/tools/guest-diag/track_waits.sh
new file mode 100644
index 0000000..4c9cd1a
--- /dev/null
+++ b/tools/guest-diag/track_waits.sh
@@ -0,0 +1,73 @@
+#!/bin/bash
+# track_waits.sh - guest-side wait sampler: while a CUDA client (cdt) runs,
+# repeatedly dump the per-channel completion counters (values), the tracker
+# wait entries (trkinfo.py, every 2nd pass) and the push pages the host could
+# not resolve (read_va.py), so the last signal before a failure is recorded.
+#
+# Written for the guest (no host paths). Run as root in the guest:
+#   sudo /tmp/track_waits.sh <pid> [samples] [sleep]
+# Logs: /tmp/trk.log, /tmp/cnt.log, /tmp/push.log
+set -u
+
+PID="${1:?usage: track_waits.sh <pid> [samples] [sleep]}"
+N="${2:-600}"
+SLEEP="${3:-1}"
+PAGE_VA="${PAGE_VA:-0x20400f000}"
+COUNT="${COUNT:-20}"
+STRIDE="${STRIDE:-16}"
+OFFSET="${OFFSET:-0xec0}"
+
+: > /tmp/trk.log
+: > /tmp/cnt.log
+: > /tmp/push.log
+
+python3 - "$PID" "$PAGE_VA" "$COUNT" "$STRIDE" "$OFFSET" <<'PY' >> /tmp/cnt.log 2>&1 &
+import os, sys, time
+pid = int(sys.argv[1]); page = int(sys.argv[2], 16)
+count = int(sys.argv[3]); stride = int(sys.argv[4], 16); off = int(sys.argv[5], 16)
+pm = os.open('/proc/%d/pagemap' % pid, os.O_RDONLY)
+mem = os.open('/proc/%d/mem' % pid, os.O_RDONLY)
+while True:
+    ts = time.strftime('%T')
+    vals = []
+    for i in range(count):
+        va = page + off + i * stride
+        try:
+            os.lseek(pm, (va // 4096) * 8, 0)
+            e = int.from_bytes(os.read(pm, 8), 'little')
+            present = (e >> 63) & 1
+            if present:
+                v = int.from_bytes(os.pread(mem, 4, va), 'little')
+                vals.append('%d:0x%x' % (i, v))
+            else:
+                vals.append('%d:-' % i)
+        except OSError:
+            vals.append('%d:ERR' % i)
+    print('%s %s' % (ts, ' '.join(vals)), flush=True)
+    time.sleep(0.5)
+PY
+CNTPID=$!
+
+for i in $(seq 1 "$N"); do
+    TS=$(date +%T.%3N)
+    kill -0 "$PID" 2>/dev/null || { echo "PROCESS GONE at $TS (sample $i)" >> /tmp/trk.log; break; }
+
+    if [ $((i % 2)) -eq 1 ]; then
+        echo "=== $TS sample $i ===" >> /tmp/trk.log
+        timeout 8 gdb -p "$PID" -batch -x /tmp/trkinfo.py 2>&1 \
+            | grep -vE "^\[|Reading symbols|^0x[0-9a-f]+ in |warning:" >> /tmp/trk.log
+    fi
+
+    {
+        echo "=== $TS push pages ==="
+        python3 /tmp/guest_pmap.py "$PID" 0x201000000 0x203400000 0x203400020 2>&1
+        for va in 0x201000000 0x203400000 0x203400020; do
+            echo "--- $va"
+            timeout 5 python3 /tmp/read_va.py "$PID" "$va" 64 --u32 2>&1
+        done
+    } >> /tmp/push.log
+
+    sleep "$SLEEP"
+done
+kill "$CNTPID" 2>/dev/null
+echo "sampler done" >> /tmp/trk.log