sema-svc: apply releases promptly and decode compute-push releases

The content sweep now:
- decodes a release whose method stream continues with any INC_METHOD,
  not only the CE LAUNCH_DMA group, so the releases at the head of a
  compute/GR channel's first push are applied too;
- remembers releases whose target VA had no translation (deferred list)
  and retries them every service pass, so a VA->PA injection lands them
  immediately instead of after the next full sweep;
- re-arms the fast sliced sweep for a few seconds after an injection;
- treats a channel whose USERD/ring cannot be resolved as stalled, so the
  sweep keeps looking for the releases its unreadable pushes carry.

tools: add per-cdt client-pin helper (sem pages + /dev/nvidia0 windows),
sem-page scanner and memprobe fixes (short-read guard, --brief).
Live-verified: injecting a client sem page applied 12 deferred releases
within a second; the service kept applying (89 -> 202) while cdt ran.

4 files changed, 252 insertions, 13 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 1f1222e..4cfed0c 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -270,6 +270,13 @@ struct sema_svc {
 	u64			fast_passes;
 	u64			fast_pass_applied;
 	u64			fast_hold_until;
+	/*
+	 * A VA->PA injection (or a batch of them) invalidates the "nothing to
+	 * apply" assumption the fast sweep backs off on: releases that missed
+	 * translation may be applicable now. Keep the fast sweep armed for a
+	 * few seconds after any injection.
+	 */
+	u64			fast_force_until;
 
 	/*
 	 * Completion hint window: after a channel completes work the client is
@@ -2019,13 +2026,88 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 /* ------------------------------------------------------------------ */
 
 /*
+ * Deferred releases.
+ *
+ * A release group can be visible in guest RAM long before its target VA has a
+ * trusted translation: the page may be scanned before the client mapping is
+ * pinned, or before the pair scan learns the VA. Dropping such a release and
+ * waiting for the next full sweep costs minutes; instead remember the
+ * (VA, payload) and retry it on every service pass. As soon as a pin or a
+ * learned pair makes the VA translatable the release lands, which is exactly
+ * what a client waiting on that semaphore needs.
+ */
+#define SVC_PENDING_MAX		512
+struct svc_pending_release {
+	u64 va;
+	u32 payload;
+	u32 op;		/* 0 = RELEASE, 1 = REDUCTION (add payload) */
+};
+
+static struct svc_pending_release svc_pending[SVC_PENDING_MAX];
+static u32 svc_pending_n;
+static u32 svc_pending_next;
+
+static void svc_pending_add(u64 va, u32 payload, u32 op)
+{
+	u32 i;
+
+	if (!payload)
+		return;
+	for (i = 0; i < svc_pending_n; i++)
+		if (svc_pending[i].va == va &&
+		    svc_pending[i].payload >= payload)
+			return;
+	if (svc_pending_n < SVC_PENDING_MAX) {
+		i = svc_pending_n++;
+	} else {
+		i = svc_pending_next++ % SVC_PENDING_MAX;
+	}
+	svc_pending[i].va = va;
+	svc_pending[i].payload = payload;
+	svc_pending[i].op = op;
+}
+
+static void svc_apply_pending(struct nv_vgpu_device *vgpu)
+{
+	u32 i = 0;
+
+	while (i < svc_pending_n) {
+		struct svc_pending_release *p = &svc_pending[i];
+		u32 cur = 0;
+
+		if (svc_sem_read(vgpu, p->va, &cur)) {
+			i++;	/* translation still missing: keep waiting */
+			continue;
+		}
+		if (p->payload <= cur) {
+			svc_pending[i] = svc_pending[--svc_pending_n];
+			continue;
+		}
+		if (p->payload - cur > 0x100000) {
+			svc_pending[i] = svc_pending[--svc_pending_n];
+			continue;
+		}
+		if (!svc_sem_write(vgpu, p->va,
+				   p->op ? cur + p->payload : p->payload)) {
+			svc.releases_applied++;
+			svc.post_hint_until = jiffies + 10 * HZ;
+			pr_info("sema-svc: deferred release va=0x%llx payload=%u (was cur=%u)\n",
+				p->va, p->payload, cur);
+			svc_pending[i] = svc_pending[--svc_pending_n];
+			continue;
+		}
+		i++;	/* unresolved: try again next pass */
+	}
+}
+
+/*
  * Look for CE semaphore-release method groups inside mapped pages:
  *
  *   [INC_METHOD header (addr>>2 == 0x240, count == 3)]
  *   [upper 32 bits of semaphore VA]
  *   [lower 32 bits of semaphore VA]
  *   [payload]
- *   [... LAUNCH_DMA ...]
+ *   [method stream continues]
  *
  * Apply by writing payload to the semaphore slot, but only in order
  * (current == payload - 1) so that UVM's tracker never sees a jump.
@@ -2058,15 +2140,29 @@ static u32 svc_apply_page_data(struct nv_vgpu_device *vgpu, const u8 *buf,
 		memcpy(&lo, buf + off + 8, 4);
 		memcpy(&payload, buf + off + 12, 4);
 
-		/* Require the trailing LAUNCH_DMA group: dramatically reduces
-		 * false positives when scanning unrelated memory. */
+		/*
+		 * The word after the release must look like the continuation of
+		 * a method stream: the CE LAUNCH_DMA this group normally
+		 * precedes, or another INC_METHOD header. Compute/GR pushes put
+		 * arbitrary methods after a release (channel setup, GPFIFO
+		 * updates), so requiring LAUNCH_DMA alone silently drops their
+		 * releases - the client tracker semaphores those pushes release
+		 * are exactly the ones clients wait on. Random
+		 * {0x20038090, hi, lo, payload} runs in unrelated data almost
+		 * never satisfy either shape.
+		 */
 		if (off + 20 <= SEMA_PAGE_SIZE) {
 			u32 dma;
 
 			memcpy(&dma, buf + off + 16, 4);
 			if (((dma >> 16) & 0xff) != CE_DMA_HDR_COUNT ||
-			    (dma & 0xff) != CE_DMA_HDR_ADDR)
-				continue;
+			    (dma & 0xff) != CE_DMA_HDR_ADDR) {
+				u32 type = (dma >> 29) & 0x7;
+				u32 cnt = (dma >> 16) & 0xfff;
+
+				if (type < 1 || type > 4 || !cnt || cnt > 16)
+					continue;
+			}
 		} else {
 			continue;
 		}
@@ -2083,6 +2179,7 @@ static u32 svc_apply_page_data(struct nv_vgpu_device *vgpu, const u8 *buf,
 		cur32 = 0;
 		if (svc_sem_read(vgpu, va, &cur32)) {
 			svc.releases_missed++;
+			svc_pending_add(va, payload, 0);
 			if (svc.releases_missed_log < 16 &&
 			    va != svc.last_miss_va) {
 				svc.last_miss_va = va;
@@ -2251,10 +2348,12 @@ static void svc_apply_sweep_slice(struct nv_vgpu_device *vgpu)
 	/*
 	 * Back off after a full pass that applied nothing: a stalled channel
 	 * with no applicable release in RAM would otherwise keep a 6 GB pass
-	 * running forever.
+	 * running forever. A recent VA->PA injection clears the hold: the
+	 * translation landscape changed, so the sweep can apply something now.
 	 */
 	if (start == 0) {
-		if (time_before(jiffies, (unsigned long)svc.fast_hold_until))
+		if (time_before(jiffies, (unsigned long)svc.fast_hold_until) &&
+		    time_after(jiffies, (unsigned long)svc.fast_force_until))
 			return;
 		svc.fast_pass_applied = svc.releases_applied;
 	}
@@ -2892,8 +2991,17 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 
 		if (!ch->userd_ok) {
 			if (svc_chan_userd(vgpu, ch, rrc ? 0 : ring_pa,
-					   rrc ? -1 : ring_ap, &put, &get))
+					   rrc ? -1 : ring_ap, &put, &get)) {
+				/*
+				 * The channel's work cannot be run (USERD not
+				 * found): its pushes stay pending, so the release
+				 * it carries will never be applied by the pump.
+				 * Treat as stalled so the content sweep looks for
+				 * the releases in RAM.
+				 */
+				stalled = true;
 				continue;
+			}
 		} else {
 			if (svc_read_ap(vgpu, ch->userd_pa + SVC_USERD_GP_PUT,
 					ch->userd_as, &put, sizeof(put)))
@@ -2926,8 +3034,10 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 			u64 va, idx = ch->last_get;
 			u32 len, e0, e1, g;
 
-			if (rrc)
-				break;	/* cannot read the ring at all */
+			if (rrc) {
+				stalled = true;	/* ring unreadable: sweep RAM */
+				break;
+			}
 
 			if (svc_read_va(vgpu,
 					ch->gpfifo_va +
@@ -3216,6 +3326,12 @@ static void svc_handle_map_add(void)
 		svc_pin_store(map_add_va + off, map_add_pa + off, ap);
 		svc_lat_store(map_add_va + off, map_add_pa + off, ap);
 	}
+	/*
+	 * Releases that missed translation before this injection may be
+	 * applicable now: keep the fast full-RAM sweep armed for a few
+	 * seconds so they land without waiting for an unrelated stall.
+	 */
+	svc.fast_force_until = jiffies + 10 * HZ;
 	pr_info("sema-svc: injected mapping va=0x%llx -> pa=0x%llx len=0x%llx (%s)\n",
 		map_add_va, map_add_pa, map_add_len,
 		ap == SVC_APERTURE_FB ? "vidmem" : "sysmem");
@@ -3290,6 +3406,21 @@ static int svc_thread(void *arg)
 		svc_pump_channels(vgpu);
 
 		/*
+		 * Releases whose target VA was not translatable when the scan
+		 * saw them: retry every pass, so a pin or a learned pair applies
+		 * them immediately instead of on the next full sweep.
+		 */
+		svc_apply_pending(vgpu);
+
+		/*
+		 * A fresh VA->PA injection changes which releases are
+		 * applicable: run the fast sliced sweep while the forced window
+		 * is open (the pump's stall path also uses it).
+		 */
+		if (time_before(jiffies, (unsigned long)svc.fast_force_until))
+			svc_apply_sweep_slice(vgpu);
+
+		/*
 		 * Push-buffer base discovery: a chunked walk of guest RAM
 		 * finds the pages whose first push names a channel, so a
 		 * channel that appears with work already waiting is pinned and
diff --git a/tools/guest-diag/memprobe.py b/tools/guest-diag/memprobe.py
index 26ed0af..96dd40b 100644
--- a/tools/guest-diag/memprobe.py
+++ b/tools/guest-diag/memprobe.py
@@ -126,8 +126,10 @@ def dump_tracker(pid, fd, p):
     if sob:
         print("  sem_obj[0:0x40]=%s" % sob.hex())
         mem = r64(sob, 0x10)
-        mvb = rd(fd, mem & ~0xfff, 0x40)
-        print("  sem_mem=0x%x val@+0x%x=0x%x" % (mem, mem & 0xfff, r32(mvb, mem & 0xfff) if mvb else -1))
+        mvb = rd(fd, mem & ~0xfff, 0x1000)
+        off = mem & 0xfff
+        val = r32(mvb, off) if mvb and off + 4 <= len(mvb) else -1
+        print("  sem_mem=0x%x val@+0x%x=0x%x" % (mem, off, val))
     q = r64(rd(fd, p + 0x48, 8) or b"\0" * 8)
     print("  prog_obj=0x%x next=0x%x cblist=0x%x idx95bc=0x%x p34=0x%x"
           % (q, r64(rd(fd, p + 0x9470, 8) or b"\0" * 8),
@@ -154,15 +156,18 @@ def main():
         if mode == "scan":
             lo, hi = 0x204800000, 0x204c0000
             limit = 8
+            brief = False
             for i, a in enumerate(sys.argv[3:]):
                 if a == "--min": lo = int(sys.argv[4 + i], 16)
                 if a == "--max": hi = int(sys.argv[4 + i], 16)
                 if a == "--limit": limit = int(sys.argv[4 + i])
+                if a == "--brief": brief = True
             res = find_trackers(pid, fd, lo, hi, limit)
             for p, sub, last, cur, sem, mem in res:
                 print("P=0x%x sub=0x%x last=0x%x cur=0x%x sem_obj=0x%x sem_mem=0x%x" %
                       (p, sub, last, cur, sem, mem or 0))
-                dump_tracker(pid, fd, p)
+                if not brief:
+                    dump_tracker(pid, fd, p)
         elif mode == "dump":
             dump_tracker(pid, fd, int(sys.argv[3], 16))
         elif mode == "hex":
diff --git a/tools/guest-diag/pin_client_sems.sh b/tools/guest-diag/pin_client_sems.sh
new file mode 100755
index 0000000..82d2789
--- /dev/null
+++ b/tools/guest-diag/pin_client_sems.sh
@@ -0,0 +1,66 @@
+#!/bin/bash
+# pin_client_sems.sh - bridge the CUDA client's device mappings into the
+# emulation for the live cdt process.
+#
+# Two things are pinned, both per-process (a new cdt maps new physical pages
+# at the same VAs, so re-run this for every cdt run):
+#   1. the /dev/nvidia0 window: the driver's channel rings/USERDs (the GR
+#      channels' GPFIFO VAs live here), so the pump can run their pushes;
+#   2. the semaphore pages libcuda's trackers poll (found by scanning the
+#      client's memory for sem references, then translated through the guest
+#      pagemap).
+#
+# Usage: tools/guest-diag/pin_client_sems.sh [iterations] [sleep-secs]
+set -u
+cd "$(dirname "$(readlink -f "$0")")/../.."
+VMEXEC=./scripts/vgpu_vm.py
+ITER=${1:-1}
+SLEEP=${2:-10}
+
+PID=$(timeout 20 $VMEXEC exec --timeout 15 'pgrep -x cdt | head -1' 2>/dev/null | tail -1)
+[ -n "$PID" ] || { echo "no cdt pid"; exit 1; }
+echo "cdt pid=$PID"
+
+B64=$(gzip -9 < tools/guest-diag/sempages.py | base64 -w0)
+timeout 30 $VMEXEC exec --timeout 25 "echo '$B64' | base64 -d | gunzip > /tmp/sempages.py" >/dev/null 2>&1
+B64=$(gzip -9 < tools/guest-diag/memprobe.py | base64 -w0)
+timeout 30 $VMEXEC exec --timeout 25 "echo '$B64' | base64 -d | gunzip > /tmp/memprobe.py" >/dev/null 2>&1
+B64=$(base64 -w0 tools/guest-diag/guest_pmap_all.py)
+timeout 30 $VMEXEC exec --timeout 25 "echo '$B64' | base64 -d > /tmp/pmapall.py" >/dev/null 2>&1
+
+pin_range() {   # pin_range <lo-dec> <hi-dec> <label>
+    local cur=$1 hi=$2 label=$3 total=0
+    while [ "$cur" -lt "$hi" ]; do
+        mapfile -t lines < <(timeout 60 $VMEXEC exec --timeout 50 "sudo python3 /tmp/pmapall.py $PID $(printf '0x%x' "$cur") 32" 2>/dev/null | grep -aE '^0x[0-9a-f]+,0x[0-9a-f]+$')
+        [ "${#lines[@]}" -gt 0 ] || break
+        for l in "${lines[@]}"; do
+            echo "$l,0x1000,1" | sudo tee /sys/module/open_iov_vfio/parameters/sema_map_add >/dev/null
+        done
+        total=$((total + ${#lines[@]}))
+        local last=${lines[-1]%%,*}
+        cur=$(( 16#${last#0x} + 0x1000 ))
+    done
+    echo "  $label: pinned $total page(s)"
+}
+
+for it in $(seq 1 "$ITER"); do
+    # 1. /dev/nvidia0 windows (channel rings/USERDs).
+    mapfile -t n0 < <(timeout 30 $VMEXEC exec --timeout 25 "awk '/\\/dev\\/nvidia0/ {print \$1}' /proc/$PID/maps" 2>/dev/null | grep -E '^[0-9a-f]+-[0-9a-f]+$')
+    for r in "${n0[@]}"; do
+        lo=$(( 16#${r%-*} ))
+        hi=$(( 16#${r#*-} ))
+        pin_range "$lo" "$hi" "nvidia0 $r"
+    done
+
+    # 2. client semaphore pages.
+    lines=$(timeout 150 $VMEXEC exec --timeout 140 "sudo timeout 120 python3 /tmp/sempages.py $PID" 2>/dev/null | grep -E '^0x[0-9a-f]+$')
+    n=0
+    for va in $lines; do
+        line=$(timeout 30 $VMEXEC exec --timeout 25 "sudo python3 /tmp/pmapall.py $PID $va 1" 2>/dev/null | grep -E '^0x[0-9a-f]+,0x[0-9a-f]+$' | head -1)
+        [ -n "$line" ] || continue
+        echo "$line,0x1000,1" | sudo tee /sys/module/open_iov_vfio/parameters/sema_map_add >/dev/null || continue
+        n=$((n + 1))
+    done
+    echo "iter $it: pinned $n sem page(s) (pid $PID)"
+    [ "$it" -lt "$ITER" ] && sleep "$SLEEP"
+done
diff --git a/tools/guest-diag/sempages.py b/tools/guest-diag/sempages.py
new file mode 100755
index 0000000..ad4dae6
--- /dev/null
+++ b/tools/guest-diag/sempages.py
@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+# Print the distinct pages of the semaphore VAs a process references
+# (reuses memprobe.py's scanner; run in the guest as root). The VA window is
+# derived from the process's /dev/nvidia* mappings unless lo/hi are given.
+# usage: sudo python3 sempages.py <pid> [lo] [hi]
+import os
+import sys
+
+sys.path.insert(0, '/tmp')
+import memprobe as m
+
+pid = int(sys.argv[1])
+if len(sys.argv) > 3:
+    lo = int(sys.argv[2], 16)
+    hi = int(sys.argv[3], 16)
+else:
+    lo = 1 << 64
+    hi = 0
+    with open('/proc/%d/maps' % pid) as f:
+        for line in f:
+            parts = line.split()
+            if len(parts) < 6 or not parts[5].startswith('/dev/nvidia'):
+                continue
+            a, b = parts[0].split('-')
+            lo = min(lo, int(a, 16))
+            hi = max(hi, int(b, 16))
+    if hi == 0:
+        print('no /dev/nvidia mappings')
+        raise SystemExit(1)
+
+fd = os.open('/proc/%d/mem' % pid, os.O_RDONLY)
+sems = m.sem_objs(pid, fd, lo, hi)
+os.close(fd)
+pages = sorted({va & ~0xfff for va in sems})
+print('sem VAs: %d pages: %d (window 0x%x..0x%x)' % (len(sems), len(pages), lo, hi))
+for pg in pages:
+    print('0x%x' % pg)