sema-svc: async VA->PA injection queue; guest-diag: fast counter+pin injector, wait sampler

4 files changed, 520 insertions, 200 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 9edb66d..72480a2 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -1088,7 +1088,7 @@ static void svc_add_map(u64 gpu_va, u64 pa);
  * finishes. VAs whose full scan found nothing are remembered and not rescanned
  * for SVC_TR_FAIL_HOLD.
  */
-#define SVC_TR_SLICE	(128ULL << 20)	/* guest RAM covered per call */
+#define SVC_TR_SLICE	(512ULL << 20)	/* guest RAM covered per call */
 #define SVC_TR_CHUNK	(64 * 1024)	/* one guest read */
 #define SVC_TR_FAIL_MAX	16
 #define SVC_TR_FAIL_HOLD	(60 * HZ)
@@ -1096,6 +1096,7 @@ static void svc_add_map(u64 gpu_va, u64 pa);
 static u64 tr_va;		/* page VA of the active scan, 0 = idle */
 static u64 tr_cursor;		/* next GPA to read */
 static unsigned long tr_start;	/* active scan start (staleness bound) */
+static unsigned long tr_last;	/* last lookup of the active scan's VA */
 static u64 tr_scans;
 static struct {
 	u64 va;
@@ -1134,13 +1135,27 @@ static int svc_targeted_resolve(struct nv_vgpu_device *vgpu, u64 gpu_va,
 	if (svc_tr_recently_failed(page_va))
 		return -ENOENT;
 
-	if (tr_va && tr_va != page_va)
-		return -EAGAIN;	/* another VA owns the active scan */
+	if (tr_va && tr_va != page_va) {
+		/*
+		 * The scan slot is owned by another page. If that page is
+		 * still being looked up (the pump retries its GPFIFO entry
+		 * every pass) let it finish; if it has not been seen for a
+		 * second, its entry was consumed or skipped and the slot
+		 * would otherwise stay wedged for the full 30 s scan window,
+		 * starving every live page behind it (observed: a skipped GR
+		 * entry left CE memops pages unresolvable until the scan
+		 * timed out).
+		 */
+		if (time_before(jiffies, tr_last + HZ))
+			return -EAGAIN;	/* another VA owns the active scan */
+		tr_va = 0;
+	}
 	if (!tr_va) {
 		tr_va = page_va;
 		tr_cursor = 0;
 		tr_start = jiffies;
 	}
+	tr_last = jiffies;
 	if (time_after(jiffies, tr_start + 30 * HZ)) {
 		/* The guest stopped giving us its memory: abandon the scan. */
 		svc_tr_note_failure(tr_va);
@@ -1213,6 +1228,22 @@ static int svc_targeted_resolve(struct nv_vgpu_device *vgpu, u64 gpu_va,
 }
 
 /*
+ * True while a full-RAM targeted scan for this VA's page owns the scan slot
+ * (and has not aged out). The pump uses it to hold an entry whose translation
+ * may land within the scan instead of consuming it as stale.
+ */
+static bool svc_resolve_in_progress(u64 va)
+{
+	u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
+
+	if (!tr_va || tr_va != page)
+		return false;
+	if (time_after(jiffies, tr_start + 30 * HZ))
+		return false;
+	return true;
+}
+
+/*
  * Translate a GPU VA using trusted sources only: a pinned translation, the
  * learned cache, the discovered map, the directories the driver published and
  * the self-discovered directory. The final vidmem window convention is
@@ -1722,6 +1753,7 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu)
 #define DMA_SEM_REDUCTION(v)		(((v) >> 14) & 0xf)
 #define DMA_SEM_REDUCTION_EN(v)		(((v) >> 19) & 0x1)
 #define SEM_RELEASE_ONE_WORD		1
+#define SEM_RELEASE_FOUR_WORD		2
 #define SEM_REDUCTION_INC		6
 
 /* Aperture targets (SET_*_PHYS_MODE). */
@@ -1947,25 +1979,6 @@ static bool svc_method_stream_plausible(const u8 *buf, u32 len)
 }
 
 /*
- * Relaxed push-prefix check for rescue candidate validation only. Real pushes
- * observed in this workload start with methods at non-zero addresses (e.g.
- * 0x2001001e = count 1, addr 0x78), which the strict check above rejects
- * because it requires addr 0. Rescue only needs the candidate page to begin
- * a plausible method stream; work execution still uses the strict check.
- */
-static bool svc_stream_prefix_ok(const u8 *buf, u32 len)
-{
-	u32 hdr = 0, op, count;
-
-	if (len < 8)
-		return false;
-	memcpy(&hdr, buf, 4);
-	op = (hdr >> 29) & 0x7;
-	count = (hdr >> 16) & 0xfff;
-	return op == 1 && count && count <= 64 && 4 + count * 4 <= len;
-}
-
-/*
  * Decode one page as a method stream and execute the pushes it contains.
  *
  * Push buffers are written back to back and pushes can span page
@@ -2320,8 +2333,18 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 				}
 			}
 
+			/*
+			 * Both release encodings write the payload word first
+			 * (NvGpuSemaphore.data[0] for the four-word format, the
+			 * single word for the one-word format), and the client's
+			 * trackers poll exactly that u32: apply the payload for
+			 * either. The four-word format's timestamp/status words
+			 * that follow are not observed by the waiters.
+			 */
 			if (DMA_SEMAPHORE_TYPE(dma) ==
-			    SEM_RELEASE_ONE_WORD) {
+			    SEM_RELEASE_ONE_WORD ||
+			    DMA_SEMAPHORE_TYPE(dma) ==
+			    SEM_RELEASE_FOUR_WORD) {
 				u64 sem_va;
 				u32 cur = 0;
 				int pending = 0;
@@ -3325,6 +3348,129 @@ static void svc_pb_pair(struct nv_vgpu_device *vgpu, struct svc_channel *ch)
 }
 
 /*
+ * Match one buffer for this channel's pool-semaphore release group and append
+ * candidate pin bases. The group is:
+ *   +0x00 SET_SEMAPHORE_A header (0x20038090, count 3, subch 4)
+ *   +0x04 hi == 1
+ *   +0x08 lo & 0xfff == this channel's pool slot
+ *   +0x0c payload (any small value; the tracked expectation can lag)
+ *   +0x10 LAUNCH_DMA semaphore-release follower (transfer none)
+ * Candidates are the page the group sits in (correct when the push starts in
+ * that page, e.g. a stale release push) and the pages implied by treating the
+ * group as the end of this entry's push, walking back a page at a time so a
+ * trailing method run after the release does not defeat the guess. Returns the
+ * updated candidate count.
+ */
+static u32 svc_rescue_match_buf(const u8 *buf, u32 len, u64 base_pa, u32 slot,
+				u64 entry_len, u64 *cand, u32 ncand)
+{
+	u32 o;
+
+	for (o = 0; o + 24 <= len && ncand < SVC_MBUF_MAX; o += 4) {
+		u32 hdr, hi, lo, payload, w5, v5;
+		u64 rel_pa, d;
+
+		memcpy(&hdr, buf + o, 4);
+		if (hdr != 0x20038090u)
+			continue;
+		memcpy(&hi, buf + o + 4, 4);
+		if (hi != 1)
+			continue;
+		memcpy(&lo, buf + o + 8, 4);
+		if ((lo & 0xfff) != slot)
+			continue;
+		memcpy(&payload, buf + o + 12, 4);
+		if (!payload || payload >= (1u << 24))
+			continue;
+		memcpy(&w5, buf + o + 16, 4);
+		memcpy(&v5, buf + o + 20, 4);
+		if ((w5 >> 29) != 1 || ((w5 >> 13) & 0x7) != 4 ||
+		    ((w5 & 0x1fff) << 2) != 0x300 ||
+		    (v5 & 0x3) != 0)
+			continue;
+
+		rel_pa = base_pa + o;
+		cand[ncand++] = rel_pa & ~(u64)(SEMA_PAGE_SIZE - 1);
+		for (d = entry_len >= 24 ? entry_len - 24 : 0;
+		     d > 0 && ncand < SVC_MBUF_MAX; d -= SEMA_PAGE_SIZE) {
+			u64 s = (rel_pa - d) & ~(u64)(SEMA_PAGE_SIZE - 1);
+
+			if (s != cand[ncand - 1])
+				cand[ncand++] = s;
+		}
+	}
+	return ncand;
+}
+
+/*
+ * Full method-stream parse: every word run must be a valid INC_METHOD header
+ * covering the entire buffer, nothing left over. Real pushes satisfy this by
+ * construction; arbitrary data (firmware pages, page tables, allocator free
+ * lists) practically never does - a false candidate must pass this test for
+ * the whole push, not just for its first header.
+ */
+static bool svc_stream_full_ok(const u8 *buf, u32 len)
+{
+	u32 off = 0, methods = 0;
+
+	while (off + 4 <= len) {
+		u32 hdr, op, count;
+
+		memcpy(&hdr, buf + off, 4);
+		if (hdr == 0)
+			break;	/* trailing zero padding */
+		op = (hdr >> 29) & 0x7;
+		count = (hdr >> 16) & 0xfff;
+		if (op != 1 || !count || count > 64 ||
+		    off + 4 + count * 4 > len)
+			return false;
+		off += 4 + count * 4;
+		if (++methods > 1024)
+			return false;
+	}
+	/* Trailing zero padding is common in recycled push slots. */
+	while (off < len && buf[off] == 0)
+		off++;
+	return methods > 0 && off == len;
+}
+
+/*
+ * Pin a candidate page as the translation of the entry VA's page and verify
+ * by reading the whole push back and parsing it as a method stream. Requires
+ * the push to end exactly on a method boundary - the stream replaces a weak
+ * prefix check that accepted page-table pages (whose first word can decode as
+ * a header). Returns 0 and leaves the pin installed on success.
+ */
+static int svc_rescue_try(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
+			  u64 entry_va, u32 entry_len, u32 pin_len, u64 cand_pa,
+			  u32 cand_i, u32 ncand)
+{
+	u8 *full;
+	u32 j;
+	int rc = -ENOENT;
+
+	for (j = 0; j < pin_len; j += SEMA_PAGE_SIZE) {
+		svc_pin_store(entry_va + j, cand_pa + j, SVC_APERTURE_SYS);
+		svc_lat_store(entry_va + j, cand_pa + j, SVC_APERTURE_SYS);
+	}
+
+	full = kmalloc(entry_len, GFP_KERNEL);
+	if (!full)
+		return -ENOMEM;
+	if (!svc_read_va_trusted(vgpu, entry_va, full, entry_len) &&
+	    svc_stream_full_ok(full, entry_len)) {
+		for (j = 0; j < pin_len; j += SEMA_PAGE_SIZE)
+			svc_mark_hot(cand_pa + j);
+		pr_info("sema-svc: content rescue ch=0x%x rank=%u slot=%u entry_va=0x%llx len=%u -> pa=0x%llx (cand %u/%u)\n",
+			ch->h_object, ch->ce_index, 4 * ch->ce_index, entry_va,
+			entry_len, cand_pa, cand_i + 1, ncand);
+		rc = 0;
+	}
+	kfree(full);
+	return rc;
+}
+
+/*
  * Content rescue for unmapped push-buffer pool VAs.
  * When the pump encounters a GPFIFO entry whose push-buffer VA is not in the
  * VA->PA map, we scan guest RAM for a page containing a push whose semaphore
@@ -3347,103 +3493,95 @@ static int svc_rescue_push_page(struct nv_vgpu_device *vgpu,
 		return -ENOENT;
 	if (entry_va < SEMA_VA_LO || entry_va >= SEMA_VA_HI)
 		return -EINVAL;
-	if (time_before(jiffies, ch->rescue_last + 5 * HZ))
-		return -EAGAIN;
-	ch->rescue_last = jiffies;
 
 	/*
 	 * Slot-aligned entries pin the whole 1 MB slot at once (all of its
 	 * pushes share the mapping); an entry in the middle of a slot only
-	 * needs the pages its own push occupies - candidate 2 below is the
-	 * PA of the entry VA's page in both cases.
+	 * needs the pages its own push occupies - the page-walking candidates
+	 * in svc_rescue_match_buf cover the push starting on a page before the
+	 * release.
 	 */
 	pin_len = (entry_va & 0xfffff) ?
 		ALIGN(entry_len + SEMA_PAGE_SIZE, SEMA_PAGE_SIZE) : 0x100000;
 
-	buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL);
+	buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL);
 	if (!buf)
 		return -ENOMEM;
 
 	/*
-	 * Scan guest RAM for the exact release group at any 4-byte offset:
-	 *   +0x00 SET_SEMAPHORE_A header (0x20038090, count 3, subch 4)
-	 *   +0x04 hi == 1
-	 *   +0x08 lo & 0xfff == this channel's pool slot
-	 *   +0x0c payload (any small value; the tracked expectation can lag)
-	 *   +0x10 LAUNCH_DMA semaphore-release follower (type 0)
-	 * The header is checked first so the inner loop is one compare for
-	 * almost every offset; the remaining exact fields make false
-	 * positives (the firmware-page matches of the old matcher)
-	 * negligible.
-	 *
-	 * Each match yields two candidate pin bases: the page the group sits
-	 * in (correct when the matched group is in the slot's first page,
-	 * e.g. a stale release push) and the page implied by treating the
-	 * group as the end of THIS entry's push (correct when the push starts
-	 * at the slot base and is longer than one page). Each candidate is
-	 * validated by pinning and reading the entry VA back; the first that
-	 * parses as a push wins.
+	 * Hot pages first: pushbuffer pages are recycled and this workload
+	 * touches only a small working set, so the page holding a just-written
+	 * push is almost always already marked hot. Scanning the hot set is
+	 * milliseconds - the client's waits expire long before the multi-
+	 * second full-RAM scan below can run, and that scan also competes
+	 * with the targeted resolver and the sweeps for the service thread.
 	 */
-	for (gpa = 0; gpa < SEMA_PA_MAX && ncand < SVC_MBUF_MAX; gpa += SVC_SCAN_CHUNK) {
-		u32 o;
+	{
+		unsigned long h;
 
-		if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK))
-			continue;
-		for (o = 0; o + 24 <= SVC_SCAN_CHUNK && ncand < SVC_MBUF_MAX; o += 4) {
-			u32 hdr, hi, lo, payload, w5, v5;
-			u64 rel_pa, s;
+		for (h = find_next_bit(svc_hot, SEMA_HOT_PAGES, 0);
+		     h < SEMA_HOT_PAGES;
+		     h = find_next_bit(svc_hot, SEMA_HOT_PAGES, h + 1)) {
+			u32 n;
 
-			memcpy(&hdr, buf + o, 4);
-			if (hdr != 0x20038090u)
-				continue;
-			memcpy(&hi, buf + o + 4, 4);
-			if (hi != 1)
-				continue;
-			memcpy(&lo, buf + o + 8, 4);
-			if ((lo & 0xfff) != slot)
-				continue;
-			memcpy(&payload, buf + o + 12, 4);
-			if (!payload || payload >= (1u << 24))
+			gpa = (u64)h << SEMA_PAGE_SHIFT;
+			if (svc_read(vgpu, gpa, buf, SEMA_PAGE_SIZE))
 				continue;
-			memcpy(&w5, buf + o + 16, 4);
-			memcpy(&v5, buf + o + 20, 4);
-			if ((w5 >> 29) != 1 || ((w5 >> 13) & 0x7) != 4 ||
-			    ((w5 & 0x1fff) << 2) != 0x300 ||
-			    (v5 & 0x3) != 0)
-				continue;
-
-			rel_pa = gpa + o;
-			cand[ncand++] = rel_pa & ~(u64)(SEMA_PAGE_SIZE - 1);
-			s = (rel_pa - (entry_len >= 24 ? entry_len - 24 : 0)) &
-			    ~(u64)(SEMA_PAGE_SIZE - 1);
-			if (ncand < SVC_MBUF_MAX &&
-			    (ncand == 0 || cand[ncand - 1] != s))
-				cand[ncand++] = s;
+			ncand = svc_rescue_match_buf(buf, SEMA_PAGE_SIZE, gpa,
+						     slot, entry_len, cand, 0);
+			for (n = 0; n < ncand; n++) {
+				if (!svc_rescue_try(vgpu, ch, entry_va,
+						    entry_len, pin_len,
+						    cand[n], n, ncand)) {
+					kfree(buf);
+					return 0;
+				}
+			}
 		}
 	}
+
+	/*
+	 * A full-RAM targeted scan for this exact page is already running (it
+	 * is triggered by the failed read above): let it finish instead of
+	 * duplicating the RAM scan, and let the pump retry the entry next pass
+	 * while the scan's result is still pending.
+	 */
+	if (svc_resolve_in_progress(entry_va)) {
+		kfree(buf);
+		return -EAGAIN;
+	}
+	if (time_before(jiffies, ch->rescue_last + 2 * HZ)) {
+		kfree(buf);
+		return -EAGAIN;
+	}
+	ch->rescue_last = jiffies;
+
 	kfree(buf);
+	buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
 
-	for (i = 0; i < ncand; i++) {
-		u8 probe[64];
-		u32 j;
-
-		for (j = 0; j < pin_len; j += SEMA_PAGE_SIZE) {
-			svc_pin_store(entry_va + j, cand[i] + j,
-				      SVC_APERTURE_SYS);
-			svc_lat_store(entry_va + j, cand[i] + j,
-				      SVC_APERTURE_SYS);
-		}
-		if (!svc_read_va_trusted(vgpu, entry_va, probe,
-					 sizeof(probe)) &&
-		    svc_stream_prefix_ok(probe, sizeof(probe))) {
-			for (j = 0; j < pin_len; j += SEMA_PAGE_SIZE)
-				svc_mark_hot(cand[i] + j);
-			pr_info("sema-svc: content rescue ch=0x%x rank=%u slot=%u entry_va=0x%llx len=%u -> pa=0x%llx (cand %u/%u)\n",
-				ch->h_object, ch->ce_index, slot, entry_va,
-				entry_len, cand[i], i + 1, ncand);
-			return 0;
+	/*
+	 * Full-RAM fallback: the header is checked first so the inner loop is
+	 * one compare for almost every offset; the remaining exact fields make
+	 * false positives (the firmware-page matches of the old matcher)
+	 * negligible. Each candidate is validated by pinning and reading the
+	 * entry VA back; the first that parses as a push wins.
+	 */
+	for (gpa = 0; gpa < SEMA_PA_MAX; gpa += SVC_SCAN_CHUNK) {
+		if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK))
+			continue;
+		ncand = svc_rescue_match_buf(buf, SVC_SCAN_CHUNK, gpa, slot,
+					     entry_len, cand, 0);
+		for (i = 0; i < ncand; i++) {
+			if (!svc_rescue_try(vgpu, ch, entry_va, entry_len,
+					    pin_len, cand[i], i, ncand)) {
+				kfree(buf);
+				return 0;
+			}
 		}
 	}
+	kfree(buf);
 
 	if (ncand)
 		pr_info("sema-svc: content rescue ch=0x%x entry_va=0x%llx len=%u: %u candidate(s) did not validate\n",
@@ -3711,7 +3849,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 							svc.gpfifo_nomap++;
 							kfree(pb);
 							break;
-						} else if (++ch->rescue_fail >= 3) {
+						} else if (++ch->rescue_fail >=
+							   (ch->engine_type ? 2 : 1)) {
 							/*
 							 * The slot no longer holds this
 							 * push (overwritten buffer): the
@@ -3719,6 +3858,21 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 							 * candidate. Consume the entry so
 							 * the pump moves on instead of
 							 * rescanning RAM every 5 s.
+							 *
+							 * GR (compute) channels have no
+							 * rescue at all - their pushes
+							 * are compute-class method
+							 * streams the CE-shaped matcher
+							 * cannot validate - and their
+							 * completion signal is the
+							 * per-channel counter we publish
+							 * from `put`. Consume on the
+							 * first failed attempt: a slow
+							 * skip delays GPGet and the
+							 * counter long enough for the
+							 * client's submission wait to
+							 * expire (observed at ~12 s with
+							 * the 3-attempt path).
 							 */
 							pr_info("sema-chan: skipping stale entry h=0x%x va=0x%llx len=%u after %u failed rescues\n",
 								ch->h_object, va, len,
@@ -3772,8 +3926,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 				 * Persist the per-entry rescue state and the
 				 * expected payload: the pump works on a snapshot
 				 * copy, so without the write-back the stale-entry
-				 * skip (rescue_fail >= 3) and the 5 s rescue rate
-				 * limit (rescue_last) can never fire and the
+				 * skip (rescue_fail >= threshold) and the rescue
+				 * rate limit (rescue_last) can never fire and the
 				 * payload expectation is reinitialized every
 				 * pass.
 				 */
@@ -4037,90 +4191,121 @@ int gsp_sema_service_dump_channels(void)
  * discoverable from the structures the service scans; this lets a session
  * feed a pair that was verified from the guest and watch push execution use
  * it.
+ *
+ * The injection is fully asynchronous: the sysfs write only enqueues the
+ * pair and returns, so a batch of mappings lands in microseconds even while
+ * the service thread is deep in a scan. Timed-out single-slot injections
+ * used to be cancelled (the pair was silently lost), and single-slot params
+ * could not take a batch at all.
  */
-static struct completion map_add_done;
-static u64 map_add_va;
-static u64 map_add_pa;
-static u64 map_add_len;
-static u32 map_add_ap;
-static bool map_add_pending;
 #define SVC_PIN_REWIND	8
+#define SVC_MAP_ADD_MAX	64
+
+struct svc_map_add_req {
+	u64 va;
+	u64 pa;
+	u64 len;
+	u32 ap;
+};
+
+static struct svc_map_add_req svc_map_add_q[SVC_MAP_ADD_MAX];
+static u32 svc_map_add_head;	/* producer */
+static u32 svc_map_add_tail;	/* consumer (service thread) */
+static DEFINE_SPINLOCK(svc_map_add_lock);
 
 static void svc_handle_map_add(void)
 {
-	u32 ap = map_add_ap ? map_add_ap : SVC_APERTURE_SYS;
 	unsigned long flags;
-	u64 off;
-	int i;
 
-	if (!map_add_pending)
-		return;
+	for (;;) {
+		struct svc_map_add_req req;
+		u32 ap, i;
+		u64 off;
 
-	for (off = 0; off < (map_add_len ? map_add_len : 1);
-	     off += SEMA_PAGE_SIZE) {
-		svc_pin_store(map_add_va + off, map_add_pa + off, ap);
-		svc_lat_store(map_add_va + off, map_add_pa + off, ap);
-	}
-	/*
-	 * A translation can be replaced after the pump already consumed
-	 * pushes that release through it (a new client maps new physical
-	 * pages at the same VA, or the first translation was wrong): the
-	 * releases resolved against the old page, compared as stale against
-	 * its values, and were dropped. Rewind every channel a few ring
-	 * entries so those pushes are re-read under the new translation;
-	 * the release/fill application paths are forward-progress guarded,
-	 * so re-executing them is idempotent.
-	 */
-	spin_lock_irqsave(&svc.chan_lock, flags);
-	for (i = 0; i < SVC_MAX_CHANNELS; i++) {
-		struct svc_channel *ch = &svc.chan[i];
-		u32 get;
+		spin_lock_irqsave(&svc_map_add_lock, flags);
+		if (svc_map_add_tail == svc_map_add_head) {
+			spin_unlock_irqrestore(&svc_map_add_lock, flags);
+			break;
+		}
+		req = svc_map_add_q[svc_map_add_tail %
+				    SVC_MAP_ADD_MAX];
+		svc_map_add_tail++;
+		spin_unlock_irqrestore(&svc_map_add_lock, flags);
+
+		ap = req.ap ? req.ap : SVC_APERTURE_SYS;
+		for (off = 0; off < (req.len ? req.len : 1);
+		     off += SEMA_PAGE_SIZE) {
+			svc_pin_store(req.va + off, req.pa + off, ap);
+			svc_lat_store(req.va + off, req.pa + off, ap);
+		}
+		/*
+		 * A translation can be replaced after the pump already
+		 * consumed pushes that release through it (a new client maps
+		 * new physical pages at the same VA, or the first translation
+		 * was wrong): the releases resolved against the old page,
+		 * compared as stale against its values, and were dropped.
+		 * Rewind every channel a few ring entries so those pushes are
+		 * re-read under the new translation; the release/fill
+		 * application paths are forward-progress guarded, so
+		 * re-executing them is idempotent.
+		 */
+		spin_lock_irqsave(&svc.chan_lock, flags);
+		for (i = 0; i < SVC_MAX_CHANNELS; i++) {
+			struct svc_channel *ch = &svc.chan[i];
+			u32 get;
+
+			if (!ch->valid || !ch->gpfifo_entries)
+				continue;
+			/*
+			 * Clamp the rewind at 0: wrapping below zero (get=0 ->
+			 * entries - N) makes the pump read ring slots past the
+			 * producer that hold stale entries, so replay only
+			 * recently consumed slots or from the start of the
+			 * ring, never around the wrap.
+			 */
+			get = ch->last_get;
+			ch->last_get = (get >= SVC_PIN_REWIND) ?
+				get - SVC_PIN_REWIND : 0;
+		}
+		spin_unlock_irqrestore(&svc.chan_lock, flags);
 
-		if (!ch->valid || !ch->gpfifo_entries)
-			continue;
 		/*
-		 * Clamp the rewind at 0: wrapping below zero (get=0 -> entries
-		 * - N) makes the pump read ring slots past the producer that
-		 * hold stale entries, so replay only recently consumed slots
-		 * or from the start of the ring, never around the wrap.
+		 * Releases that missed translation before this injection may
+		 * be applicable now: keep the fast full-RAM sweep armed for a
+		 * few seconds so they land without waiting for an unrelated
+		 * stall.
 		 */
-		get = ch->last_get;
-		ch->last_get = (get >= SVC_PIN_REWIND) ? get - SVC_PIN_REWIND : 0;
+		svc.fast_force_until = jiffies + 10 * HZ;
+		pr_info("sema-svc: injected mapping va=0x%llx -> pa=0x%llx len=0x%llx (%s), rewound channels by %u\n",
+			req.va, req.pa, req.len,
+			ap == SVC_APERTURE_FB ? "vidmem" : "sysmem",
+			SVC_PIN_REWIND);
 	}
-	spin_unlock_irqrestore(&svc.chan_lock, flags);
-
-	/*
-	 * Releases that missed translation before this injection may be
-	 * applicable now: keep the fast full-RAM sweep armed for a few
-	 * seconds so they land without waiting for an unrelated stall.
-	 */
-	svc.fast_force_until = jiffies + 10 * HZ;
-	pr_info("sema-svc: injected mapping va=0x%llx -> pa=0x%llx len=0x%llx (%s), rewound channels by %u\n",
-		map_add_va, map_add_pa, map_add_len,
-		ap == SVC_APERTURE_FB ? "vidmem" : "sysmem", SVC_PIN_REWIND);
-	map_add_pending = false;
-	complete(&map_add_done);
 }
 
 int gsp_sema_service_add_map(u64 gpu_va, u64 pa, u64 len, u32 ap)
 {
+	unsigned long flags;
+	u32 next;
+
 	if (!svc.task)
 		return -ENODEV;
 
 	if (len > (64ULL << 20) || ap > SVC_APERTURE_FB)
 		return -EINVAL;
 
-	init_completion(&map_add_done);
-	map_add_va = gpu_va;
-	map_add_pa = pa;
-	map_add_len = len;
-	map_add_ap = ap;
-	map_add_pending = true;
-
-	if (!wait_for_completion_timeout(&map_add_done, 3 * HZ)) {
-		map_add_pending = false;
-		return -ETIMEDOUT;
+	spin_lock_irqsave(&svc_map_add_lock, flags);
+	next = svc_map_add_head + 1;
+	if (next - svc_map_add_tail > SVC_MAP_ADD_MAX) {
+		spin_unlock_irqrestore(&svc_map_add_lock, flags);
+		return -ENOSPC;
 	}
+	svc_map_add_q[svc_map_add_head % SVC_MAP_ADD_MAX].va = gpu_va;
+	svc_map_add_q[svc_map_add_head % SVC_MAP_ADD_MAX].pa = pa;
+	svc_map_add_q[svc_map_add_head % SVC_MAP_ADD_MAX].len = len;
+	svc_map_add_q[svc_map_add_head % SVC_MAP_ADD_MAX].ap = ap;
+	svc_map_add_head = next;
+	spin_unlock_irqrestore(&svc_map_add_lock, flags);
 	return 0;
 }
 
diff --git a/tools/guest-diag/inject_counters.sh b/tools/guest-diag/inject_counters.sh
index b4e20bd..8a56558 100755
--- a/tools/guest-diag/inject_counters.sh
+++ b/tools/guest-diag/inject_counters.sh
@@ -1,27 +1,43 @@
 #!/bin/bash
 # inject_counters.sh - register the running CUDA client's per-channel GPU
 # completion counters with the host sema service (the PART 45/46 manual
-# procedure, made durable). The service then publishes each GR channel's
-# consumed GPFIFO count into the client's counter, which is the completion
-# signal its trackers poll.
+# procedure, made durable). The host then tracks each GR channel's consumed
+# GPFIFO progress in the counter the client polls.
 #
 # Usage: tools/guest-diag/inject_counters.sh [guest-pid] [page-va] [count]
 #   guest-pid  default: pgrep -x cdt in the guest
 #   page-va    default: 0x20400f000 (observed counter page VA)
 #   count      default: 20 (one per GR channel)
 #
-# Handle order: GR channels (eng=0) from the host channel dump, ascending, are
-# zipped with ascending counter index (the observed 1:1 layout). Override the
-# list with GRS="0x...,0x...,...".
-set -euo pipefail
+# Env:
+#   WATCH=<seconds>  keep watching for new GR channels and register them,
+#                    instead of a single pass (default 0 = single pass).
+#   MAP=1            also enqueue the sema_map_add pin per channel (default
+#                    on). The pin teaches the host the counter page's VA->PA,
+#                    which is what lets the client's own release pushes into
+#                    the counter page apply; without it the host logs
+#                    "release miss ... (no translation)" and the counters
+#                    never move. The sysfs write is async now (queue), so
+#                    this is cheap.
+#   MAP=0            skip the pins (counters only advance from the host's
+#                    fallback put publishing).
+#   GRS=0x..,0x..    override the channel handle list.
+#
+# Handle order: GR channels (eng=0) from the host channel dump, descending,
+# are zipped with ascending counter index (the observed 1:1 layout; the
+# last-created channel owns counter 0).
+set -uo pipefail
 
 HERE="$(cd "$(dirname "$0")" && pwd)"
 VMEXEC="$HERE/../../scripts/vgpu_vm.py"
 PARAM=/sys/module/open_iov_vfio/parameters
+DONE=/tmp/inject_counters.done
 
 PID="${1:-}"
 PAGE_VA="${2:-0x20400f000}"
 COUNT="${3:-20}"
+WATCH="${WATCH:-0}"
+MAP="${MAP:-1}"
 
 if [ -z "$PID" ]; then
     PID=$(timeout 20 "$VMEXEC" exec "pgrep -x cdt" 2>/dev/null \
@@ -36,41 +52,86 @@ fi
 b64=$(base64 -w0 "$HERE/sem_counters.py")
 timeout 30 "$VMEXEC" exec "echo '$b64' | base64 -d > /tmp/sem_counters.py" >/dev/null
 
-# Channel handles: GR channels from the live channel dump, ascending.
-if [ -z "${GRS:-}" ]; then
+all_grs() {
     echo 1 | sudo tee "$PARAM/chan_dump" >/dev/null
     sleep 1
-    GRS=$(sudo dmesg | grep -o 'sema-chan-dump: h=0x[0-9a-f]* eng=0' \
-        | grep -o '0x[0-9a-f]*' | sort -u | paste -sd,)
-fi
-if [ -z "$GRS" ]; then
-    echo "ERROR: no GR channel handles (chan_dump empty; is a VM attached?)" >&2
-    exit 1
-fi
+    sudo dmesg | grep -o 'sema-chan-dump: h=0x[0-9a-f]* eng=0' \
+        | grep -o '0x[0-9a-f]*' | sort -u | paste -sd, || true
+}
+
+register_handles() {
+    local grs="$1"
+    [ -z "$grs" ] && return 0
+
+    # Counter index order: observed DESCENDING by channel handle (the client's
+    # last-created channel owns counter 0). ORDER=asc restores creation order.
+    if [ "${ORDER:-desc}" = "desc" ]; then
+        grs=$(echo "$grs" | tr ',' '\n' | sort -r | paste -sd,)
+    fi
 
-# Counter index order: observed DESCENDING by channel handle (the client's
-# last-created channel owns counter 0; verified live: trackers waiting on
-# counter idx2/idx3 belong to the channels 2nd/3rd from the top). ORDER=asc
-# restores creation order.
-if [ "${ORDER:-desc}" = "desc" ]; then
-    GRS=$(echo "$GRS" | tr ',' '\n' | sort -r | paste -sd,)
+    local pairs
+    pairs=$(timeout 30 "$VMEXEC" exec \
+        "python3 /tmp/sem_counters.py $PID $PAGE_VA --count=$COUNT --pairs $grs" \
+        2>/dev/null | grep -E '^0x[0-9a-f]+,0x[0-9a-f]+,[0-9]+$' || true)
+    if [ -z "$pairs" ]; then
+        echo "WARN: enumeration returned no pairs for: $grs" >&2
+        return 1
+    fi
+
+    local h va pa
+    while IFS=, read -r h va pa; do
+        # sema_counter is a direct state write (no service-thread round trip),
+        # so this loop completes in milliseconds.
+        if printf '%s,%s' "$h" "$pa" | sudo tee "$PARAM/sema_counter" >/dev/null; then
+            echo "$h" >> "$DONE"
+            echo "registered h=$h counter va=$va pa=$pa"
+        else
+            echo "WARN: sema_counter write failed for h=$h pa=$pa" >&2
+        fi
+        if [ "$MAP" = "1" ]; then
+            printf '%s,%s' "$va" "$pa" | sudo tee "$PARAM/sema_map_add" >/dev/null \
+                || echo "WARN: sema_map_add timed out for h=$h (pin skipped)" >&2
+        fi
+    done <<< "$pairs"
+    return 0
+}
+
+: > "$DONE"
+
+if [ -n "${GRS:-}" ]; then
+    register_handles "$GRS"
+    exit 0
 fi
 
-PAIRS=$(timeout 30 "$VMEXEC" exec \
-    "python3 /tmp/sem_counters.py $PID $PAGE_VA --count=$COUNT --pairs $GRS" \
-    2>/dev/null | grep -E '^0x[0-9a-f]+,0x[0-9a-f]+,[0-9]+$')
-if [ -z "$PAIRS" ]; then
-    echo "ERROR: enumeration returned no pairs (pagemap access?)" >&2
-    exit 1
+GRS=$(all_grs)
+
+if [ "$WATCH" = "0" ]; then
+    if [ -z "$GRS" ]; then
+        echo "ERROR: no GR channel handles (chan_dump empty; is a VM attached?)" >&2
+        exit 1
+    fi
+    register_handles "$GRS"
+    exit 0
 fi
 
-while IFS=, read -r h va pa; do
-    # printf: sysfs writes carry a trailing newline that kstrtoull tolerates
-    # for both fields; use it anyway (no newline is also fine).
-    printf '%s,%s' "$h" "$pa" | sudo tee "$PARAM/sema_counter" >/dev/null
-    # Pin the counter VA->PA too: a push that releases the counter then
-    # resolves and applies through the normal release path, and pins survive
-    # the periodic map rebuilds.
-    printf '%s,%s' "$va" "$pa" | sudo tee "$PARAM/sema_map_add" >/dev/null
-    echo "registered h=$h counter va=$va pa=$pa"
-done <<< "$PAIRS"
+# Watch mode: register newly appearing channels until the deadline passes and
+# no new handles show up for a few consecutive polls.
+deadline=$(( $(date +%s) + WATCH ))
+quiet=0
+while [ "$(date +%s)" -lt "$deadline" ]; do
+    GRS=$(all_grs)
+    new=""
+    for h in $(echo "$GRS" | tr ',' ' '); do
+        grep -qx "$h" "$DONE" 2>/dev/null || new="$new,$h"
+    done
+    new="${new#,}"
+    if [ -n "$new" ]; then
+        register_handles "$new" || true
+        quiet=0
+    else
+        quiet=$((quiet + 1))
+        [ "$quiet" -ge 30 ] && break
+    fi
+    sleep 2
+done
+echo "watch done: $(wc -l < "$DONE") channel(s) registered"
diff --git a/tools/guest-diag/push_guest_tools.sh b/tools/guest-diag/push_guest_tools.sh
index b0711d4..93f7e6a 100755
--- a/tools/guest-diag/push_guest_tools.sh
+++ b/tools/guest-diag/push_guest_tools.sh
@@ -33,6 +33,7 @@ push_b64 "$HERE/sem_counters.py" /tmp/sem_counters.py
 push_b64 "$HERE/sem_slots.py"   /tmp/sem_slots.py
 push_b64 "$HERE/read_va.py"     /tmp/read_va.py
 push_b64 "$HERE/guest_pmap.py"  /tmp/guest_pmap.py
+push_b64 "$HERE/track_waits.sh" /tmp/track_waits.sh
 
 echo "[*] compiling in guest..."
 "$VMEXEC" exec "gcc -O1 -w -o /tmp/ptio /tmp/ptio.c && \
diff --git a/tools/guest-diag/track_waits.sh b/tools/guest-diag/track_waits.sh
new file mode 100644
index 0000000..4c9cd1a
--- /dev/null
+++ b/tools/guest-diag/track_waits.sh
@@ -0,0 +1,73 @@
+#!/bin/bash
+# track_waits.sh - guest-side wait sampler: while a CUDA client (cdt) runs,
+# repeatedly dump the per-channel completion counters (values), the tracker
+# wait entries (trkinfo.py, every 2nd pass) and the push pages the host could
+# not resolve (read_va.py), so the last signal before a failure is recorded.
+#
+# Written for the guest (no host paths). Run as root in the guest:
+#   sudo /tmp/track_waits.sh <pid> [samples] [sleep]
+# Logs: /tmp/trk.log, /tmp/cnt.log, /tmp/push.log
+set -u
+
+PID="${1:?usage: track_waits.sh <pid> [samples] [sleep]}"
+N="${2:-600}"
+SLEEP="${3:-1}"
+PAGE_VA="${PAGE_VA:-0x20400f000}"
+COUNT="${COUNT:-20}"
+STRIDE="${STRIDE:-16}"
+OFFSET="${OFFSET:-0xec0}"
+
+: > /tmp/trk.log
+: > /tmp/cnt.log
+: > /tmp/push.log
+
+python3 - "$PID" "$PAGE_VA" "$COUNT" "$STRIDE" "$OFFSET" <<'PY' >> /tmp/cnt.log 2>&1 &
+import os, sys, time
+pid = int(sys.argv[1]); page = int(sys.argv[2], 16)
+count = int(sys.argv[3]); stride = int(sys.argv[4], 16); off = int(sys.argv[5], 16)
+pm = os.open('/proc/%d/pagemap' % pid, os.O_RDONLY)
+mem = os.open('/proc/%d/mem' % pid, os.O_RDONLY)
+while True:
+    ts = time.strftime('%T')
+    vals = []
+    for i in range(count):
+        va = page + off + i * stride
+        try:
+            os.lseek(pm, (va // 4096) * 8, 0)
+            e = int.from_bytes(os.read(pm, 8), 'little')
+            present = (e >> 63) & 1
+            if present:
+                v = int.from_bytes(os.pread(mem, 4, va), 'little')
+                vals.append('%d:0x%x' % (i, v))
+            else:
+                vals.append('%d:-' % i)
+        except OSError:
+            vals.append('%d:ERR' % i)
+    print('%s %s' % (ts, ' '.join(vals)), flush=True)
+    time.sleep(0.5)
+PY
+CNTPID=$!
+
+for i in $(seq 1 "$N"); do
+    TS=$(date +%T.%3N)
+    kill -0 "$PID" 2>/dev/null || { echo "PROCESS GONE at $TS (sample $i)" >> /tmp/trk.log; break; }
+
+    if [ $((i % 2)) -eq 1 ]; then
+        echo "=== $TS sample $i ===" >> /tmp/trk.log
+        timeout 8 gdb -p "$PID" -batch -x /tmp/trkinfo.py 2>&1 \
+            | grep -vE "^\[|Reading symbols|^0x[0-9a-f]+ in |warning:" >> /tmp/trk.log
+    fi
+
+    {
+        echo "=== $TS push pages ==="
+        python3 /tmp/guest_pmap.py "$PID" 0x201000000 0x203400000 0x203400020 2>&1
+        for va in 0x201000000 0x203400000 0x203400020; do
+            echo "--- $va"
+            timeout 5 python3 /tmp/read_va.py "$PID" "$va" 64 --u32 2>&1
+        done
+    } >> /tmp/push.log
+
+    sleep "$SLEEP"
+done
+kill "$CNTPID" 2>/dev/null
+echo "sampler done" >> /tmp/trk.log