sema-svc: deliver UVM CE tracking releases without host-side pins

The runtime client's registration stalled when no guest-side pin scanner was
running: the content rescue refused to run before the pool semaphore was
translated (its expected payload was unlearned), and unreadable CE entries
were consumed after two failed rescues ~40 ms apart, dropping the client's
wait. The push pages then could not be found within the entry's grace window.

- rescue: drop the learned-payload gate (the matcher only needs the channel
  rank and the release shape); keep unreadable CE entries pending for
  sema_ce_grace_ms instead of consuming after a fixed attempt count
- expected payload: re-learn from the pool whenever it advanced (a deferred
  release could land after the first read, leaving the expectation stale)
- matcher: require the expected payload exactly and validate the read-back
  stream carries it, so a recycled page of an older push cannot be pinned
- scan: descending full-range pass (the work allocations live in the high
  RAM block) plus a +/-512 MB near pass around the verified push-buffer base
- releases decoded with an untranslated semaphore are queued and applied
  when the page appears (svc_pending_add on both decode paths)
- a CE push that reads as a plausible stream but carries no release for the
  channel is re-routed through the rescue instead of silently consumed
- new module param sema_rel_recover (default off): after grace + rescue,
  apply the channel's expected tracking value so the client's wait clears;
  counter-visible as releases_recovered, distinct from sema_fake_rel

Evidence: fresh boot, real releases, no host watcher, no kprobes - the six
UVM CE first pushes are found by content rescue, their releases applied, and
the client reaches cudaGetDeviceCount rc=0 (it then waits for the third
push, which this change routes to the rescue).

3 files changed, 367 insertions, 58 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 75c5697..1ea8038 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -355,6 +355,7 @@ struct sema_svc {
 	u64			memsets_fail;
 	u64			ring_ops;		/* op runs from ring-decoded pushes */
 	u64			lda_log;		/* LAUNCH_DMA inventory budget */
+	u64			rescue_rej_log;		/* rescue rejection budget */
 	u64			sem256_log;		/* 256-byte op trace budget */
 	u64			op_tr_log;		/* operand targeted-resolve budget */
 	u32			op_tr_pending;		/* an operand's targeted scan is in progress */
@@ -372,6 +373,7 @@ struct sema_svc {
 	u64			gpfifo_skipped;		/* stale entries consumed */
 	u64			releases_seen;
 	u64			releases_applied;
+	u64			releases_recovered;	/* expected-value fallback */
 	u64			releases_missed;	/* no VA->PA entry */
 	u64			releases_missed_log;
 	u64			last_miss_va;
@@ -1840,12 +1842,12 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu)
 		svc.releases_stale, svc.releases_jumped,
 		svc.walks_ok, svc.walks_ok + svc.walks_miss, svc.lat_hits,
 		svc_scan_cursor);
-	pr_info("sema-svc:   pages=%llu pushes=%llu pending=%llu nomap=%llu memset=%llu/%llu/%llu ringops=%llu denied=%llu false=%llu skip=%llu faked=%llu crel=%llu\n",
+	pr_info("sema-svc:   pages=%llu pushes=%llu pending=%llu nomap=%llu memset=%llu/%llu/%llu ringops=%llu denied=%llu false=%llu skip=%llu faked=%llu rec=%llu crel=%llu\n",
 		svc.pages_decoded, svc.pushes_seen, svc.pushes_pending,
 		svc.pushes_nomap, svc.memsets_done, svc.memsets_nomap,
 		svc.memsets_fail, svc.ring_ops, svc.ops_denied,
 		svc.false_trans, svc.gpfifo_skipped, svc.releases_faked,
-		svc.counter_rel_applied);
+		svc.releases_recovered, svc.counter_rel_applied);
 	pr_info("sema-svc:   counter pub_calls=%llu gr_rel=%llu\n",
 		svc.counter_pub_calls, svc.gr_rel_applied);
 	pr_info("sema-svc:   gpfifo done=%llu nomap=%llu vidmem_writes=%llu ring_auto=%u pb=%u/%u pins=%u/%llu ipins=%u/%llu\n",
@@ -2466,6 +2468,10 @@ static u64 svc_rel_va_seen;
 static u32 svc_dbg_streams;
 
 static void svc_counter_learn(u32 h, u64 sem_va);
+static void svc_pending_add(u64 va, u32 payload, u32 op);
+static int svc_rescue_push_page(struct nv_vgpu_device *vgpu,
+				struct svc_channel *ch, u64 entry_va,
+				u32 entry_len);
 
 /* GR per-channel completion counters: one 16-byte-strided u32 slot per GR
  * channel in a client semaphore page. Releases to this range must be written
@@ -3020,8 +3026,15 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 									  st.rs_payload);
 						break;
 					}
-					if (svc_sem_read(vgpu, va, &cur))
+					if (svc_sem_read(vgpu, va, &cur)) {
+						/*
+						 * No translation yet: queue
+						 * the release so it lands when
+						 * the page is discovered.
+						 */
+						svc_pending_add(va, st.rs_payload, 0);
 						break;
+					}
 					if (st.rs_payload <= cur)
 						break;
 					svc.pushes_pending++;
@@ -3237,9 +3250,25 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 
 				if (st.have_sem && st.sem_payload &&
 				    sem_va >= SEMA_VA_LO && sem_va < SEMA_VA_HI) {
-					if (svc_sem_read(vgpu, sem_va, &cur))
+					if (svc_sem_read(vgpu, sem_va, &cur)) {
 						svc.pushes_nomap++;
-					else {
+						/*
+						 * The semaphore page has no
+						 * translation yet (UVM's pool
+						 * page can land a moment after
+						 * its first push): defer the
+						 * release so it is applied when
+						 * the translation arrives
+						 * instead of silently dropping
+						 * the wait the client polls
+						 * forever.
+						 */
+						svc_pending_add(sem_va,
+								st.sem_payload,
+								(DMA_SEM_REDUCTION_EN(dma) &&
+								 DMA_SEM_REDUCTION(dma) ==
+								 SEM_REDUCTION_INC) ? 1 : 0);
+					} else {
 						have_cur = true;
 						if (st.sem_payload > cur)
 							pending = 1;
@@ -4147,6 +4176,16 @@ unsigned gsp_sema_gr_grace_ms = 8000;
 bool gsp_sema_fake_rel;
 
 /*
+ * Bounded recovery for CE release pushes whose page cannot be read or found
+ * by the content rescue within the CE grace window: apply the channel's
+ * expected tracking value so the client's wait clears. Same write as
+ * sema_fake_rel, but only after the grace window and a full rescue attempt;
+ * the push's data writes (if any) are not executed in that case. Off by
+ * default; counter-visible as releases_recovered.
+ */
+bool gsp_sema_rel_recover;
+
+/*
  * Master diagnostic switch for the per-push/per-doorbell/per-op tracing.
  * Off by default: a healthy boot must leave dmesg quiet (boot stats and
  * state changes only). Enable per run through the module parameter when
@@ -4387,7 +4426,8 @@ static void svc_pb_pair(struct nv_vgpu_device *vgpu, struct svc_channel *ch)
  * updated candidate count.
  */
 static u32 svc_rescue_match_buf(const u8 *buf, u32 len, u64 base_pa, u32 slot,
-				u64 entry_len, u64 *cand, u32 ncand)
+				u32 want_payload, u64 entry_len, u64 *cand,
+				u32 ncand)
 {
 	u32 o;
 
@@ -4407,6 +4447,16 @@ static u32 svc_rescue_match_buf(const u8 *buf, u32 len, u64 base_pa, u32 slot,
 		memcpy(&payload, buf + o + 12, 4);
 		if (!payload || payload >= (1u << 24))
 			continue;
+		/*
+		 * When the channel's next expected release payload is known
+		 * (learned from the pool), require it exactly: a recycled or
+		 * stale copy of an older push for the same channel rank has a
+		 * lower payload and must not be pinned as this entry's page
+		 * (observed: the second push of a channel re-pinned to the
+		 * page of its first push, so the real push was never read).
+		 */
+		if (want_payload && payload != want_payload)
+			continue;
 		memcpy(&w5, buf + o + 16, 4);
 		memcpy(&v5, buf + o + 20, 4);
 		if ((w5 >> 29) != 1 || ((w5 >> 13) & 0x7) != 4 ||
@@ -4460,6 +4510,21 @@ static bool svc_stream_full_ok(const u8 *buf, u32 len)
 }
 
 /*
+ * Does the read-back push carry the release this channel's entry should?
+ * When the expected payload is known it must appear verbatim; a recycled or
+ * stale page otherwise validates as a method stream but belongs to a previous
+ * push (observed: the second push of a channel accepted its own first push's
+ * page, whose release payload was one behind).
+ */
+static bool svc_rescue_payload_ok(u32 slot, u32 want, const u8 *buf, u32 len)
+{
+	u64 dummy[SVC_MBUF_MAX];
+
+	return svc_rescue_match_buf(buf, len, 0, slot, want, len,
+				    dummy, 0) > 0;
+}
+
+/*
  * Pin a candidate page as the translation of the entry VA's page and verify
  * by reading the whole push back and parsing it as a method stream. Requires
  * the push to end exactly on a method boundary - the stream replaces a weak
@@ -4482,20 +4547,76 @@ static int svc_rescue_try(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 	full = kmalloc(entry_len, GFP_KERNEL);
 	if (!full)
 		return -ENOMEM;
-	if (!svc_read_va_trusted(vgpu, entry_va, full, entry_len) &&
-	    svc_stream_full_ok(full, entry_len)) {
+	rc = svc_read_va_trusted(vgpu, entry_va, full, entry_len);
+	if (!rc && svc_stream_full_ok(full, entry_len) &&
+	    svc_rescue_payload_ok(4 * ch->ce_index,
+				  ch->expected_sem_payload, full,
+				  entry_len)) {
 		for (j = 0; j < pin_len; j += SEMA_PAGE_SIZE)
 			svc_mark_hot(cand_pa + j);
-		pr_info("sema-svc: content rescue ch=0x%x rank=%u slot=%u entry_va=0x%llx len=%u -> pa=0x%llx (cand %u/%u)\n",
+		pr_info("sema-svc: content rescue ch=0x%x rank=%u slot=%u entry_va=0x%llx len=%u want=%u -> pa=0x%llx (cand %u/%u)\n",
 			ch->h_object, ch->ce_index, 4 * ch->ce_index, entry_va,
-			entry_len, cand_pa, cand_i + 1, ncand);
+			entry_len, ch->expected_sem_payload, cand_pa, cand_i + 1, ncand);
 		rc = 0;
+	} else if (gsp_sema_dbg && svc.rescue_rej_log < 32) {
+		u32 w0 = 0;
+
+		svc.rescue_rej_log++;
+		if (!rc && entry_len >= 4)
+			memcpy(&w0, full, 4);
+		pr_info("sema-svc: rescue reject cand=0x%llx va=0x%llx read_rc=%d w0=0x%08x\n",
+			cand_pa, entry_va, rc, w0);
+		rc = -ENOENT;
 	}
 	kfree(full);
 	return rc;
 }
 
 /*
+ * Scan a guest-RAM range descending for a page that validates as this entry's
+ * push (method stream + expected release payload), pinning it and returning 0
+ * on the first hit. `hi` and `lo` are inclusive chunk start addresses.
+ */
+static int svc_rescue_scan_range(struct nv_vgpu_device *vgpu,
+				 struct svc_channel *ch, u64 entry_va,
+				 u32 entry_len, u32 pin_len, u32 slot,
+				 u64 hi, u64 lo, u8 *buf, u64 *cand)
+{
+	u64 gpa = hi;
+	unsigned long t0 = jiffies;
+	u32 total = 0;
+	u64 bytes = 0;
+
+	for (;;) {
+		if (!svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK)) {
+			u32 ncand, n;
+
+			bytes += SVC_SCAN_CHUNK;
+			ncand = svc_rescue_match_buf(buf, SVC_SCAN_CHUNK, gpa,
+						     slot,
+						     ch->expected_sem_payload,
+						     entry_len, cand, 0);
+			total += ncand;
+			for (n = 0; n < ncand; n++) {
+				if (!svc_rescue_try(vgpu, ch, entry_va,
+						    entry_len, pin_len,
+						    cand[n], n, ncand))
+					return 0;
+			}
+		}
+		if (gpa <= lo)
+			break;
+		gpa = (gpa - lo > SVC_SCAN_CHUNK) ?
+			gpa - SVC_SCAN_CHUNK : lo;
+	}
+	if (gsp_sema_dbg)
+		pr_info("sema-svc: rescue scan h=0x%x va=0x%llx [0x%llx..0x%llx] cand=%u %llu MB %u ms\n",
+			ch->h_object, entry_va, hi, lo, total, bytes >> 20,
+			jiffies_to_msecs(jiffies - t0));
+	return -ENOENT;
+}
+
+/*
  * Content rescue for unmapped push-buffer pool VAs.
  * When the pump encounters a GPFIFO entry whose push-buffer VA is not in the
  * VA->PA map, we scan guest RAM for a page containing a push whose semaphore
@@ -4509,12 +4630,21 @@ static int svc_rescue_push_page(struct nv_vgpu_device *vgpu,
 	u8 *buf;
 	u64 gpa;
 	u32 slot = 4 * ch->ce_index;
-	u32 want_payload = ch->expected_sem_payload;
 	u64 cand[SVC_MBUF_MAX];
 	u32 ncand = 0, i;
 	u32 pin_len;
 
-	if (!ch->engine_type || ch->ce_index >= 32 || !want_payload)
+	/*
+	 * The matcher needs only the channel rank (slot) and the release-push
+	 * shape; it does NOT need the pool semaphore translation that
+	 * expected_sem_payload is learned from. Requiring the learned payload
+	 * here made an unreadable first push fail instantly when the pool page
+	 * was still untranslated (observed: entry consumed after two -ENOENT
+	 * "rescues" 40 ms apart, release lost, client waits forever). Let the
+	 * content scan run in that case: a validated candidate pins the push
+	 * page, the pump re-reads the entry and applies/queues the release.
+	 */
+	if (!ch->engine_type || ch->ce_index >= 32)
 		return -ENOENT;
 	if (entry_va < SEMA_VA_LO || entry_va >= SEMA_VA_HI)
 		return -EINVAL;
@@ -4529,6 +4659,26 @@ static int svc_rescue_push_page(struct nv_vgpu_device *vgpu,
 	pin_len = (entry_va & 0xfffff) ?
 		ALIGN(entry_len + SEMA_PAGE_SIZE, SEMA_PAGE_SIZE) : 0x100000;
 
+	/*
+	 * Fast path: once a channel's push-buffer window base is verified
+	 * (svc_pb_pair pinned it from the first push), a later entry's page
+	 * sits at the same window-relative offset in guest RAM. Validate the
+	 * candidate by content (method stream + this entry's release payload)
+	 * before pinning - a recycled or non-contiguous window must not be
+	 * trusted blindly.
+	 */
+	for (i = 0; i < svc_pb_n; i++) {
+		u64 cand_pa;
+
+		if (svc_pb[i].slot != slot || !svc_pb[i].va ||
+		    entry_va < svc_pb[i].va)
+			continue;
+		cand_pa = svc_pb[i].pa_page + (entry_va - svc_pb[i].va);
+		if (!svc_rescue_try(vgpu, ch, entry_va, entry_len, pin_len,
+				    cand_pa, i, svc_pb_n))
+			return 0;
+	}
+
 	buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL);
 	if (!buf)
 		return -ENOMEM;
@@ -4553,7 +4703,8 @@ static int svc_rescue_push_page(struct nv_vgpu_device *vgpu,
 			if (svc_read(vgpu, gpa, buf, SEMA_PAGE_SIZE))
 				continue;
 			ncand = svc_rescue_match_buf(buf, SEMA_PAGE_SIZE, gpa,
-						     slot, entry_len, cand, 0);
+						     slot, ch->expected_sem_payload,
+						     entry_len, cand, 0);
 			for (n = 0; n < ncand; n++) {
 				if (!svc_rescue_try(vgpu, ch, entry_va,
 						    entry_len, pin_len,
@@ -4566,15 +4717,12 @@ static int svc_rescue_push_page(struct nv_vgpu_device *vgpu,
 	}
 
 	/*
-	 * A full-RAM targeted scan for this exact page is already running (it
-	 * is triggered by the failed read above): let it finish instead of
-	 * duplicating the RAM scan, and let the pump retry the entry next pass
-	 * while the scan's result is still pending.
+	 * The full-RAM targeted resolver (rm_mem record search) runs
+	 * incrementally on lookups and can be busy for minutes on a 6 GB
+	 * range; do not let it suppress the content rescue, which looks for
+	 * the push bytes themselves and is the only path that can pin a push
+	 * page fast. The 2 s rate limit below bounds the duplicate work.
 	 */
-	if (svc_resolve_in_progress(entry_va)) {
-		kfree(buf);
-		return -EAGAIN;
-	}
 	if (time_before(jiffies, ch->rescue_last + 2 * HZ)) {
 		kfree(buf);
 		return -EAGAIN;
@@ -4587,30 +4735,52 @@ static int svc_rescue_push_page(struct nv_vgpu_device *vgpu,
 		return -ENOMEM;
 
 	/*
-	 * Full-RAM fallback: the header is checked first so the inner loop is
-	 * one compare for almost every offset; the remaining exact fields make
-	 * false positives (the firmware-page matches of the old matcher)
-	 * negligible. Each candidate is validated by pinning and reading the
-	 * entry VA back; the first that parses as a push wins.
+	 * Near scan: a channel's push pages are allocated in the slab around
+	 * its verified first-push page (observed ~350 MB away), while the
+	 * entry's grace window rarely covers a full descending pass over the
+	 * 6 GB range. Scan +/-512 MB around every known push-buffer base for
+	 * this rank before the full-range fallback.
 	 */
-	for (gpa = 0; gpa < SEMA_PA_MAX; gpa += SVC_SCAN_CHUNK) {
-		if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK))
+	for (i = 0; i < svc_pb_n; i++) {
+		u64 hi, lo;
+
+		if (svc_pb[i].slot != slot || !svc_pb[i].pa_page)
 			continue;
-		ncand = svc_rescue_match_buf(buf, SVC_SCAN_CHUNK, gpa, slot,
-					     entry_len, cand, 0);
-		for (i = 0; i < ncand; i++) {
-			if (!svc_rescue_try(vgpu, ch, entry_va, entry_len,
-					    pin_len, cand[i], i, ncand)) {
-				kfree(buf);
-				return 0;
-			}
+		hi = ALIGN_DOWN(svc_pb[i].pa_page + (512ULL << 20),
+				SVC_SCAN_CHUNK);
+		lo = ALIGN(svc_pb[i].pa_page > (512ULL << 20) ?
+			   svc_pb[i].pa_page - (512ULL << 20) :
+			   (u64)SVC_SCAN_CHUNK, SVC_SCAN_CHUNK);
+		if (hi > SEMA_PA_MAX - SVC_SCAN_CHUNK)
+			hi = SEMA_PA_MAX - SVC_SCAN_CHUNK;
+		if (lo <= hi &&
+		    !svc_rescue_scan_range(vgpu, ch, entry_va, entry_len,
+					   pin_len, slot, hi, lo, buf, cand)) {
+			kfree(buf);
+			return 0;
 		}
 	}
-	kfree(buf);
 
-	if (ncand)
-		pr_info("sema-svc: content rescue ch=0x%x entry_va=0x%llx len=%u: %u candidate(s) did not validate\n",
-			ch->h_object, entry_va, entry_len, ncand);
+	/*
+	 * Full-range fallback, descending from the top of the scanned range.
+	 * The header is checked first so the inner loop is one compare for
+	 * almost every offset; the remaining exact fields make false positives
+	 * (the firmware-page matches of the old matcher) negligible. Each
+	 * candidate is validated by pinning and reading the entry VA back; the
+	 * first that parses as a push wins.
+	 *
+	 * Descending order matters for latency: the guest's work allocations
+	 * (push buffers, page directories, semaphore pools) live in the high
+	 * RAM block (observed 0x1054fe000..0x1221e050) while the entry's
+	 * grace window is a few seconds - an ascending scan from 0 needs
+	 * 4.4+ GB to reach them, a descending one finds them in the first
+	 * few hundred MB.
+	 */
+	i = svc_rescue_scan_range(vgpu, ch, entry_va, entry_len, pin_len, slot,
+				  SEMA_PA_MAX - SVC_SCAN_CHUNK, 0, buf, cand);
+	kfree(buf);
+	if (!i)
+		return 0;
 	return -ENOENT;
 }
 
@@ -4625,9 +4795,7 @@ static void svc_init_expected_payload(struct nv_vgpu_device *vgpu,
 	u64 pa = 0;
 	int ap = SVC_APERTURE_SYS;
 	u32 cur = 0;
-
-	if (ch->expected_sem_payload)
-		return; /* already initialized */
+	u32 want;
 
 	if (svc_lookup_trusted(vgpu, pool_va, &pa, &ap))
 		return; /* pool page not mapped yet */
@@ -4636,14 +4804,27 @@ static void svc_init_expected_payload(struct nv_vgpu_device *vgpu,
 			&cur, sizeof(cur)))
 		return;
 
-	if (cur < 0x100000) {
-		ch->expected_sem_payload = cur + 1;
-		if (gsp_sema_dbg && ch->logged_expected != cur) {
-			ch->logged_expected = cur;
-			pr_info("sema-svc: ch=0x%x rank=%u pool sem @0x%llx cur=%u -> expected=%u\n",
-				ch->h_object, ch->ce_index, pool_va, cur,
-				ch->expected_sem_payload);
-		}
+	if (cur >= 0x100000)
+		return;
+
+	/*
+	 * Track the pool value, do not freeze on the first readable sample. A
+	 * release we deferred (no translation at decode time) can land after
+	 * this ran and advanced the pool, leaving the expectation one behind;
+	 * the next unreadable push is then matched against a stale payload and
+	 * pins the wrong page (observed: the second push rescued to the first
+	 * push's page because the pool read 0 before the deferred write).
+	 * Recompute whenever the pool has advanced beyond our expectation.
+	 */
+	want = cur + 1;
+	if (want <= ch->expected_sem_payload)
+		return;
+	ch->expected_sem_payload = want;
+	if (gsp_sema_dbg && ch->logged_expected != cur) {
+		ch->logged_expected = cur;
+		pr_info("sema-svc: ch=0x%x rank=%u pool sem @0x%llx cur=%u -> expected=%u\n",
+			ch->h_object, ch->ce_index, pool_va, cur,
+			ch->expected_sem_payload);
 	}
 }
 
@@ -4942,6 +5123,32 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 							}
 							ch->op_retry = 0;
 							rel_seen = (svc_rel_va_seen != rel_before);
+							/*
+							 * A CE push that should carry this
+							 * channel's next tracking release but
+							 * decoded none read from a
+							 * stale/wrong page (its content only
+							 * resembles a method stream). Treat
+							 * it like an unreadable push: content
+							 * rescue first, then hold the entry
+							 * for the CE grace window instead of
+							 * consuming it - consuming dropped
+							 * the client's wait (observed after
+							 * the second push: client polls for
+							 * value 3 forever).
+							 */
+							if (!rel_seen && ch->engine_type &&
+							    ch->expected_sem_payload) {
+								u64 rel2 = svc_rel_va_seen;
+
+								if (!svc_rescue_push_page(vgpu, ch, va, len) &&
+								    !svc_read_va_trusted(vgpu, va, pb, len))
+									svc_decode_stream(vgpu, pb,
+											  len, 0, false,
+											  ch, idx, va, true);
+								rel_seen =
+									(svc_rel_va_seen != rel2);
+							}
 							if (rel_seen)
 								ch->last_rel_va =
 									svc_rel_va_seen;
@@ -4950,8 +5157,33 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 							if (ch->engine_type && ch->expected_sem_payload) {
 								if (!rel_seen && gsp_sema_fake_rel)
 									svc_fake_release(vgpu, ch);
-								else
+								else if (!rel_seen) {
+									if (!ch->defer_since)
+										ch->defer_since =
+											jiffies;
+									if (time_before(jiffies,
+											ch->defer_since +
+											msecs_to_jiffies(gsp_sema_ce_grace_ms))) {
+										svc.gpfifo_nomap++;
+										kfree(pb);
+										break;
+									}
+									pr_info("sema-chan: skipping stale entry h=0x%x va=0x%llx len=%u after %u ms grace\n",
+										ch->h_object, va, len,
+										gsp_sema_ce_grace_ms);
+									if ((gsp_sema_fake_rel ||
+									     gsp_sema_rel_recover) &&
+									    !svc_fake_release(vgpu, ch)) {
+										if (!ch->defer_since)
+											ch->defer_since =
+												jiffies;
+										svc.gpfifo_nomap++;
+										kfree(pb);
+										break;
+									}
+								} else {
 									ch->expected_sem_payload++;
+								}
 							}
 						}
 					} else {
@@ -5075,13 +5307,87 @@ if (gsp_sema_fake_rel &&
 							svc.gpfifo_nomap++;
 							kfree(pb);
 							break;
-						} else if (++ch->rescue_fail >=
-							   (ch->engine_type ? 2 : 1) &&
-							   (ch->engine_type ||
-							    (ch->defer_since &&
-							     time_after(jiffies,
+						} else if (ch->engine_type) {
+							/*
+							 * CE channel and the rescue
+							 * validated no candidate yet.
+							 * The push page's translation
+							 * (content scan, targeted
+							 * resolve, walk) can land
+							 * seconds after the channel
+							 * appears, and this entry
+							 * carries the tracking-
+							 * semaphore release the client
+							 * waits on. Keep the entry
+							 * pending for the CE grace
+							 * window (retrying the rescue)
+							 * instead of consuming it
+							 * after a fixed attempt count
+							 * - that consumed the client's
+							 * never-delivered wait.
+							 */
+							if (!ch->defer_since)
+								ch->defer_since =
+									jiffies;
+							if (time_before(jiffies,
 									ch->defer_since +
-									msecs_to_jiffies(gsp_sema_gr_grace_ms))))) {
+									msecs_to_jiffies(gsp_sema_ce_grace_ms))) {
+								svc.gpfifo_nomap++;
+								kfree(pb);
+								break;
+							}
+							pr_info("sema-chan: skipping stale entry h=0x%x va=0x%llx len=%u after %u ms grace\n",
+								ch->h_object, va, len,
+								gsp_sema_ce_grace_ms);
+							if (gsp_sema_fake_rel ||
+							    gsp_sema_rel_recover) {
+								u64 faked_before =
+									svc.releases_faked;
+
+								if (!svc_fake_release(vgpu, ch)) {
+									/*
+									 * The release could not be
+									 * delivered (pool page not
+									 * translated yet): keep the
+									 * entry pending so a pin that
+									 * lands shortly still applies
+									 * it instead of silently
+									 * dropping the client's wait.
+									 */
+									if (!ch->defer_since)
+										ch->defer_since =
+											jiffies;
+									svc.gpfifo_nomap++;
+									kfree(pb);
+									break;
+								}
+								if (!gsp_sema_fake_rel &&
+								    svc.releases_faked !=
+								    faked_before) {
+									svc.releases_recovered++;
+									pr_info("sema-chan: recovery release h=0x%x rank=%u expected=%u (push page unreadable after rescue)\n",
+										ch->h_object,
+										ch->ce_index,
+										ch->expected_sem_payload);
+								}
+							}
+							ch->rescue_fail = 0;
+							ch->defer_since = 0;
+							ch->last_get = (idx + 1) %
+								       ch->gpfifo_entries;
+							svc.gpfifo_skipped++;
+							/* The entry carried a release we
+							 * could not apply; run the fast
+							 * content sweep so the client's
+							 * wait still clears. */
+							stalled = true;
+							kfree(pb);
+							continue;
+						} else if (++ch->rescue_fail >= 1 &&
+							   ch->defer_since &&
+							   time_after(jiffies,
+								      ch->defer_since +
+								      msecs_to_jiffies(gsp_sema_gr_grace_ms))) {
 							/*
 							 * The slot no longer holds this
 							 * push (overwritten buffer): the
diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h
index baed3cf..7778443 100644
--- a/kernel/gsp_sema_service.h
+++ b/kernel/gsp_sema_service.h
@@ -78,6 +78,7 @@ extern bool gsp_sema_gr_rings;
  * driver's wait clears immediately instead of after the content sweep.
  */
 extern bool gsp_sema_fake_rel;
+extern bool gsp_sema_rel_recover;
 
 /*
  * Master diagnostic switch for the per-push/per-doorbell/per-op tracing
diff --git a/kernel/main.c b/kernel/main.c
index dd38402..fa88d19 100644
--- a/kernel/main.c
+++ b/kernel/main.c
@@ -2443,6 +2443,8 @@ MODULE_PARM_DESC(sema_gr_rings, "Auto-discover+pin GR channel rings (default 1)"
  */
 module_param_named(sema_fake_rel, gsp_sema_fake_rel, bool, 0644);
 MODULE_PARM_DESC(sema_fake_rel, "Acknowledge unreadable CE releases directly (default 0)");
+module_param_named(sema_rel_recover, gsp_sema_rel_recover, bool, 0644);
+MODULE_PARM_DESC(sema_rel_recover, "Recover unreadable CE releases after grace + rescue (default 0)");
 
 /*
  * Skip executing the pushed work (CE memsets/copies) while still applying the