vgsp: m163 lost-H2D copy integrity (drain injected pins before CE op resolution; hold failed device-copy ops) + m164 CLASS-2 GR no-capture entry hold + m165 pool release applies at fresh-pin time + m166 timeout-ladder warm-sem fail-fast; scanner: first-class pool VA resolution + periodic re-assert + demand routing

4 files changed, 250 insertions, 4 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index f54a0dc..d21c5ea 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -17192,6 +17192,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
 	u8 *qmd_dst_mem = NULL;
 	u32 qmd_rel_off = 0, ch_rel_off = 0;
 	u32 poll = 0, w0 = 0, w1 = 0, grpoll_n = 0;
+	bool sem_gated = false;
 	static u32 sked_prev[11];
 	static u32 sked_sample[11];
 	static u32 sked_tick;
@@ -18277,6 +18278,32 @@ gr_page_unmapped:
 					rr);
 			}
 			/*
+			 * m166: liveness gate before spending the fresh-entry
+			 * retries. The m156 warm sem push (the stage-4 GR sem
+			 * block) is the only proven "engine alive" probe; if
+			 * it times out, the engine is wedged for the boot and
+			 * every further retry is futile - fail fast instead of
+			 * burning ~20 s with the single-inflight slot held
+			 * (r167 memset2d: 8 internal ladder timeouts over 31 s
+			 * for an entry the front-end never fetched). The probe
+			 * itself is bounded by the sem push's own poll.
+			 */
+			if (!sem_gated) {
+				int sp_ret;
+
+				sem_gated = true;
+				mutex_lock(&gsp_phys_rm_lock);
+				sp_ret = gsp_phys_rmcall_gr_sem_push_locked(bar0);
+				mutex_unlock(&gsp_phys_rm_lock);
+				if (sp_ret) {
+					gsp_phys_gr_sem_wedged++;
+					pr_info("vgsp-gr: timeout ladder: warm sem WEDGED (rc=%d) - failing fast instead of fresh-entry retries\n",
+						sp_ret);
+					break;
+				}
+				pr_info("vgsp-gr: timeout ladder: warm sem ALIVE - continuing fresh-entry retries\n");
+			}
+			/*
 			 * The re-arm above only replays the schedule verbs; if the
 			 * entry was already fetched or the front-end never latched
 			 * it, that cannot make the QMD run. Re-publish a FRESH ring
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index c9c36b3..d6b9617 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -355,6 +355,12 @@ struct svc_channel {
 				 * the capture; forcing it lets a deferred
 				 * launch be forwarded once the engine's
 				 * in-flight work completes) */
+	u32	gr_nocap_idx;	/* m164 CLASS-2: slot in the no-capture hold
+				 * (a readable launch push whose decode saw
+				 * QMD words but built no candidate) */
+	u32	gr_nocap_replay;/* 1 = window opened on a replay decode */
+	u64	gr_nocap_va;	/* push VA the window was opened for */
+	unsigned long gr_nocap_since;	/* absolute first-seen stamp */
 };
 
 /*
@@ -775,6 +781,11 @@ struct sema_svc {
 						 * expectation advanced, write queued */
 	u64			gpfifo_held;		/* passes an entry was left pending
 						 * for a translation/grace window */
+	u64			gr_nocap_held;		/* m164: GR holds opened for a
+						 * launch push whose decode saw QMD
+						 * words but built no candidate */
+	u64			gr_nocap_consumed;	/* m164: no-capture windows that
+						 * expired and consumed the entry */
 	u64			gpfifo_starved;		/* entries consumed without a
 						 * deliverable exact slot */
 	u64			counter_rel_applied;	/* releases to registered counters */
@@ -1194,6 +1205,7 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc)
 		memset(ch, 0, sizeof(*ch));
 		ch->gr_hold_idx = ~0u;	/* m156: no hold armed yet */
 		ch->gr_rehold_idx = ~0u;
+		ch->gr_nocap_idx = ~0u;	/* m164: no no-capture window armed */
 		if (same)
 			bitmap_copy(ch->rel_done, keep_rel_done,
 				    SVC_GPFIFO_MAX_BITS);
@@ -2699,6 +2711,29 @@ static inline bool svc_pool_hold_active(void)
 }
 
 /*
+ * m165 (D-F2a): the blanket pool hold exists to stop a release decoded
+ * before the guest's own injection lands from being applied through a
+ * stale translation. Once a host-verified injected pin for THIS VA is
+ * present, that premise is gone: the pin source is the resolver's first
+ * choice (read-back proven) and the ipin table is wiped at session flush,
+ * so it is this session's ground truth. r156 measured every pool release
+ * waiting the full registration+5.00 s with the pin ready all along.
+ */
+static bool svc_pool_hold_active_va(u64 va)
+{
+	u64 ipa = 0;
+	int iap = 0;
+
+	if (!svc_pool_hold_active())
+		return false;
+	if (!gsp_sema_pool_hold_fresh)
+		return true;
+	if (svc_ipin_lookup(va, &ipa, &iap) == 0)
+		return false;
+	return true;
+}
+
+/*
  * Verified pool-semaphore translations, keyed by page VA. An entry is only
  * created from an injected pin or from a targeted rm_mem resolve whose page
  * was read back successfully; the learned cache and the discovered map are
@@ -3591,11 +3626,12 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu)
 		svc.false_trans, svc.gpfifo_skipped, svc.gpfifo_rescued,
 		svc.releases_faked,
 		svc.releases_recovered, svc.counter_rel_applied);
-	pr_info("sema-svc:   drain held=%llu starved=%llu forwarded=%llu faked=%llu rec=%llu ops_once=%llu reident=%llu opdef=%llu/%llu/%llu\n",
+	pr_info("sema-svc:   drain held=%llu starved=%llu forwarded=%llu faked=%llu rec=%llu ops_once=%llu reident=%llu opdef=%llu/%llu/%llu gr_nocap=%llu/%llu\n",
 		svc.gpfifo_held, svc.gpfifo_starved, svc.releases_forwarded,
 		svc.releases_faked, svc.releases_recovered,
 		svc.ops_once_skipped, svc.ops_reident,
-		svc.opdef_held, svc.opdef_applied, svc.opdef_expired);
+		svc.opdef_held, svc.opdef_applied, svc.opdef_expired,
+		svc.gr_nocap_held, svc.gr_nocap_consumed);
 	pr_info("sema-svc:   counter pub_calls=%llu gr_rel=%llu reresolve=%llu pushpg_fb=%llu altw=%llu residue=%llu\n",
 		svc.counter_pub_calls, svc.gr_rel_applied,
 		svc.sem_write_reresolve, svc.pushpg_fallback,
@@ -4836,6 +4872,11 @@ static int svc_grce_inline_write(struct nv_vgpu_device *vgpu, u64 dst_va,
 	return 0;
 }
 
+/* defined below (svc_handle_map_add); the CE decode path drains the queued
+ * scanner injections before resolving copy operands, exactly like the GR
+ * path's m143 drain. */
+static void svc_handle_map_add(void);
+
 /* Execute a decoded operation list (fills and copies). `ops_done` (may be
  * NULL) is the per-entry bitmap of ops that already ran for the entry's
  * identity; `ops_base` is the sequence number of ops[0] within the entry. */
@@ -4846,6 +4887,17 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops,
 	static u64 bigop_log;
 
 	svc.cur_ch = ch_h;
+	/*
+	 * m163: apply every queued scanner injection before resolving this
+	 * pass's copy operands. Without this, a CE copy decoded in the window
+	 * between the client's push and its pins being processed resolves
+	 * through the previous client's stale translation (or fails), while
+	 * its release is still applied - the client's cudaMemcpy "completes"
+	 * with no data and the next GR launch computes from the stale vidmem
+	 * page (streams2 r163: a1 held the previous client's h_a; the vadd
+	 * output was a_stale + b_fresh).
+	 */
+	svc_handle_map_add();
 	for (i = 0; i < n_ops; i++) {
 		struct svc_memset_op *m = &ops[i];
 		u32 seq = ops_base + i;
@@ -4917,9 +4969,18 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops,
 		 * (observed r72 c2) must not withhold a release that gates a GR
 		 * launch with a 5 s poll deadline. On expiry the release is
 		 * flushed and the op stays skipped (the pre-r62 fallback).
+		 * m163 extends the hold to copy ops whose destination is a
+		 * device buffer: a lost small H2D that silently "completes"
+		 * corrupts the next launch's input (streams2 r163: the 128 KB
+		 * a1 copy's release was applied with no copy op ever run; the
+		 * vadd computed a_stale + b and the client read the wrong c).
+		 * A small failing device copy now costs a bounded hold and the
+		 * retried decode runs it once the pins land - never silent
+		 * corruption. Small failing memsets keep the old trade.
 		 */
 		if (rc != 0 && rc != SVC_COPY_FORWARDED &&
-		    m->bytes >= (256u << 10))
+		    (m->bytes >= (256u << 10) ||
+		     (m->type == 1 && m->dst >= SVC_VAS_FLOOR)))
 			svc.op_tr_pending = 1;
 		if (gsp_sema_dbg && m->bytes == 256 && svc.sem256_log < 64) {
 			svc.sem256_log++;
@@ -5906,6 +5967,15 @@ static void svc_fwd_rel_direct_note(u32 h, u64 va, u32 payload)
  * mapped to its register offset. Each completed QMD resets the accumulator,
  * so a push may carry more than one launch.
  */
+/*
+ * m164: per-decode QMD witness for the CLASS-2 no-capture hold. Set to the
+ * max number of QMD words any accumulator saw during the current
+ * svc_decode_stream() call; reset at its top and read by the entry loop's
+ * no-capture check. Service-thread only.
+ */
+static u32 svc_gr_qmd_seen;
+static u64 svc_gr_nocap_log;
+
 static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 			  struct svc_push_state *st, u32 addr, u32 count,
 			  const u8 *data, const u8 *push, u32 push_len,
@@ -5935,6 +6005,8 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 			}
 		}
 	}
+	if (st->qmd_n > svc_gr_qmd_seen)
+		svc_gr_qmd_seen = st->qmd_n;
 	if (st->qmd_have == 3)
 		st->qmd_dev_va = (((u64)st->qmd_hi << 32) | st->qmd_lo) << 8;
 	if (st->qmd_n == SVC_QMD_WORDS) {
@@ -6001,6 +6073,9 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 	u32 ops_seq = 0;	/* ops appended this pass (sequence numbers) */
 	u32 off = 0;
 
+	/* m164: fresh CLASS-2 QMD witness for this decode pass. */
+	svc_gr_qmd_seen = 0;
+
 	/*
 	 * A false VA->PA pair (the discovered map is content-scanned) can
 	 * point a ring entry's push VA at an unrelated page - observed:
@@ -7440,7 +7515,7 @@ static void svc_apply_pending(struct nv_vgpu_device *vgpu)
 		 * a pool VA the base may not have landed: applying while
 		 * the word is still 0 is absorbed, so hold and retry.
 		 */
-		if (svc_pool_sem_va(p->va) && svc_pool_hold_active()) {
+		if (svc_pool_sem_va(p->va) && svc_pool_hold_active_va(p->va)) {
 			p->tries++;
 			svc_pending_note_starved(p);
 			i++;
@@ -9339,6 +9414,27 @@ bool gsp_sema_counter_derive = true;
 unsigned gsp_sema_gr_grace_ms = 8000;
 
 /*
+ * m164 CLASS-2: hold window (ms) for a readable GR launch push whose decode
+ * saw QMD words but built no launch candidate for its channel (a replay-
+ * suppressed capture, a partial/split QMD, or a push page the pin stream has
+ * not reached yet). The entry is left unconsumed for this window so the next
+ * pass re-decodes it with a forced fresh capture (gr_rehold_idx); on expiry
+ * the historical consume applies. Without this, such entries were consumed
+ * silently and the client's launch never ran (streams2 second vadd: 56
+ * consumes / 2 captures on one channel; the client read zeros).
+ */
+unsigned gsp_sema_gr_nocap_hold_ms = 6000;
+
+/*
+ * m165 (D-F2a): when set (default), a pool release is only deferred by the
+ * post-registration hold while NO host-verified injected pin exists for its
+ * VA; a fresh pin is this session's ground truth and lets the apply land at
+ * pin time (~0.5-2 s) instead of registration+5 s. Set 0 to restore the
+ * legacy blanket 5 s hold.
+ */
+bool gsp_sema_pool_hold_fresh = 1;
+
+/*
  * CE release fallback: when a push carrying a tracking-semaphore release
  * cannot be read (and the content rescue failed), advance the channel's
  * tracking semaphore to the expected payload directly. The push's data
@@ -14776,6 +14872,84 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 								}
 							}
 							/*
+							 * m164 CLASS-2 no-capture hold: a
+							 * readable GR launch push whose
+							 * decode saw QMD words but built
+							 * no launch candidate for this
+							 * channel (replay-suppressed
+							 * capture, partial/split QMD, or
+							 * a page the pin stream has not
+							 * reached) must not be consumed
+							 * at the tail - consuming drops
+							 * the client's launch and only
+							 * the RAM sweep's release reaches
+							 * it (r161/r167 streams2 second
+							 * vadd: 56 consumes / 2 captures
+							 * on channel 0x5c000023). Hold
+							 * the entry and force a fresh
+							 * capture on the next pass
+							 * (gr_rehold_idx makes the replay
+							 * decode rebuild the candidate);
+							 * one bounded window per
+							 * (slot, push VA), then fall
+							 * through to the historical
+							 * consume so the channel can
+							 * never livelock.
+							 */
+							if (!ch->engine_type && va && len &&
+							    gsp_phys_gr_forward_enabled() &&
+							    svc_gr_qmd_seen > 0 &&
+							    !(svc_gr_cand.valid &&
+							      svc_gr_cand.h_object ==
+									ch->h_object &&
+							      svc_gr_cand.push_va == va)) {
+								if (ch->gr_nocap_idx !=
+									(u32)idx ||
+								    ch->gr_nocap_va != va) {
+									ch->gr_nocap_idx =
+										(u32)idx;
+									ch->gr_nocap_va = va;
+									ch->gr_nocap_replay =
+										entry_replay ?
+											1 : 0;
+									ch->gr_nocap_since =
+										jiffies;
+								}
+								if (gsp_sema_gr_nocap_hold_ms &&
+								    time_before(jiffies,
+										ch->gr_nocap_since +
+										msecs_to_jiffies(
+										 gsp_sema_gr_nocap_hold_ms))) {
+									/* Force a fresh capture on
+									 * the retry: a replay decode
+									 * would suppress it. */
+									ch->gr_rehold_idx =
+										(u32)idx;
+									svc.gpfifo_held++;
+									svc.gr_nocap_held++;
+									if (svc_gr_nocap_log < 16) {
+										svc_gr_nocap_log++;
+										pr_info("sema-chan: GR no-capture hold h=0x%x entry=%u va=0x%llx len=%u replay=%d qmd_seen=%u (forcing capture; window %u ms)\n",
+											ch->h_object,
+											(u32)idx, va,
+											len,
+											ch->gr_nocap_replay,
+											svc_gr_qmd_seen,
+											gsp_sema_gr_nocap_hold_ms);
+									}
+									kfree(pb);
+									break;
+								}
+								/* Window expired: consume
+								 * exactly as the pre-change
+								 * tail does, with a proof
+								 * line. */
+								svc.gr_nocap_consumed++;
+								pr_info_ratelimited("sema-chan: GR no-capture hold expired h=0x%x entry=%u va=0x%llx len=%u (consuming without a launch)\n",
+									ch->h_object,
+									(u32)idx, va, len);
+							}
+							/*
 							 * A CE push that should carry this
 							 * channel's next tracking release but
 							 * decoded none read from a
@@ -15355,6 +15529,12 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 			ch->last_get = (idx + 1) % ch->gpfifo_entries;
 			ch->rescue_fail = 0;
 			ch->defer_since = 0;
+			/* m164: the entry was consumed (or fell through the
+			 * no-capture window); clear the hold identity so a
+			 * future push on this slot opens a fresh window. */
+			ch->gr_nocap_idx = ~0u;
+			ch->gr_nocap_since = 0;
+			ch->gr_nocap_va = 0;
 			ch->unread_va = 0;
 			ch->unread_until = 0;
 
@@ -15472,6 +15652,15 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 				 */
 				svc.chan[k].gr_hold_until = ch->gr_hold_until;
 				svc.chan[k].gr_hold_idx = ch->gr_hold_idx;
+				/* m164: the CLASS-2 no-capture window is snapshot
+				 * state too - without the write-back every pass
+				 * would re-arm it and rehold the entry forever
+				 * (the m150 lesson). */
+				svc.chan[k].gr_nocap_idx = ch->gr_nocap_idx;
+				svc.chan[k].gr_nocap_replay =
+					ch->gr_nocap_replay;
+				svc.chan[k].gr_nocap_va = ch->gr_nocap_va;
+				svc.chan[k].gr_nocap_since = ch->gr_nocap_since;
 				svc.chan[k].pushpg_va = ch->pushpg_va;
 				svc.chan[k].pushpg_n = ch->pushpg_n;
 				svc.chan[k].content_max = ch->content_max;
diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h
index c1dd763..ede8fa0 100644
--- a/kernel/gsp_sema_service.h
+++ b/kernel/gsp_sema_service.h
@@ -263,4 +263,16 @@ extern unsigned gsp_sema_push_hold_ms;
  */
 extern unsigned gsp_sema_gr_grace_ms;
 
+/*
+ * m164 CLASS-2: hold window (ms) for a GR launch push whose decode saw QMD
+ * words but built no launch candidate; 0 disables the hold.
+ */
+extern unsigned gsp_sema_gr_nocap_hold_ms;
+
+/*
+ * m165 (D-F2a): skip the post-registration pool hold when a verified
+ * injected pin exists for the VA (default on); 0 restores the blanket 5 s.
+ */
+extern bool gsp_sema_pool_hold_fresh;
+
 #endif /* _GSP_SEMA_SERVICE_H_ */
diff --git a/kernel/main.c b/kernel/main.c
index 257638a..c4e4792 100644
--- a/kernel/main.c
+++ b/kernel/main.c
@@ -2771,6 +2771,24 @@ module_param_named(sema_gr_grace_ms, gsp_sema_gr_grace_ms, uint, 0644);
 MODULE_PARM_DESC(sema_gr_grace_ms, "GR pending-entry grace window in ms (default 8000)");
 
 /*
+ * m164 CLASS-2: hold window (ms) for a readable GR launch push whose decode
+ * saw QMD words but built no launch candidate (replay-suppressed capture,
+ * partial/split QMD, or an unreadable push page). The entry stays unconsumed
+ * for this window and the next pass forces a fresh capture; on expiry the
+ * historical consume applies. 0 disables.
+ */
+module_param_named(sema_gr_nocap_hold_ms, gsp_sema_gr_nocap_hold_ms, uint, 0644);
+MODULE_PARM_DESC(sema_gr_nocap_hold_ms, "GR no-capture entry hold in ms (default 6000, 0 disables)");
+
+/*
+ * m165 (D-F2a): skip the post-registration pool-release hold when a
+ * host-verified injected pin for the VA exists (default on). 0 restores the
+ * legacy blanket 5 s hold.
+ */
+module_param_named(sema_pool_hold_fresh, gsp_sema_pool_hold_fresh, bool, 0644);
+MODULE_PARM_DESC(sema_pool_hold_fresh, "apply pool releases at fresh-pin time instead of registration+5s (default 1)");
+
+/*
  * Debug: inject a verified {GPU VA -> guest PA} translation into the
  * semaphore service (used when the client's mapping cannot be discovered
  * from guest structures):