sema/boot: r383 kill the blanket bounce copy-out (alias+staleness write surface) + write guards; REPLAY-CAPTURE closes the silent launch-drop hole; CE operand misses feed the pin-demand channel

- bounce copy-out now records staged_hpa[bi] per launch and skips: (a) recs whose hpa != staged slot content (alias/leftover), (b) pages the engine did not change (~955 of 968 per boot launch); the remaining changed pages re-validate via gsp_sema_gr_write_guard before the 4 KiB memcpy. Removes the blanket 968-page write-back that was the remaining boot-corruption lottery vector (guest slab freelist clobbered by a foreign page write).
- new bounded WGUARD-SKIP guards: gr_write_guard (pin-first re-resolution) on copy-out, release-word zero/restore (rel_va plumbed through the submit), stale-recover-zero, counter release; svc_write_guard pin-agreement on 4/8-byte SYS/FB service writes; svc_copy SYS-dst hold on a fresh-pin contradiction.
- svc_qmd_accum replay hole: a trusted replay of an unseen push now captures (REPLAY-CAPTURE) instead of silently dropping; known push -> LAUNCH-SKIP witness; capture history ring prevents double-capture.
- svc_op_translate miss path emits the stalled-VA demand signal for CE operands.

1 files changed, 246 insertions, 7 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index c9de53a..3c837b3 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -334,6 +334,10 @@ struct svc_channel {
 				 * publish targets counter_pa instead, so both
 				 * families of the counter page get written
 				 * (SA-COUNTER FORENSICS-grv7-counter F1) */
+	u64	release_va;	/* r378: the release VA release_pa was learned
+				 * from, kept so a late write (stale-recover
+				 * zero) can re-validate the target through
+				 * gsp_sema_write_guard before it lands */
 	u32	counter_last;	/* last value published to counter_pa */
 	u64	pushpg_va;	/* last push page logged as pin-vs-window
 				 * divergent (rate limit: one line per page) */
@@ -1290,6 +1294,7 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc)
 		int k;
 		u64 keep_counter_pa = 0;
 		u64 keep_release_pa = 0;
+		u64 keep_release_va = 0;
 		u8 keep_gr_index = 0, keep_ce_index = 0;
 		bool keep_pub_armed = false;
 		u32 keep_counter_gen = 0;
@@ -1315,6 +1320,7 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc)
 		if (same_gen) {
 			keep_counter_pa = ch->counter_pa;
 			keep_release_pa = ch->release_pa;
+			keep_release_va = ch->release_va;
 			keep_gr_index = ch->gr_index;
 			keep_ce_index = ch->ce_index;
 			/*
@@ -1396,6 +1402,7 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc)
 		if (same) {
 			ch->counter_pa = keep_counter_pa;
 			ch->release_pa = keep_release_pa;
+			ch->release_va = keep_release_va;
 			ch->gr_index = keep_gr_index;
 			ch->ce_index = keep_ce_index;
 		}
@@ -4512,7 +4519,18 @@ static int svc_op_translate(struct nv_vgpu_device *vgpu, u64 va, u64 *addr,
 	 * withholding the release corrupted the next session (r59: the late
 	 * memset overwrote refilled data after the client had passed the
 	 * release), so the two must move together.
+	 *
+	 * r383 (streams2 big-copy class): name the missed VA through the
+	 * same stalled-VA demand channel the GR fill uses. The userspace pin
+	 * stream watches the "stalled VA" row and pushes the VA to the
+	 * in-guest scanner, so the pin lands on the next scanner pass
+	 * instead of waiting out a full scan cadence. The hold itself is
+	 * unchanged; this only shortens the window in which the entry sits
+	 * op-pending (r377: a held H2D copy executed ~1.5 s late and read a
+	 * staging page the client had since refilled with the next copy's
+	 * bytes - the neighbor-data class).
 	 */
+	svc_log_stalled_va(va & ~(u64)(SEMA_PAGE_SIZE - 1));
 	svc.op_tr_pending = 1;
 	return -ENOENT;
 }
@@ -4938,6 +4956,16 @@ struct svc_gr_cand_rec {
 static struct svc_gr_cand_rec svc_gr_cand_ring[SVC_GR_CAND_RING];
 static u32 svc_gr_cand_pos;
 #define svc_gr_cand (svc_gr_cand_ring[svc_gr_cand_pos])
+/*
+ * r383: push-VA history of recent captures, beyond the (four-slot)
+ * candidate ring's lifetime. A replayed decode of a complete QMD asks
+ * "has this launch ever been captured?"; the ring alone can have rotated
+ * past it, and a wrong "no" would re-capture (and re-execute) an already
+ * forwarded launch. Bounded, service thread only.
+ */
+#define SVC_GR_CAP_HIST		8
+static u64 svc_gr_cap_hist[SVC_GR_CAP_HIST];
+static u32 svc_gr_cap_hist_pos;
 
 static_assert(SVC_GR_QMD_WORDS == GSP_PHYS_GR_QMD_WORDS,
 	      "the sema GR QMD word count must match the backend ABI");
@@ -5048,6 +5076,9 @@ static void svc_gr_cand_capture(struct svc_channel *ch,
 	svc_gr_cand.h_object = ch->h_object;
 	svc_gr_cand.push_va = push_va;
 	svc_gr_cand.push_len = push_len;
+	/* r383: remember the identity beyond this ring slot's lifetime. */
+	svc_gr_cap_hist[svc_gr_cap_hist_pos % SVC_GR_CAP_HIST] = push_va;
+	svc_gr_cap_hist_pos++;
 	if (push && push_len)
 		svc_gr_cand.push_hash = svc_push_hash(push, push_len);
 	if (push && push_len >= 4) {
@@ -5668,6 +5699,36 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total,
 		return -ENOENT;
 	}
 	/*
+	 * r378: the same stale-table class for SYS destinations resolved by
+	 * the walk/scan (prov >= 3): a FRESH injected pin that disagrees is
+	 * the guest's newest statement, so writing through the resolved page
+	 * lands on a page the guest has freed/re-homed (the guest-corruption
+	 * class). Hold through the same bounded re-decode path instead of
+	 * writing; a missing pin keeps the historical behaviour (the
+	 * resolution is then the write authority).
+	 */
+	if (dst_virt && dst_aperture == CE_APERTURE_SYS &&
+	    dst_prov >= 3 && dst_orig >= SVC_VAS_FLOOR &&
+	    gsp_sema_copy_dst_pin_guard) {
+		u64 ipa = 0;
+		int iap = SVC_APERTURE_SYS;
+
+		if (!svc_ipin_lookup(dst_orig, &ipa, &iap) &&
+		    (((ipa & ~(u64)(SEMA_PAGE_SIZE - 1)) !=
+		      (dst & ~(u64)(SEMA_PAGE_SIZE - 1))) ||
+		     svc_ap_of(iap) != SVC_APERTURE_SYS) &&
+		    svc_dst_guard_hold(dst_orig)) {
+			svc.copy_dst_hold++;
+			if (svc.copy_dst_hold <= 16)
+				pr_info("sema-svc: device-dst copy held: dst=0x%llx prov=%u pin=0x%llx (stale SYS target)\n",
+					dst_orig, dst_prov, ipa);
+			if (!src_phys && src_orig >= SVC_VAS_FLOOR)
+				svc_copy_snap_capture(vgpu, dst_orig,
+						      src_orig, total);
+			return -ENOENT;
+		}
+	}
+	/*
 	 * r231 READ-ORDER guard/witness: a device-src copy whose source VA
 	 * has a PENDING device-dst write hold reads the PRE-write content -
 	 * the writer's bytes land only when the guard resolves while the
@@ -7323,6 +7384,15 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h,
 		return;
 	}
 	nv = inc ? cur + payload : payload;
+	/*
+	 * r378: the slot PA was learned from the release VA (or registered
+	 * from the client's pagemap) earlier; re-validate it against the
+	 * current pin before the 4-byte release lands, so a slot the guest
+	 * has since freed/re-homed is never written through the stale page.
+	 */
+	if (sem_va && !gsp_sema_write_guard(sem_va, pa, SVC_APERTURE_SYS,
+					    "counter-rel"))
+		return;
 	if (svc_write_ap(vgpu, pa, SVC_APERTURE_SYS, &nv, sizeof(nv))) {
 		if (gsp_sema_dbg && svc.counter_rel_dbg < 256) {
 			svc.counter_rel_dbg++;
@@ -7756,6 +7826,13 @@ static void svc_counter_learn(struct nv_vgpu_device *vgpu, u32 h, u64 sem_va)
 					pr_info("sema-chan: release learned h=0x%x pa=0x%llx (from release va=0x%llx)\n",
 						h, pa & ~0x3ULL, sem_va);
 			}
+			/*
+			 * r378: keep the VA the slot was learned from so a
+			 * late write to release_pa (the stale-recover zero)
+			 * can re-validate the target through
+			 * gsp_sema_write_guard before it lands.
+			 */
+			svc.chan[i].release_va = sem_va;
 			break;
 		}
 	}
@@ -7991,6 +8068,29 @@ static void svc_fwd_rel_direct_note(u32 h, u64 va, u32 payload)
 static u32 svc_gr_qmd_seen;
 static u64 svc_gr_nocap_log;
 
+/*
+ * r383: has a launch with this push VA ever been captured? Scans the live
+ * candidate ring and the recent-capture history. Used by the replayed-QMD
+ * dispatch below: a replay whose push was never captured is a launch that
+ * would otherwise be silently dropped (the streams2 second-vadd class); a
+ * replay of a captured/offered push is the exactly-once no-op.
+ */
+static bool svc_gr_cand_push_seen(u64 push_va)
+{
+	u32 i;
+
+	if (!push_va)
+		return false;
+	for (i = 0; i < SVC_GR_CAND_RING; i++)
+		if (svc_gr_cand_ring[i].valid &&
+		    svc_gr_cand_ring[i].push_va == push_va)
+			return true;
+	for (i = 0; i < SVC_GR_CAP_HIST; i++)
+		if (svc_gr_cap_hist[i] == push_va)
+			return true;
+	return false;
+}
+
 static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 			  struct svc_push_state *st, u32 addr, u32 count,
 			  const u8 *data, const u8 *push, u32 push_len,
@@ -8073,13 +8173,47 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 			pr_info("sema-gr: inline recs n=%u dup_dst=%u\n",
 				n, dup);
 		}
-		if (gsp_phys_gr_forward_enabled() && trusted && !replay)
-			svc_gr_cand_capture(ch, st, recs, n, push, push_len,
-					    st->push_va);
-		else if (!replay || !gsp_phys_gr_forward_enabled())
+		if (gsp_phys_gr_forward_enabled()) {
+			if (trusted && !replay) {
+				svc_gr_cand_capture(ch, st, recs, n, push,
+						    push_len, st->push_va);
+			} else if (trusted && replay &&
+				   !svc_gr_cand_push_seen(st->push_va)) {
+				/*
+				 * r383 (streams2 second-vadd class): a
+				 * complete QMD first seen on a replay pass
+				 * fell through BOTH arms below - the
+				 * forward path skips replays and the
+				 * bridge arm needs !replay - so the
+				 * launch was silently dropped (zero
+				 * engine writes, client read stale/zero
+				 * outputs). The push VA was never
+				 * captured, so capture it now (trusted
+				 * content only, as the first-sight path);
+				 * a replay of an already captured/offered
+				 * launch stays the exactly-once no-op.
+				 */
+				pr_info("sema-gr: REPLAY-CAPTURE h=0x%x push=0x%llx (complete QMD first seen on a replay pass; no capture on record)\n",
+					ch->h_object, st->push_va);
+				svc_gr_cand_capture(ch, st, recs, n, push,
+						    push_len, st->push_va);
+			} else if (replay && svc_gr_nocap_log < 64) {
+				svc_gr_nocap_log++;
+				pr_info("sema-gr: LAUNCH-SKIP h=0x%x push=0x%llx (replay; already captured=%d trusted=%d)\n",
+					ch->h_object, st->push_va,
+					svc_gr_cand_push_seen(st->push_va),
+					trusted);
+			} else if (!replay) {
+				gsp_cuda_bridge_launch(vgpu, ch->h_object,
+						       st->qmd,
+						       st->qmd_dev_va,
+						       st->push_va, recs, n);
+			}
+		} else if (!replay) {
 			gsp_cuda_bridge_launch(vgpu, ch->h_object, st->qmd,
 					       st->qmd_dev_va, st->push_va,
 					       recs, n);
+		}
 		st->qmd_n = 0;
 		st->qmd_bits = 0;
 		st->qmd_have = 0;
@@ -16197,6 +16331,12 @@ static void svc_fwd_gr_work(void)
 	gs.rel_hpa = rel_hpa;
 	gs.rel_off = (u32)(svc_fwd_req.gr_rel_va & (SEMA_PAGE_SIZE - 1));
 	gs.rel_payload = svc_fwd_req.gr_rel_payload;
+	/*
+	 * r378: carry the release VA so the backend can re-validate the
+	 * release-word write targets (zero-on-retry, restore-on-accept)
+	 * through gsp_sema_gr_write_guard before they land.
+	 */
+	gs.rel_va = svc_fwd_req.gr_rel_va;
 	gs.sem_hpa = 0;
 	gs.sem_off = 0;
 	gs.sem_payload = 0;
@@ -16929,6 +17069,90 @@ int gsp_sema_gr_reloc_resolve(u64 va, struct gsp_sema_gr_reloc *out)
 }
 
 /*
+ * r378 write guard, GR backend half (see gsp_sema_service.h). The GR submit
+ * resolves every operand page once and the copy-out/release writes run
+ * seconds later (the bounded poll, the retire wait, the outcome retries); a
+ * guest buffer freed and re-homed inside that window would receive a full
+ * page of another buffer's content through the stale HPA - the guest
+ * boot-corruption class (slab freelist clobbered by a foreign page write).
+ *
+ * Authority order mirrors the fill's own choice (r372/r376 pin-wins): a
+ * FRESH injected pin is the guest's newest statement, so the guard compares
+ * the write target against the pin's page first; only a pin-less VA falls
+ * back to the trusted resolver chain. The landing table is deliberately NOT
+ * the authority here - our own launches feed it, so it would confirm the
+ * target by construction. A miss on every source keeps the historical
+ * behaviour (the recorded landing is the write authority), so the guard can
+ * only remove provably stale writes. Bounded diagnostic budget.
+ */
+static u32 svc_wguard_log;	/* r378 WGUARD-SKIP budget (256/boot) */
+bool gsp_sema_gr_write_guard(u64 va, u64 hpa, const char *path)
+{
+	u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
+	u64 ipa = 0, cur_hpa = 0;
+	int iap = SVC_APERTURE_SYS;
+	u32 off = 0;
+	int ret;
+
+	if (!va || !hpa)
+		return true;
+	if (!svc_ipin_lookup(page, &ipa, &iap)) {
+		u32 kind = svc_ap_of(iap) == SVC_APERTURE_FB ?
+			   SVC_APERTURE_FB : SVC_APERTURE_SYS;
+
+		ret = svc_fwd_operand_hpa(svc_fwd_req.vgpu, kind,
+					  ipa & ~(u64)(SEMA_PAGE_SIZE - 1),
+					  0, SEMA_PAGE_SIZE, &cur_hpa, &off);
+		if (ret || off)
+			return true;	/* pin page unresolvable: keep */
+	} else {
+		struct gsp_sema_gr_reloc r;
+
+		if (gsp_sema_gr_reloc_resolve(va, &r))
+			return true;	/* no current answer: keep */
+		cur_hpa = r.hpa;
+	}
+	if ((cur_hpa & ~(u64)(SEMA_PAGE_SIZE - 1)) ==
+	    (hpa & ~(u64)(SEMA_PAGE_SIZE - 1)))
+		return true;
+	if (svc_wguard_log < 256) {
+		svc_wguard_log++;
+		pr_warn("vgsp-sema: WGUARD-SKIP path=%s va=0x%llx used=0x%llx now=0x%llx (stale write target)\n",
+			path, va, hpa, cur_hpa);
+	}
+	return false;
+}
+
+/*
+ * r378 write guard, service half (see gsp_sema_service.h). A fresh injected
+ * pin is the guest's newest statement about a VA; a pin that disagrees with
+ * the recorded write target means the page was freed/re-homed (the r372/r376
+ * pin-wins rule applied at the write instead of only at the resolution). A
+ * missing pin keeps the historical behaviour - the recorded landing is then
+ * the write authority - so no green path loses its write.
+ */
+bool gsp_sema_write_guard(u64 va, u64 pa, u32 ap, const char *path)
+{
+	u64 ipa = 0;
+	int iap = SVC_APERTURE_SYS;
+
+	if (!va || !pa || va < SVC_VAS_FLOOR)
+		return true;
+	if (svc_ipin_lookup(va, &ipa, &iap))
+		return true;	/* no fresh pin: nothing newer to obey */
+	if ((ipa & ~(u64)(SEMA_PAGE_SIZE - 1)) ==
+	    (pa & ~(u64)(SEMA_PAGE_SIZE - 1)) &&
+	    svc_ap_of(iap) == svc_ap_of((int)ap))
+		return true;
+	if (svc_wguard_log < 256) {
+		svc_wguard_log++;
+		pr_warn("vgsp-sema: WGUARD-SKIP path=%s va=0x%llx used=0x%llx/%u pin=0x%llx/%d (stale write target)\n",
+			path, va, pa, ap, ipa, iap);
+	}
+	return false;
+}
+
+/*
  * m144 (second-client launch-state divergence): collect device-VA-shaped
  * candidate pointers from the bytes the ENGINE will actually execute.
  *
@@ -18624,9 +18848,24 @@ static void svc_gr_stale_wedge_check(struct nv_vgpu_device *vgpu,
 				}
 				ch->counter_last = 0;
 			} else if (upa) {
-				int zrc = svc_write_ap(vgpu, upa,
-						       SVC_APERTURE_SYS,
-						       &zero, sizeof(zero));
+				int zrc;
+
+				/*
+				 * r378: the zero is authored from a PA
+				 * learned earlier (release_va -> release_pa);
+				 * re-validate it before the 4-byte write
+				 * lands so a freed/re-homed slot is never
+				 * zeroed through the stale page.
+				 */
+				if (!gsp_sema_write_guard(ch->release_va,
+							  upa,
+							  SVC_APERTURE_SYS,
+							  "stale-recover-zero"))
+					zrc = -ESTALE;
+				else
+					zrc = svc_write_ap(vgpu, upa,
+							   SVC_APERTURE_SYS,
+							   &zero, sizeof(zero));
 				if (gsp_sema_dbg && svc.pend_dbg_log < 1024) {
 					svc.pend_dbg_log++;
 					pr_info("sema-diag: stale-recover-zero h=0x%x upa=0x%llx last=%u zrc=%d\n",