vgsp: CONT-92 - sema_gl_prog_guest_va=0 knob (keep program operands at guest VAs; DECISIVE NEGATIVE: MMU Fault ENGINE GRAPHICS HUBCLIENT_PD @ 0x04350000 FAULT_PTE VIRT_READ - guest VAs not engine-fetchable even with Tier-1 PDE install, knob default 0); gsp_phys_rmcall_gr_pte_leaf applies pte_kind bits 63:56 to scratch-pool/Tier-1 leaves; per-window DS-ESR W1C clear + DS-ESR pre-clear log; POSITIVE: region write LANDS - sema_sph_region_base=0x600000 changed same retrying window latch a6000e20->a6000e66 (+SPH bits 1,2,6), deterministic per-draw re-validation; pte_kind=6 refuted; client init-push decoded: no program-region write exists, Turing hasProgramRegion=0 per NVIDIA caps table; next: sweep sema_sph_region_base 0x600900/0x601000/0x610000/0x1e0000 + both encodings vs per-window latch for SPH set below {5,9,10,11}; fallback SASS tail crossing 0x1000 never contig-mapped at SPH_VA+0x1000 - SESSION 153 r1307-r1310

1 files changed, 194 insertions, 16 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index 454c3e7..15a344d 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -10393,6 +10393,32 @@ static u64 gsp_phys_rmcall_pde(u64 pa)
 }
 
 /*
+ * SESSION 153-CONTINUED-92: leaf PTE for the engine-visible guest/scratch
+ * mappings with the configurable v2 KIND field (bits 63:56).
+ *
+ * m96 documented the symptom that motivates this: on Turing a kind=0 PTE is
+ * what the SM I-fetch refuses with MMU_NACK while the FE tolerates it, and
+ * `gsp_phys_rmcall_gr_pte_kind` was introduced to plant the win0/identity
+ * pool PTEs with a kind (UVM turing uses GENERIC_MEMORY 0x06). The knob was
+ * never applied to the scratch-PT pool mappings - every shadow VA the 3D
+ * window's program operands are relocated to (the VAs the DS fetches the
+ * SPH through) carried kind=0. The DS validates a program header, i.e. runs
+ * a program fetch; if that path is the SM-side one, kind=0 is a candidate
+ * cause for the SPH rejection that is independent of content (and explains
+ * why no content A/B ever moved the wall). With the knob 0 this is
+ * byte-identical to the previous PTE encoding; nonzero = experiment.
+ */
+static u64 gsp_phys_rmcall_gr_pte_leaf(u64 hpa)
+{
+	u64 pte = gsp_phys_rmcall_pte(hpa);
+	int kind = gsp_phys_rmcall_gr_pte_kind_value();
+
+	if (kind)
+		pte |= (u64)(u32)kind << 56;
+	return pte;
+}
+
+/*
  * GP10X leaf PTE for a frame-buffer (VIDEO_MEMORY aperture 0) address:
  * VALID=1 (bit 0), APERTURE=0 (bits 2:1), VOL=1 (bit 3), frame-buffer byte
  * offset >> 12 in the VIDEO address field (bits 32:8; dev_mmu.h:592-595,
@@ -17998,6 +18024,44 @@ bool gsp_phys_rmcall_gr_sph_bind_value(void)
 }
 
 /*
+ * SESSION 153-CONTINUED-92: leave program-family operands at their ORIGINAL
+ * guest VAs when the operand's page is resolved and identity-mappable.
+ *
+ * The DS I-fetch reaches the delivered images through the shadow VAs the
+ * window pass picks (r1298-r1306: stages fetched at 0x620900/0x620200/
+ * 0x620000) and rejects them with the bytes provably correct at those VAs
+ * (INLINE-RB rb=020a0481; canary proves the fetch follows the mapped page's
+ * content). The one variable left is the address space the DS validates in:
+ * the client programmed its own guest-VA program heap, and the shadow VAs
+ * (0x2xxxxx..0x6xxxxx pools) sit outside any region the client's state
+ * describes - a region/space mismatch rejects all bound stages regardless
+ * of content (rr=0/0, ds=a400e20 at every retire). With this knob the
+ * window pass first tries the Tier-1 identity mapper on the operand's own
+ * guest VA (launch page set first, the sema resolver else): when the guest
+ * VA is mapped to the resolved HPA, the operand is left verbatim, so the
+ * DS fetches and validates the SPH at the same VA space the client's heap
+ * uses. A page without an identity mapping keeps the shadow rewrite exactly
+ * as before (fail-open), and the identity-kept pairs still count toward the
+ * pass's relocated census so the SPH-BIND guard (orel == nrun) is stable.
+ * r1307 RESULT (experiment executed, negative): with the knob on, the
+ * beef lane's program pairs stayed at their guest VAs (0x4350900/0x4350f00/
+ * 0x4350200/0x4350000, PROG-GUEST-VA receipts) and the engine faulted
+ * `MMU Fault: ENGINE GRAPHICS HUBCLIENT_PD faulted @ 0x0_04350000. FAULT_PTE
+ * ACCESS_TYPE_VIRT_READ` within 2.5 s of every such launch - the Tier-1
+ * identity map installs (the PDE window is walked: FAULT_PTE, not
+ * FAULT_PDE) yet the engine still cannot fetch at the client's own VA.
+ * This reconfirms the r929 v2 lesson: guest VAs are not engine-fetchable on
+ * this replay channel, so the "validate in guest VA space" hypothesis is
+ * not testable by operand rewriting (the fetch dies before any DS check).
+ * Default 0 (baseline shadow rewrite); 1 reproduces the FAULT_PTE experiment.
+ */
+static bool gsp_phys_rmcall_gr_prog_guest_va;
+module_param_named(sema_gl_prog_guest_va, gsp_phys_rmcall_gr_prog_guest_va,
+		   bool, 0644);
+MODULE_PARM_DESC(sema_gl_prog_guest_va,
+	"Leave program-family (0x2014-family) operands at their original guest VAs when identity-mappable (default 0; 1 = the r1307 FAULT_PTE experiment - guest VAs are not engine-fetchable, do not keep)");
+
+/*
  * SESSION 153-CONTINUED-84: SET_MME_SHADOW_RAM_CONTROL mode before the
  * window. The vendor comment (mesh shader bind path) is explicit:
  * "MME filtering swallows them all if old and new state values are the
@@ -18693,6 +18757,31 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb,
 					iowrite32(0u, ds_bar0 + 0x00405844u);
 					iowrite32(0u, ds_bar0 + 0x0040584cu);
 				}
+				/*
+				 * SESSION 153-CONTINUED-92: clear the DS HWW
+				 * ESR latch (W1C) ahead of every window so the
+				 * retire readback names EXACTLY this launch's
+				 * failed SPH checks instead of the union of
+				 * every prior draw's. The latch was observed
+				 * sticky across launches (r1308: a6000e20
+				 * constant over 25 s of draws), which made
+				 * "did this window's checks change?" unreadable.
+				 * Log the value being cleared (the previous
+				 * window's set) so the A/B sequence is in the
+				 * ring either way.
+				 */
+				{
+					u32 esr = ioread32(ds_bar0 + 0x00405840u);
+
+					if (esr)
+						iowrite32(esr, ds_bar0 + 0x00405840u);
+					if (ds_mask_log < 24u) {
+						ds_mask_log++;
+						pr_info("vgsp-gr: DS-ESR pre-clear=%08x (prev window's latch; this window starts clean) h=0x%x push=0x%llx\n",
+							esr, req->h_object,
+							req->push_va);
+					}
+				}
 			}
 			if (ds_mask_log < 24u) {
 				ds_mask_log++;
@@ -19486,7 +19575,7 @@ static int gsp_phys_rmcall_gr_map_va_locked(void __iomem *bar0, u64 guest_va,
 	pt_va = gsp_phys_chunk_page_va(pt, 0);
 	if (!pt_va)
 		return -EIO;
-	put_unaligned_le64(gsp_phys_rmcall_pte(hpa),
+	put_unaligned_le64(gsp_phys_rmcall_gr_pte_leaf(hpa),
 			   pt_va + (size_t)pte_idx * 8);
 	dma_wmb();
 	boot_stage.rmcall_gr_win[slot].pte[i].idx = pte_idx;
@@ -19654,7 +19743,7 @@ static int gsp_phys_rmcall_gr_map_shadow_locked(u32 slot, u64 hpa,
 	pt = gsp_phys_rmcall_guest_pt_page_va(page);
 	if (!pt)
 		return -EIO;
-	put_unaligned_le64(gsp_phys_rmcall_pte(hpa),
+	put_unaligned_le64(gsp_phys_rmcall_gr_pte_leaf(hpa),
 			   pt + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES *
 					    line_in_page) * 8);
 	dma_wmb();
@@ -19701,7 +19790,7 @@ static int gsp_phys_rmcall_gr_map_shadow_contig_locked(u32 line, u32 pte,
 	pt = gsp_phys_rmcall_guest_pt_page_va(page);
 	if (!pt)
 		return -EIO;
-	put_unaligned_le64(gsp_phys_rmcall_pte(hpa),
+	put_unaligned_le64(gsp_phys_rmcall_gr_pte_leaf(hpa),
 			   pt + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES *
 					    line_in_page + pte) * 8);
 	dma_wmb();
@@ -22034,6 +22123,53 @@ static u64 gsp_phys_gr_sph_canary_hpa(void)
 	return (u64)gsp_phys_chunk_page_dma(&sph_canary_chunk, 0);
 }
 
+/*
+ * SESSION 153-CONTINUED-92: identity-fetch keep test (see the
+ * sema_gl_prog_guest_va knob above). Resolve the operand's page (the
+ * launch's own page set first - the pages the copy-in authored - then the
+ * sema service's trusted resolver), install the Tier-1 identity PTE at the
+ * guest VA itself and report whether the operand may stay verbatim. The
+ * receipt names the exact VA the DS will now fetch through (the guest VA,
+ * shadow column = the guest VA) and reads the fetch words back through the
+ * host direct map of the mapped HPA, so "the DS reads the authored page at
+ * its own heap VA" is provable from the log alone. Anything the identity
+ * mapper refuses (no window slot, stale fetch line, unresolved HPA) returns
+ * false and the caller falls back to the shadow rewrite. Called with
+ * gsp_phys_rm_lock held.
+ */
+static bool gsp_phys_rmcall_gr_prog_keep_locked(void __iomem *bar0,
+						u32 epoch, u32 up_w, u32 lo_w,
+						const struct gsp_phys_gr_submit *req)
+{
+	u64 va = ((u64)(up_w & 0xffu) << 32) | (u64)(lo_w & 0xfffffffcu);
+	u64 page = va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1);
+	u64 hpa = 0;
+	u32 i;
+
+	if (!va || !page)
+		return false;
+	for (i = 0; i < req->n_pages; i++) {
+		if ((req->pages[i].guest_va &
+		     ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) == page &&
+		    req->pages[i].hpa) {
+			hpa = req->pages[i].hpa;
+			break;
+		}
+	}
+	if (!hpa) {
+		struct gsp_sema_gr_reloc r;
+
+		if (!gsp_sema_gr_reloc_resolve(page, &r) && r.hpa)
+			hpa = r.hpa;
+	}
+	if (!hpa)
+		return false;
+	if (gsp_phys_rmcall_gr_map_va_locked(bar0, page, hpa, epoch))
+		return false;
+	gsp_phys_gr_sph_alias_receipt(page, va, va, hpa, "prog-guest");
+	return true;
+}
+
 static bool gsp_phys_rmcall_gr_3d_reloc(u32 *up_w, u32 *lo_w,
 					const struct gsp_phys_gr_submit *req,
 					u32 *n_shadow)
@@ -22622,13 +22758,15 @@ static void gsp_phys_gr_3d_mme_miss(u32 wv)
 }
 
 
+static u32 gsp_phys_rmcall_gr_3d_last_progkeep;
 static u32 gsp_phys_rmcall_gr_3d_patch_window(const u32 *in, u32 n, u32 *out,
 					      u32 *out_n, u32 *n_dropped,
 					      const struct gsp_phys_gr_submit *req,
-					      u32 *n_shadow)
+					      u32 *n_shadow,
+					      void __iomem *bar0, u32 epoch)
 {
 	u32 off = 0, wout = 0, reloc = 0, dropped = 0, subdrop = 0, sdm_drop = 0;
-	u32 wp_match = 0, wp_lowrun = 0;
+	u32 wp_match = 0, wp_lowrun = 0, prog_keep = 0;
 
 	*out_n = n;
 	*n_dropped = 0;
@@ -22928,14 +23066,49 @@ static u32 gsp_phys_rmcall_gr_3d_patch_window(const u32 *in, u32 n, u32 *out,
 			if (pair) {
 				u64 pva = ((u64)(d[lo_i] & 0xffu) << 32) |
 					  (u64)(d[lo_i + 1] & 0xfffffffcu);
-				bool ok = gsp_phys_rmcall_gr_3d_reloc(
-					&d[lo_i], &d[lo_i + 1], req,
-					n_shadow);
 				static u32 pair_log;
+				bool ok = false, kept = false;
 
-				if (ok)
+				/*
+				 * SESSION 153-CONTINUED-92: the program-family
+				 * space experiment (see sema_gl_prog_guest_va).
+				 * The DS rejects every bound stage whose SPH it
+				 * fetches through a relocated shadow VA while
+				 * the bytes at those VAs are provably correct;
+				 * leave the operand at its ORIGINAL guest VA
+				 * whenever the page resolves + identity-maps,
+				 * so the DS fetches/validates in the same VA
+				 * space the client's program heap describes.
+				 * Kept pairs count as relocated so the SPH-BIND
+				 * guard (orel == nrun) is unaffected; a page the
+				 * identity mapper refuses falls through to the
+				 * shadow rewrite exactly as before.
+				 */
+				if (READ_ONCE(gsp_phys_rmcall_gr_prog_guest_va) &&
+				    bar0 && req->prog_va &&
+				    addr >= 0x2014u && addr < 0x4000u &&
+				    (addr & 0x3fu) == 0x14u &&
+				    gsp_phys_rmcall_gr_prog_keep_locked(
+					    bar0, epoch, d[lo_i], d[lo_i + 1],
+					    req)) {
+					static u32 progva_log;
+
+					kept = true;
+					prog_keep++;
+					if (progva_log < 24u) {
+						progva_log++;
+						pr_info("vgsp-gr: PROG-GUEST-VA keep addr=0x%x va=0x%llx (Tier-1 identity; operand left verbatim) h=0x%x\n",
+							addr, pva,
+							req->h_object);
+					}
+				} else {
+					ok = gsp_phys_rmcall_gr_3d_reloc(
+						&d[lo_i], &d[lo_i + 1], req,
+						n_shadow);
+				}
+				if (ok || kept)
 					reloc++;
-				if (pair_log < 48) {
+				if (!kept && pair_log < 48) {
 					pair_log++;
 					pr_info("vgsp-gr: pair addr=0x%x va=0x%llx -> %s0x%llx\n",
 						addr, pva, ok ? "" :
@@ -23104,6 +23277,7 @@ static u32 gsp_phys_rmcall_gr_3d_patch_window(const u32 *in, u32 n, u32 *out,
 	*n_dropped = dropped;
 	gsp_phys_rmcall_gr_3d_last_subdrop = subdrop;
 	gsp_phys_rmcall_gr_3d_last_sdmdrop = sdm_drop;
+	gsp_phys_rmcall_gr_3d_last_progkeep = prog_keep;
 	return reloc;
 }
 
@@ -24764,7 +24938,7 @@ gr_page_unmapped:
 		gsp_phys_rmcall_gr_3d_miss_n = 0;
 		nrel = gsp_phys_rmcall_gr_3d_patch_window(
 			req->stream, req->stream_n, stream_patch, &win_n,
-			&ndrop, req, &n_shadow);
+			&ndrop, req, &n_shadow, bar0, epoch);
 
 		win3d = stream_patch;
 		win3d_n = win_n;
@@ -24794,10 +24968,11 @@ gr_page_unmapped:
 					req->h_object, req->push_va);
 			}
 		}
-		pr_info("vgsp-gr: 3D window relocated %u operand pair(s) to shadow VAs (window=%u shadow=%u dropped=%u subdrop=%u sdmdrop=%u)\n",
+		pr_info("vgsp-gr: 3D window relocated %u operand pair(s) to shadow VAs (window=%u shadow=%u dropped=%u subdrop=%u sdmdrop=%u progkeep=%u)\n",
 			nrel, win3d_n, n_shadow, ndrop,
 			gsp_phys_rmcall_gr_3d_last_subdrop,
-			gsp_phys_rmcall_gr_3d_last_sdmdrop);
+			gsp_phys_rmcall_gr_3d_last_sdmdrop,
+			gsp_phys_rmcall_gr_3d_last_progkeep);
 		/*
 		 * SESSION 153-CONTINUED-87: emit the client's missing
 		 * pipeline-stage program binds (see the sema_gl_sph_bind
@@ -25003,7 +25178,8 @@ gr_page_unmapped:
 				nrun += 2;
 			}
 			orel = gsp_phys_rmcall_gr_3d_patch_window(
-				in, nfin, out, &on, &odrop, req, &n_shadow);
+				in, nfin, out, &on, &odrop, req, &n_shadow,
+				bar0, epoch);
 			if (orel == nrun && on == nfin) {
 				memmove(&stream_patch[nfin], stream_patch,
 					(size_t)win3d_n * 4u);
@@ -25058,7 +25234,8 @@ gr_page_unmapped:
 						gsp_phys_gr_surf_pre_in, sn,
 						gsp_phys_gr_surf_pre,
 						&gsp_phys_gr_surf_pre_n,
-						&sdrop, req, &n_shadow);
+						&sdrop, req, &n_shadow,
+						bar0, epoch);
 					WRITE_ONCE(gsp_phys_gr_surf_pre_gen,
 						   spg);
 					pr_info("vgsp-gr: SURF-PRELUDE patched gen=%u raw=%u kept=%u dropped=%u reloc=%u h=0x%x push=0x%llx\n",
@@ -25117,7 +25294,8 @@ gr_page_unmapped:
 				gsp_phys_rmcall_gr_3d_patch_window(ap, an,
 					gsp_phys_gr_attr_pre,
 					&gsp_phys_gr_attr_pre_n,
-					&adrop, req, &n_shadow);
+					&adrop, req, &n_shadow,
+					bar0, epoch);
 				if (gsp_phys_gr_attr_pre_n) {
 					WRITE_ONCE(gsp_phys_gr_attr_pre_gen,
 						   apg);