vgsp: CONT-91 - svc_gr_inline_retarget re-roots captured program-family inline records (page not in launch prog_va family nor prog_bind_va[]) onto launch family offset-preserved, page-crossing tails follow head, wired into svc_gr_req_fill; svc_gr_page_inline_covers widened; sema_ds_mask_clear=1 clears DS_HWW_REPORT_MASK/_2 (0x405844/c) ahead of every 3D window; gsp-phys-ds-spy debugfs; GR-STATE retire logs sph= (STAGE0..5 START/UPPER) + rr= (PROGRAM_REGION) + MME shadow readback; sema_gl_sph_region + sema_sph_region_base emit Volta+ program region (SET_MME_SHADOW_SCRATCH 0x78/0x79 = 0x35e0/0x35e4) - r1298 real V4 image at exact bind VA (INLINE-APPLY len=1536 first=020a0481 + INLINE-RB rb=020a0481); same launch DS flags sph=0,00620900,00620200,0,0,00620000 rr=0,0 ds=a4000e20 w/ correct bytes + region A/B 0/0 unchanged; DS rejects all bound stages -> unresolved variable = address space DS validates in (our shadow VAs 0x6xxxxx vs client guest VA region); next A/B: knob leaving program operands at ORIGINAL guest VAs when page identity-mappable (skip shadow rewrite for that pair only) - SESSION 153 r1298-r1306

1 files changed, 278 insertions, 1 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index fe1f7b0..454c3e7 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -17930,6 +17930,47 @@ MODULE_PARM_DESC(sema_gl_shader_exc,
 	"Emit SET_SHADER_EXCEPTIONS (0x1528) <- 0 after SET_OBJECT like _nv3dInitChannelTuring (default 1)");
 
 /*
+ * SESSION 153-CONTINUED-91: clear DS_HWW_REPORT_MASK/_2 (0x405844/0x40584c)
+ * ahead of every replayed 3D window (host write + in-stream MME PRI R/W
+ * macro 5) so a DS SPH validation miss latches without poisoning the PRI
+ * block and the GSP-RM exception handler killing the channel. See the
+ * emission-site comment. Default 1; A/B with 0 restores the old behavior
+ * (every SPH miss = priv-error poison + channel wedge).
+ */
+static bool gsp_phys_rmcall_gr_ds_mask_clear = true;
+module_param_named(sema_ds_mask_clear, gsp_phys_rmcall_gr_ds_mask_clear,
+		   bool, 0644);
+MODULE_PARM_DESC(sema_ds_mask_clear,
+	"Clear DS_HWW_REPORT_MASK/_2 ahead of replayed 3D windows (default 1; 0 = leave the all-report mask)");
+
+/*
+ * SESSION 153-CONTINUED-91: emit the Turing program-region pair (the MME
+ * shadow scratch 0x78/0x79 = NVC397_SET_MME_PROGRAM_REGION) with the
+ * launch's own program family page as the shader-heap base, so the DS's
+ * SPH validation has a region to accept the fetched SPHs against. See the
+ * emission-site comment. Default 1; A/B with 0.
+ */
+static bool gsp_phys_rmcall_gr_sph_region = true;
+module_param_named(sema_gl_sph_region, gsp_phys_rmcall_gr_sph_region,
+		   bool, 0644);
+MODULE_PARM_DESC(sema_gl_sph_region,
+	"Emit the Volta+ program-region pair (SET_MME_SHADOW_SCRATCH 0x78/0x79 <- program family page) ahead of replayed 3D windows (default 1)");
+
+/*
+ * Runtime-selectable program-region base. 0 = auto (the launch's own
+ * program family page, the guest-VA heap base the real driver would use);
+ * a nonzero value forces that base - the engine-side shadow-VA space needs
+ * its own experiment (the SPH_INFO snapshot shows the DS fetched through
+ * shadow VAs 0x620900/0x620200/0x620000, so a guest-VA base may be the
+ * mismatch). Writable at runtime for a live A/B.
+ */
+static u64 gsp_phys_rmcall_gr_sph_region_base;
+module_param_named(sema_sph_region_base, gsp_phys_rmcall_gr_sph_region_base,
+		   ullong, 0644);
+MODULE_PARM_DESC(sema_sph_region_base,
+	"Program-region base override (0 = launch program family page; nonzero = explicit engine-side base)");
+
+/*
  * SESSION 153-CONTINUED-87: the missing TESS_CONTROL/PIXEL program binds.
  * The client's real channel state binds every pipeline stage once per
  * context; the captured init-push image (guest RAM scrape) carries
@@ -18616,6 +18657,53 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb,
 			}
 		}
 		/*
+		 * SESSION 153-CONTINUED-91: DS HWW report suppression.
+		 *
+		 * The DS validates the SPH of every bound pipeline program at
+		 * draw time and reports failures through DS_HWW_REPORT_MASK
+		 * (0x405844, observed live = 0x00ffffff = every SPH<n>_ERR
+		 * reported; mask_2 = 0x77). A reported failure poisons the
+		 * whole PRI block (reads return 0xbadf1201 - the priv-error
+		 * marker already documented for the MME latch) and the GSP-RM
+		 * exception handler kills the channel: every draw attempt
+		 * wedges at the first SPH error storm even when the SPH bytes
+		 * at the bound VAs are the client's own real V4 images (r1298:
+		 * INLINE-RB rb=020a0481 at the bind VA, DS still 5/9/10/11).
+		 * The reference driver's own diagnostic path (Aftermath)
+		 * explicitly ENABLES these reports to capture them, i.e. the
+		 * reports are a diagnostic tap, not the shader-execute gate.
+		 * Clear the mask so a SPH miss latches (readable between
+		 * faults) without poisoning the channel. Host write only:
+		 * an in-stream CALL_MME_MACRO(5) (the NVN PRI R/W macro)
+		 * faulted with MME ILLEGAL_OPCODE (pc=513 mac=0xf1) on
+		 * r1300 - the Turing MME's loaded macro program does not
+		 * carry that GF100-era slot - so the register is written
+		 * directly (it persists: the next beef launch reads 0/0).
+		 * A/B with sema_ds_mask_clear=0.
+		 */
+		if (READ_ONCE(gsp_phys_rmcall_gr_ds_mask_clear)) {
+			static u32 ds_mask_log;
+			void __iomem *ds_bar0 = gsp_phys_gpu_bar0();
+			u32 m0 = 0, m2 = 0;
+
+			if (ds_bar0) {
+				m0 = ioread32(ds_bar0 + 0x00405844u);
+				m2 = ioread32(ds_bar0 + 0x0040584cu);
+				if (m0 || m2) {
+					iowrite32(0u, ds_bar0 + 0x00405844u);
+					iowrite32(0u, ds_bar0 + 0x0040584cu);
+				}
+			}
+			if (ds_mask_log < 24u) {
+				ds_mask_log++;
+				pr_info("vgsp-gr: DS-MASK clear mask=%08x/%08x rb=%08x/%08x h=0x%x push=0x%llx\n",
+					m0, m2,
+					ds_bar0 ? ioread32(ds_bar0 + 0x00405844u) : 0u,
+					ds_bar0 ? ioread32(ds_bar0 + 0x0040584cu) : 0u,
+					req->h_object, req->push_va);
+			}
+		}
+		/*
 		 * SESSION 153-CONTINUED-86: the Turing channel-init SPH version
 		 * pair (see the sema_gl_sph_version knob). Emitted on EVERY
 		 * replayed 3D window (3 words, idempotent): the DS validates
@@ -18649,6 +18737,47 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb,
 			*p++ = val;
 		}
 		/*
+		 * SESSION 153-CONTINUED-91: the Turing program region.
+		 *
+		 * The DS's SPH validation checks each fetched SPH against the
+		 * channel's PROGRAM REGION. On pre-Volta that was
+		 * SET_PROGRAM_REGION_A/B (0x1608/0x160c); on Volta+ those
+		 * methods are obsolete and the driver writes the same 64-bit
+		 * shader-code-heap base through the MME shadow scratch pair
+		 * (nvTuring: NVC397_SET_MME_PROGRAM_REGION = SET_MME_SHADOW_
+		 * SCRATCH(MME_SCRATCH_PROGRAM_REGION_A=0x78/B=0x79),
+		 * gv100ResendProgramAddresses/`gf100Init`; method 0x3400 +
+		 * i*4 -> 0x35e0/0x35e4). Observed live at the faulting draw:
+		 * DS_HWW_ESR_SPH_INFO_PROGRAM_REGION_A/B = 0/0 and DS latches
+		 * SPH errors for every bound stage whose bytes are provably
+		 * correct (r1298 INLINE-RB rb=020a0481 at the bind VA;
+		 * r1303 GR-STATE sph stages 1/2/5 = 0x620900/0x620200/0x620000
+		 * with ds=a4000e20 and rr=0/0) - a region mismatch rejects all
+		 * of them regardless of content. Emit the pair (A = upper
+		 * 7:0, B = lower 31:0) with the launch's own program family
+		 * page as the heap base. A/B with sema_gl_sph_region=0.
+		 */
+		if (READ_ONCE(gsp_phys_rmcall_gr_sph_region) && req->prog_va) {
+			static u32 sph_reg_log;
+			u64 base = READ_ONCE(gsp_phys_rmcall_gr_sph_region_base);
+
+			if (!base)
+				base = req->prog_va &
+				       ~(u64)(GSP_PHYS_PAGE_SIZE - 1);
+
+			*p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) |
+			       (1u << 16) | (0x35e0u >> 2);
+			*p++ = (u32)(base >> 32) & 0xffu;
+			*p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) |
+			       (1u << 16) | (0x35e4u >> 2);
+			*p++ = (u32)base;
+			if (sph_reg_log < 16u) {
+				sph_reg_log++;
+				pr_info("vgsp-gr: SPH-REGION emitted base=0x%llx (scratch 0x78/0x79) h=0x%x push=0x%llx\n",
+					base, req->h_object, req->push_va);
+			}
+		}
+		/*
 		 * SESSION 153-CONTINUED-84: SET_MME_VERSION=1 (0x234).
 		 *
 		 * The Turing channel-init defaults push
@@ -21096,6 +21225,17 @@ static noinline void gsp_phys_gr_state_rb_log(u32 h_object)
 	void __iomem *bar0 = gsp_phys_gpu_bar0();
 	bool beef = (h_object & 0xfffff000u) == 0xbeef0000u;
 	u32 fed0, fed1, fed2, fed3, esr, info, ir2, trap, data, ds, gf;
+	/*
+	 * SESSION 153-CONTINUED-91: with the DS HWW report mask cleared
+	 * (sema_ds_mask_clear) a SPH validation miss latches in DS_HWW_ESR
+	 * without the GSP-RM fault handler poisoning the PRI block, so the
+	 * per-stage SPH_INFO snapshot is finally readable: STAGE0..5 START
+	 * (0x405c00+si*4) / UPPER (0x405c24+si*4) name the exact SPH VA each
+	 * pipeline stage fetched, and PROGRAM_REGION_A/B (0x405c1c/c20) the
+	 * region the DS validated against (the r1277+ wall's missing datum).
+	 */
+	u32 sph[6], sphu[6], rr_a, rr_b;
+	u32 si;
 
 	if (!bar0)
 		return;
@@ -21105,6 +21245,12 @@ static noinline void gsp_phys_gr_state_rb_log(u32 h_object)
 		gs_beef_n++;
 	else
 		gs_other_n++;
+	for (si = 0; si < 6u; si++) {
+		sph[si] = ioread32(bar0 + 0x00405c00u + si * 4u);
+		sphu[si] = ioread32(bar0 + 0x00405c24u + si * 4u);
+	}
+	rr_a = ioread32(bar0 + 0x00405c1cu);
+	rr_b = ioread32(bar0 + 0x00405c20u);
 	fed0 = ioread32(bar0 + 0x0040a630u);
 	fed1 = ioread32(bar0 + 0x0040a634u);
 	fed2 = ioread32(bar0 + 0x0040a638u);
@@ -21121,6 +21267,25 @@ static noinline void gsp_phys_gr_state_rb_log(u32 h_object)
 		fed0 & 0xfffu, (fed0 >> 12) & 0x3ffu, (fed0 >> 31) & 1u,
 		esr, info, ir2, trap, trap & 0x3ffcu, (trap >> 20) & 1u,
 		data, ds, gf);
+	pr_info("vgsp-gr: GR-STATE sph h=0x%x ds2=%08x sph=%08x,%08x,%08x,%08x,%08x,%08x sphU=%02x,%02x,%02x,%02x,%02x,%02x rr=%08x,%08x\n",
+		h_object, ioread32(bar0 + 0x00405848u),
+		sph[0], sph[1], sph[2], sph[3], sph[4], sph[5],
+		sphu[0] & 0xffu, sphu[1] & 0xffu, sphu[2] & 0xffu,
+		sphu[3] & 0xffu, sphu[4] & 0xffu, sphu[5] & 0xffu,
+		rr_a, rr_b);
+	/*
+	 * SESSION 153-CONTINUED-91: did the emitted program-region pair
+	 * (SET_MME_SHADOW_SCRATCH 0x78/0x79 = method 0x35e0/0x35e4 through
+	 * class 0xc597) actually land in the MME shadow? The shadow read
+	 * port is the same one the store-side probe uses; a zero readback
+	 * means the write never took effect and every region experiment so
+	 * far was a no-op.
+	 */
+	if (beef)
+		pr_info("vgsp-gr: GR-STATE region-shadow h=0x%x scratch[0x78]=%08x scratch[0x79]=%08x\n",
+			h_object,
+			gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x35e0u),
+			gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x35e4u));
 }
 
 /*
@@ -29814,6 +29979,109 @@ static const struct file_operations gsp_phys_reg_probe_fops = {
 	.release = single_release,
 };
 
+/*
+ * SESSION 153-CONTINUED-91: DS SPH fault spy.
+ *
+ * The DS_HWW_ESR SPH<n>_ERR bits (bit n = "the SPH with index n failed
+ * validation", hwref NV_PGRAPH_PRI_DS_HWW_ESR_SPH0_ERR..SPH23_ERR) and the
+ * per-stage SPH_INFO snapshot (STAGE0..STAGE5 START/UPPER at 0x405c00+si*4 /
+ * 0x405c24+si*4, PROGRAM_REGION_A/B at 0x405c1c/c20) are latched by the DS
+ * at fault time and consumed+cleared by the real GSP-RM exception handler
+ * before our relayed fault packet arrives (the fault-time register read in
+ * gsp_phys_fault_regs_log sees ESR=0x80000000 and all-zero stage starts
+ * one millisecond later). The RM's own diagnostic prints those STAGE
+ * registers (grErrorPrintDsSphInfo_GM200), so they name the exact SPH VAs
+ * the DS rejected - the missing datum behind the r1277+ Shader Program
+ * Header wall (CONT-90/91 could only move the fault SET by content, never
+ * identify the SPH entries).
+ *
+ * Reading this file runs a bounded tight poll of DS_HWW_ESR and snapshots
+ * the ESR pair, the six stage starts and the region pair on every sample
+ * that has any SPH bit set. Development-only, read-only, no state touched;
+ * run `cat /sys/kernel/debug/vgsp/gsp-phys-ds-spy` while the client draws.
+ * Duration via sema_ds_spy_ms (default 20 s).
+ */
+static u32 gsp_phys_ds_spy_ms = 20000;
+module_param_named(sema_ds_spy_ms, gsp_phys_ds_spy_ms, uint, 0644);
+MODULE_PARM_DESC(sema_ds_spy_ms,
+	"gsp-phys-ds-spy sampling duration in milliseconds (default 20000)");
+
+#define GSP_PHYS_DS_ESR			0x00405840u
+#define GSP_PHYS_DS_ESR_2		0x00405848u
+#define GSP_PHYS_DS_REPORT_MASK		0x00405844u
+#define GSP_PHYS_DS_REPORT_MASK_2	0x0040584cu
+#define GSP_PHYS_DS_SPH_STAGE_LO	0x00405c00u
+#define GSP_PHYS_DS_SPH_STAGE_UP	0x00405c24u
+#define GSP_PHYS_DS_SPH_REGION_A	0x00405c1cu
+#define GSP_PHYS_DS_SPH_REGION_B	0x00405c20u
+
+static int gsp_phys_ds_spy_show(struct seq_file *m, void *unused)
+{
+	void __iomem *bar0 = gsp_phys_gpu_bar0();
+	u64 t0;
+	u32 hits = 0, samples = 0;
+	u32 ms = READ_ONCE(gsp_phys_ds_spy_ms);
+
+	if (!bar0)
+		return -ENODEV;
+	if (!ms)
+		ms = 20000;
+	seq_printf(m, "# DS-SPY begin duration_ms=%u (poll DS_HWW_ESR 0x%08x)\n",
+		   ms, GSP_PHYS_DS_ESR);
+	seq_printf(m, "# DS-SPY masks esr=%08x esr2=%08x mask=%08x mask2=%08x\n",
+		   readl(bar0 + GSP_PHYS_DS_ESR),
+		   readl(bar0 + GSP_PHYS_DS_ESR_2),
+		   readl(bar0 + GSP_PHYS_DS_REPORT_MASK),
+		   readl(bar0 + GSP_PHYS_DS_REPORT_MASK_2));
+	t0 = ktime_get_ns();
+	while (ktime_get_ns() - t0 < (u64)ms * 1000000ull) {
+		u32 esr = readl(bar0 + GSP_PHYS_DS_ESR);
+
+		samples++;
+		if (esr & 0x7fffffffu) {
+			u32 esr2 = readl(bar0 + GSP_PHYS_DS_ESR_2);
+			u32 esr_a, s[6], u[6], ra, rb;
+			u32 si;
+
+			for (si = 0; si < 6u; si++) {
+				s[si] = readl(bar0 +
+					      GSP_PHYS_DS_SPH_STAGE_LO +
+					      si * 4u);
+				u[si] = readl(bar0 +
+					      GSP_PHYS_DS_SPH_STAGE_UP +
+					      si * 4u);
+			}
+			ra = readl(bar0 + GSP_PHYS_DS_SPH_REGION_A);
+			rb = readl(bar0 + GSP_PHYS_DS_SPH_REGION_B);
+			esr_a = readl(bar0 + GSP_PHYS_DS_ESR);
+			hits++;
+			seq_printf(m,
+				   "DS-SPY hit %u t=%lldus esr=%08x esr2=%08x esr_after=%08x region=%08x/%08x stage=%08x,%08x,%08x,%08x,%08x,%08x upper=%02x,%02x,%02x,%02x,%02x,%02x\n",
+				   hits, div_u64(ktime_get_ns() - t0, 1000),
+				   esr, esr2, esr_a, ra, rb,
+				   s[0], s[1], s[2], s[3], s[4], s[5],
+				   u[0] & 0xffu, u[1] & 0xffu, u[2] & 0xffu,
+				   u[3] & 0xffu, u[4] & 0xffu, u[5] & 0xffu);
+			udelay(20);
+		}
+	}
+	seq_printf(m, "# DS-SPY end samples=%u hits=%u\n", samples, hits);
+	return 0;
+}
+
+static int gsp_phys_ds_spy_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, gsp_phys_ds_spy_show, NULL);
+}
+
+static const struct file_operations gsp_phys_ds_spy_fops = {
+	.owner = THIS_MODULE,
+	.open = gsp_phys_ds_spy_open,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = single_release,
+};
+
 int gsp_phys_boot_debugfs_init(void)
 {
 	struct dentry *file;
@@ -29976,7 +30244,16 @@ int gsp_phys_boot_debugfs_init(void)
 			err);
 		return (int)err;
 	}
-	pr_info("vgsp-boot: debugfs files /sys/kernel/debug/vgsp/{gsp-phys-state read-only, gsp-phys-log-dump read-only, gsp-phys-status-drain read-only, gsp-phys-reg-probe read-only, gsp-phys-unload-rpc write-only, gsp-phys-rm-probe write-only, gsp-phys-rm-call write-only, gsp-phys-guest-gpa write-only, gsp-phys-guest-ram write-only} ready\n");
+	/*
+	 * SESSION 153-CONTINUED-91: the DS SPH fault spy (see the show fn).
+	 * Diagnostic only; failure to create it must not fail the init.
+	 */
+	file = debugfs_create_file("gsp-phys-ds-spy", 0444,
+				   gsp_phys_debugfs_root, NULL,
+				   &gsp_phys_ds_spy_fops);
+	if (IS_ERR_OR_NULL(file))
+		pr_warn("vgsp-boot: debugfs file gsp-phys-ds-spy creation failed (DS SPH fault spy unavailable)\n");
+	pr_info("vgsp-boot: debugfs files /sys/kernel/debug/vgsp/{gsp-phys-state read-only, gsp-phys-log-dump read-only, gsp-phys-status-drain read-only, gsp-phys-reg-probe read-only, gsp-phys-ds-spy read-only, gsp-phys-unload-rpc write-only, gsp-phys-rm-probe write-only, gsp-phys-rm-call write-only, gsp-phys-guest-gpa write-only, gsp-phys-guest-ram write-only} ready\n");
 	return 0;
 }