vgsp: m69 - window-0 shadow pool for the QMD program/param pages

m68 proved the program page's shadow copy contains real SASS (SASS_version
0x75) and the prefetch pair is fixed, yet the engine still never completes
and never writes its release anywhere - including a window-0 module sem.
Best remaining suspect: the window-1 scratch VA range (PT chain installed by
DMA_UPDATE_PDE_2 pdeIndex=1) has never been proven readable by the GR engine
core, unlike the window-0 GPFIFO/PB/QMDWIN small-PT line. m69 plants 8
module pages at VAs 0x108000..0x10f000 (slots 0x108..0x10f, same 128 B PT
fetch line, planted before first fetch like qmdwin), copies the resolved
program/param page content into them at launch and rewrites the QMD pointer
pairs there (param=32, program=48). Param: gsp_phys_rmcall_gr_shadow_win0.

3 files changed, 124 insertions, 6 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index 8de8408..611a128 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -996,6 +996,17 @@ bool gsp_phys_boot_requested(void)
 #define GSP_PHYS_RMCALL_VA_SRC		(GSP_PHYS_RMCALL_VA_BASE + 2 * GSP_PHYS_PAGE_SIZE)
 #define GSP_PHYS_RMCALL_VA_DST		(GSP_PHYS_RMCALL_VA_BASE + 3 * GSP_PHYS_PAGE_SIZE)
 #define GSP_PHYS_RMCALL_VA_SEM		(GSP_PHYS_RMCALL_VA_BASE + 4 * GSP_PHYS_PAGE_SIZE)
+/*
+ * m69: window-0 shadow pool - 8 module pages at VAs 0x108000..0x10f000
+ * (small-PT slots 0x108..0x10f, the same 128 B PTE fetch line as the
+ * GPFIFO/PB/QMDWIN slots, planted before the engine's first fetch). The
+ * Tier-2 copies of the QMD's program/param pages are placed here so the GR
+ * engine never has to read through the window-1 scratch VA range whose PT
+ * chain (DMA_UPDATE_PDE_2 pdeIndex=1) has never been proven engine-readable.
+ */
+#define GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE \
+	(GSP_PHYS_RMCALL_VA_BASE + 8 * GSP_PHYS_PAGE_SIZE)
+#define GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS	8u
 
 /*
  * Rank-2 qmdwin diagnostic (gsp_phys_rmcall_gr_qmdwin=1): dedicated module
@@ -2107,6 +2118,17 @@ static struct gsp_phys_boot_stage {
 	struct gsp_phys_boot_chunk rmcall_gr_pt[GSP_PHYS_RMCALL_GR_PT_PAGES];
 	struct gsp_phys_boot_chunk rmcall_gr_shadow;	/* Tier-2 pages */
 	/*
+	 * m69: window-0 shadow pool (gsp_phys_rmcall_gr_shadow_win0=1).
+	 * Eight module pages whose PTEs are planted in the proven window-0
+	 * PT line at slots 0x108..0x10f (VAs 0x108000..0x10f000) by
+	 * gsp_phys_rmcall_pdb_build, before the engine's first fetch (same
+	 * pattern as the qmdwin slot). The launch copies the QMD's
+	 * program/param page content into them and rewrites the QMD pointers
+	 * there, so the GR engine never reads the window-1 scratch range.
+	 */
+	struct gsp_phys_boot_chunk rmcall_gr_shadow_win0;
+	u32 rmcall_gr_shadow_win0_used;
+	/*
 	 * Rank-2 qmdwin: dedicated module page for the forwarded inline-QMD
 	 * destination, planted (PTE written) by gsp_phys_rmcall_pdb_build when
 	 * gsp_phys_rmcall_gr_qmdwin=1; rmcall_gr_qmdwin_va stays 0 when off or
@@ -2747,6 +2769,9 @@ static void gsp_phys_rmcall_reset(void)
 	memset(&boot_stage.rmcall_gr_qmdwin, 0,
 	       sizeof(boot_stage.rmcall_gr_qmdwin));
 	boot_stage.rmcall_gr_qmdwin_va = 0;
+	memset(&boot_stage.rmcall_gr_shadow_win0, 0,
+	       sizeof(boot_stage.rmcall_gr_shadow_win0));
+	boot_stage.rmcall_gr_shadow_win0_used = 0;
 	memset(boot_stage.rmcall_gr_win, 0, sizeof(boot_stage.rmcall_gr_win));
 	for (i = 0; i < GSP_PHYS_RMCALL_BUF_COUNT; i++) {
 		boot_stage.rmcall_mem_handles[i] = 0;
@@ -9205,6 +9230,47 @@ static void gsp_phys_rmcall_pdb_build(void)
 		}
 	}
 	/*
+	 * m69: plant the window-0 shadow pool PTEs (slots 0x108..0x10f) in the
+	 * same proven small-PT line, before the engine's first fetch. The
+	 * launch copies the QMD's program/param page content into these pages
+	 * and rewrites the QMD pointers to these VAs, so the GR engine never
+	 * reads the window-1 scratch range (whose DMA_UPDATE_PDE_2 PT chain
+	 * has never been proven engine-readable).
+	 */
+	if (gsp_phys_rmcall_gr_shadow_win0_value()) {
+		struct gsp_phys_boot_chunk *w0 =
+			&boot_stage.rmcall_gr_shadow_win0;
+
+		if (!w0->pages && boot_stage.pdev) {
+			if (!gsp_phys_chunk_alloc(boot_stage.pdev, w0,
+						  GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS))
+				gsp_phys_chunk_zero(w0);
+			else
+				pr_warn("vgsp-boot: rm-call: shadow-win0 pool allocation failed; the launch keeps the window-1 shadow path\n");
+		}
+		if (w0->pages) {
+			u32 s;
+
+			for (s = 0; s < GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS; s++) {
+				u64 va = GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE +
+					 (u64)s * GSP_PHYS_PAGE_SIZE;
+				u32 idx = (u32)((va >> 12) & 0x1ffu);
+
+				put_unaligned_le64(gsp_phys_rmcall_pte(
+						(u64)gsp_phys_chunk_page_dma(w0, s)),
+					(u8 *)gsp_phys_chunk_page_va(pt, 0) +
+					(size_t)idx * 8);
+			}
+			dma_wmb();
+			pr_info("vgsp-boot: rm-call: shadow-win0 pool planted %u pages at VA 0x%llx..0x%llx (small-PT slots 0x108..0x10f, before first fetch)\n",
+				GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS,
+				GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE,
+				GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE +
+					(GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS - 1) *
+						GSP_PHYS_PAGE_SIZE);
+		}
+	}
+	/*
 	 * P6a guest windows: start from explicit zero (invalid PTE) for both
 	 * slots, then, when the guest GPA is already pinned (it must be pinned
 	 * before stage 3 for this), pre-build both slots with the translated
@@ -15964,12 +16030,39 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
 			if (copied) {
 				u32 sh_line =
 					boot_stage.rmcall_guest_pt_next_line;
-
-				ret = gsp_phys_rmcall_gr_map_shadow_locked(
-					n_shadow,
-					(u64)gsp_phys_chunk_page_dma(sh,
-								      n_shadow),
-					&sh_va);
+				bool win0 = (pg->qmd_word == 32u ||
+					     pg->qmd_word == 48u) &&
+					gsp_phys_rmcall_gr_shadow_win0_value() &&
+					boot_stage.rmcall_gr_shadow_win0.pages &&
+					boot_stage.rmcall_gr_shadow_win0_used <
+						GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS;
+
+				if (win0) {
+					u8 *dst = gsp_phys_chunk_page_va(
+						&boot_stage.rmcall_gr_shadow_win0,
+						boot_stage.rmcall_gr_shadow_win0_used);
+
+					if (!dst) {
+						ret = -EIO;
+					} else {
+						memcpy(dst, sh_page,
+						       GSP_PHYS_PAGE_SIZE);
+						sh_va =
+							GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE +
+							(u64)boot_stage.rmcall_gr_shadow_win0_used *
+								GSP_PHYS_PAGE_SIZE;
+						boot_stage.rmcall_gr_shadow_win0_used++;
+						ret = 0;
+						pr_info("vgsp-gr: qmd_word %u page copied to window-0 shadow VA=0x%llx (proven PT line)\n",
+							pg->qmd_word, sh_va);
+					}
+				} else {
+					ret = gsp_phys_rmcall_gr_map_shadow_locked(
+						n_shadow,
+						(u64)gsp_phys_chunk_page_dma(sh,
+									      n_shadow),
+						&sh_va);
+				}
 				if (!ret) {
 					/*
 					 * Preserve the original pointer's page
diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c
index 3429507..1029b2e 100644
--- a/kernel/gsp_phys_gpu.c
+++ b/kernel/gsp_phys_gpu.c
@@ -746,6 +746,27 @@ int gsp_phys_rmcall_faultbuf_sys_value(void)
 EXPORT_SYMBOL_GPL(gsp_phys_rmcall_faultbuf_sys_value);
 
 /*
+ * m69 window-0 shadow pool (gsp_phys_rmcall_gr_shadow_win0; default 0). When
+ * set, the forwarded GR launch copies the QMD's program/param pages into the
+ * window-0 shadow pool (module pages at VAs 0x108000..0x10f000, PTEs planted
+ * at stage 3 in the proven GPFIFO/PB/QMDWIN small-PT line) and rewrites the
+ * QMD pointer pairs there, instead of mapping them through the window-1
+ * scratch range whose DMA_UPDATE_PDE_2 PT chain has never been proven
+ * readable by the GR engine core.
+ */
+static int gsp_phys_rmcall_gr_shadow_win0_param;
+module_param_named(gsp_phys_rmcall_gr_shadow_win0,
+		   gsp_phys_rmcall_gr_shadow_win0_param, int, 0644);
+MODULE_PARM_DESC(gsp_phys_rmcall_gr_shadow_win0,
+		 "Copy forwarded GR program/param pages into the window-0 shadow pool (proven PT line; default 0)");
+
+int gsp_phys_rmcall_gr_shadow_win0_value(void)
+{
+	return gsp_phys_rmcall_gr_shadow_win0_param;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_shadow_win0_value);
+
+/*
  * Ladder VAS ownership selector (gsp_phys_rmcall_vas_shared; default 0).
  * The H1 GR promote wall is a direct consequence of the ladder's
  * externally-owned FERMI_VASPACE_A: with IS_EXTERNALLY_OWNED the RM treats
diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h
index 5c1207c..196e844 100644
--- a/kernel/gsp_phys_gpu.h
+++ b/kernel/gsp_phys_gpu.h
@@ -333,6 +333,10 @@ int gsp_phys_rmcall_gr_rel_mod_value(void);
  * entries are host-readable (default 0 = server FBMEM first). */
 int gsp_phys_rmcall_faultbuf_sys_value(void);
 
+/* m69: copy forwarded GR program/param pages into the window-0 shadow pool
+ * (proven small-PT line) instead of the window-1 scratch range (default 0). */
+int gsp_phys_rmcall_gr_shadow_win0_value(void);
+
 /*
  * Ladder VAS ownership selector (gsp_phys_rmcall_vas_shared; default 0).
  * 0 = FERMI_VASPACE_A is allocated IS_EXTERNALLY_OWNED (historical ladder