vgsp: m91 - gsp_phys_rmcall_gr_ptr_guest: keep QMD pointer pairs at the guest VAs

The m90/m91 trap decoded the SM I-fetch failure precisely: GPC0/GPC1 TPC0
SM0 WARP_ESR ERROR=0x20 ADDR_ERROR_TYPE=0xb = MMU_NACK, PC=0x109880 (the
rewritten window-0 shadow program VA). The FE reads the same small-PT line
fine (the forwarded inline QMD in the qmdwin page is processed to CWD
ACTIVE), so the SM walk is not using the channel/ladder VAS the shadow
lives in - it appears to use the guest subctx page directory (instance
SC_PAGE_DIRECTORY_BASE 0x10956b000), which maps the guest VAs the real
client's captured QMD points at, and not our window.

Add gr_ptr_guest: when set, the forwarded launch leaves the QMD pointer
pairs (param w32/33, program w48/49, prefetch w8/w31) exactly as the guest
driver wrote them (as the real driver's captured QMD carries), and only
relocates the QMD page itself (qmdwin, a proven FE read). Shadows are still
copied for logging but unused.

3 files changed, 103 insertions, 50 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index c70a79f..9778385 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -16471,58 +16471,84 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
 						&sh_va);
 				}
 				if (!ret) {
+					bool keep_guest =
+						gsp_phys_rmcall_gr_ptr_guest_value();
+
 					/*
-					 * Preserve the original pointer's page
-					 * offset: pg->guest_va is page-based,
-					 * but the QMD pair (still the guest's
-					 * value here) carries the exact entry
-					 * offset (e.g. program +0x800). Without
-					 * it the engine would fetch the wrong
-					 * slot of the copied page.
-					 */
-					u64 ptr_va = sh_va |
-						((u64)qmd[pg->qmd_word] &
-						 (GSP_PHYS_PAGE_SIZE - 1));
-
-					n_shadow++;
-					qmd[pg->qmd_word] = (u32)ptr_va;
-					qmd[pg->qmd_word + 1] =
-						(qmd[pg->qmd_word + 1] &
-						 ~0x1ffffu) |
-						((u32)(ptr_va >> 32) & 0x1ffffu);
-					/*
-					 * m68: the program-prefetch pair (w8
-					 * LO32(prefetch>>8), w31[8:0]
-					 * HI32(prefetch>>8)) still carried the
-					 * guest VA; a prefetch into our unmapped
-					 * VAS may be what cancels the QMD after
-					 * SKED schedules it. If the original
-					 * prefetch address lies in the page we
-					 * just shadowed, point it at the same
-					 * shadow copy (preserving SASS_VERSION /
-					 * prefetch size in w31).
+					 * m91: the SM I-fetch at the shadow VA
+					 * NACKs (ESR MMU_NACK, PC=shadow) while
+					 * the FE reads the same PT line fine -
+					 * the SMs appear to walk the guest
+					 * subctx PDB, which maps the guest VAs
+					 * but not our window VAs. When
+					 * keep_guest is set, leave every QMD
+					 * pointer pair at the guest's original
+					 * value (as the real driver's captured
+					 * QMD carries them) and use only the
+					 * relocated QMD page (qmdwin, a proven
+					 * FE/ladder-VAS read).
 					 */
-					if (pg->qmd_word == 48) {
-						u64 orig_pref =
-							((u64)(qmd[31] &
-							       0x1ffu) << 40) |
-							((u64)qmd[8] << 8);
-
-						if ((orig_pref &
-						     ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) ==
-						    pg_va) {
-							u64 new_pref = ptr_va |
-								(orig_pref &
-								 (GSP_PHYS_PAGE_SIZE - 1));
-
-							qmd[8] = (u32)(new_pref >> 8);
-							qmd[31] = (qmd[31] &
-								   ~0x1ffu) |
-								  ((u32)((new_pref >> 40)) &
-								   0x1ffu);
-							pr_info("vgsp-gr: program prefetch rewritten to shadow 0x%llx (was 0x%llx)\n",
-								new_pref,
-								orig_pref);
+					if (keep_guest) {
+						pr_info("vgsp-gr: qmd_word %u pointer kept at guest VA 0x%08x%08x (shadow copy at 0x%llx unused)\n",
+							pg->qmd_word,
+							qmd[pg->qmd_word + 1],
+							qmd[pg->qmd_word],
+							sh_va);
+					} else {
+						/*
+						 * Preserve the original pointer's
+						 * page offset: pg->guest_va is
+						 * page-based, but the QMD pair
+						 * (still the guest's value here)
+						 * carries the exact entry offset
+						 * (e.g. program +0x800). Without
+						 * it the engine would fetch the
+						 * wrong slot of the copied page.
+						 */
+						u64 ptr_va = sh_va |
+							((u64)qmd[pg->qmd_word] &
+							 (GSP_PHYS_PAGE_SIZE - 1));
+
+						n_shadow++;
+						qmd[pg->qmd_word] = (u32)ptr_va;
+						qmd[pg->qmd_word + 1] =
+							(qmd[pg->qmd_word + 1] &
+							 ~0x1ffffu) |
+							((u32)(ptr_va >> 32) & 0x1ffffu);
+						/*
+						 * m68: the program-prefetch pair (w8
+						 * LO32(prefetch>>8), w31[8:0]
+						 * HI32(prefetch>>8)) still carried the
+						 * guest VA; a prefetch into our unmapped
+						 * VAS may be what cancels the QMD after
+						 * SKED schedules it. If the original
+						 * prefetch address lies in the page we
+						 * just shadowed, point it at the same
+						 * shadow copy (preserving SASS_VERSION /
+						 * prefetch size in w31).
+						 */
+						if (pg->qmd_word == 48) {
+							u64 orig_pref =
+								((u64)(qmd[31] &
+								       0x1ffu) << 40) |
+								((u64)qmd[8] << 8);
+
+							if ((orig_pref &
+							     ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) ==
+							    pg_va) {
+								u64 new_pref = ptr_va |
+									(orig_pref &
+									 (GSP_PHYS_PAGE_SIZE - 1));
+
+								qmd[8] = (u32)(new_pref >> 8);
+								qmd[31] = (qmd[31] &
+									   ~0x1ffu) |
+									  ((u32)((new_pref >> 40)) &
+									   0x1ffu);
+								pr_info("vgsp-gr: program prefetch rewritten to shadow 0x%llx (was 0x%llx)\n",
+									new_pref,
+									orig_pref);
+							}
 						}
 					}
 					mapped = true;
diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c
index ee6625c..629bd3a 100644
--- a/kernel/gsp_phys_gpu.c
+++ b/kernel/gsp_phys_gpu.c
@@ -970,6 +970,30 @@ int gsp_phys_rmcall_gr_shadow_win0_value(void)
 EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_shadow_win0_value);
 
 /*
+ * m91: keep the forwarded launch's QMD pointer pairs (param w32/33, program
+ * w48/49, prefetch w8/w31) at the guest's original VAs instead of rewriting
+ * them to the window-0 shadow copies (gsp_phys_rmcall_gr_ptr_guest; default
+ * 0). Rationale: the m90/m91 trap shows the SM I-fetch at the shadow VA is
+ * refused with WARP_ESR=MMU_NACK while the FE reads the same PT line fine,
+ * so the SM walk appears to use the guest subctx page directory (which maps
+ * the guest VAs) rather than the channel/ladder VAS. With this set, the
+ * shadow copies are still made (for logs/bookkeeping) but unused, and the
+ * engine sees exactly the pointer values the real driver's captured QMD
+ * carries.
+ */
+static int gsp_phys_rmcall_gr_ptr_guest_param;
+module_param_named(gsp_phys_rmcall_gr_ptr_guest,
+		   gsp_phys_rmcall_gr_ptr_guest_param, int, 0644);
+MODULE_PARM_DESC(gsp_phys_rmcall_gr_ptr_guest,
+		 "Keep forwarded QMD pointer pairs at the guest VAs (SM walk appears to use the guest subctx PDB; default 0)");
+
+int gsp_phys_rmcall_gr_ptr_guest_value(void)
+{
+	return gsp_phys_rmcall_gr_ptr_guest_param;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_ptr_guest_value);
+
+/*
  * Ladder VAS ownership selector (gsp_phys_rmcall_vas_shared; default 0).
  * The H1 GR promote wall is a direct consequence of the ladder's
  * externally-owned FERMI_VASPACE_A: with IS_EXTERNALLY_OWNED the RM treats
diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h
index 0b85758..62cc16d 100644
--- a/kernel/gsp_phys_gpu.h
+++ b/kernel/gsp_phys_gpu.h
@@ -337,6 +337,9 @@ int gsp_phys_rmcall_faultbuf_sys_value(void);
  * (proven small-PT line) instead of the window-1 scratch range (default 0). */
 int gsp_phys_rmcall_gr_shadow_win0_value(void);
 
+/* m91: keep the forwarded QMD pointer pairs at the guest VAs (default 0). */
+int gsp_phys_rmcall_gr_ptr_guest_value(void);
+
 /* m72: append the vendor local-memory context block to the GR prelude so
  * SKEDCHECK05 LOCAL_MEMORY_TOTAL_SIZE passes (default 0); base VA and
  * per-TPC size are overridable for A/B runs. */