vgsp: m95 - program GPC MMU_CTRL from PFB MMU_CTRL (RM grInitGPCMmuPRIRegs copy)

The real RM initialises every GPC's NV_PGPC_PRI_MMU_CTRL (0x500880 +
gpc*0x8000) from NV_PFB_PRI_MMU_CTRL (0x100C80) using the TU102
hub-to-GPC matching fields (VM_PG_SIZE, COMP_FAULT, MISS_GRAN, CACHE_MODE,
USE_PDB_BIG_PAGE_SIZE, ATOMIC_CAPABILITY_MODE, ATOMIC_CAPABILITY_SYS_NCOH;
ext-tree gr_gk104.c:3340-3380 + gr_tu102.c:229-238). Our ladder never
programmed it (grep-verified). The m93 INST dump shows the server's SC(0)
PDB carries USE_VER2_PT_FORMAT + BIG_PAGE_SIZE=1, so a GPC/hub MMU_CTRL
mismatch is a candidate for the GPC-side walker refusing the SM I-fetch
with MMU_NACK while hub-side reads resolve. Gated by
gsp_phys_rmcall_gr_gpc_mmu_init (default 0); sampler now reads
GPC0/GPC1_MMU_CTRL (0x500880/0x508880) and PFB_MMU_CTRL (0x100C80).

4 files changed, 61 insertions, 0 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index 1057e16..51d6685 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -16064,6 +16064,41 @@ static bool gsp_phys_gr_fault_active(void)
 }
 
 /*
+ * m95: program each GPC's MMU_CTRL from the hub's PFB MMU_CTRL, exactly like
+ * the real RM's grInitGPCMmuPRIRegs (read NV_PFB_PRI_MMU_CTRL, mask with
+ * the TU102 hub-to-GPC matching fields, write NV_PGPC_PRI_MMU_CTRL per GPC
+ * with the masked value). The matching fields are VM_PG_SIZE(0), COMP_FAULT(2), MISS_GRAN(4:3),
+ * CACHE_MODE(6:5), USE_PDB_BIG_PAGE_SIZE(11), ATOMIC_CAPABILITY_MODE(25:24),
+ * ATOMIC_CAPABILITY_SYS_NCOH_MODE(27). The SC_PDB the server programmed
+ * carries USE_VER2_PT_FORMAT + BIG_PAGE_SIZE (m93 INST dump), so a GPC-side
+ * MMU_CTRL that does not match the hub would make the GPC walker misread the
+ * page tables and refuse the SM I-fetch with MMU_NACK while hub-side reads
+ * still resolve. Gated by gsp_phys_rmcall_gr_gpc_mmu_init.
+ */
+#define GSP_PHYS_GR_PFB_PRI_MMU_CTRL	0x00100C80u
+#define GSP_PHYS_GR_GPC0_MMU_CTRL	0x00500880u
+#define GSP_PHYS_GR_GPC_MMU_STRIDE	0x8000u
+#define GSP_PHYS_GR_GPC_COUNT		6u
+#define GSP_PHYS_GR_GPC_MMU_MATCH_MASK	0x0b00087du
+
+static void gsp_phys_rmcall_gr_gpc_mmu_init(void __iomem *bar0)
+{
+	u32 hub = readl(bar0 + GSP_PHYS_GR_PFB_PRI_MMU_CTRL);
+	u32 wr = hub & GSP_PHYS_GR_GPC_MMU_MATCH_MASK;
+	unsigned int g;
+
+	for (g = 0; g < GSP_PHYS_GR_GPC_COUNT; g++) {
+		void __iomem *reg = bar0 + GSP_PHYS_GR_GPC0_MMU_CTRL +
+				    (unsigned long)g * GSP_PHYS_GR_GPC_MMU_STRIDE;
+		u32 old = readl(reg);
+
+		writel(wr, reg);
+		pr_info("vgsp-gr: GPC%u MMU_CTRL 0x%08x -> 0x%08x (hub 0x%08x, mask 0x%08x)\n",
+			g, old, wr, hub, GSP_PHYS_GR_GPC_MMU_MATCH_MASK);
+	}
+}
+
+/*
  * One forwarded GR launch on the ladder channel (see gsp_phys_booter.h). The
  * caller has resolved the launch's pages; every mapping is fresh or proven
  * unchanged, the QMD release0 and the channel release are authored either by
@@ -16745,6 +16780,8 @@ gr_page_unmapped:
 				le32_to_cpu(iv[172]), le32_to_cpu(iv[173]));
 		}
 	}
+	if (gsp_phys_rmcall_gr_gpc_mmu_init_value())
+		gsp_phys_rmcall_gr_gpc_mmu_init(bar0);
 	pr_info("vgsp-gr: GR LAUNCH submitted h=0x%x push=0x%llx len=%u pages=%u shadow=%u qmd_va=0x%llx slot=%u words=%u prelude=%d qmdvirt=%d token=0x%x; polling qmd_rel=%u ch_rel=0x%08x (<=%u ms)%s\n",
 		req->h_object, req->push_va, req->push_len, req->n_pages,
 		n_shadow, req->qmd_va, sub.slot, words,
diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c
index 629bd3a..3b2feb4 100644
--- a/kernel/gsp_phys_gpu.c
+++ b/kernel/gsp_phys_gpu.c
@@ -994,6 +994,24 @@ int gsp_phys_rmcall_gr_ptr_guest_value(void)
 EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_ptr_guest_value);
 
 /*
+ * m95: program each GPC's MMU_CTRL from the hub's PFB MMU_CTRL (the real RM's
+ * grInitGPCMmuPRIRegs matching-fields copy) before a forwarded launch. The
+ * GPC-side walker refuses the SM I-fetch with MMU_NACK while hub-side reads
+ * resolve; a GPC/hub MMU_CTRL mismatch is one candidate. Default 0.
+ */
+static int gsp_phys_rmcall_gr_gpc_mmu_init_param;
+module_param_named(gsp_phys_rmcall_gr_gpc_mmu_init,
+		   gsp_phys_rmcall_gr_gpc_mmu_init_param, int, 0644);
+MODULE_PARM_DESC(gsp_phys_rmcall_gr_gpc_mmu_init,
+		 "Program GPC MMU_CTRL from PFB MMU_CTRL matching fields before a forwarded launch (default 0)");
+
+int gsp_phys_rmcall_gr_gpc_mmu_init_value(void)
+{
+	return gsp_phys_rmcall_gr_gpc_mmu_init_param;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_gpc_mmu_init_value);
+
+/*
  * Ladder VAS ownership selector (gsp_phys_rmcall_vas_shared; default 0).
  * The H1 GR promote wall is a direct consequence of the ladder's
  * externally-owned FERMI_VASPACE_A: with IS_EXTERNALLY_OWNED the RM treats
diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h
index 62cc16d..9bd5df3 100644
--- a/kernel/gsp_phys_gpu.h
+++ b/kernel/gsp_phys_gpu.h
@@ -340,6 +340,9 @@ int gsp_phys_rmcall_gr_shadow_win0_value(void);
 /* m91: keep the forwarded QMD pointer pairs at the guest VAs (default 0). */
 int gsp_phys_rmcall_gr_ptr_guest_value(void);
 
+/* m95: program GPC MMU_CTRL from PFB MMU_CTRL before a forwarded launch. */
+int gsp_phys_rmcall_gr_gpc_mmu_init_value(void);
+
 /* m72: append the vendor local-memory context block to the GR prelude so
  * SKEDCHECK05 LOCAL_MEMORY_TOTAL_SIZE passes (default 0); base VA and
  * per-TPC size are overridable for A/B runs. */
diff --git a/tools/gr-regs/reg_sampler.py b/tools/gr-regs/reg_sampler.py
index acd7e00..24c8296 100644
--- a/tools/gr-regs/reg_sampler.py
+++ b/tools/gr-regs/reg_sampler.py
@@ -47,6 +47,9 @@ REGS = [
     (0x504830, "G0T0S2_WARP_ESR"),
     (0x5048B0, "G0T0S3_WARP_ESR"),
     (0x500324, "GPC0_GPCMMU_ESR"),
+    (0x500880, "GPC0_MMU_CTRL"),
+    (0x508880, "GPC1_MMU_CTRL"),
+    (0x100C80, "PFB_MMU_CTRL"),
     (0x50AC90, "GPC1_EXCEPTION"),
     (0x50C730, "G1T0S0_WARP_ESR"),
     (0x50C734, "G1T0S0_ESR_MASK"),