vgsp: m32 rank-2 qmdwin - relocate the inline-QMD destination to a proven ring-chunk page (compile-verified, untested)m32

3 files changed, 144 insertions, 5 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index dc8e641..f15a117 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -888,6 +888,20 @@ bool gsp_phys_boot_requested(void)
 #define GSP_PHYS_RMCALL_VA_SEM		(GSP_PHYS_RMCALL_VA_BASE + 4 * GSP_PHYS_PAGE_SIZE)
 
 /*
+ * Rank-2 qmdwin diagnostic (gsp_phys_rmcall_gr_qmdwin=1): dedicated module
+ * page for the forwarded inline-QMD destination inside the proven ladder
+ * window-0 small-PT line (slots 0x100..0x104 are GPFIFO/PB/SRC/DST/SEM).
+ * Slot 0x105/0x106 (VA 0x105000/0x106000) are owned by the legacy P6a
+ * guest-mem pre-build whenever a guest GPA is pinned, so the unused slot 0x107
+ * is used; it lies in the same 128 B GMMU PTE fetch line already proven
+ * engine-reachable (and engine-writable: the REPORT_SEM writes land at
+ * 0x104000). Because the GMMU caches PTE fetch lines with no invalidate, the
+ * PTE is planted by gsp_phys_rmcall_pdb_build() at stage 3 - before the
+ * engine's first fetch of the line - and never at launch time.
+ */
+#define GSP_PHYS_RMCALL_VA_GR_QMDWIN	(GSP_PHYS_RMCALL_VA_BASE + 0x7000)
+
+/*
  * P6a guest-memory windows (gsp_phys_guest_mem=1). Two more slots in the same
  * 2 MB PD0 slot / small-PT page as the five ladder pages, so VA_GUEST_W takes
  * PT slot 0x105 and VA_GUEST_R slot 0x106. These VAs are legacy: triggers 6/7
@@ -1898,6 +1912,14 @@ static struct gsp_phys_boot_stage {
 	u32 rmcall_guest_pt_win_mask;	/* Tier-2 scratch windows installed */
 	struct gsp_phys_boot_chunk rmcall_gr_pt[GSP_PHYS_RMCALL_GR_PT_PAGES];
 	struct gsp_phys_boot_chunk rmcall_gr_shadow;	/* Tier-2 pages */
+	/*
+	 * Rank-2 qmdwin: dedicated module page for the forwarded inline-QMD
+	 * destination, planted (PTE written) by gsp_phys_rmcall_pdb_build when
+	 * gsp_phys_rmcall_gr_qmdwin=1; rmcall_gr_qmdwin_va stays 0 when off or
+	 * when the plant failed.
+	 */
+	struct gsp_phys_boot_chunk rmcall_gr_qmdwin;
+	u64 rmcall_gr_qmdwin_va;	/* planted VA; 0 = guest destination */
 	struct {
 		u32 pde_index;
 		u8 pt_slot;
@@ -2528,6 +2550,9 @@ static void gsp_phys_rmcall_reset(void)
 	memset(boot_stage.rmcall_gr_pt, 0, sizeof(boot_stage.rmcall_gr_pt));
 	memset(&boot_stage.rmcall_gr_shadow, 0,
 	       sizeof(boot_stage.rmcall_gr_shadow));
+	memset(&boot_stage.rmcall_gr_qmdwin, 0,
+	       sizeof(boot_stage.rmcall_gr_qmdwin));
+	boot_stage.rmcall_gr_qmdwin_va = 0;
 	memset(boot_stage.rmcall_gr_win, 0, sizeof(boot_stage.rmcall_gr_win));
 	for (i = 0; i < GSP_PHYS_RMCALL_BUF_COUNT; i++) {
 		boot_stage.rmcall_mem_handles[i] = 0;
@@ -8823,6 +8848,48 @@ static void gsp_phys_rmcall_pdb_build(void)
 			   (u8 *)gsp_phys_chunk_page_va(pt, 0) +
 			   (size_t)sem_idx * 8);
 	/*
+	 * Rank-2 qmdwin: plant the dedicated module page for the forwarded
+	 * inline-QMD destination in this same small-PT line (slot 0x107, VA
+	 * 0x107000; slots 0x105/0x106 stay the P6a guest slots below). This
+	 * runs at stage 3, before the engine's first fetch of the 128 B PT
+	 * line: the GMMU caches PTE fetch lines with no invalidate, so a
+	 * launch-time PTE write into an already-fetched line would be
+	 * invisible. Gated by the module parameter; with 0 the PT page is
+	 * byte-identical (slot 0x107 stays zero from the stage-2 zeroing).
+	 */
+	if (gsp_phys_rmcall_gr_qmdwin_value()) {
+		struct gsp_phys_boot_chunk *qw =
+			&boot_stage.rmcall_gr_qmdwin;
+		u32 qw_idx = (u32)((GSP_PHYS_RMCALL_VA_GR_QMDWIN >> 12) &
+				   0x1ffu);
+
+		if (!qw->pages) {
+			if (!boot_stage.pdev) {
+				pr_warn("vgsp-boot: rm-call: qmdwin plant skipped: no claimed pci_dev; launches keep the guest QMD destination\n");
+			} else if (!gsp_phys_chunk_alloc(boot_stage.pdev, qw, 1u)) {
+				gsp_phys_chunk_zero(qw);
+			} else {
+				pr_warn("vgsp-boot: rm-call: qmdwin page allocation failed; launches keep the guest QMD destination\n");
+			}
+		}
+		if (qw->pages) {
+			put_unaligned_le64(gsp_phys_rmcall_pte(
+					(u64)gsp_phys_chunk_page_dma(qw, 0)),
+				(u8 *)gsp_phys_chunk_page_va(pt, 0) +
+				(size_t)qw_idx * 8);
+			if (!boot_stage.rmcall_gr_qmdwin_va)
+				pr_info("vgsp-boot: rm-call: qmdwin module page planted VA=0x%llx ptSlot=0x%x pte=0x%016llx dma=0x%llx (before the engine's first fetch)\n",
+					GSP_PHYS_RMCALL_VA_GR_QMDWIN, qw_idx,
+					get_unaligned_le64(
+						(u8 *)gsp_phys_chunk_page_va(
+							pt, 0) +
+						(size_t)qw_idx * 8),
+					(u64)gsp_phys_chunk_page_dma(qw, 0));
+			boot_stage.rmcall_gr_qmdwin_va =
+				GSP_PHYS_RMCALL_VA_GR_QMDWIN;
+		}
+	}
+	/*
 	 * P6a guest windows: start from explicit zero (invalid PTE) for both
 	 * slots, then, when the guest GPA is already pinned (it must be pinned
 	 * before stage 3 for this), pre-build both slots with the translated
@@ -14151,6 +14218,8 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
 	u32 qmd_win_pde = 0, qmd_win_installed = 0;
 	int qmd_win_ret = 0;
 	bool qmd_win_diag = false;
+	bool qmdwin_on = false;
+	char qmdwin_note[32] = "";
 	int ret = 0;
 
 	if (out)
@@ -14477,6 +14546,32 @@ gr_page_unmapped:
 	}
 
 	/*
+	 * Rank-2 qmdwin (gsp_phys_rmcall_gr_qmdwin=1): relocate the inline-QMD
+	 * destination to the dedicated module page planted in the ladder's
+	 * window-0 small-PT line (VA 0x107000). The PTE was written by
+	 * gsp_phys_rmcall_pdb_build before the engine's first fetch of that
+	 * 128 B fetch line; the page is module-owned, so the prewrite and the
+	 * post-mortem involve no guest page, and it shares the proven line the
+	 * GPFIFO/PB/REPORT_SEM legs already use. The 2 MiB window is installed
+	 * by the ladder itself, so no window ensure is sent for it (a
+	 * pdeIndex-0 Tier-1 install would touch the ladder's own PDE).
+	 */
+	if (gsp_phys_rmcall_gr_qmdwin_value()) {
+		u8 *qw = boot_stage.rmcall_gr_qmdwin.pages ?
+			gsp_phys_chunk_page_va(&boot_stage.rmcall_gr_qmdwin, 0) :
+			NULL;
+
+		if (boot_stage.rmcall_gr_qmdwin_va && qw) {
+			qmdwin_on = true;
+			qmd_dst = boot_stage.rmcall_gr_qmdwin_va;
+			qmd_dst_mem = qw;
+			pr_info("vgsp-gr: qmdwin: relocating inline-QMD dest 0x%llx -> 0x%llx (module ring chunk page)\n",
+				req->qmd_va, qmd_dst);
+		} else {
+			pr_warn("vgsp-gr: qmdwin=1 but no planted module page (stage-3 plant skipped or failed); using the guest QMD destination\n");
+		}
+	}
+	/*
 	 * Inline-QMD destination: the CUDA driver points ADDRESS_A/B at its
 	 * own QMD buffer (turing_launch.c:1629) and the IQ2M copies the 64
 	 * inline words there before scheduling. Map the guest QMD page and
@@ -14485,7 +14580,7 @@ gr_page_unmapped:
 	 * alias keeps the launch runnable, and an unresolved page keeps the
 	 * module scratch VA.
 	 */
-	if (req->qmd_hpa) {
+	if (!qmdwin_on && req->qmd_hpa) {
 		u64 qmd_page = req->qmd_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1);
 		u32 qmd_off = (u32)(req->qmd_va & (GSP_PHYS_PAGE_SIZE - 1));
 		bool dst_ok = false;
@@ -14531,6 +14626,9 @@ gr_page_unmapped:
 		pr_info("vgsp-gr: inline-QMD dest va=0x%llx hpa=0x%llx -> 0x%llx (shadow=%u)\n",
 			req->qmd_va, req->qmd_hpa, qmd_dst, n_shadow);
 	}
+	if (qmdwin_on)
+		scnprintf(qmdwin_note, sizeof(qmdwin_note),
+			  " qmdwin=0x%llx", qmd_dst);
 
 	/*
 	 * Channel release: author it into the guest semaphore page when the
@@ -14596,13 +14694,13 @@ gr_page_unmapped:
 		goto out;
 	}
 
-	pr_info("vgsp-gr: GR LAUNCH submitted h=0x%x push=0x%llx len=%u pages=%u shadow=%u qmd_va=0x%llx slot=%u words=%u prelude=%d qmdvirt=%d token=0x%x; polling qmd_rel=%u ch_rel=0x%08x (<=%u ms)\n",
+	pr_info("vgsp-gr: GR LAUNCH submitted h=0x%x push=0x%llx len=%u pages=%u shadow=%u qmd_va=0x%llx slot=%u words=%u prelude=%d qmdvirt=%d token=0x%x; polling qmd_rel=%u ch_rel=0x%08x (<=%u ms)%s\n",
 		req->h_object, req->push_va, req->push_len, req->n_pages,
 		n_shadow, req->qmd_va, sub.slot, words,
 		gsp_phys_rmcall_gr_prelude_value(),
 		gsp_phys_rmcall_gr_qmdvirt_value(),
 		boot_stage.rmcall_token,
-		qmd_rel, want_ch, GSP_PHYS_RMCALL_GR_POLL_MS);
+		qmd_rel, want_ch, GSP_PHYS_RMCALL_GR_POLL_MS, qmdwin_note);
 	if (qmd_win_diag)
 		pr_info("vgsp-gr: qmd dest va=0x%llx pdeIndex=%u installed=%u win_mask=0x%x ensure_ret=%d\n",
 			qmd_dst, qmd_win_pde, qmd_win_installed,
@@ -14689,12 +14787,13 @@ gr_page_unmapped:
 					relw = READ_ONCE(
 						*(u32 *)((u8 *)__va(qmd_rel_hpa) +
 							 qmd_rel_off));
-				pr_info("vgsp-gr: post-mortem gpget=%u->%u qmd_page w0=%08x w1=%08x w23..25=%08x %08x %08x w63=%08x relw=%08x iq2m=%s\n",
+				pr_info("vgsp-gr: post-mortem gpget=%u->%u qmd_page w0=%08x w1=%08x w23..25=%08x %08x %08x w63=%08x relw=%08x iq2m=%s%s\n",
 					sub.gpget, gpget_post, q0, q1, q23, q24,
 					q25, q63, relw,
 					q63 == GSP_PHYS_GR_QMD_SENTINEL ?
 						"not-run (sentinel intact)" :
-						"ran (page overwritten)");
+						"ran (page overwritten)",
+					qmdwin_note);
 			}
 			poll = w1;
 			ret = -ETIMEDOUT;
@@ -15535,6 +15634,7 @@ static void gsp_phys_rmcall_fini(struct pci_dev *pdev)
 	for (i = 0; i < GSP_PHYS_RMCALL_GR_PT_PAGES; i++)
 		gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_pt[i]);
 	gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_shadow);
+	gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_qmdwin);
 	gsp_phys_chunk_free(pdev, &boot_stage.rmcall_patch);
 	gsp_phys_chunk_free(pdev, &boot_stage.rmcall_pm);
 	gsp_phys_chunk_free(pdev, &boot_stage.rmcall_fecs);
diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c
index 4848f4b..0c7e1e5 100644
--- a/kernel/gsp_phys_gpu.c
+++ b/kernel/gsp_phys_gpu.c
@@ -619,6 +619,35 @@ int gsp_phys_rmcall_gr_qmdvirt_value(void)
 EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_qmdvirt_value);
 
 /*
+ * Rank-2 qmdwin diagnostic (gsp_phys_rmcall_gr_qmdwin; default 0). When set,
+ * a forwarded GR launch relocates its inline-QMD destination from the guest
+ * QMD VA to a dedicated module page mapped at the ladder's own window-0 VA
+ * 0x107000 (GSP_PHYS_RMCALL_VA_GR_QMDWIN): the same 2 MiB window and small-PT
+ * page whose PTEs the GPFIFO/PB/REPORT_SEM legs have already proven
+ * engine-reachable and engine-writable. The PTE is planted by
+ * gsp_phys_rmcall_pdb_build() at stage 3, before the engine's first fetch of
+ * that 128 B PTE fetch line (the GMMU caches fetch lines with no invalidate,
+ * so a runtime PTE write into an already-fetched line would be invisible).
+ * Slot 0x106 (VA 0x106000) is avoided because the legacy P6a guest-mem
+ * pre-build owns it whenever a guest GPA is pinned; slot 0x107 is unused.
+ * With 1, the prewrite and post-mortem use the module page only (no guest
+ * page); 0 keeps the historical guest-VA destination byte-identical. Read at
+ * stage 3 (PTE plant) and at each launch (destination choice); enabling it
+ * after stage 3 falls back to the guest destination with a warning.
+ */
+static int gsp_phys_rmcall_gr_qmdwin_param;
+module_param_named(gsp_phys_rmcall_gr_qmdwin,
+		   gsp_phys_rmcall_gr_qmdwin_param, int, 0644);
+MODULE_PARM_DESC(gsp_phys_rmcall_gr_qmdwin,
+		 "Relocate the forwarded GR inline-QMD destination to a dedicated module page in the ladder window-0 PT line (default 0; 1 enables the Rank-2 qmdwin diagnostic)");
+
+int gsp_phys_rmcall_gr_qmdwin_value(void)
+{
+	return gsp_phys_rmcall_gr_qmdwin_param;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_qmdwin_value);
+
+/*
  * Ladder VAS ownership selector (gsp_phys_rmcall_vas_shared; default 0).
  * The H1 GR promote wall is a direct consequence of the ladder's
  * externally-owned FERMI_VASPACE_A: with IS_EXTERNALLY_OWNED the RM treats
diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h
index 3dce1a6..2ebe167 100644
--- a/kernel/gsp_phys_gpu.h
+++ b/kernel/gsp_phys_gpu.h
@@ -294,6 +294,16 @@ int gsp_phys_rmcall_gr_prelude_value(void);
 int gsp_phys_rmcall_gr_qmdvirt_value(void);
 
 /*
+ * gsp_phys_rmcall_gr_qmdwin (default 0): 1 relocates the forwarded GR
+ * inline-QMD destination from the guest QMD VA to a dedicated module page
+ * mapped at the proven engine-reachable window-0 VA 0x107000 (the PTE is
+ * planted in the ladder small-PT line at stage 3, before the engine's first
+ * fetch); 0 keeps the guest destination exactly as before. Rank-2
+ * diagnostic: prewrite/post-mortem use the module page only.
+ */
+int gsp_phys_rmcall_gr_qmdwin_value(void);
+
+/*
  * Ladder VAS ownership selector (gsp_phys_rmcall_vas_shared; default 0).
  * 0 = FERMI_VASPACE_A is allocated IS_EXTERNALLY_OWNED (historical ladder
  * behaviour: the client owns page tables, VA bookings and engine-context