vgsp: golden9g - fabricated golden-VAS page-table levels + walker ctx window

1 files changed, 124 insertions, 61 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index f7cb150..c7abdaf 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -924,6 +924,15 @@ enum gsp_phys_rmcall_buf {
 	GSP_PHYS_RMCALL_BUF_GUEST_PT13,
 	GSP_PHYS_RMCALL_BUF_GUEST_PT14,
 	GSP_PHYS_RMCALL_BUF_GUEST_PT15,
+	/*
+	 * Golden9g: the three client page-table levels the real golden VAS
+	 * hands to the server with VASPACE_COPY_SERVER_RESERVED_PDES
+	 * (0x90f10106): PD2 (0x2f, size 0x20), PD1 (0x26) and PD0 (0x1d),
+	 * chained 0 -> 1 -> 2 with the proven GP10X sysmem PDEs.
+	 */
+	GSP_PHYS_RMCALL_BUF_GOLDEN_PT0,
+	GSP_PHYS_RMCALL_BUF_GOLDEN_PT1,
+	GSP_PHYS_RMCALL_BUF_GOLDEN_PT2,
 	GSP_PHYS_RMCALL_BUF_COUNT
 };
 
@@ -6646,6 +6655,7 @@ static int gsp_phys_rmcall_control(void __iomem *bar0, u32 h_object, u32 cmd,
 				   u32 *out_rpc_result, u32 *out_status,
 				   const char *what);
 static void gsp_phys_rmcall_pdb_build(void);
+static u64 gsp_phys_rmcall_pde(u64 pa);
 
 /*
  * Shared-management VAS: install the ladder's page-table window through the
@@ -7795,44 +7805,138 @@ static void gsp_phys_rmcall_golden_fail(const char *step, u32 handle,
 
 /*
  * Mode-25 VAS construct control (0x90f10106), gated by fwclient mask bit 8.
- * The captured real-client control copied three server-RM-internal page-table
- * levels; the ladder has no safe source for those PAs, so the only form this
- * path can send is the zero-level no-op range (numLevelsToCopy=0) and it does
- * so only when the gate is set. With the default mask the control is skipped
- * and the deviation is logged; this sub-step still needs a live bisection
- * (report open item 3).
+ * golden9g: the real client's VAS construction (gvaspaceReserveSplitVaSpace ->
+ * _gvaspaceReserveVaForClientRm) allocates the three client page-table levels
+ * for the server range [4GB, 4.5GB] and hands their PAs to the server with
+ * this control; the server mirrors them with mmuWalkModifyLevelInstance. The
+ * ladder now fabricates the identical chain in host sysmem (pageShifts
+ * 0x2f/0x26/0x1d, sizes 0x20/0x1000/0x1000, aperture SYS_COH) with the proven
+ * GP10X sysmem PDEs, so the golden VAS has the same server-side walker state
+ * as the real client's. Set gsp_phys_rmcall_fwclient_mask bit 8 (0x100) to
+ * send it; without the bit the control is skipped and the deviation logged.
  */
 static int gsp_phys_rmcall_golden9_vas_control(void __iomem *bar0)
 {
 	struct gsp_phys_rmcall_vas_copy_pdes_params *p;
+	u64 pa0, pa1, pa2;
 	u32 rpc_result = 0, status = 0;
 	int ret;
 
 	if (!(gsp_phys_rmcall_fwclient_mask_value() &
 	      GSP_PHYS_RMCALL_GOLDEN9_MASK_VAS_COPY_PDES)) {
-		pr_info("vgsp-boot: rm-call: GOLDEN9 VAS COPY_SERVER_RESERVED_PDES (0x90f10106) skipped: the captured three level PAs are server-RM-internal and the ladder has no safe source (uncertain, report open item 3); set gsp_phys_rmcall_fwclient_mask bit 8 (0x%x) to send the zero-level no-op-range form for bisection\n",
+		pr_info("vgsp-boot: rm-call: GOLDEN9 VAS COPY_SERVER_RESERVED_PDES (0x90f10106) skipped (set gsp_phys_rmcall_fwclient_mask bit 8 (0x%x) to send the fabricated three-level chain)\n",
 			GSP_PHYS_RMCALL_GOLDEN9_MASK_VAS_COPY_PDES);
 		return 0;
 	}
 
+	/* Build the client page-table chain: PT0(PD2[0]) -> PT1(PD1[0]) -> PT2(PD0). */
+	gsp_phys_chunk_zero(
+		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GOLDEN_PT0]);
+	gsp_phys_chunk_zero(
+		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GOLDEN_PT1]);
+	gsp_phys_chunk_zero(
+		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GOLDEN_PT2]);
+	pa0 = (u64)gsp_phys_chunk_page_dma(
+		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GOLDEN_PT0], 0);
+	pa1 = (u64)gsp_phys_chunk_page_dma(
+		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GOLDEN_PT1], 0);
+	pa2 = (u64)gsp_phys_chunk_page_dma(
+		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GOLDEN_PT2], 0);
+	put_unaligned_le64(gsp_phys_rmcall_pde(pa1),
+		(u8 *)gsp_phys_chunk_page_va(
+			&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GOLDEN_PT0], 0));
+	put_unaligned_le64(gsp_phys_rmcall_pde(pa2),
+		(u8 *)gsp_phys_chunk_page_va(
+			&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GOLDEN_PT1], 0));
+
 	p = kvzalloc(sizeof(*p), GFP_KERNEL);
 	if (!p)
 		return -ENOMEM;
 	p->page_size = cpu_to_le64(1ULL << 29);	/* GMMU_PD1_VADDR_BIT_LO */
+	p->virt_addr_lo = cpu_to_le64(0x100000000ULL);	/* SPLIT_VAS_SERVER_RM_MANAGED_VA_START */
 	p->virt_addr_hi = cpu_to_le64(0x11fffffffULL);
-	p->num_levels_to_copy = cpu_to_le32(0);
+	p->num_levels_to_copy = cpu_to_le32(3);
+	p->levels[0].phys_addr = cpu_to_le64(pa0);
+	p->levels[0].size = cpu_to_le64(0x20);
+	p->levels[0].aperture = cpu_to_le32(4);	/* GMMU_APERTURE_SYS_COH */
+	p->levels[0].page_shift = cpu_to_le32(0x2f);
+	p->levels[1].phys_addr = cpu_to_le64(pa1);
+	p->levels[1].size = cpu_to_le64(0x1000);
+	p->levels[1].aperture = cpu_to_le32(4);
+	p->levels[1].page_shift = cpu_to_le32(0x26);
+	p->levels[2].phys_addr = cpu_to_le64(pa2);
+	p->levels[2].size = cpu_to_le64(0x1000);
+	p->levels[2].aperture = cpu_to_le32(4);
+	p->levels[2].page_shift = cpu_to_le32(0x1d);
 	/* The captured RPC targets the VAS itself (hObject=0xbaba0042). */
 	ret = gsp_phys_rmcall_control(bar0,
 			GSP_PHYS_RMCALL_HANDLE_GOLDEN_VAS,
 			GSP_PHYS_RMCALL_CTRL_VASPACE_COPY_SERVER_RESERVED_PDES,
 			p, sizeof(*p), 24, NULL, 0, &rpc_result, &status,
-			"VASPACE_COPY_SERVER_RESERVED_PDES (no-op range, gated)");
-	pr_info("vgsp-boot: rm-call: GOLDEN9 VAS COPY_SERVER_RESERVED_PDES: no-op range numLevels=0 pageSize=0x20000000 virtAddrHi=0x11fffffff status=0x%x rpc_result=0x%x ret=%d\n",
-		status, rpc_result, ret);
+			"VASPACE_COPY_SERVER_RESERVED_PDES (3 fabricated levels)");
+	pr_info("vgsp-boot: rm-call: GOLDEN9G VAS COPY_SERVER_RESERVED_PDES: numLevels=3 pageSize=0x20000000 range=[0x%llx,0x%llx] levels={0x%llx/0x20/0x2f, 0x%llx/0x1000/0x26, 0x%llx/0x1000/0x1d} status=0x%x rpc_result=0x%x ret=%d\n",
+		le64_to_cpu(p->virt_addr_lo), le64_to_cpu(p->virt_addr_hi),
+		pa0, pa1, pa2, status, rpc_result, ret);
 	kvfree(p);
 	return ret;
 }
 
+/*
+ * golden9g: install the golden VAS context window (VA_GR_CTX >> 21) in the
+ * server's own MMU walker via NV0080_CTRL_CMD_DMA_UPDATE_PDE_2, the supported
+ * client map path for a non-external VAS. The PT pages hold the golden9e
+ * VIDMEM leaf PTEs (built by gsp_phys_rmcall_pdb_build) for the FBMEM
+ * MAIN/PATCH/FECS/PRIV/UNR buffers, so the GR engine can fetch the promoted
+ * context VAs through the golden VAS exactly as it did through the ladder
+ * VAS's raw PDB. Gated by fwclient mask bit 8 together with the copy control.
+ */
+static int gsp_phys_rmcall_golden9_vas_windows(void __iomem *bar0)
+{
+	struct gsp_phys_rmcall_update_pde2_params p;
+	u32 i;
+	int ret;
+
+	if (!(gsp_phys_rmcall_fwclient_mask_value() &
+	      GSP_PHYS_RMCALL_GOLDEN9_MASK_VAS_COPY_PDES))
+		return 0;
+	if (!boot_stage.rmcall_ctx_pt_pages) {
+		pr_err("vgsp-boot: rm-call: GOLDEN9G ctx window install: no ctx PT pages built; aborting the golden replay with no retry\n");
+		return -EINVAL;
+	}
+	for (i = 0; i < boot_stage.rmcall_ctx_pt_pages; i++) {
+		u32 ctx_index = (u32)((GSP_PHYS_RMCALL_VA_GR_CTX >> 21) + i);
+		u64 ctx_pt_pa = (u64)gsp_phys_chunk_page_dma(
+			&boot_stage.rmcall_ctx_pt[i], 0);
+		u32 rpc_result = 0, status = 0;
+
+		memset(&p, 0, sizeof(p));
+		p.pde_index = cpu_to_le32(ctx_index);
+		p.flags = cpu_to_le32(GSP_PHYS_RMCALL_PDE2_FLAG_FLUSH);
+		p.pt[0].phys_addr = cpu_to_le64(ctx_pt_pa);
+		p.pt[0].aperture =
+			cpu_to_le32(GSP_PHYS_RMCALL_PDE2_APERTURE_SYS_COH);
+		p.pt[1].aperture =
+			cpu_to_le32(GSP_PHYS_RMCALL_PDE2_APERTURE_INVALID);
+		p.h_va_space = cpu_to_le32(GSP_PHYS_RMCALL_HANDLE_GOLDEN_VAS);
+		p.sub_device_id = 0;
+		ret = gsp_phys_rmcall_control(bar0,
+				boot_stage.rmcall_device,
+				GSP_PHYS_RMCALL_CTRL_DMA_UPDATE_PDE_2,
+				&p, sizeof(p), sizeof(p), &p, sizeof(p),
+				&rpc_result, &status,
+				"DMA_UPDATE_PDE_2 (golden9g golden VAS ctx window)");
+		if (ret) {
+			pr_err("vgsp-boot: rm-call: GOLDEN9G golden VAS ctx window %u rejected: status=0x%x rpc_result=0x%x ret=%d; aborting the golden replay with no retry\n",
+			       ctx_index, status, rpc_result, ret);
+			return ret;
+		}
+		pr_info("vgsp-boot: rm-call: GOLDEN9G golden VAS ctx window accepted: pdeIndex=%u ptPa=0x%llx hVas=0x%x status=0x%x\n",
+			ctx_index, ctx_pt_pa,
+			GSP_PHYS_RMCALL_HANDLE_GOLDEN_VAS, status);
+	}
+	return 0;
+}
+
 static int gsp_phys_rmcall_golden9_promote(void __iomem *bar0);
 
 static int gsp_phys_rmcall_golden_mem(void __iomem *bar0,
@@ -7854,45 +7958,6 @@ static int gsp_phys_rmcall_golden_mem(void __iomem *bar0,
 	return ret;
 }
 
-/*
- * Golden9f: install the client PDB hierarchy on the golden VAS. The golden
- * VAS is PTETABLE_HEAP_MANAGED (client-owned page tables on this config, no
- * split range), so the supported client mechanism is
- * DMA_SET_PAGE_DIRECTORY with the ladder's root PDB (the same tree the
- * ladder VAS uses; its window maps the promoted ctx VAs). Without this the
- * channel alloc's pVAS validation rejects the empty golden VAS (the
- * golden9a/b NOCAT chain 0x51a8f10/0x52008f4/0x51a89a4).
- */
-static int gsp_phys_rmcall_golden9_vas_pdb(void __iomem *bar0)
-{
-	struct gsp_phys_rmcall_pdb_params pdb;
-	u32 pdb_rpc = 0;
-	int ret;
-
-	memset(&pdb, 0, sizeof(pdb));
-	pdb.phys_address = cpu_to_le64((u64)gsp_phys_chunk_page_dma(
-		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_VASPDB], 0));
-	pdb.num_entries = cpu_to_le32(GSP_PHYS_RMCALL_PDB_NUM_ENTRIES);
-	pdb.flags = cpu_to_le32(GSP_PHYS_RMCALL_PDB_FLAG_APERTURE_SYSMEM_NONCOH |
-				GSP_PHYS_RMCALL_PDB_FLAG_ALL_CHANNELS);
-	pdb.h_va_space = cpu_to_le32(GSP_PHYS_RMCALL_HANDLE_GOLDEN_VAS);
-	pdb.ch_id = 0;
-	pdb.sub_device_id = cpu_to_le32(GSP_PHYS_RMCALL_PDB_SUBDEVICE_ID);
-	pdb.pasid = 0;
-	ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_device,
-			GSP_PHYS_RMCALL_CTRL_DMA_SET_PAGE_DIRECTORY,
-			&pdb, sizeof(pdb), 16, NULL, 0,
-			&pdb_rpc, NULL,
-			"DMA_SET_PAGE_DIRECTORY (golden9f golden VAS)");
-	pr_info("vgsp-boot: rm-call: GOLDEN9F golden VAS PDB install: hVas=0x%x root=0x%llx numEntries=%u ret=%d rpc_result=0x%x\n",
-		GSP_PHYS_RMCALL_HANDLE_GOLDEN_VAS,
-		(u64)gsp_phys_chunk_page_dma(
-			&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_VASPDB],
-			0),
-		le32_to_cpu(pdb.num_entries), ret, pdb_rpc);
-	return ret;
-}
-
 static int gsp_phys_rmcall_golden_replay(void __iomem *bar0)
 {
 	struct gsp_phys_rmcall_vas_params va;
@@ -7926,9 +7991,11 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0)
 	}
 
 	/*
-	 * Mode 25: the captured VAS construct control follows the VAS alloc.
-	 * It is gated off by default and aborts the replay only when the
-	 * operator enabled the gate and the server rejected it.
+	 * Mode 25: the captured VAS construct control follows the VAS alloc,
+	 * then golden9g installs the context window in the golden VAS's
+	 * server walker. Both are gated off by default (fwclient mask bit 8)
+	 * and abort the replay only when the gate is enabled and the server
+	 * rejects a step.
 	 */
 	if (boot_stage.rmcall_gr_promote ==
 			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 &&
@@ -7939,13 +8006,9 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0)
 			       ret);
 			return ret;
 		}
-		/*
-		 * Golden9f: give the golden VAS real page tables (the ladder's
-		 * root PDB) before the channel alloc validates its pVAS.
-		 */
-		ret = gsp_phys_rmcall_golden9_vas_pdb(bar0);
+		ret = gsp_phys_rmcall_golden9_vas_windows(bar0);
 		if (ret) {
-			pr_err("vgsp-boot: rm-call: GOLDEN9F golden VAS PDB install failed (%d); the golden replay aborts with no retry\n",
+			pr_err("vgsp-boot: rm-call: GOLDEN9G golden VAS ctx window install failed (%d); the golden replay aborts with no retry\n",
 			       ret);
 			return ret;
 		}
@@ -15152,8 +15215,8 @@ static_assert(offsetof(struct gsp_phys_rmcall_channel_params, process_id) == 296
 	      "NV_CHANNEL_ALLOC_PARAMS.ProcessID must be at offset 296");
 static_assert(offsetof(struct gsp_phys_rmcall_channel_params, tpc_config_id) == 360,
 	      "NV_CHANNEL_ALLOC_PARAMS.tpcConfigID must be at offset 360");
-static_assert(GSP_PHYS_RMCALL_BUF_COUNT == 30,
-	      "the rm-call ladder registers exactly thirty DMA buffers (14 ladder + 16 single-page guest scratch PTs)");
+static_assert(GSP_PHYS_RMCALL_BUF_COUNT == 33,
+	      "the rm-call ladder registers exactly thirty-three DMA buffers (14 ladder + 16 single-page guest scratch PTs + 3 golden9g VAS levels)");
 static_assert((GSP_PHYS_RMCALL_HANDLE_MEM_BASE +
 	       (GSP_PHYS_RMCALL_BUF_COUNT - 1) *
 	       GSP_PHYS_RMCALL_HANDLE_MEM_STRIDE) < 0xcaf80000u,