vgsp: m99 - identity-map the forwarded kernel's guest data pages into the engine tree

The m97/m98 decode proved the SM instruction fetch and const reads work;
the wall is the kernel's first global data access (prologue instruction at
+0x80) to a guest-VA device pointer (param page words 6..9 carry the demo's
two 16MB buffers at 0x763dXXXXXXXX), which is not mapped in the engine's
page tables. The walker walks OUR root->PD2->PD1->PD0->PT tree (SC PDB ==
our VASPDB chunk, proven by matching the same-run INST base to the PDB-root
DMA log), and fresh direct-planted tables are visible at first fetch.

Service side (gsp_sema_service.c): svc_gr_data_fill() scans the param
(c[0x0]) page for plausible device VAs, resolves the containing 2MB windows
(-1..+16 windows, early stop when a window resolves nothing) via
svc_resolve_va on the service thread, and carries {va, pa} in
svc_fwd_req.gr_data; svc_fwd_gr_work() converts PA->HPA and attaches the
list to the submit.

Backend (gsp_phys_booter.c): gsp_phys_rmcall_gr_data_map() plants VA->HPA
leaf PTEs (GENERIC_MEMORY kind 0x06, like UVM/RM on Turing) into lazily
allocated PD1/PD0/PT chunk branches under root[0] (VAs < 2^47), caching
branches per index; the submit maps all req->data pages before publish and
logs the count.

1 files changed, 166 insertions, 0 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index 19fb67e..6db26a4 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -1009,6 +1009,18 @@ bool gsp_phys_boot_requested(void)
 #define GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE \
 	(GSP_PHYS_RMCALL_VA_BASE + 8 * GSP_PHYS_PAGE_SIZE)
 #define GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS	8u
+/*
+ * m99: engine-tree branches for identity-mapping the forwarded kernel's
+ * guest data pages. The walker uses our root[0] -> PD2 -> PD1 -> PD0 dual ->
+ * small PT hierarchy (root[48:47] -> PD2[46:38] -> PD1[37:29] -> PD0[28:21]
+ * -> PT[20:12]). PD1 pages are cached per PD2 index, PD0 pages per
+ * (PD2,PD1) index, PT pages per (PD2,PD1,PD0) index; the tables are fresh
+ * (never fetched) when planted, so the first access at kernel runtime sees
+ * them even though the GMMU has no PTE-line invalidate.
+ */
+#define GSP_PHYS_RMCALL_GR_DATA_PD1_MAX	4u
+#define GSP_PHYS_RMCALL_GR_DATA_PD0_MAX	8u
+#define GSP_PHYS_RMCALL_GR_DATA_PT_MAX	64u
 
 /*
  * Rank-2 qmdwin diagnostic (gsp_phys_rmcall_gr_qmdwin=1): dedicated module
@@ -2138,6 +2150,18 @@ static struct gsp_phys_boot_stage {
 	 */
 	struct gsp_phys_boot_chunk rmcall_gr_qmdwin;
 	u64 rmcall_gr_qmdwin_va;	/* planted VA; 0 = guest destination */
+	/* m99: guest data identity-map table branches (see the PD*_MAX block). */
+	struct gsp_phys_boot_chunk
+		rmcall_gr_data_pd1[GSP_PHYS_RMCALL_GR_DATA_PD1_MAX];
+	struct gsp_phys_boot_chunk
+		rmcall_gr_data_pd0[GSP_PHYS_RMCALL_GR_DATA_PD0_MAX];
+	struct gsp_phys_boot_chunk
+		rmcall_gr_data_pt[GSP_PHYS_RMCALL_GR_DATA_PT_MAX];
+	u32 rmcall_gr_data_pd1_idx[GSP_PHYS_RMCALL_GR_DATA_PD1_MAX];
+	u32 rmcall_gr_data_pd0_idx[GSP_PHYS_RMCALL_GR_DATA_PD0_MAX];
+	u32 rmcall_gr_data_pt_idx[GSP_PHYS_RMCALL_GR_DATA_PT_MAX];
+	u32 rmcall_gr_data_pd1_n, rmcall_gr_data_pd0_n, rmcall_gr_data_pt_n;
+	u64 rmcall_gr_data_pages;
 	struct {
 		u32 pde_index;
 		u8 pt_slot;
@@ -16119,6 +16143,135 @@ static void gsp_phys_rmcall_gr_gpc_mmu_init(void __iomem *bar0)
 }
 
 /*
+ * m99: guest data identity mapping. Plant (VA -> HPA) into the engine page
+ * tree our walker uses. Only root[0] VAs (< 2^47) are reachable; the tables
+ * along the path are allocated lazily from module chunks and are fresh, so
+ * planting before the launch is visible to the engine's first walk. The PTE
+ * carries GENERIC_MEMORY (0x06) in KIND[63:56] like the Turing UVM/RM
+ * mappings (PITCH does not exist on Turing).
+ */
+static struct gsp_phys_boot_chunk *gsp_phys_rmcall_gr_data_pd1_get(u32 i2)
+{
+	struct gsp_phys_boot_chunk *pd2 =
+		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PD2];
+	u32 k;
+
+	for (k = 0; k < boot_stage.rmcall_gr_data_pd1_n; k++)
+		if (boot_stage.rmcall_gr_data_pd1_idx[k] == i2)
+			return &boot_stage.rmcall_gr_data_pd1[k];
+	if (boot_stage.rmcall_gr_data_pd1_n >= GSP_PHYS_RMCALL_GR_DATA_PD1_MAX)
+		return NULL;
+	k = boot_stage.rmcall_gr_data_pd1_n;
+	if (gsp_phys_chunk_alloc(boot_stage.pdev,
+				 &boot_stage.rmcall_gr_data_pd1[k], 1))
+		return NULL;
+	gsp_phys_chunk_zero(&boot_stage.rmcall_gr_data_pd1[k]);
+	boot_stage.rmcall_gr_data_pd1_idx[k] = i2;
+	boot_stage.rmcall_gr_data_pd1_n++;
+	put_unaligned_le64(gsp_phys_rmcall_pde((u64)gsp_phys_chunk_page_dma(
+				   &boot_stage.rmcall_gr_data_pd1[k], 0)),
+			   (u8 *)gsp_phys_chunk_page_va(pd2, 0) +
+			   (size_t)i2 * 8);
+	dma_wmb();
+	pr_info("vgsp-gr: data-map PD2[%u] -> pd1 dma=0x%llx\n", i2,
+		(u64)gsp_phys_chunk_page_dma(&boot_stage.rmcall_gr_data_pd1[k], 0));
+	return &boot_stage.rmcall_gr_data_pd1[k];
+}
+
+static struct gsp_phys_boot_chunk *gsp_phys_rmcall_gr_data_pd0_get(u32 i2,
+								   u32 i1)
+{
+	struct gsp_phys_boot_chunk *pd1 = gsp_phys_rmcall_gr_data_pd1_get(i2);
+	u32 key = (i2 << 9) | i1;
+	u32 k;
+
+	if (!pd1)
+		return NULL;
+	for (k = 0; k < boot_stage.rmcall_gr_data_pd0_n; k++)
+		if (boot_stage.rmcall_gr_data_pd0_idx[k] == key)
+			return &boot_stage.rmcall_gr_data_pd0[k];
+	if (boot_stage.rmcall_gr_data_pd0_n >= GSP_PHYS_RMCALL_GR_DATA_PD0_MAX)
+		return NULL;
+	k = boot_stage.rmcall_gr_data_pd0_n;
+	if (gsp_phys_chunk_alloc(boot_stage.pdev,
+				 &boot_stage.rmcall_gr_data_pd0[k], 1))
+		return NULL;
+	gsp_phys_chunk_zero(&boot_stage.rmcall_gr_data_pd0[k]);
+	boot_stage.rmcall_gr_data_pd0_idx[k] = key;
+	boot_stage.rmcall_gr_data_pd0_n++;
+	put_unaligned_le64(gsp_phys_rmcall_pde((u64)gsp_phys_chunk_page_dma(
+				   &boot_stage.rmcall_gr_data_pd0[k], 0)),
+			   (u8 *)gsp_phys_chunk_page_va(pd1, 0) +
+			   (size_t)i1 * 8);
+	dma_wmb();
+	pr_info("vgsp-gr: data-map PD1[%u] -> pd0 dma=0x%llx (i2=%u)\n", i1,
+		(u64)gsp_phys_chunk_page_dma(&boot_stage.rmcall_gr_data_pd0[k], 0),
+		i2);
+	return &boot_stage.rmcall_gr_data_pd0[k];
+}
+
+static struct gsp_phys_boot_chunk *gsp_phys_rmcall_gr_data_pt_get(u32 i2,
+								  u32 i1,
+								  u32 i0)
+{
+	struct gsp_phys_boot_chunk *pd0 =
+		gsp_phys_rmcall_gr_data_pd0_get(i2, i1);
+	u32 key = (i2 << 18) | (i1 << 9) | i0;
+	u32 k;
+
+	if (!pd0)
+		return NULL;
+	for (k = 0; k < boot_stage.rmcall_gr_data_pt_n; k++)
+		if (boot_stage.rmcall_gr_data_pt_idx[k] == key)
+			return &boot_stage.rmcall_gr_data_pt[k];
+	if (boot_stage.rmcall_gr_data_pt_n >= GSP_PHYS_RMCALL_GR_DATA_PT_MAX)
+		return NULL;
+	k = boot_stage.rmcall_gr_data_pt_n;
+	if (gsp_phys_chunk_alloc(boot_stage.pdev,
+				 &boot_stage.rmcall_gr_data_pt[k], 1))
+		return NULL;
+	gsp_phys_chunk_zero(&boot_stage.rmcall_gr_data_pt[k]);
+	boot_stage.rmcall_gr_data_pt_idx[k] = key;
+	boot_stage.rmcall_gr_data_pt_n++;
+	/* 16 B dual PDE: big half unused, small half -> 4 KB PT. */
+	put_unaligned_le64(0,
+			   (u8 *)gsp_phys_chunk_page_va(pd0, 0) +
+			   (size_t)i0 * 16);
+	put_unaligned_le64(gsp_phys_rmcall_pde((u64)gsp_phys_chunk_page_dma(
+				   &boot_stage.rmcall_gr_data_pt[k], 0)),
+			   (u8 *)gsp_phys_chunk_page_va(pd0, 0) +
+			   (size_t)i0 * 16 + 8);
+	dma_wmb();
+	pr_info("vgsp-gr: data-map PD0[%u] -> pt dma=0x%llx (i2=%u i1=%u)\n", i0,
+		(u64)gsp_phys_chunk_page_dma(&boot_stage.rmcall_gr_data_pt[k], 0),
+		i2, i1);
+	return &boot_stage.rmcall_gr_data_pt[k];
+}
+
+static int gsp_phys_rmcall_gr_data_map(u64 va, u64 hpa)
+{
+	struct gsp_phys_boot_chunk *pt;
+	u32 r = (u32)((va >> 47) & 0x1u);
+	u32 i2 = (u32)((va >> 38) & 0x1ffu);
+	u32 i1 = (u32)((va >> 29) & 0x1ffu);
+	u32 i0 = (u32)((va >> 21) & 0x1ffu);
+	u32 ip = (u32)((va >> 12) & 0x1ffu);
+	u64 pte;
+
+	if (r != 0)
+		return -ERANGE;	/* root[0] only; the demo VAs are < 2^47 */
+	pt = gsp_phys_rmcall_gr_data_pt_get(i2, i1, i0);
+	if (!pt)
+		return -ENOSPC;
+	pte = gsp_phys_rmcall_pte(hpa) | (6ULL << 56);
+	put_unaligned_le64(pte, (u8 *)gsp_phys_chunk_page_va(pt, 0) +
+			   (size_t)ip * 8);
+	dma_wmb();
+	boot_stage.rmcall_gr_data_pages++;
+	return 0;
+}
+
+/*
  * One forwarded GR launch on the ladder channel (see gsp_phys_booter.h). The
  * caller has resolved the launch's pages; every mapping is fresh or proven
  * unchanged, the QMD release0 and the channel release are authored either by
@@ -16831,6 +16984,19 @@ gr_page_unmapped:
 				le32_to_cpu(iv[172]), le32_to_cpu(iv[173]));
 		}
 	}
+	if (req->n_data) {
+		u32 dmapped = 0, dfailed = 0, di;
+
+		for (di = 0; di < req->n_data; di++) {
+			if (gsp_phys_rmcall_gr_data_map(req->data[di].va,
+							req->data[di].hpa))
+				dfailed++;
+			else
+				dmapped++;
+		}
+		pr_info("vgsp-gr: guest data identity-map: %u pages in the engine tree, %u failed (pt pages=%u)\n",
+			dmapped, dfailed, boot_stage.rmcall_gr_data_pt_n);
+	}
 	if (gsp_phys_rmcall_gr_gpc_mmu_init_value())
 		gsp_phys_rmcall_gr_gpc_mmu_init(bar0);
 	pr_info("vgsp-gr: GR LAUNCH submitted h=0x%x push=0x%llx len=%u pages=%u shadow=%u qmd_va=0x%llx slot=%u words=%u prelude=%d qmdvirt=%d token=0x%x; polling qmd_rel=%u ch_rel=0x%08x (<=%u ms)%s\n",