vgsp: m99 - identity-map the forwarded kernel's guest data pages into the engine tree

The m97/m98 decode proved the SM instruction fetch and const reads work;
the wall is the kernel's first global data access (prologue instruction at
+0x80) to a guest-VA device pointer (param page words 6..9 carry the demo's
two 16MB buffers at 0x763dXXXXXXXX), which is not mapped in the engine's
page tables. The walker walks OUR root->PD2->PD1->PD0->PT tree (SC PDB ==
our VASPDB chunk, proven by matching the same-run INST base to the PDB-root
DMA log), and fresh direct-planted tables are visible at first fetch.

Service side (gsp_sema_service.c): svc_gr_data_fill() scans the param
(c[0x0]) page for plausible device VAs, resolves the containing 2MB windows
(-1..+16 windows, early stop when a window resolves nothing) via
svc_resolve_va on the service thread, and carries {va, pa} in
svc_fwd_req.gr_data; svc_fwd_gr_work() converts PA->HPA and attaches the
list to the submit.

Backend (gsp_phys_booter.c): gsp_phys_rmcall_gr_data_map() plants VA->HPA
leaf PTEs (GENERIC_MEMORY kind 0x06, like UVM/RM on Turing) into lazily
allocated PD1/PD0/PT chunk branches under root[0] (VAs < 2^47), caching
branches per index; the submit maps all req->data pages before publish and
logs the count.

3 files changed, 303 insertions, 0 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index 19fb67e..6db26a4 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -1009,6 +1009,18 @@ bool gsp_phys_boot_requested(void)
 #define GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE \
 	(GSP_PHYS_RMCALL_VA_BASE + 8 * GSP_PHYS_PAGE_SIZE)
 #define GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS	8u
+/*
+ * m99: engine-tree branches for identity-mapping the forwarded kernel's
+ * guest data pages. The walker uses our root[0] -> PD2 -> PD1 -> PD0 dual ->
+ * small PT hierarchy (root[48:47] -> PD2[46:38] -> PD1[37:29] -> PD0[28:21]
+ * -> PT[20:12]). PD1 pages are cached per PD2 index, PD0 pages per
+ * (PD2,PD1) index, PT pages per (PD2,PD1,PD0) index; the tables are fresh
+ * (never fetched) when planted, so the first access at kernel runtime sees
+ * them even though the GMMU has no PTE-line invalidate.
+ */
+#define GSP_PHYS_RMCALL_GR_DATA_PD1_MAX	4u
+#define GSP_PHYS_RMCALL_GR_DATA_PD0_MAX	8u
+#define GSP_PHYS_RMCALL_GR_DATA_PT_MAX	64u
 
 /*
  * Rank-2 qmdwin diagnostic (gsp_phys_rmcall_gr_qmdwin=1): dedicated module
@@ -2138,6 +2150,18 @@ static struct gsp_phys_boot_stage {
 	 */
 	struct gsp_phys_boot_chunk rmcall_gr_qmdwin;
 	u64 rmcall_gr_qmdwin_va;	/* planted VA; 0 = guest destination */
+	/* m99: guest data identity-map table branches (see the PD*_MAX block). */
+	struct gsp_phys_boot_chunk
+		rmcall_gr_data_pd1[GSP_PHYS_RMCALL_GR_DATA_PD1_MAX];
+	struct gsp_phys_boot_chunk
+		rmcall_gr_data_pd0[GSP_PHYS_RMCALL_GR_DATA_PD0_MAX];
+	struct gsp_phys_boot_chunk
+		rmcall_gr_data_pt[GSP_PHYS_RMCALL_GR_DATA_PT_MAX];
+	u32 rmcall_gr_data_pd1_idx[GSP_PHYS_RMCALL_GR_DATA_PD1_MAX];
+	u32 rmcall_gr_data_pd0_idx[GSP_PHYS_RMCALL_GR_DATA_PD0_MAX];
+	u32 rmcall_gr_data_pt_idx[GSP_PHYS_RMCALL_GR_DATA_PT_MAX];
+	u32 rmcall_gr_data_pd1_n, rmcall_gr_data_pd0_n, rmcall_gr_data_pt_n;
+	u64 rmcall_gr_data_pages;
 	struct {
 		u32 pde_index;
 		u8 pt_slot;
@@ -16119,6 +16143,135 @@ static void gsp_phys_rmcall_gr_gpc_mmu_init(void __iomem *bar0)
 }
 
 /*
+ * m99: guest data identity mapping. Plant (VA -> HPA) into the engine page
+ * tree our walker uses. Only root[0] VAs (< 2^47) are reachable; the tables
+ * along the path are allocated lazily from module chunks and are fresh, so
+ * planting before the launch is visible to the engine's first walk. The PTE
+ * carries GENERIC_MEMORY (0x06) in KIND[63:56] like the Turing UVM/RM
+ * mappings (PITCH does not exist on Turing).
+ */
+static struct gsp_phys_boot_chunk *gsp_phys_rmcall_gr_data_pd1_get(u32 i2)
+{
+	struct gsp_phys_boot_chunk *pd2 =
+		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PD2];
+	u32 k;
+
+	for (k = 0; k < boot_stage.rmcall_gr_data_pd1_n; k++)
+		if (boot_stage.rmcall_gr_data_pd1_idx[k] == i2)
+			return &boot_stage.rmcall_gr_data_pd1[k];
+	if (boot_stage.rmcall_gr_data_pd1_n >= GSP_PHYS_RMCALL_GR_DATA_PD1_MAX)
+		return NULL;
+	k = boot_stage.rmcall_gr_data_pd1_n;
+	if (gsp_phys_chunk_alloc(boot_stage.pdev,
+				 &boot_stage.rmcall_gr_data_pd1[k], 1))
+		return NULL;
+	gsp_phys_chunk_zero(&boot_stage.rmcall_gr_data_pd1[k]);
+	boot_stage.rmcall_gr_data_pd1_idx[k] = i2;
+	boot_stage.rmcall_gr_data_pd1_n++;
+	put_unaligned_le64(gsp_phys_rmcall_pde((u64)gsp_phys_chunk_page_dma(
+				   &boot_stage.rmcall_gr_data_pd1[k], 0)),
+			   (u8 *)gsp_phys_chunk_page_va(pd2, 0) +
+			   (size_t)i2 * 8);
+	dma_wmb();
+	pr_info("vgsp-gr: data-map PD2[%u] -> pd1 dma=0x%llx\n", i2,
+		(u64)gsp_phys_chunk_page_dma(&boot_stage.rmcall_gr_data_pd1[k], 0));
+	return &boot_stage.rmcall_gr_data_pd1[k];
+}
+
+static struct gsp_phys_boot_chunk *gsp_phys_rmcall_gr_data_pd0_get(u32 i2,
+								   u32 i1)
+{
+	struct gsp_phys_boot_chunk *pd1 = gsp_phys_rmcall_gr_data_pd1_get(i2);
+	u32 key = (i2 << 9) | i1;
+	u32 k;
+
+	if (!pd1)
+		return NULL;
+	for (k = 0; k < boot_stage.rmcall_gr_data_pd0_n; k++)
+		if (boot_stage.rmcall_gr_data_pd0_idx[k] == key)
+			return &boot_stage.rmcall_gr_data_pd0[k];
+	if (boot_stage.rmcall_gr_data_pd0_n >= GSP_PHYS_RMCALL_GR_DATA_PD0_MAX)
+		return NULL;
+	k = boot_stage.rmcall_gr_data_pd0_n;
+	if (gsp_phys_chunk_alloc(boot_stage.pdev,
+				 &boot_stage.rmcall_gr_data_pd0[k], 1))
+		return NULL;
+	gsp_phys_chunk_zero(&boot_stage.rmcall_gr_data_pd0[k]);
+	boot_stage.rmcall_gr_data_pd0_idx[k] = key;
+	boot_stage.rmcall_gr_data_pd0_n++;
+	put_unaligned_le64(gsp_phys_rmcall_pde((u64)gsp_phys_chunk_page_dma(
+				   &boot_stage.rmcall_gr_data_pd0[k], 0)),
+			   (u8 *)gsp_phys_chunk_page_va(pd1, 0) +
+			   (size_t)i1 * 8);
+	dma_wmb();
+	pr_info("vgsp-gr: data-map PD1[%u] -> pd0 dma=0x%llx (i2=%u)\n", i1,
+		(u64)gsp_phys_chunk_page_dma(&boot_stage.rmcall_gr_data_pd0[k], 0),
+		i2);
+	return &boot_stage.rmcall_gr_data_pd0[k];
+}
+
+static struct gsp_phys_boot_chunk *gsp_phys_rmcall_gr_data_pt_get(u32 i2,
+								  u32 i1,
+								  u32 i0)
+{
+	struct gsp_phys_boot_chunk *pd0 =
+		gsp_phys_rmcall_gr_data_pd0_get(i2, i1);
+	u32 key = (i2 << 18) | (i1 << 9) | i0;
+	u32 k;
+
+	if (!pd0)
+		return NULL;
+	for (k = 0; k < boot_stage.rmcall_gr_data_pt_n; k++)
+		if (boot_stage.rmcall_gr_data_pt_idx[k] == key)
+			return &boot_stage.rmcall_gr_data_pt[k];
+	if (boot_stage.rmcall_gr_data_pt_n >= GSP_PHYS_RMCALL_GR_DATA_PT_MAX)
+		return NULL;
+	k = boot_stage.rmcall_gr_data_pt_n;
+	if (gsp_phys_chunk_alloc(boot_stage.pdev,
+				 &boot_stage.rmcall_gr_data_pt[k], 1))
+		return NULL;
+	gsp_phys_chunk_zero(&boot_stage.rmcall_gr_data_pt[k]);
+	boot_stage.rmcall_gr_data_pt_idx[k] = key;
+	boot_stage.rmcall_gr_data_pt_n++;
+	/* 16 B dual PDE: big half unused, small half -> 4 KB PT. */
+	put_unaligned_le64(0,
+			   (u8 *)gsp_phys_chunk_page_va(pd0, 0) +
+			   (size_t)i0 * 16);
+	put_unaligned_le64(gsp_phys_rmcall_pde((u64)gsp_phys_chunk_page_dma(
+				   &boot_stage.rmcall_gr_data_pt[k], 0)),
+			   (u8 *)gsp_phys_chunk_page_va(pd0, 0) +
+			   (size_t)i0 * 16 + 8);
+	dma_wmb();
+	pr_info("vgsp-gr: data-map PD0[%u] -> pt dma=0x%llx (i2=%u i1=%u)\n", i0,
+		(u64)gsp_phys_chunk_page_dma(&boot_stage.rmcall_gr_data_pt[k], 0),
+		i2, i1);
+	return &boot_stage.rmcall_gr_data_pt[k];
+}
+
+static int gsp_phys_rmcall_gr_data_map(u64 va, u64 hpa)
+{
+	struct gsp_phys_boot_chunk *pt;
+	u32 r = (u32)((va >> 47) & 0x1u);
+	u32 i2 = (u32)((va >> 38) & 0x1ffu);
+	u32 i1 = (u32)((va >> 29) & 0x1ffu);
+	u32 i0 = (u32)((va >> 21) & 0x1ffu);
+	u32 ip = (u32)((va >> 12) & 0x1ffu);
+	u64 pte;
+
+	if (r != 0)
+		return -ERANGE;	/* root[0] only; the demo VAs are < 2^47 */
+	pt = gsp_phys_rmcall_gr_data_pt_get(i2, i1, i0);
+	if (!pt)
+		return -ENOSPC;
+	pte = gsp_phys_rmcall_pte(hpa) | (6ULL << 56);
+	put_unaligned_le64(pte, (u8 *)gsp_phys_chunk_page_va(pt, 0) +
+			   (size_t)ip * 8);
+	dma_wmb();
+	boot_stage.rmcall_gr_data_pages++;
+	return 0;
+}
+
+/*
  * One forwarded GR launch on the ladder channel (see gsp_phys_booter.h). The
  * caller has resolved the launch's pages; every mapping is fresh or proven
  * unchanged, the QMD release0 and the channel release are authored either by
@@ -16831,6 +16984,19 @@ gr_page_unmapped:
 				le32_to_cpu(iv[172]), le32_to_cpu(iv[173]));
 		}
 	}
+	if (req->n_data) {
+		u32 dmapped = 0, dfailed = 0, di;
+
+		for (di = 0; di < req->n_data; di++) {
+			if (gsp_phys_rmcall_gr_data_map(req->data[di].va,
+							req->data[di].hpa))
+				dfailed++;
+			else
+				dmapped++;
+		}
+		pr_info("vgsp-gr: guest data identity-map: %u pages in the engine tree, %u failed (pt pages=%u)\n",
+			dmapped, dfailed, boot_stage.rmcall_gr_data_pt_n);
+	}
 	if (gsp_phys_rmcall_gr_gpc_mmu_init_value())
 		gsp_phys_rmcall_gr_gpc_mmu_init(bar0);
 	pr_info("vgsp-gr: GR LAUNCH submitted h=0x%x push=0x%llx len=%u pages=%u shadow=%u qmd_va=0x%llx slot=%u words=%u prelude=%d qmdvirt=%d token=0x%x; polling qmd_rel=%u ch_rel=0x%08x (<=%u ms)%s\n",
diff --git a/kernel/gsp_phys_booter.h b/kernel/gsp_phys_booter.h
index e46fb86..29cf07f 100644
--- a/kernel/gsp_phys_booter.h
+++ b/kernel/gsp_phys_booter.h
@@ -362,6 +362,12 @@ struct gsp_phys_gr_inline {
 	u8 data[GSP_PHYS_GR_INLINE_BYTES];
 };
 
+/* m99: one guest data page identity-mapped into the engine's page tree. */
+struct gsp_phys_gr_data_page {
+	u64 va;		/* guest VA, page aligned */
+	u64 hpa;	/* host PA, page aligned */
+};
+
 struct gsp_phys_gr_submit {
 	const u32 *qmd;		/* 64 rewritten-QMD source words */
 	u32 qmd_words;		/* must be GSP_PHYS_GR_QMD_WORDS */
@@ -382,6 +388,8 @@ struct gsp_phys_gr_submit {
 	u32 n_pages;
 	const struct gsp_phys_gr_inline *inl;
 	u32 n_inl;
+	const struct gsp_phys_gr_data_page *data;	/* m99: identity pages */
+	u32 n_data;
 	u64 rel_hpa;		/* guest-visible QMD release0 page, or 0 */
 	u32 rel_off;
 	u32 rel_payload;
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 0605692..0e93058 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -3218,6 +3218,14 @@ static struct svc_fwd_cand svc_fwd_cand;
 #define SVC_GR_INLINE_BYTES	64
 #define SVC_GR_POINTER_MAX	8
 #define SVC_GR_PAGES_MAX	16
+/*
+ * m99: guest data identity-map capacity. The forwarded kernel's global data
+ * pages (discovered by scanning the param/const-0 page for device VAs) are
+ * resolved on the service thread and identity-mapped into the engine tree.
+ * 16K pages = 64 MB of window coverage, comfortably above the demo buffers
+ * (2 x 16 MB observed) with headroom.
+ */
+#define SVC_GR_DATA_MAX		16384
 #define SVC_GR_PUSH_WORDS	256
 #define SVC_GR_QMD_WORD_NONE	0xffffffffu
 /* svc_gr_cand.rel_flags bits (decoded from qmd[24] and qmd[4]). */
@@ -3239,6 +3247,13 @@ struct svc_gr_page {
 	u32 qmd_word;	/* QMD pointer pair word index, or NONE */
 };
 
+/* m99: one guest data page resolved on the service thread (VA -> guest PA). */
+struct svc_gr_data {
+	u64 va;		/* guest VA, page aligned */
+	u64 pa;		/* guest PA, page aligned (0 = unresolved) */
+	u8 ap;		/* SVC_APERTURE_* of pa */
+};
+
 struct svc_gr_cand {
 	bool valid;
 	bool offered;		/* consumed by a request or a fallback */
@@ -8260,6 +8275,8 @@ struct svc_fwd_req {
 	struct svc_gr_inline gr_inline[SVC_GR_INLINE_MAX];
 	u32 gr_n_pages;
 	struct svc_gr_page gr_pages[SVC_GR_PAGES_MAX];
+	u32 gr_n_data;
+	struct svc_gr_data gr_data[SVC_GR_DATA_MAX];
 };
 
 static struct svc_fwd_req svc_fwd_req;
@@ -9069,6 +9086,7 @@ static void svc_fwd_gr_work(void)
 	 * in svc_decode_stream).
 	 */
 	static struct gsp_phys_gr_page pages[SVC_GR_PAGES_MAX];
+	static struct gsp_phys_gr_data_page data[SVC_GR_DATA_MAX];
 	struct gsp_phys_gr_submit gs;
 	struct gsp_phys_backend_copy_result cp = { 0 };
 	u64 rel_hpa = 0;
@@ -9139,6 +9157,26 @@ static void svc_fwd_gr_work(void)
 					 SEMA_PAGE_SIZE, &qhpa, &qoff) && !qoff)
 			gs.qmd_hpa = qhpa;
 	}
+	gs.n_data = 0;
+	for (i = 0; i < svc_fwd_req.gr_n_data; i++) {
+		const struct svc_gr_data *dg = &svc_fwd_req.gr_data[i];
+		u8 dkind = (dg->ap == SVC_APERTURE_FB) ?
+			   SVC_APERTURE_FB : SVC_APERTURE_SYS;
+		u64 dhpa = 0;
+		u32 doff = 0;
+
+		if (!dg->pa)
+			continue;
+		if (svc_fwd_operand_hpa(svc_fwd_req.vgpu, dkind, dg->pa, 0,
+					SEMA_PAGE_SIZE, &dhpa, &doff) || doff)
+			continue;
+		if (gs.n_data >= SVC_GR_DATA_MAX)
+			break;
+		data[gs.n_data].va = dg->va;
+		data[gs.n_data].hpa = dhpa;
+		gs.n_data++;
+	}
+	gs.data = data;
 	gs.pages = pages;
 	gs.n_pages = svc_fwd_req.gr_n_pages;
 	gs.inl = (const struct gsp_phys_gr_inline *)svc_fwd_req.gr_inline;
@@ -9480,6 +9518,96 @@ static bool svc_gr_page_inline_covers(u64 page)
  * it (the backend's Tier 2 relocation). Returns 0 or an errno the caller
  * falls back on.
  */
+/*
+ * m99: discover and resolve the forwarded kernel's guest data pages. The
+ * kernel's device pointers live as values inside the param buffer (the QMD
+ * word 32 page, c[0x0]); the engine executes the kernel against the guest's
+ * real memory, so every data page the kernel touches must be present in the
+ * engine's page tree. Scan the param page for plausible device VAs, then
+ * resolve the containing 2 MB windows (one window back for negative indexing,
+ * up to sixteen forward with an early stop once a window resolves nothing).
+ * Resolution only happens on the service thread; the work item turns the
+ * guest PAs into HPAs and the backend plants the identity PTEs.
+ */
+static void svc_gr_data_fill(struct nv_vgpu_device *vgpu)
+{
+	const struct svc_gr_page *pp = NULL;
+	u64 wins[8];
+	u32 n_win = 0;
+	u64 param_hpa = 0;
+	u32 off = 0, i, w;
+	const u8 *page;
+	u8 kind;
+
+	svc_fwd_req.gr_n_data = 0;
+	for (i = 0; i < svc_fwd_req.gr_n_pages; i++) {
+		if (svc_fwd_req.gr_pages[i].qmd_word == 32 &&
+		    svc_fwd_req.gr_pages[i].pa) {
+			pp = &svc_fwd_req.gr_pages[i];
+			break;
+		}
+	}
+	if (!pp)
+		return;
+	kind = (pp->ap == SVC_APERTURE_FB) ? SVC_APERTURE_FB :
+					     SVC_APERTURE_SYS;
+	if (svc_fwd_operand_hpa(vgpu, kind, pp->pa, 0, SEMA_PAGE_SIZE,
+				&param_hpa, &off) || off)
+		return;
+	page = (const u8 *)__va(param_hpa);
+	if (!page)
+		return;
+	for (i = 0; i < 256; i++) {
+		u64 v = get_unaligned_le64(page + i * 8);
+		u64 wv;
+
+		if (v < 0x10000000ULL || v >= (1ULL << 47) || (v & 3))
+			continue;
+		wv = v & ~0x1fffffULL;
+		for (w = 0; w < n_win; w++)
+			if (wins[w] == wv)
+				break;
+		if (w == n_win && n_win < ARRAY_SIZE(wins))
+			wins[n_win++] = wv;
+	}
+	for (w = 0; w < n_win; w++) {
+		int wi;
+
+		for (wi = -1; wi <= 16; wi++) {
+			u64 wv = wins[w] + (u64)wi * 0x200000ULL;
+			u32 got = 0, p;
+
+			if (wi < 0 && wins[w] < 0x200000ULL)
+				continue;
+			if (wv >= (1ULL << 47))
+				break;
+			for (p = 0; p < 512; p++) {
+				u64 va = wv + (u64)p * SEMA_PAGE_SIZE;
+				u64 pa = 0;
+				u32 ap = 0;
+				struct svc_gr_data *rec;
+
+				if (svc_resolve_va(vgpu, va, &pa, &ap))
+					continue;
+				if (svc_fwd_req.gr_n_data >= SVC_GR_DATA_MAX)
+					return;
+				rec = &svc_fwd_req.gr_data
+					       [svc_fwd_req.gr_n_data++];
+				rec->va = va;
+				rec->pa = pa & ~(u64)(SEMA_PAGE_SIZE - 1);
+				rec->ap = (u8)svc_ap_of(ap);
+				got++;
+			}
+			if (!got && wi > 0)
+				break;
+			if (svc_fwd_req.gr_n_data >= SVC_GR_DATA_MAX)
+				return;
+		}
+	}
+	pr_info("vgsp-sema: svc-gr data fill: %u windows, %u pages resolved\n",
+		n_win, svc_fwd_req.gr_n_data);
+}
+
 static int svc_gr_req_fill(struct nv_vgpu_device *vgpu,
 			   const struct svc_channel *ch)
 {
@@ -9593,6 +9721,7 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu,
 		if (!svc_gr_page_inline_covers(pg->va))
 			return -ENOENT;
 	}
+	svc_gr_data_fill(vgpu);
 	return 0;
 }