vgsp: m99 - identity-map the forwarded kernel's guest data pages into the engine tree

The m97/m98 decode proved the SM instruction fetch and const reads work;
the wall is the kernel's first global data access (prologue instruction at
+0x80) to a guest-VA device pointer (param page words 6..9 carry the demo's
two 16MB buffers at 0x763dXXXXXXXX), which is not mapped in the engine's
page tables. The walker walks OUR root->PD2->PD1->PD0->PT tree (SC PDB ==
our VASPDB chunk, proven by matching the same-run INST base to the PDB-root
DMA log), and fresh direct-planted tables are visible at first fetch.

Service side (gsp_sema_service.c): svc_gr_data_fill() scans the param
(c[0x0]) page for plausible device VAs, resolves the containing 2MB windows
(-1..+16 windows, early stop when a window resolves nothing) via
svc_resolve_va on the service thread, and carries {va, pa} in
svc_fwd_req.gr_data; svc_fwd_gr_work() converts PA->HPA and attaches the
list to the submit.

Backend (gsp_phys_booter.c): gsp_phys_rmcall_gr_data_map() plants VA->HPA
leaf PTEs (GENERIC_MEMORY kind 0x06, like UVM/RM on Turing) into lazily
allocated PD1/PD0/PT chunk branches under root[0] (VAs < 2^47), caching
branches per index; the submit maps all req->data pages before publish and
logs the count.

1 files changed, 129 insertions, 0 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 0605692..0e93058 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -3218,6 +3218,14 @@ static struct svc_fwd_cand svc_fwd_cand;
 #define SVC_GR_INLINE_BYTES	64
 #define SVC_GR_POINTER_MAX	8
 #define SVC_GR_PAGES_MAX	16
+/*
+ * m99: guest data identity-map capacity. The forwarded kernel's global data
+ * pages (discovered by scanning the param/const-0 page for device VAs) are
+ * resolved on the service thread and identity-mapped into the engine tree.
+ * 16K pages = 64 MB of window coverage, comfortably above the demo buffers
+ * (2 x 16 MB observed) with headroom.
+ */
+#define SVC_GR_DATA_MAX		16384
 #define SVC_GR_PUSH_WORDS	256
 #define SVC_GR_QMD_WORD_NONE	0xffffffffu
 /* svc_gr_cand.rel_flags bits (decoded from qmd[24] and qmd[4]). */
@@ -3239,6 +3247,13 @@ struct svc_gr_page {
 	u32 qmd_word;	/* QMD pointer pair word index, or NONE */
 };
 
+/* m99: one guest data page resolved on the service thread (VA -> guest PA). */
+struct svc_gr_data {
+	u64 va;		/* guest VA, page aligned */
+	u64 pa;		/* guest PA, page aligned (0 = unresolved) */
+	u8 ap;		/* SVC_APERTURE_* of pa */
+};
+
 struct svc_gr_cand {
 	bool valid;
 	bool offered;		/* consumed by a request or a fallback */
@@ -8260,6 +8275,8 @@ struct svc_fwd_req {
 	struct svc_gr_inline gr_inline[SVC_GR_INLINE_MAX];
 	u32 gr_n_pages;
 	struct svc_gr_page gr_pages[SVC_GR_PAGES_MAX];
+	u32 gr_n_data;
+	struct svc_gr_data gr_data[SVC_GR_DATA_MAX];
 };
 
 static struct svc_fwd_req svc_fwd_req;
@@ -9069,6 +9086,7 @@ static void svc_fwd_gr_work(void)
 	 * in svc_decode_stream).
 	 */
 	static struct gsp_phys_gr_page pages[SVC_GR_PAGES_MAX];
+	static struct gsp_phys_gr_data_page data[SVC_GR_DATA_MAX];
 	struct gsp_phys_gr_submit gs;
 	struct gsp_phys_backend_copy_result cp = { 0 };
 	u64 rel_hpa = 0;
@@ -9139,6 +9157,26 @@ static void svc_fwd_gr_work(void)
 					 SEMA_PAGE_SIZE, &qhpa, &qoff) && !qoff)
 			gs.qmd_hpa = qhpa;
 	}
+	gs.n_data = 0;
+	for (i = 0; i < svc_fwd_req.gr_n_data; i++) {
+		const struct svc_gr_data *dg = &svc_fwd_req.gr_data[i];
+		u8 dkind = (dg->ap == SVC_APERTURE_FB) ?
+			   SVC_APERTURE_FB : SVC_APERTURE_SYS;
+		u64 dhpa = 0;
+		u32 doff = 0;
+
+		if (!dg->pa)
+			continue;
+		if (svc_fwd_operand_hpa(svc_fwd_req.vgpu, dkind, dg->pa, 0,
+					SEMA_PAGE_SIZE, &dhpa, &doff) || doff)
+			continue;
+		if (gs.n_data >= SVC_GR_DATA_MAX)
+			break;
+		data[gs.n_data].va = dg->va;
+		data[gs.n_data].hpa = dhpa;
+		gs.n_data++;
+	}
+	gs.data = data;
 	gs.pages = pages;
 	gs.n_pages = svc_fwd_req.gr_n_pages;
 	gs.inl = (const struct gsp_phys_gr_inline *)svc_fwd_req.gr_inline;
@@ -9480,6 +9518,96 @@ static bool svc_gr_page_inline_covers(u64 page)
  * it (the backend's Tier 2 relocation). Returns 0 or an errno the caller
  * falls back on.
  */
+/*
+ * m99: discover and resolve the forwarded kernel's guest data pages. The
+ * kernel's device pointers live as values inside the param buffer (the QMD
+ * word 32 page, c[0x0]); the engine executes the kernel against the guest's
+ * real memory, so every data page the kernel touches must be present in the
+ * engine's page tree. Scan the param page for plausible device VAs, then
+ * resolve the containing 2 MB windows (one window back for negative indexing,
+ * up to sixteen forward with an early stop once a window resolves nothing).
+ * Resolution only happens on the service thread; the work item turns the
+ * guest PAs into HPAs and the backend plants the identity PTEs.
+ */
+static void svc_gr_data_fill(struct nv_vgpu_device *vgpu)
+{
+	const struct svc_gr_page *pp = NULL;
+	u64 wins[8];
+	u32 n_win = 0;
+	u64 param_hpa = 0;
+	u32 off = 0, i, w;
+	const u8 *page;
+	u8 kind;
+
+	svc_fwd_req.gr_n_data = 0;
+	for (i = 0; i < svc_fwd_req.gr_n_pages; i++) {
+		if (svc_fwd_req.gr_pages[i].qmd_word == 32 &&
+		    svc_fwd_req.gr_pages[i].pa) {
+			pp = &svc_fwd_req.gr_pages[i];
+			break;
+		}
+	}
+	if (!pp)
+		return;
+	kind = (pp->ap == SVC_APERTURE_FB) ? SVC_APERTURE_FB :
+					     SVC_APERTURE_SYS;
+	if (svc_fwd_operand_hpa(vgpu, kind, pp->pa, 0, SEMA_PAGE_SIZE,
+				&param_hpa, &off) || off)
+		return;
+	page = (const u8 *)__va(param_hpa);
+	if (!page)
+		return;
+	for (i = 0; i < 256; i++) {
+		u64 v = get_unaligned_le64(page + i * 8);
+		u64 wv;
+
+		if (v < 0x10000000ULL || v >= (1ULL << 47) || (v & 3))
+			continue;
+		wv = v & ~0x1fffffULL;
+		for (w = 0; w < n_win; w++)
+			if (wins[w] == wv)
+				break;
+		if (w == n_win && n_win < ARRAY_SIZE(wins))
+			wins[n_win++] = wv;
+	}
+	for (w = 0; w < n_win; w++) {
+		int wi;
+
+		for (wi = -1; wi <= 16; wi++) {
+			u64 wv = wins[w] + (u64)wi * 0x200000ULL;
+			u32 got = 0, p;
+
+			if (wi < 0 && wins[w] < 0x200000ULL)
+				continue;
+			if (wv >= (1ULL << 47))
+				break;
+			for (p = 0; p < 512; p++) {
+				u64 va = wv + (u64)p * SEMA_PAGE_SIZE;
+				u64 pa = 0;
+				u32 ap = 0;
+				struct svc_gr_data *rec;
+
+				if (svc_resolve_va(vgpu, va, &pa, &ap))
+					continue;
+				if (svc_fwd_req.gr_n_data >= SVC_GR_DATA_MAX)
+					return;
+				rec = &svc_fwd_req.gr_data
+					       [svc_fwd_req.gr_n_data++];
+				rec->va = va;
+				rec->pa = pa & ~(u64)(SEMA_PAGE_SIZE - 1);
+				rec->ap = (u8)svc_ap_of(ap);
+				got++;
+			}
+			if (!got && wi > 0)
+				break;
+			if (svc_fwd_req.gr_n_data >= SVC_GR_DATA_MAX)
+				return;
+		}
+	}
+	pr_info("vgsp-sema: svc-gr data fill: %u windows, %u pages resolved\n",
+		n_win, svc_fwd_req.gr_n_data);
+}
+
 static int svc_gr_req_fill(struct nv_vgpu_device *vgpu,
 			   const struct svc_channel *ch)
 {
@@ -9593,6 +9721,7 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu,
 		if (!svc_gr_page_inline_covers(pg->va))
 			return -ENOENT;
 	}
+	svc_gr_data_fill(vgpu);
 	return 0;
 }