vgsp: m102 - relocate guest data into the server-managed VA region and rewrite bank pointers

The server refuses UPDATE_PDE_2 (status 0x1f) for guest CUDA VAs: they are
client-managed, while the golden VAS reserves the server-managed region
[0x100000000, 0x11fffffff) (gsp_phys_booter.c:8455). Map each resolved data
page there at an affine-relocated VA (identity HPA, no copies), rewrite every
8-byte pointer value inside the shadowed const-bank pages (banks 0..7) that
falls inside a mapped guest window to the relocated VA, and raise the Tier-1
window pool to 32 windows x 512 PTEs so the relocated pages fit. The kernel
now reaches its real data through the server-installed windows.

3 files changed, 115 insertions, 11 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index 04788d2..53932d4 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -1104,8 +1104,8 @@ bool gsp_phys_boot_requested(void)
  * inline writes. Both are deliberately outside rmcall_chunks so the COPY0
  * stage-2 registration walk stays byte-identical.
  */
-#define GSP_PHYS_RMCALL_GR_PT_PAGES	8u
-#define GSP_PHYS_RMCALL_GR_PT_ENTRIES	32u
+#define GSP_PHYS_RMCALL_GR_PT_PAGES	32u
+#define GSP_PHYS_RMCALL_GR_PT_ENTRIES	512u
 #define GSP_PHYS_RMCALL_GR_SHADOW_PAGES	24u
 
 /*
@@ -16151,6 +16151,44 @@ static void gsp_phys_rmcall_gr_gpc_mmu_init(void __iomem *bar0)
  * mappings (PITCH does not exist on Turing).
  */
 /*
+ * m102: rewrite the data pointers inside a shadowed const-bank page: any
+ * 8-byte value that falls inside a mapped guest data window is replaced by
+ * the affine-relocated server-managed VA (the engine cannot walk the
+ * client-managed guest VAs: UPDATE_PDE_2 refuses them with 0x1f).
+ */
+static void gsp_phys_gr_rewrite_data_vars(u32 *w, u32 n_words,
+					  const struct gsp_phys_gr_submit *req,
+					  u32 bank)
+{
+	u32 rew = 0, k;
+
+	if (!req->n_data_wins)
+		return;
+	for (k = 0; k + 1 < n_words; k += 2) {
+		u64 v = (u64)w[k] | ((u64)w[k + 1] << 32);
+		u64 wb;
+		u32 x;
+
+		if (v < 0x10000000ULL || v >= (1ULL << 47) || (v & 7))
+			continue;
+		wb = v & ~0x1fffffULL;
+		for (x = 0; x < req->n_data_wins; x++) {
+			if (req->data_wins[x].va_base != wb)
+				continue;
+			v = req->data_wins[x].reloc_base |
+			    (v & 0x1fffffULL);
+			w[k] = (u32)v;
+			w[k + 1] = (u32)(v >> 32);
+			rew++;
+			break;
+		}
+	}
+	if (rew)
+		pr_info("vgsp-gr: bank %u: %u data pointers relocated\n",
+			bank, rew);
+}
+
+/*
  * m101: map one guest data page through the server walker (Tier-1
  * DMA_UPDATE_PDE_2), exactly like the launch request pages. On the
  * shared-management VAS the client PD0 entries are not preserved by the
@@ -16500,6 +16538,11 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
 				       GSP_PHYS_PAGE_SIZE);
 				copied++;
 			}
+			if (pg->hpa && pg->qmd_word >= 32 && pg->qmd_word <= 47 &&
+			    !(pg->qmd_word & 1))
+				gsp_phys_gr_rewrite_data_vars((u32 *)sh_page,
+					GSP_PHYS_PAGE_SIZE / 4, req,
+					(pg->qmd_word - 32) / 2);
 			/*
 			 * m98: dump the program prologue (qmd_word 48, from
 			 * the QMD's in-page entry offset) and the param/const-0
@@ -16892,7 +16935,9 @@ gr_page_unmapped:
 
 		for (di = 0; di < req->n_data; di++) {
 			if (gsp_phys_rmcall_gr_data_map(bar0,
-							req->data[di].va,
+							req->data[di].reloc_va ?
+							 req->data[di].reloc_va :
+							 req->data[di].va,
 							req->data[di].hpa,
 							boot_stage.rmcall_gr_submit_epoch))
 				dfailed++;
diff --git a/kernel/gsp_phys_booter.h b/kernel/gsp_phys_booter.h
index 29cf07f..896a436 100644
--- a/kernel/gsp_phys_booter.h
+++ b/kernel/gsp_phys_booter.h
@@ -366,6 +366,13 @@ struct gsp_phys_gr_inline {
 struct gsp_phys_gr_data_page {
 	u64 va;		/* guest VA, page aligned */
 	u64 hpa;	/* host PA, page aligned */
+	u64 reloc_va;	/* m102: server-managed VA this hpa is mapped at */
+};
+
+/* m102: guest 2 MB window -> relocated (server-managed) window base. */
+struct gsp_phys_gr_data_win {
+	u64 va_base;	/* guest VA window base (2 MB aligned) */
+	u64 reloc_base;	/* relocated VA window base (2 MB aligned) */
 };
 
 struct gsp_phys_gr_submit {
@@ -390,6 +397,8 @@ struct gsp_phys_gr_submit {
 	u32 n_inl;
 	const struct gsp_phys_gr_data_page *data;	/* m99: identity pages */
 	u32 n_data;
+	const struct gsp_phys_gr_data_win *data_wins;	/* m102: window map */
+	u32 n_data_wins;
 	u64 rel_hpa;		/* guest-visible QMD release0 page, or 0 */
 	u32 rel_off;
 	u32 rel_payload;
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 6dd0c7e..dfc9922 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -3251,9 +3251,24 @@ struct svc_gr_page {
 struct svc_gr_data {
 	u64 va;		/* guest VA, page aligned */
 	u64 pa;		/* guest PA, page aligned (0 = unresolved) */
+	u64 reloc_va;	/* m102: server-managed VA the page maps to */
 	u8 ap;		/* SVC_APERTURE_* of pa */
 };
 
+/*
+ * m102: guest window -> relocated window base. The server only installs
+ * page-directory entries in its managed region (the golden VAS reserves
+ * [0x100000000, 0x11fffffff), 256 x 2 MB windows); guest CUDA VAs are
+ * client-managed and refused with 0x1f. Data pages are mapped in the
+ * managed region at an affine-relocated VA and the pointers in the bank
+ * pages are rewritten to match.
+ */
+#define SVC_GR_DATA_WINS_MAX	64
+struct svc_gr_data_win {
+	u64 va_base;		/* guest window base (2 MB aligned) */
+	u64 reloc_base;		/* relocated window base */
+};
+
 struct svc_gr_cand {
 	bool valid;
 	bool offered;		/* consumed by a request or a fallback */
@@ -8277,6 +8292,8 @@ struct svc_fwd_req {
 	struct svc_gr_page gr_pages[SVC_GR_PAGES_MAX];
 	u32 gr_n_data;
 	struct svc_gr_data gr_data[SVC_GR_DATA_MAX];
+	u32 gr_n_data_wins;
+	struct svc_gr_data_win gr_data_wins[SVC_GR_DATA_WINS_MAX];
 };
 
 static struct svc_fwd_req svc_fwd_req;
@@ -9087,6 +9104,7 @@ static void svc_fwd_gr_work(void)
 	 */
 	static struct gsp_phys_gr_page pages[SVC_GR_PAGES_MAX];
 	static struct gsp_phys_gr_data_page data[SVC_GR_DATA_MAX];
+	static struct gsp_phys_gr_data_win data_wins[SVC_GR_DATA_WINS_MAX];
 	struct gsp_phys_gr_submit gs;
 	struct gsp_phys_backend_copy_result cp = { 0 };
 	u64 rel_hpa = 0;
@@ -9174,9 +9192,17 @@ static void svc_fwd_gr_work(void)
 			break;
 		data[gs.n_data].va = dg->va;
 		data[gs.n_data].hpa = dhpa;
+		data[gs.n_data].reloc_va = dg->reloc_va;
 		gs.n_data++;
 	}
 	gs.data = data;
+	gs.n_data_wins = svc_fwd_req.gr_n_data_wins;
+	for (i = 0; i < svc_fwd_req.gr_n_data_wins; i++) {
+		data_wins[i].va_base = svc_fwd_req.gr_data_wins[i].va_base;
+		data_wins[i].reloc_base =
+			svc_fwd_req.gr_data_wins[i].reloc_base;
+	}
+	gs.data_wins = data_wins;
 	gs.pages = pages;
 	gs.n_pages = svc_fwd_req.gr_n_pages;
 	gs.inl = (const struct gsp_phys_gr_inline *)svc_fwd_req.gr_inline;
@@ -9529,20 +9555,24 @@ static bool svc_gr_page_inline_covers(u64 page)
  * Resolution only happens on the service thread; the work item turns the
  * guest PAs into HPAs and the backend plants the identity PTEs.
  */
+#define SVC_GR_DATA_RELOC_BASE	0x100000000ULL
+
 static void svc_gr_data_fill(struct nv_vgpu_device *vgpu)
 {
 	u64 wins[16];
-	u32 n_win = 0;
+	u32 n_win = 0, n_cand;
 	u32 off = 0, i, w;
 	u8 kind;
 
 	svc_fwd_req.gr_n_data = 0;
+	svc_fwd_req.gr_n_data_wins = 0;
 	/*
 	 * m100: scan every const-bank page the launch carries (QMD words
 	 * 32..47 are the bank LO/HI pairs; the demo's LDG.E base comes from
 	 * c[0x4][0x0], not from the c[0x0] param page). Each page's first
 	 * 2 KB is scanned for plausible device VAs.
 	 */
+	n_cand = 0;
 	for (i = 0; i < svc_fwd_req.gr_n_pages; i++) {
 		const struct svc_gr_page *pp = &svc_fwd_req.gr_pages[i];
 		u64 hpa = 0;
@@ -9568,24 +9598,43 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu)
 			    (v & 3))
 				continue;
 			wv = v & ~0x1fffffULL;
-			for (w = 0; w < n_win; w++)
+			for (w = 0; w < n_cand; w++)
 				if (wins[w] == wv)
 					break;
-			if (w == n_win && n_win < ARRAY_SIZE(wins))
-				wins[n_win++] = wv;
+			if (w == n_cand && n_cand < ARRAY_SIZE(wins))
+				wins[n_cand++] = wv;
 		}
 	}
-	for (w = 0; w < n_win; w++) {
+	for (w = 0; w < n_cand; w++) {
 		int wi;
 
 		for (wi = -1; wi <= 16; wi++) {
 			u64 wv = wins[w] + (u64)wi * 0x200000ULL;
-			u32 got = 0, p;
+			u64 rbase;
+			u32 got = 0, p, x;
 
 			if (wi < 0 && wins[w] < 0x200000ULL)
 				continue;
 			if (wv >= (1ULL << 47))
 				break;
+			for (x = 0; x < svc_fwd_req.gr_n_data_wins; x++)
+				if (svc_fwd_req.gr_data_wins[x].va_base == wv)
+					break;
+			if (x == svc_fwd_req.gr_n_data_wins) {
+				if (svc_fwd_req.gr_n_data_wins >=
+				    SVC_GR_DATA_WINS_MAX)
+					return;
+				svc_fwd_req.gr_data_wins[x].va_base = wv;
+				rbase = SVC_GR_DATA_RELOC_BASE +
+					((u64)svc_fwd_req.gr_n_data_wins <<
+					 21);
+				svc_fwd_req.gr_data_wins[x].reloc_base =
+					rbase;
+				svc_fwd_req.gr_n_data_wins++;
+			} else {
+				rbase =
+				 svc_fwd_req.gr_data_wins[x].reloc_base;
+			}
 			for (p = 0; p < 512; p++) {
 				u64 va = wv + (u64)p * SEMA_PAGE_SIZE;
 				u64 pa = 0;
@@ -9600,6 +9649,7 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu)
 					       [svc_fwd_req.gr_n_data++];
 				rec->va = va;
 				rec->pa = pa & ~(u64)(SEMA_PAGE_SIZE - 1);
+				rec->reloc_va = rbase + (va & 0x1fffffULL);
 				rec->ap = (u8)svc_ap_of(ap);
 				got++;
 			}
@@ -9609,8 +9659,8 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu)
 				return;
 		}
 	}
-	pr_info("vgsp-sema: svc-gr data fill: %u windows, %u pages resolved\n",
-		n_win, svc_fwd_req.gr_n_data);
+	pr_info("vgsp-sema: svc-gr data fill: %u candidate ptrs, %u windows, %u pages resolved\n",
+		n_cand, svc_fwd_req.gr_n_data_wins, svc_fwd_req.gr_n_data);
 }
 
 static int svc_gr_req_fill(struct nv_vgpu_device *vgpu,