vgsp-gr: B-2 planted 64 MiB data-bounce region for large GR operands

Replace the 205-page / 68-pages-per-pointer GR operand window with a
boot-planted 16384-page (64 MiB) region at VA 0x0a000000 (PD0 80..111):
one dual PD0 entry per 2 MB window backed by one fresh 4 KB PT page,
every leaf PTE written at stage 3 before any engine fetch (the GMMU has
no PTE invalidate, so no leaf is rewritten at runtime). The 208-page
window, mail slot 206 and marker 207 stay as-is (mail injection and
copy diagnostics unchanged) but carry no operand data when the grown
region is on.

Sema service: reloc base and page cap derive from the shared header
constants; per-pointer span becomes a 4096-page runaway guard; a total
cap hit is fail-closed (launch stalled for the bounded hold, loud
CAP-HIT log, no silent truncation); the launch's own param/bank pages
are skipped as data candidates (budget hygiene). Copy-in, copy-out,
retirement wait and the window diagnostic index the active region
through one helper/chunk selector; checksum and dedup scratch move off
the kernel stack.

gsp_phys_gr_data_region=0 (insmod, boot-fixed) flips the whole path
back to the legacy 205-page window for A/B trialing.

1 files changed, 101 insertions, 44 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index f8925df..baac8d4 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -3924,11 +3924,11 @@ static struct svc_fwd_cand svc_fwd_cand;
 /*
  * m99: guest data identity-map capacity. The forwarded kernel's global data
  * pages (discovered by scanning the param/const-0 page for device VAs) are
- * resolved on the service thread and identity-mapped into the engine tree.
- * 16K pages = 64 MB of window coverage, comfortably above the demo buffers
- * (2 x 16 MB observed) with headroom.
+ * resolved on the service thread and relocated into the engine-visible
+ * region. B-2: 32K records = 2x the grown region's 16384 pages of headroom,
+ * so the hand-off array is never the binding limit.
  */
-#define SVC_GR_DATA_MAX		16384
+#define SVC_GR_DATA_MAX		32768
 #define SVC_GR_PUSH_WORDS	256
 #define SVC_GR_QMD_WORD_NONE	0xffffffffu
 /* svc_gr_cand.rel_flags bits (decoded from qmd[24] and qmd[4]). */
@@ -12360,32 +12360,36 @@ static bool svc_gr_page_inline_covers(u64 page)
  * Resolution only happens on the service thread; the work item turns the
  * guest PAs into HPAs and the backend plants the identity PTEs.
  */
-#define SVC_GR_DATA_RELOC_BASE	0x110000ULL
+/*
+ * B-2 (large-operand data path): the SVC assigns each resolved operand page
+ * a reloc VA in the grown boot-planted region (the layout constants live in
+ * gsp_phys_booter.h so the two sides cannot drift; gsp_phys_gr_data_region
+ * selects grown vs legacy, default grown). The legacy 0x110000 window below
+ * the mail page (205 data pages max, 68 per pointer) remains selectable with
+ * gsp_phys_gr_data_region=0 for A/B trialing.
+ */
+#define SVC_GR_DATA_RELOC_BASE		GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0
+#define SVC_GR_DATA_RELOC_BASE_OLD	0x110000ULL
 /*
  * Per candidate pointer, map at most this many pages from the pointer's
- * page onward (the engine-visible main-PT window has ~220 free 4 KB slots;
- * the demo buffers are small on the native path).
- * m120: lowered 64 -> 32 so every pointer page fits under the 100-page
- * bounce cap (mail 2x32=64, vadd 3x32=96); at 64/ptr the second/third
- * candidate run was truncated by the cap and its pointer word never
- * rewritten (m119: first LDG base c[0x0][0x168] possibly untouched).
- * m140: 32 -> 33. A cudaMalloc'd 128 KB buffer whose pointer is 0x400
- * into its page spans 33 pages (32 full + the 1 KB tail), and the kernel
- * accesses the tail: with 32 pages the tail read the NEXT candidate's
- * relocated page (wrong values; vadd tails). 3x33=99 <= the 100-page cap.
- */
-/*
- * m150 (large buffers): the bounce window was grown to 208 pages (mail page
- * at slot 206). 68 pages per candidate/pointer covers a 256 KB cudaMalloc'd
- * operand whose pointer sits 0x400 into its page (64 full + offset + tail),
- * and up to three such operands (3x68 = 204 <= 205) fit below the mail page.
- * Larger-than-256 KB operands still hit the cap; they need a chunked data
- * path (future work).
- */
-#define SVC_GR_DATA_PAGES_PER_PTR	68u
-/* Total pages the pre-commit bounce window can carry below the mail page
- * (GSP_PHYS_RMCALL_GR_MAIL_SLOT = 206). */
-#define SVC_GR_DATA_BOUNCE_MAX	205u
+ * page onward. With the grown region this is a runaway guard, not a budget:
+ * a cold page still stops the run (stop-on-miss below), so a real operand's
+ * span is bounded by its extent. History: 64 -> 32 (m120) -> 33 (m140) under
+ * the old 100-page cap -> 68 (m150, 256 KB operands under the 205-page
+ * window); the old value is kept for the legacy path.
+ */
+#define SVC_GR_DATA_PAGES_PER_PTR	4096u
+#define SVC_GR_DATA_PAGES_PER_PTR_OLD	68u
+/*
+ * Total operand pages the engine-visible data region can carry; derived from
+ * the booter's shared constant (no drift). A cap hit is FAIL-CLOSED: the
+ * fill marks the launch stalled for the hold/retry path and returns instead
+ * of silently truncating later pointers - a truncated operand must never
+ * reach the engine. The legacy value is the pre-B-2 window below the mail
+ * page (GSP_PHYS_RMCALL_GR_MAIL_SLOT = 206).
+ */
+#define SVC_GR_DATA_BOUNCE_MAX		GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES
+#define SVC_GR_DATA_BOUNCE_MAX_OLD	205u
 
 static u32 svc_gr_data_log;	/* m142: bounded per-page attribution log */
 static u32 svc_gr_src_log;	/* m150: bounded per-candidate source log */
@@ -12515,10 +12519,23 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu)
 {
 	u64 wins[16];
 	bool is_arg[16];	/* m150: candidate came from the argument scan */
-	u32 n_win = 0, n_cand;
+	u32 n_cand;
 	u32 off = 0, i, w;
 	u8 kind;
 	/*
+	 * B-2: active data-region geometry. gsp_phys_gr_data_region=1 (the
+	 * default) selects the grown 16384-page region planted by the booter
+	 * at GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0; 0 keeps the legacy 205-page
+	 * window below the mail page for A/B trialing.
+	 */
+	bool region_on = gsp_phys_gr_data_region_value() != 0;
+	u64 reloc_base = region_on ? SVC_GR_DATA_RELOC_BASE :
+				     SVC_GR_DATA_RELOC_BASE_OLD;
+	u32 per_ptr = region_on ? SVC_GR_DATA_PAGES_PER_PTR :
+				  SVC_GR_DATA_PAGES_PER_PTR_OLD;
+	u32 bounce_max = region_on ? SVC_GR_DATA_BOUNCE_MAX :
+				     SVC_GR_DATA_BOUNCE_MAX_OLD;
+	/*
 	 * m150: high (user/device-space) argument candidates and how many of
 	 * them produced at least one resolved page. None resolving is the
 	 * second-client zeros signature: the engine would run the kernel
@@ -12623,9 +12640,11 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu)
 		 * inline-captured bytes as well (see svc_gr_scan_inline). */
 		svc_gr_scan_inline(wins, &n_cand, pp->va);
 	}
-	/* m110: one contiguous run of pages per candidate pointer, relocated
-	 * into the engine-walkable main-PT window (0x110000 onward, fresh
-	 * PTE lines; the commit-time snapshot only preserves that window). */
+	/* m110/B-2: one contiguous run of pages per candidate pointer,
+	 * relocated into the active engine-visible data region (grown region
+	 * 0x0a000000+ with gsp_phys_gr_data_region=1 - the default - or the
+	 * legacy 0x110000 window; both are boot-planted, never-rewritten PTE
+	 * ranges the commit-time snapshot preserves). */
 	for (w = 0; w < n_cand; w++) {
 		u64 base = wins[w];
 		u32 p;
@@ -12640,17 +12659,46 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu)
 		 * operands (high, cudaMalloc-shaped VAs) get the full per-pointer
 		 * span. Low-VA candidates (the launch's own QMD/release/reloc
 		 * pointers) need a single page at most, and the launch's QMD VA
-		 * is not a kernel operand at all - relocating 68 pages for it
-		 * starved the operands that follow and blew the 205-page cap
+		 * is not a kernel operand at all - relocating a full span for it
+		 * starved the operands that follow and blew the old window cap
 		 * (r141: a 256 KB operand resolved 0 pages because the QMD and
-		 * param candidates consumed the window). Excluding them lets
-		 * three 256 KB operands (3x68 = 204) fit.
+		 * param candidates consumed the window).
 		 */
 		if ((base & ~(u64)(SEMA_PAGE_SIZE - 1)) ==
 		    (svc_fwd_req.gr_qmd_dev_va & ~(u64)(SEMA_PAGE_SIZE - 1)))
 			continue;
-		span = (base >= 0x40000000000ULL) ?
-			SVC_GR_DATA_PAGES_PER_PTR : 1u;
+		/*
+		 * R3 hygiene (large-buffer budget discipline): the launch's own
+		 * param/bank pages (the qmd_word 32..47 gr_pages entries) and
+		 * the param VA are driver bookkeeping, not kernel operands - the
+		 * observed fills resolved a full 68-page span for the param page
+		 * in every launch and starved the real operands under the old
+		 * window. Skip them as data candidates; they are
+		 * shadowed/rewritten through gr_pages/Tier 2, not this path.
+		 */
+		if (svc_fwd_req.gr_param_va &&
+		    (base & ~(u64)(SEMA_PAGE_SIZE - 1)) ==
+		    (svc_fwd_req.gr_param_va & ~(u64)(SEMA_PAGE_SIZE - 1)))
+			continue;
+		{
+			bool launch_internal = false;
+
+			for (i = 0; i < svc_fwd_req.gr_n_pages; i++) {
+				const struct svc_gr_page *pp =
+					&svc_fwd_req.gr_pages[i];
+
+				if (pp->qmd_word >= 32u &&
+				    pp->qmd_word <= 47u &&
+				    (base & ~(u64)(SEMA_PAGE_SIZE - 1)) ==
+				    (pp->va & ~(u64)(SEMA_PAGE_SIZE - 1))) {
+					launch_internal = true;
+					break;
+				}
+			}
+			if (launch_internal)
+				continue;
+		}
+		span = (base >= 0x40000000000ULL) ? per_ptr : 1u;
 
 		if (is_arg[w] && base >= 0x40000000000ULL)
 			n_arg_hi++;
@@ -12699,16 +12747,26 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu)
 					break;
 				}
 			}
-			if (svc_fwd_req.gr_n_data >= SVC_GR_DATA_BOUNCE_MAX) {
-				pr_info("vgsp-sema: svc-gr data fill: %u candidate ptrs, %u pages resolved (reloc 0x%llx+) CAP-HIT\n",
+			if (svc_fwd_req.gr_n_data >= bounce_max) {
+				/*
+				 * B-2 fail-closed cap: the active region is
+				 * full and later pointers would be silently
+				 * dropped (the old CAP-HIT return forwarded
+				 * a truncated operand set). Mark the launch
+				 * stalled so svc_gr_request holds/retries it
+				 * via the existing bounded hold instead of
+				 * forwarding incomplete data.
+				 */
+				svc_gr_data_stalled = true;
+				pr_err("vgsp-sema: svc-gr data fill: %u candidate ptrs, %u pages resolved (reloc 0x%llx+) CAP-HIT: region full, launch held fail-closed (no silent truncation)\n",
 					n_cand, svc_fwd_req.gr_n_data,
-					(u64)SVC_GR_DATA_RELOC_BASE);
+					reloc_base);
 				return;
 			}
 			rec = &svc_fwd_req.gr_data[svc_fwd_req.gr_n_data];
 			rec->va = va;
 			rec->pa = pa & ~(u64)(SEMA_PAGE_SIZE - 1);
-			rec->reloc_va = SVC_GR_DATA_RELOC_BASE +
+			rec->reloc_va = reloc_base +
 					((u64)svc_fwd_req.gr_n_data << 12);
 			rec->ap = (u8)svc_ap_of(ap);
 			svc_fwd_req.gr_n_data++;
@@ -12773,8 +12831,7 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu)
 			n_cand, svc_fwd_req.gr_n_data, n_arg_hi);
 	else
 		pr_info("vgsp-sema: svc-gr data fill: %u candidate ptrs, %u pages resolved (reloc 0x%llx+)\n",
-			n_cand, svc_fwd_req.gr_n_data,
-			(u64)SVC_GR_DATA_RELOC_BASE);
+			n_cand, svc_fwd_req.gr_n_data, reloc_base);
 }
 
 static int svc_gr_req_fill(struct nv_vgpu_device *vgpu,