vgsp: replay/block capacity 1024->2048 words (r1126 drop was arithmetic: window 390+prelude 901+sa 47 > 1024), two-page module pushbuffer (VA_PB2 0x102000 small-PT 0x102, CE SRC->0x109000, warm-sem probe page 0x10a000), gsp_phys_gr_pb_copy page-wise, static_asserts - planted verified (r1127 pb page2+probe planted before first fetch); >4KB path never exercised (5 boots beef never forwarded; walk PA 0x23921d000=8.89GB outside 8GB image); guest setup push NOT in guest RAM (ram_content_pin accepted 0/74); next: drop+re-walk src=4 all-zero reads, reject PA>guest-RAM top - SESSION 153 r1127

1 files changed, 192 insertions, 25 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index cf6e33c..d099a7e 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -1131,18 +1131,35 @@ bool gsp_phys_boot_requested(void)
  * from gpFifoOffset, the pushbuffer from the GPFIFO ring entry GET, the CE
  * operands from the method block): SET_SEMAPHORE_A/B are always GPU VAs with
  * no mode bit (clc5b5.h:39-44; uvm_pascal_ce.c:42-63) and the user channel's
- * copy is virtual-mode on both sides (uvm_maxwell_ce.c:179-205). All five
- * pages live in one 2 MB PD0 slot below the VAS vaSize limit: with
+ * copy is virtual-mode on both sides (uvm_maxwell_ce.c:179-205). All the
+ * ladder pages live in one 2 MB PD0 slot below the VAS vaSize limit: with
  * VA_BASE = 0x00100000 every parent index (48:47, 46:38, 37:29, 28:21) is 0
- * and the five 4 KB pages take small-PT slots 0x100..0x104
- * (kern_gmmu_fmt_gp10x.c:16-83; uvm_pascal_mmu.c:25-32,172-204).
+ * and the 4 KB pages take small-PT slots in the single 128 B PTE line
+ * 0x100..0x10f (kern_gmmu_fmt_gp10x.c:16-83; uvm_pascal_mmu.c:25-32,172-204).
+ *
+ * SESSION 153-CONTINUED-66: the module pushbuffer is TWO pages. The MME
+ * context-setup prelude matured to 901 words while the beef window runs 390
+ * words and the SA-RAM stubs add 29..47, so the single-launch block needs
+ * ~1400 words = 5.6 KB: the 1-page pushbuffer made every prelude-bearing
+ * launch print "MME-PRELUDE deferred (window=390 prelude=901 sa=47 exceeds
+ * block; replaying window alone)" and go out prelude=0. The block now spans
+ * VA_PB..VA_PB+0x2000 with the second page at small-PT slot 0x102. The
+ * slot 0x102 page used to be CE SRC; SRC moved to 0x109 (it is only read by
+ * the CE copy path and the GR inline-QMD destination default). Slot 0x10a
+ * is the dedicated warm-semaphore probe page: the probe used to share the
+ * pushbuffer's upper half, which a >0x800-byte forwarded block now occupies.
  */
 #define GSP_PHYS_RMCALL_VA_BASE		0x00100000ULL
 #define GSP_PHYS_RMCALL_VA_GPFIFO	GSP_PHYS_RMCALL_VA_BASE
 #define GSP_PHYS_RMCALL_VA_PB		(GSP_PHYS_RMCALL_VA_BASE + GSP_PHYS_PAGE_SIZE)
-#define GSP_PHYS_RMCALL_VA_SRC		(GSP_PHYS_RMCALL_VA_BASE + 2 * GSP_PHYS_PAGE_SIZE)
+/* SESSION 153-CONTINUED-66: page 2 of the module pushbuffer (same PT line). */
+#define GSP_PHYS_RMCALL_VA_PB2		(GSP_PHYS_RMCALL_VA_BASE + 2 * GSP_PHYS_PAGE_SIZE)
 #define GSP_PHYS_RMCALL_VA_DST		(GSP_PHYS_RMCALL_VA_BASE + 3 * GSP_PHYS_PAGE_SIZE)
 #define GSP_PHYS_RMCALL_VA_SEM		(GSP_PHYS_RMCALL_VA_BASE + 4 * GSP_PHYS_PAGE_SIZE)
+/* SESSION 153-CONTINUED-66: CE SRC moved off the pushbuffer's second page. */
+#define GSP_PHYS_RMCALL_VA_SRC		(GSP_PHYS_RMCALL_VA_BASE + 0x9000ULL)
+/* SESSION 153-CONTINUED-66: warm-semaphore probe page (own PTE, no sharing). */
+#define GSP_PHYS_RMCALL_VA_GR_PROBE	(GSP_PHYS_RMCALL_VA_BASE + 0xA000ULL)
 /*
  * m69: window-0 shadow pool - 16 module pages holding the Tier-2 copies of
  * the QMD's program/param pages, so the GR engine never has to read through
@@ -2483,6 +2500,18 @@ static struct gsp_phys_boot_stage {
 	 */
 	struct gsp_phys_boot_chunk rmcall_gr_qmdwin;
 	u64 rmcall_gr_qmdwin_va;	/* planted VA; 0 = guest destination */
+	/*
+	 * SESSION 153-CONTINUED-66: page 2 of the module pushbuffer plus the
+	 * dedicated warm-semaphore probe page. Allocated lazily by
+	 * gsp_phys_rmcall_pdb_build() and planted in the ladder's 128 B PTE
+	 * line at slots 0x102 (VA_PB2) and 0x10a (VA_GR_PROBE), before the
+	 * engine's first fetch. Page 0 continues the registered BUF_PB chunk
+	 * so stage 2's registration and every existing pushbuffer user are
+	 * unchanged; only the >4 KB GR launch block reaches into these pages.
+	 */
+	struct gsp_phys_boot_chunk rmcall_gr_pb2;
+	struct gsp_phys_boot_chunk rmcall_gr_probe;
+	bool rmcall_gr_pb2_planted;
 	/* m104: module page holding the injected mail_kernel message. */
 	struct gsp_phys_boot_chunk rmcall_gr_mail;
 	/*
@@ -3252,6 +3281,11 @@ static void gsp_phys_rmcall_reset(void)
 	memset(&boot_stage.rmcall_gr_qmdwin, 0,
 	       sizeof(boot_stage.rmcall_gr_qmdwin));
 	boot_stage.rmcall_gr_qmdwin_va = 0;
+	memset(&boot_stage.rmcall_gr_pb2, 0,
+	       sizeof(boot_stage.rmcall_gr_pb2));
+	memset(&boot_stage.rmcall_gr_probe, 0,
+	       sizeof(boot_stage.rmcall_gr_probe));
+	boot_stage.rmcall_gr_pb2_planted = false;
 	memset(&boot_stage.rmcall_gr_shadow_win0, 0,
 	       sizeof(boot_stage.rmcall_gr_shadow_win0));
 	boot_stage.rmcall_gr_shadow_win0_used = 0;
@@ -10206,6 +10240,51 @@ static void gsp_phys_rmcall_pdb_build(void)
 		}
 	}
 	/*
+	 * SESSION 153-CONTINUED-66: plant page 2 of the module pushbuffer
+	 * (VA_PB2, small-PT slot 0x102) and the dedicated warm-semaphore
+	 * probe page (VA_GR_PROBE, slot 0x10a) in the same 128 B PTE line,
+	 * at stage 3 before the engine's first fetch (no invalidate exists).
+	 * Page 0 stays the registered BUF_PB chunk (stage 2 unchanged); this
+	 * chunk is engine-reached purely through these PTEs, exactly like
+	 * the qmdwin pair above. Without it the GR launch block keeps the
+	 * single-page bound (<= 4 KB).
+	 */
+	if (!boot_stage.rmcall_gr_pb2.pages && boot_stage.pdev) {
+		if (!gsp_phys_chunk_alloc(boot_stage.pdev,
+					  &boot_stage.rmcall_gr_pb2, 1u))
+			gsp_phys_chunk_zero(&boot_stage.rmcall_gr_pb2);
+		else
+			pr_warn("vgsp-boot: rm-call: pushbuffer page2 allocation failed; >4 KB launches keep the single-page block\n");
+	}
+	if (!boot_stage.rmcall_gr_probe.pages && boot_stage.pdev) {
+		if (!gsp_phys_chunk_alloc(boot_stage.pdev,
+					  &boot_stage.rmcall_gr_probe, 1u))
+			gsp_phys_chunk_zero(&boot_stage.rmcall_gr_probe);
+		else
+			pr_warn("vgsp-boot: rm-call: warm-sem probe page allocation failed; the probe keeps the pushbuffer upper half\n");
+	}
+	if (boot_stage.rmcall_gr_pb2.pages && boot_stage.rmcall_gr_probe.pages) {
+		u32 pb2_idx = (u32)((GSP_PHYS_RMCALL_VA_PB2 >> 12) & 0x1ffu);
+		u32 pr_idx = (u32)((GSP_PHYS_RMCALL_VA_GR_PROBE >> 12) &
+				   0x1ffu);
+
+		put_unaligned_le64(gsp_phys_rmcall_pte(
+				(u64)gsp_phys_chunk_page_dma(
+					&boot_stage.rmcall_gr_pb2, 0)),
+			(u8 *)gsp_phys_chunk_page_va(pt, 0) +
+			(size_t)pb2_idx * 8);
+		put_unaligned_le64(gsp_phys_rmcall_pte(
+				(u64)gsp_phys_chunk_page_dma(
+					&boot_stage.rmcall_gr_probe, 0)),
+			(u8 *)gsp_phys_chunk_page_va(pt, 0) +
+			(size_t)pr_idx * 8);
+		dma_wmb();
+		boot_stage.rmcall_gr_pb2_planted = true;
+		pr_info("vgsp-boot: rm-call: GR launch pb page2 + warm-sem probe planted VA=0x%llx/0x%llx ptSlots=0x%x/0x%x (two-page block; before first fetch)\n",
+			GSP_PHYS_RMCALL_VA_PB2, GSP_PHYS_RMCALL_VA_GR_PROBE,
+			pb2_idx, pr_idx);
+	}
+	/*
 	 * m69: plant the window-0 shadow pool PTEs (current win0 slot range) in the
 	 * same proven small-PT line, before the engine's first fetch. The
 	 * launch copies the QMD's program/param page content into these pages
@@ -15486,20 +15565,26 @@ static int gsp_phys_rmcall_publish_locked(void __iomem *bar0, u64 pb_gpu_va,
 /*
  * SESSION 153-CONTINUED-12: pb_off selects where in the module pushbuffer
  * page the method block is written and fetched from. The warm-semaphore
- * probe (gsp_phys_rmcall_gr_sem_push_locked) uses the upper half so it can
- * never overwrite a forwarded launch's pending method block: measured with
- * the PBDMA dump, the probe's 10-dword block landing on words 0..9 of a
- * pending 139-word 3D replay made the PBDMA walk the probe's
- * SET_OBJECT/REPORT_SEM tail and then decode the replay window's operand
- * 0x00230200 as an instruction -> PBENTRY on chid 4 and no retire. Every
- * forwarded GR/CE block stays at offset 0 (<= 512 dwords = 0x800 bytes by
- * the GSP_PHYS_RMCALL_GR_METHOD_WORDS bound).
+ * probe (gsp_phys_rmcall_gr_sem_push_locked) used the upper half so it
+ * could never overwrite a forwarded launch's pending method block:
+ * measured with the PBDMA dump, the probe's 10-dword block landing on
+ * words 0..9 of a pending 139-word 3D replay made the PBDMA walk the
+ * probe's SET_OBJECT/REPORT_SEM tail and then decode the replay window's
+ * operand 0x00230200 as an instruction -> PBENTRY on chid 4 and no
+ * retire. Every forwarded GR/CE block stays at offset 0.
+ *
+ * SESSION 153-CONTINUED-66: pb_alt/pb_alt_va point the block at a
+ * dedicated one-page module buffer (the probe's VA_GR_PROBE page) instead
+ * of the registered pushbuffer; a forwarded GR block may now span both
+ * pushbuffer pages, so no fixed offset of that page is safe for the probe.
  */
 static int gsp_phys_rmcall_ce_submit_off_locked(void __iomem *bar0,
 						u64 src_va, u64 dst_va,
 						u64 sem_va, u32 bytes,
 						u32 sem_payload, bool gr,
 						u32 pb_off,
+						struct gsp_phys_boot_chunk *pb_alt,
+						u64 pb_alt_va,
 						struct gsp_phys_rmcall_ce_submit *out)
 {
 	struct gsp_phys_boot_chunk *pb =
@@ -15534,6 +15619,18 @@ static int gsp_phys_rmcall_ce_submit_off_locked(void __iomem *bar0,
 	pb_gpu_va = boot_stage.rmcall_va_pb;
 	if (!pb_gpu_va || (pb_gpu_va & 3))
 		return -EIO;
+	/*
+	 * SESSION 153-CONTINUED-66: pb_alt selects a dedicated one-page module
+	 * buffer instead of the registered pushbuffer page (the warm-semaphore
+	 * probe uses its own PTE-planted page now that a forwarded GR block may
+	 * span BOTH pushbuffer pages). pb_off stays relative to that page.
+	 */
+	if (pb_alt && pb_alt->pages) {
+		pb = pb_alt;
+		pb_gpu_va = pb_alt_va;
+		if (!pb_gpu_va || (pb_gpu_va & 3))
+			return -EIO;
+	}
 
 	if (!gsp_phys_chunk_page_va(pb, 0) || pb_off > GSP_PHYS_PAGE_SIZE)
 		return -EIO;
@@ -15572,7 +15669,7 @@ static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va,
 {
 	return gsp_phys_rmcall_ce_submit_off_locked(bar0, src_va, dst_va,
 						    sem_va, bytes, sem_payload,
-						    gr, 0u, out);
+						    gr, 0u, NULL, 0, out);
 }
 
 /*
@@ -15638,8 +15735,13 @@ static int gsp_phys_rmcall_gr_sem_push_locked(void __iomem *bar0)
 	 * on words 0.. of that page: measured PB-FAULT-DUMP shows the probe's
 	 * 10-dword block overwriting a pending 139-word 3D replay's head, so
 	 * the PBDMA walks the probe then decodes a replay operand as an
-	 * instruction (PBENTRY, no retire). Use the upper half (0x800); the
-	 * forwarded block is bounded to 512 dwords = 0x800 bytes.
+	 * instruction (PBENTRY, no retire).
+	 *
+	 * SESSION 153-CONTINUED-66: the forwarded GR block now spans BOTH
+	 * pushbuffer pages (up to 2048 dwords = 0x2000 bytes), so the old
+	 * upper-half placement (0x800) would clobber it. The probe has its
+	 * own PTE-planted page (VA_GR_PROBE, small-PT slot 0x10a); when that
+	 * plant is unavailable the probe falls back to 0x800 as before.
 	 */
 	ret = gsp_phys_rmcall_ce_submit_off_locked(bar0,
 					       GSP_PHYS_RMCALL_VA_SRC,
@@ -15647,7 +15749,14 @@ static int gsp_phys_rmcall_gr_sem_push_locked(void __iomem *bar0)
 					       GSP_PHYS_RMCALL_VA_SEM,
 					       GSP_PHYS_RMCALL_COPY_SIZE,
 					       GSP_PHYS_RMCALL_SEM_PAYLOAD, true,
-					       0x800u, &sub);
+					       boot_stage.rmcall_gr_probe.pages ?
+						0u : 0x800u,
+					       boot_stage.rmcall_gr_probe.pages ?
+						&boot_stage.rmcall_gr_probe :
+						NULL,
+					       boot_stage.rmcall_gr_probe.pages ?
+						GSP_PHYS_RMCALL_VA_GR_PROBE : 0,
+					       &sub);
 	if (ret) {
 		pr_err("vgsp-boot: rm-call: GR submission refused before the doorbell (%d)\n",
 		       ret);
@@ -16833,9 +16942,12 @@ EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_ce_copy);
  * the optional QMD-virt pair(2). Raised 512 -> 1024 (SESSION 153-
  * CONTINUED-53) so a 3D stream replay can carry the captured MME setup
  * prelude (<=512 words) ahead of the largest chunked window (<=448) in one
- * block; the block still lands inside the one 4 KB chunk page it is copied
- * to. */
-#define GSP_PHYS_RMCALL_GR_METHOD_WORDS		1024u
+ * block. SESSION 153-CONTINUED-66: 1024 -> 2048 because the matured prelude
+ * reached 901 words and the deferral test (window=390 + prelude=901 + sa=47
+ * vs the bound) dropped it on every launch; the block now spans the module
+ * pushbuffer's TWO 4 KB pages (VA_PB..VA_PB2) and is copied page-wise by
+ * gsp_phys_gr_pb_copy(). */
+#define GSP_PHYS_RMCALL_GR_METHOD_WORDS		2048u
 
 static u64 gsp_phys_gr_submitted;
 static u64 gsp_phys_gr_ok;
@@ -21272,6 +21384,45 @@ next:
 	}
 }
 
+/*
+ * SESSION 153-CONTINUED-66: write a built method block into the module
+ * pushbuffer. The block may exceed one 4 KB page (GSP_PHYS_RMCALL_GR_METHOD_
+ * WORDS is 2048), so copy page-wise: page 0 is the registered BUF_PB chunk,
+ * page 1 is the PTE-planted rmcall_gr_pb2 page at VA_PB2. The two pages are
+ * contiguous in GPU VA (VA_PB..VA_PB+0x2000), so the GPFIFO entry's single
+ * fetch address + length walks both. A short/absent page-2 plant leaves the
+ * tail unwritten and the caller's bound check keeps such launches <= 4 KB.
+ */
+static void gsp_phys_gr_pb_copy(const u32 *methods, u32 words)
+{
+	struct gsp_phys_boot_chunk *pb =
+		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB];
+	u32 per = GSP_PHYS_PAGE_SIZE / 4u;
+	u32 done = 0;
+
+	while (done < words) {
+		struct gsp_phys_boot_chunk *c;
+		u32 page = done / per;
+		u32 off = done % per;
+		u32 n = words - done;
+		u8 *dst;
+
+		if (n > per - off)
+			n = per - off;
+		if (!page)
+			c = pb;
+		else if (page == 1u)
+			c = &boot_stage.rmcall_gr_pb2;
+		else
+			return;
+		dst = gsp_phys_chunk_page_va(c, 0);
+		if (!dst)
+			return;
+		memcpy(dst + (size_t)off * 4u, methods + done, (size_t)n * 4u);
+		done += n;
+	}
+}
+
 int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
 				     struct gsp_phys_backend_copy_result *out)
 {
@@ -22826,7 +22977,7 @@ gr_page_unmapped:
 						win3d, win3d_n,
 						gsp_phys_gr_attr_pre,
 						gsp_phys_gr_attr_pre_n);
-	memcpy(gsp_phys_chunk_page_va(pb, 0), methods, words * 4);
+	gsp_phys_gr_pb_copy(methods, words);
 	/* SESSION 153-CONTINUED-12: identify this block to a later PBDMA dump. */
 	WRITE_ONCE(gsp_phys_gr_last_words, words);
 	WRITE_ONCE(gsp_phys_gr_last_h, req->h_object);
@@ -24360,8 +24511,8 @@ gr_page_unmapped:
 							win3d_n,
 							gsp_phys_gr_attr_pre,
 							gsp_phys_gr_attr_pre_n);
-						memcpy(gsp_phys_chunk_page_va(pb, 0),
-						       methods, words * 4);
+						gsp_phys_gr_pb_copy(methods,
+								    words);
 					}
 					if (qmdwin_swap)
 						pr_info("vgsp-gr: outcome retry %u: release words zeroed + fresh qmdwin bank %u dest 0x%llx + win0 bank %u%s + PB rebuilt\n",
@@ -25958,6 +26109,8 @@ static void gsp_phys_rmcall_fini(struct pci_dev *pdev)
 		gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_pt[i]);
 	gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_shadow);
 	gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_qmdwin);
+	gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_pb2);
+	gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_probe);
 	/*
 	 * B-2: the old 208-page bounce window (previously leaked here) and the
 	 * grown data-bounce region chunks. The region's PT chunk and data chunk
@@ -28016,6 +28169,20 @@ static_assert(((GSP_PHYS_RMCALL_VA_GPFIFO >> 12) & 0x1ff) !=
 	      (((GSP_PHYS_RMCALL_VA_GPFIFO >> 12) & 0x1ff) + 4) ==
 	      ((GSP_PHYS_RMCALL_VA_SEM >> 12) & 0x1ff),
 	      "the GPFIFO, pushbuffer and CE src/dst/semaphore VAs must take the five consecutive small-PT slots 0x100..0x104");
+/*
+ * SESSION 153-CONTINUED-66: the two-page pushbuffer and the moved CE SRC /
+ * probe pages must stay inside the same 2 MB PD0 slot and the same 128 B PTE
+ * fetch line as the ladder, and must not collide with each other or with the
+ * QMD-window pair (0x107/0x108).
+ */
+static_assert(((GSP_PHYS_RMCALL_VA_PB2 >> 12) & 0x1ff) ==
+	      ((GSP_PHYS_RMCALL_VA_PB >> 12) & 0x1ff) + 1,
+	      "the pushbuffer's second page must sit directly above the first (one contiguous GPU VA fetch)");
+static_assert(((GSP_PHYS_RMCALL_VA_SRC >> 21) & 0xff) == 0 &&
+	      ((GSP_PHYS_RMCALL_VA_GR_PROBE >> 21) & 0xff) == 0 &&
+	      ((GSP_PHYS_RMCALL_VA_SRC >> 12) & 0x1ff) == 0x109 &&
+	      ((GSP_PHYS_RMCALL_VA_GR_PROBE >> 12) & 0x1ff) == 0x10a,
+	      "the moved CE SRC page and the warm-semaphore probe page must take the free slots 0x109/0x10a in the ladder's PTE line");
 static_assert(((GSP_PHYS_RMCALL_VA_GUEST_PT_BASE >> 47) & 0x3) == 0 &&
 	      ((GSP_PHYS_RMCALL_VA_GUEST_PT_BASE >> 38) & 0x1ff) == 0 &&
 	      ((GSP_PHYS_RMCALL_VA_GUEST_PT_BASE >> 29) & 0x1ff) == 0 &&
@@ -28064,8 +28231,8 @@ static_assert(GSP_PHYS_RMCALL_GR_INIT_WORDS +
 	      GSP_PHYS_RMCALL_GR_METHOD_WORDS,
 	      "the verbatim GR launch block minimum (compute init + window + PCAS + REPORT_SEMAPHORE) must fit the bound; the per-launch runtime check bounds raw_qmd_hdr for larger preludes");
 static_assert(GSP_PHYS_RMCALL_GR_METHOD_WORDS * 4u <=
-	      GSP_PHYS_RMCALL_COPY_SIZE,
-	      "the GR launch method block must fit one module pushbuffer page");
+	      2u * GSP_PHYS_RMCALL_COPY_SIZE,
+	      "the GR launch method block may span the two module pushbuffer pages (SESSION 153-CONTINUED-66: the matured MME prelude no longer fits one page)");
 
 /*
  * GspSystemInfo wire ABI, pinned against the reference header compile