From cc6c198a234cf3a42ae565328fe83d490faf1f02 Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Sat, 31 Oct 2026 10:50:08 -0400 Subject: vgsp: replay/block capacity 1024->2048 words (r1126 drop was arithmetic: window 390+prelude 901+sa 47 > 1024), two-page module pushbuffer (VA_PB2 0x102000 small-PT 0x102, CE SRC->0x109000, warm-sem probe page 0x10a000), gsp_phys_gr_pb_copy page-wise, static_asserts - planted verified (r1127 pb page2+probe planted before first fetch); >4KB path never exercised (5 boots beef never forwarded; walk PA 0x23921d000=8.89GB outside 8GB image); guest setup push NOT in guest RAM (ram_content_pin accepted 0/74); next: drop+re-walk src=4 all-zero reads, reject PA>guest-RAM top - SESSION 153 r1127 --- kernel/gsp_phys_booter.c | 217 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 192 insertions(+), 25 deletions(-) (limited to 'kernel') diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index cf6e33c..d099a7e 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -1131,18 +1131,35 @@ bool gsp_phys_boot_requested(void) * from gpFifoOffset, the pushbuffer from the GPFIFO ring entry GET, the CE * operands from the method block): SET_SEMAPHORE_A/B are always GPU VAs with * no mode bit (clc5b5.h:39-44; uvm_pascal_ce.c:42-63) and the user channel's - * copy is virtual-mode on both sides (uvm_maxwell_ce.c:179-205). All five - * pages live in one 2 MB PD0 slot below the VAS vaSize limit: with + * copy is virtual-mode on both sides (uvm_maxwell_ce.c:179-205). All the + * ladder pages live in one 2 MB PD0 slot below the VAS vaSize limit: with * VA_BASE = 0x00100000 every parent index (48:47, 46:38, 37:29, 28:21) is 0 - * and the five 4 KB pages take small-PT slots 0x100..0x104 - * (kern_gmmu_fmt_gp10x.c:16-83; uvm_pascal_mmu.c:25-32,172-204). + * and the 4 KB pages take small-PT slots in the single 128 B PTE line + * 0x100..0x10f (kern_gmmu_fmt_gp10x.c:16-83; uvm_pascal_mmu.c:25-32,172-204). + * + * SESSION 153-CONTINUED-66: the module pushbuffer is TWO pages. The MME + * context-setup prelude matured to 901 words while the beef window runs 390 + * words and the SA-RAM stubs add 29..47, so the single-launch block needs + * ~1400 words = 5.6 KB: the 1-page pushbuffer made every prelude-bearing + * launch print "MME-PRELUDE deferred (window=390 prelude=901 sa=47 exceeds + * block; replaying window alone)" and go out prelude=0. The block now spans + * VA_PB..VA_PB+0x2000 with the second page at small-PT slot 0x102. The + * slot 0x102 page used to be CE SRC; SRC moved to 0x109 (it is only read by + * the CE copy path and the GR inline-QMD destination default). Slot 0x10a + * is the dedicated warm-semaphore probe page: the probe used to share the + * pushbuffer's upper half, which a >0x800-byte forwarded block now occupies. */ #define GSP_PHYS_RMCALL_VA_BASE 0x00100000ULL #define GSP_PHYS_RMCALL_VA_GPFIFO GSP_PHYS_RMCALL_VA_BASE #define GSP_PHYS_RMCALL_VA_PB (GSP_PHYS_RMCALL_VA_BASE + GSP_PHYS_PAGE_SIZE) -#define GSP_PHYS_RMCALL_VA_SRC (GSP_PHYS_RMCALL_VA_BASE + 2 * GSP_PHYS_PAGE_SIZE) +/* SESSION 153-CONTINUED-66: page 2 of the module pushbuffer (same PT line). */ +#define GSP_PHYS_RMCALL_VA_PB2 (GSP_PHYS_RMCALL_VA_BASE + 2 * GSP_PHYS_PAGE_SIZE) #define GSP_PHYS_RMCALL_VA_DST (GSP_PHYS_RMCALL_VA_BASE + 3 * GSP_PHYS_PAGE_SIZE) #define GSP_PHYS_RMCALL_VA_SEM (GSP_PHYS_RMCALL_VA_BASE + 4 * GSP_PHYS_PAGE_SIZE) +/* SESSION 153-CONTINUED-66: CE SRC moved off the pushbuffer's second page. */ +#define GSP_PHYS_RMCALL_VA_SRC (GSP_PHYS_RMCALL_VA_BASE + 0x9000ULL) +/* SESSION 153-CONTINUED-66: warm-semaphore probe page (own PTE, no sharing). */ +#define GSP_PHYS_RMCALL_VA_GR_PROBE (GSP_PHYS_RMCALL_VA_BASE + 0xA000ULL) /* * m69: window-0 shadow pool - 16 module pages holding the Tier-2 copies of * the QMD's program/param pages, so the GR engine never has to read through @@ -2483,6 +2500,18 @@ static struct gsp_phys_boot_stage { */ struct gsp_phys_boot_chunk rmcall_gr_qmdwin; u64 rmcall_gr_qmdwin_va; /* planted VA; 0 = guest destination */ + /* + * SESSION 153-CONTINUED-66: page 2 of the module pushbuffer plus the + * dedicated warm-semaphore probe page. Allocated lazily by + * gsp_phys_rmcall_pdb_build() and planted in the ladder's 128 B PTE + * line at slots 0x102 (VA_PB2) and 0x10a (VA_GR_PROBE), before the + * engine's first fetch. Page 0 continues the registered BUF_PB chunk + * so stage 2's registration and every existing pushbuffer user are + * unchanged; only the >4 KB GR launch block reaches into these pages. + */ + struct gsp_phys_boot_chunk rmcall_gr_pb2; + struct gsp_phys_boot_chunk rmcall_gr_probe; + bool rmcall_gr_pb2_planted; /* m104: module page holding the injected mail_kernel message. */ struct gsp_phys_boot_chunk rmcall_gr_mail; /* @@ -3252,6 +3281,11 @@ static void gsp_phys_rmcall_reset(void) memset(&boot_stage.rmcall_gr_qmdwin, 0, sizeof(boot_stage.rmcall_gr_qmdwin)); boot_stage.rmcall_gr_qmdwin_va = 0; + memset(&boot_stage.rmcall_gr_pb2, 0, + sizeof(boot_stage.rmcall_gr_pb2)); + memset(&boot_stage.rmcall_gr_probe, 0, + sizeof(boot_stage.rmcall_gr_probe)); + boot_stage.rmcall_gr_pb2_planted = false; memset(&boot_stage.rmcall_gr_shadow_win0, 0, sizeof(boot_stage.rmcall_gr_shadow_win0)); boot_stage.rmcall_gr_shadow_win0_used = 0; @@ -10205,6 +10239,51 @@ static void gsp_phys_rmcall_pdb_build(void) GSP_PHYS_RMCALL_VA_GR_QMDWIN; } } + /* + * SESSION 153-CONTINUED-66: plant page 2 of the module pushbuffer + * (VA_PB2, small-PT slot 0x102) and the dedicated warm-semaphore + * probe page (VA_GR_PROBE, slot 0x10a) in the same 128 B PTE line, + * at stage 3 before the engine's first fetch (no invalidate exists). + * Page 0 stays the registered BUF_PB chunk (stage 2 unchanged); this + * chunk is engine-reached purely through these PTEs, exactly like + * the qmdwin pair above. Without it the GR launch block keeps the + * single-page bound (<= 4 KB). + */ + if (!boot_stage.rmcall_gr_pb2.pages && boot_stage.pdev) { + if (!gsp_phys_chunk_alloc(boot_stage.pdev, + &boot_stage.rmcall_gr_pb2, 1u)) + gsp_phys_chunk_zero(&boot_stage.rmcall_gr_pb2); + else + pr_warn("vgsp-boot: rm-call: pushbuffer page2 allocation failed; >4 KB launches keep the single-page block\n"); + } + if (!boot_stage.rmcall_gr_probe.pages && boot_stage.pdev) { + if (!gsp_phys_chunk_alloc(boot_stage.pdev, + &boot_stage.rmcall_gr_probe, 1u)) + gsp_phys_chunk_zero(&boot_stage.rmcall_gr_probe); + else + pr_warn("vgsp-boot: rm-call: warm-sem probe page allocation failed; the probe keeps the pushbuffer upper half\n"); + } + if (boot_stage.rmcall_gr_pb2.pages && boot_stage.rmcall_gr_probe.pages) { + u32 pb2_idx = (u32)((GSP_PHYS_RMCALL_VA_PB2 >> 12) & 0x1ffu); + u32 pr_idx = (u32)((GSP_PHYS_RMCALL_VA_GR_PROBE >> 12) & + 0x1ffu); + + put_unaligned_le64(gsp_phys_rmcall_pte( + (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_gr_pb2, 0)), + (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)pb2_idx * 8); + put_unaligned_le64(gsp_phys_rmcall_pte( + (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_gr_probe, 0)), + (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)pr_idx * 8); + dma_wmb(); + boot_stage.rmcall_gr_pb2_planted = true; + pr_info("vgsp-boot: rm-call: GR launch pb page2 + warm-sem probe planted VA=0x%llx/0x%llx ptSlots=0x%x/0x%x (two-page block; before first fetch)\n", + GSP_PHYS_RMCALL_VA_PB2, GSP_PHYS_RMCALL_VA_GR_PROBE, + pb2_idx, pr_idx); + } /* * m69: plant the window-0 shadow pool PTEs (current win0 slot range) in the * same proven small-PT line, before the engine's first fetch. The @@ -15486,20 +15565,26 @@ static int gsp_phys_rmcall_publish_locked(void __iomem *bar0, u64 pb_gpu_va, /* * SESSION 153-CONTINUED-12: pb_off selects where in the module pushbuffer * page the method block is written and fetched from. The warm-semaphore - * probe (gsp_phys_rmcall_gr_sem_push_locked) uses the upper half so it can - * never overwrite a forwarded launch's pending method block: measured with - * the PBDMA dump, the probe's 10-dword block landing on words 0..9 of a - * pending 139-word 3D replay made the PBDMA walk the probe's - * SET_OBJECT/REPORT_SEM tail and then decode the replay window's operand - * 0x00230200 as an instruction -> PBENTRY on chid 4 and no retire. Every - * forwarded GR/CE block stays at offset 0 (<= 512 dwords = 0x800 bytes by - * the GSP_PHYS_RMCALL_GR_METHOD_WORDS bound). + * probe (gsp_phys_rmcall_gr_sem_push_locked) used the upper half so it + * could never overwrite a forwarded launch's pending method block: + * measured with the PBDMA dump, the probe's 10-dword block landing on + * words 0..9 of a pending 139-word 3D replay made the PBDMA walk the + * probe's SET_OBJECT/REPORT_SEM tail and then decode the replay window's + * operand 0x00230200 as an instruction -> PBENTRY on chid 4 and no + * retire. Every forwarded GR/CE block stays at offset 0. + * + * SESSION 153-CONTINUED-66: pb_alt/pb_alt_va point the block at a + * dedicated one-page module buffer (the probe's VA_GR_PROBE page) instead + * of the registered pushbuffer; a forwarded GR block may now span both + * pushbuffer pages, so no fixed offset of that page is safe for the probe. */ static int gsp_phys_rmcall_ce_submit_off_locked(void __iomem *bar0, u64 src_va, u64 dst_va, u64 sem_va, u32 bytes, u32 sem_payload, bool gr, u32 pb_off, + struct gsp_phys_boot_chunk *pb_alt, + u64 pb_alt_va, struct gsp_phys_rmcall_ce_submit *out) { struct gsp_phys_boot_chunk *pb = @@ -15534,6 +15619,18 @@ static int gsp_phys_rmcall_ce_submit_off_locked(void __iomem *bar0, pb_gpu_va = boot_stage.rmcall_va_pb; if (!pb_gpu_va || (pb_gpu_va & 3)) return -EIO; + /* + * SESSION 153-CONTINUED-66: pb_alt selects a dedicated one-page module + * buffer instead of the registered pushbuffer page (the warm-semaphore + * probe uses its own PTE-planted page now that a forwarded GR block may + * span BOTH pushbuffer pages). pb_off stays relative to that page. + */ + if (pb_alt && pb_alt->pages) { + pb = pb_alt; + pb_gpu_va = pb_alt_va; + if (!pb_gpu_va || (pb_gpu_va & 3)) + return -EIO; + } if (!gsp_phys_chunk_page_va(pb, 0) || pb_off > GSP_PHYS_PAGE_SIZE) return -EIO; @@ -15572,7 +15669,7 @@ static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va, { return gsp_phys_rmcall_ce_submit_off_locked(bar0, src_va, dst_va, sem_va, bytes, sem_payload, - gr, 0u, out); + gr, 0u, NULL, 0, out); } /* @@ -15638,8 +15735,13 @@ static int gsp_phys_rmcall_gr_sem_push_locked(void __iomem *bar0) * on words 0.. of that page: measured PB-FAULT-DUMP shows the probe's * 10-dword block overwriting a pending 139-word 3D replay's head, so * the PBDMA walks the probe then decodes a replay operand as an - * instruction (PBENTRY, no retire). Use the upper half (0x800); the - * forwarded block is bounded to 512 dwords = 0x800 bytes. + * instruction (PBENTRY, no retire). + * + * SESSION 153-CONTINUED-66: the forwarded GR block now spans BOTH + * pushbuffer pages (up to 2048 dwords = 0x2000 bytes), so the old + * upper-half placement (0x800) would clobber it. The probe has its + * own PTE-planted page (VA_GR_PROBE, small-PT slot 0x10a); when that + * plant is unavailable the probe falls back to 0x800 as before. */ ret = gsp_phys_rmcall_ce_submit_off_locked(bar0, GSP_PHYS_RMCALL_VA_SRC, @@ -15647,7 +15749,14 @@ static int gsp_phys_rmcall_gr_sem_push_locked(void __iomem *bar0) GSP_PHYS_RMCALL_VA_SEM, GSP_PHYS_RMCALL_COPY_SIZE, GSP_PHYS_RMCALL_SEM_PAYLOAD, true, - 0x800u, &sub); + boot_stage.rmcall_gr_probe.pages ? + 0u : 0x800u, + boot_stage.rmcall_gr_probe.pages ? + &boot_stage.rmcall_gr_probe : + NULL, + boot_stage.rmcall_gr_probe.pages ? + GSP_PHYS_RMCALL_VA_GR_PROBE : 0, + &sub); if (ret) { pr_err("vgsp-boot: rm-call: GR submission refused before the doorbell (%d)\n", ret); @@ -16833,9 +16942,12 @@ EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_ce_copy); * the optional QMD-virt pair(2). Raised 512 -> 1024 (SESSION 153- * CONTINUED-53) so a 3D stream replay can carry the captured MME setup * prelude (<=512 words) ahead of the largest chunked window (<=448) in one - * block; the block still lands inside the one 4 KB chunk page it is copied - * to. */ -#define GSP_PHYS_RMCALL_GR_METHOD_WORDS 1024u + * block. SESSION 153-CONTINUED-66: 1024 -> 2048 because the matured prelude + * reached 901 words and the deferral test (window=390 + prelude=901 + sa=47 + * vs the bound) dropped it on every launch; the block now spans the module + * pushbuffer's TWO 4 KB pages (VA_PB..VA_PB2) and is copied page-wise by + * gsp_phys_gr_pb_copy(). */ +#define GSP_PHYS_RMCALL_GR_METHOD_WORDS 2048u static u64 gsp_phys_gr_submitted; static u64 gsp_phys_gr_ok; @@ -21272,6 +21384,45 @@ next: } } +/* + * SESSION 153-CONTINUED-66: write a built method block into the module + * pushbuffer. The block may exceed one 4 KB page (GSP_PHYS_RMCALL_GR_METHOD_ + * WORDS is 2048), so copy page-wise: page 0 is the registered BUF_PB chunk, + * page 1 is the PTE-planted rmcall_gr_pb2 page at VA_PB2. The two pages are + * contiguous in GPU VA (VA_PB..VA_PB+0x2000), so the GPFIFO entry's single + * fetch address + length walks both. A short/absent page-2 plant leaves the + * tail unwritten and the caller's bound check keeps such launches <= 4 KB. + */ +static void gsp_phys_gr_pb_copy(const u32 *methods, u32 words) +{ + struct gsp_phys_boot_chunk *pb = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB]; + u32 per = GSP_PHYS_PAGE_SIZE / 4u; + u32 done = 0; + + while (done < words) { + struct gsp_phys_boot_chunk *c; + u32 page = done / per; + u32 off = done % per; + u32 n = words - done; + u8 *dst; + + if (n > per - off) + n = per - off; + if (!page) + c = pb; + else if (page == 1u) + c = &boot_stage.rmcall_gr_pb2; + else + return; + dst = gsp_phys_chunk_page_va(c, 0); + if (!dst) + return; + memcpy(dst + (size_t)off * 4u, methods + done, (size_t)n * 4u); + done += n; + } +} + int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, struct gsp_phys_backend_copy_result *out) { @@ -22826,7 +22977,7 @@ gr_page_unmapped: win3d, win3d_n, gsp_phys_gr_attr_pre, gsp_phys_gr_attr_pre_n); - memcpy(gsp_phys_chunk_page_va(pb, 0), methods, words * 4); + gsp_phys_gr_pb_copy(methods, words); /* SESSION 153-CONTINUED-12: identify this block to a later PBDMA dump. */ WRITE_ONCE(gsp_phys_gr_last_words, words); WRITE_ONCE(gsp_phys_gr_last_h, req->h_object); @@ -24360,8 +24511,8 @@ gr_page_unmapped: win3d_n, gsp_phys_gr_attr_pre, gsp_phys_gr_attr_pre_n); - memcpy(gsp_phys_chunk_page_va(pb, 0), - methods, words * 4); + gsp_phys_gr_pb_copy(methods, + words); } if (qmdwin_swap) pr_info("vgsp-gr: outcome retry %u: release words zeroed + fresh qmdwin bank %u dest 0x%llx + win0 bank %u%s + PB rebuilt\n", @@ -25958,6 +26109,8 @@ static void gsp_phys_rmcall_fini(struct pci_dev *pdev) gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_pt[i]); gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_shadow); gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_qmdwin); + gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_pb2); + gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_probe); /* * B-2: the old 208-page bounce window (previously leaked here) and the * grown data-bounce region chunks. The region's PT chunk and data chunk @@ -28016,6 +28169,20 @@ static_assert(((GSP_PHYS_RMCALL_VA_GPFIFO >> 12) & 0x1ff) != (((GSP_PHYS_RMCALL_VA_GPFIFO >> 12) & 0x1ff) + 4) == ((GSP_PHYS_RMCALL_VA_SEM >> 12) & 0x1ff), "the GPFIFO, pushbuffer and CE src/dst/semaphore VAs must take the five consecutive small-PT slots 0x100..0x104"); +/* + * SESSION 153-CONTINUED-66: the two-page pushbuffer and the moved CE SRC / + * probe pages must stay inside the same 2 MB PD0 slot and the same 128 B PTE + * fetch line as the ladder, and must not collide with each other or with the + * QMD-window pair (0x107/0x108). + */ +static_assert(((GSP_PHYS_RMCALL_VA_PB2 >> 12) & 0x1ff) == + ((GSP_PHYS_RMCALL_VA_PB >> 12) & 0x1ff) + 1, + "the pushbuffer's second page must sit directly above the first (one contiguous GPU VA fetch)"); +static_assert(((GSP_PHYS_RMCALL_VA_SRC >> 21) & 0xff) == 0 && + ((GSP_PHYS_RMCALL_VA_GR_PROBE >> 21) & 0xff) == 0 && + ((GSP_PHYS_RMCALL_VA_SRC >> 12) & 0x1ff) == 0x109 && + ((GSP_PHYS_RMCALL_VA_GR_PROBE >> 12) & 0x1ff) == 0x10a, + "the moved CE SRC page and the warm-semaphore probe page must take the free slots 0x109/0x10a in the ladder's PTE line"); static_assert(((GSP_PHYS_RMCALL_VA_GUEST_PT_BASE >> 47) & 0x3) == 0 && ((GSP_PHYS_RMCALL_VA_GUEST_PT_BASE >> 38) & 0x1ff) == 0 && ((GSP_PHYS_RMCALL_VA_GUEST_PT_BASE >> 29) & 0x1ff) == 0 && @@ -28064,8 +28231,8 @@ static_assert(GSP_PHYS_RMCALL_GR_INIT_WORDS + GSP_PHYS_RMCALL_GR_METHOD_WORDS, "the verbatim GR launch block minimum (compute init + window + PCAS + REPORT_SEMAPHORE) must fit the bound; the per-launch runtime check bounds raw_qmd_hdr for larger preludes"); static_assert(GSP_PHYS_RMCALL_GR_METHOD_WORDS * 4u <= - GSP_PHYS_RMCALL_COPY_SIZE, - "the GR launch method block must fit one module pushbuffer page"); + 2u * GSP_PHYS_RMCALL_COPY_SIZE, + "the GR launch method block may span the two module pushbuffer pages (SESSION 153-CONTINUED-66: the matured MME prelude no longer fits one page)"); /* * GspSystemInfo wire ABI, pinned against the reference header compile -- cgit v1.2.3