diff options
| -rw-r--r-- | kernel/gsp_phys_booter.c | 283 | ||||
| -rw-r--r-- | kernel/gsp_phys_booter.h | 19 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.c | 22 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.h | 4 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 145 |
5 files changed, 383 insertions, 90 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 9eb1179..b2302bc 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -2304,6 +2304,11 @@ static struct gsp_phys_boot_stage { * walker resolves the commit-time tree only). The launch copies the * kernel's data pages in, rewrites the pointers to these VAs (reloc * base 0x110000), and copies the results back after the release. + * B-2: with the grown data region on (gsp_phys_gr_data_region=1, the + * default) this window no longer carries operand data - only the mail + * slot and marker - but it stays allocated so the mail injection and + * the copy diagnostics are unchanged, and it remains the whole data + * region with the parameter off. * * The window is 208 pages so the data region below the mail page can * carry the operand pages of kernels whose buffers exceed the old @@ -2316,6 +2321,18 @@ static struct gsp_phys_boot_stage { */ struct gsp_phys_boot_chunk rmcall_gr_bounce; u32 rmcall_gr_bounce_used; + /* + * B-2: grown boot-fixed data-bounce region (gsp_phys_gr_data_region=1, + * the default; see the constants in gsp_phys_booter.h). One 16384-page + * (64 MiB) data chunk plus GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PT_PAGES 4 KB + * PT pages (one per 2 MB window), all planted once at stage 3 before + * any engine fetch. The old 208-page window above keeps its mail slot + * 206 and marker 207 exactly as-is (the mail injection and copy + * diagnostics rely on them); when the grown region is on it no longer + * carries operand data. + */ + struct gsp_phys_boot_chunk rmcall_gr_data_bounce; + struct gsp_phys_boot_chunk rmcall_gr_data_bounce_pt; #define GSP_PHYS_RMCALL_GR_BOUNCE_PAGES 208u #define GSP_PHYS_RMCALL_GR_BOUNCE_SLOT0 0x110u #define GSP_PHYS_RMCALL_GR_BOUNCE_VA0 0x110000ULL @@ -2345,6 +2362,51 @@ static_assert(GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE + static_assert(GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS == 2u * GSP_PHYS_RMCALL_GR_SHADOW0_BANK_SLOTS, "win0 shadow pool must be exactly two banks"); +/* + * B-2: the grown data-bounce region is a hard VA-layout contract. One fresh + * PT page per 2 MB window; PD0 slots immediately above the m152 extension + * pool's 32..79 under the same root/PD2/PD1 chain as the ladder; the whole + * region below the non-shared VAS vaSize (a non-shared VAS addresses PD0 + * entries 0..127 = 256 MB, so the region's 64 MB is a 32 MB-slack fit); and + * disjoint from every other planted window in the PDB (old bounce 0x110000.. + * 0x1dffff, win0 pool 0x1e0000.., guest scratch windows 0x200000..). Any + * future move of VA0/SLOT0/size fails at compile time. + */ +static_assert(GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES == + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PT_PAGES * 512u, + "the grown data-bounce region must be exactly one PT page per 2 MB window"); +static_assert(GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 == + (u64)GSP_PHYS_RMCALL_GR_DATA_BOUNCE_SLOT0 * (2ULL << 20), + "the grown data-bounce VA0 must equal its PD0 slot base"); +static_assert(((GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 >> 47) & 0x3) == 0 && + ((GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 >> 38) & 0x1ff) == 0 && + ((GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 >> 29) & 0x1ff) == 0 && + ((GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 >> 21) & 0xff) == + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_SLOT0 && + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_SLOT0 + + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PT_PAGES <= 128, + "the grown data-bounce region must use PD0 entries 80..111 under the ladder's root/PD2/PD1 chain"); +static_assert(GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 + + (u64)GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES * + GSP_PHYS_PAGE_SIZE <= + GSP_PHYS_RMCALL_VAS_SIZE, + "the grown data-bounce region must stay below the VAS vaSize limit"); +static_assert(GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 >= + GSP_PHYS_RMCALL_VA_GUEST_PT_EXT_BASE + + (u64)GSP_PHYS_RMCALL_GUEST_PT_EXT_PAGES * (2ULL << 20), + "the grown data-bounce region must start above the m152 extension scratch pool"); +static_assert(GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 >= + GSP_PHYS_RMCALL_GR_BOUNCE_VA0 + + (u64)GSP_PHYS_RMCALL_GR_BOUNCE_PAGES * + GSP_PHYS_PAGE_SIZE && + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 >= + GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE + + (u64)GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS * + GSP_PHYS_PAGE_SIZE && + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 >= + GSP_PHYS_RMCALL_VA_GUEST_PT_BASE + + (u64)GSP_PHYS_RMCALL_GUEST_PT_PAGES * (2ULL << 20), + "the grown data-bounce region must be disjoint from the old bounce window, the win0 pool and the guest scratch window"); /* m99: guest data identity-map table branches (see the PD*_MAX block). */ struct gsp_phys_boot_chunk rmcall_gr_data_pd1[GSP_PHYS_RMCALL_GR_DATA_PD1_MAX]; @@ -9705,6 +9767,77 @@ static void gsp_phys_rmcall_pdb_build(void) pr_err("vgsp-boot: rm-call: bounce window allocation failed\n"); } } + /* + * B-2: grown data-bounce region (gsp_phys_gr_data_region=1, default). + * One 16384-page (64 MiB) data chunk plus one fresh 4 KB PT page per + * 2 MB window; PD0 dual entries 80..111 in the client PDB (immediately + * above the m152 extension pool's 32..79), every leaf PTE pre-written + * with the chunk page's DMA address. The whole map is final at stage 3 + * - before the engine's first fetch - so the 128 B PTE fetch-line + * granule is fully populated at plant time (the GMMU has no invalidate + * path; a runtime PTE write would be invisible). One-shot like the + * bounce plant above: no later pdb_build() call re-zeroes the PD0 page, + * and the region's other writers (ext pool, ctx, win0) target disjoint + * slots. Planted only when the parameter is on; with 0 the SVC and the + * copy path stay on the legacy 205-page window. + */ + if (gsp_phys_gr_data_region_value() && + (!boot_stage.rmcall_gr_data_bounce.pages || + !boot_stage.rmcall_gr_data_bounce_pt.pages)) { + struct gsp_phys_boot_chunk *ptd = + &boot_stage.rmcall_gr_data_bounce_pt; + u32 i; + + if (!gsp_phys_chunk_alloc(boot_stage.pdev, + &boot_stage.rmcall_gr_data_bounce, + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES)) + gsp_phys_chunk_zero(&boot_stage.rmcall_gr_data_bounce); + if (!gsp_phys_chunk_alloc(boot_stage.pdev, ptd, + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PT_PAGES)) + gsp_phys_chunk_zero(ptd); + if (!boot_stage.rmcall_gr_data_bounce.pages || !ptd->pages) { + pr_err("vgsp-boot: rm-call: data-bounce region allocation failed (data=%u pt=%u pages); launches fail closed on the grown region\n", + boot_stage.rmcall_gr_data_bounce.pages, + ptd->pages); + } else { + for (i = 0; i < GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PT_PAGES; i++) { + u32 pd0_idx = + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_SLOT0 + i; + u64 pde = gsp_phys_rmcall_pde( + (u64)gsp_phys_chunk_page_dma(ptd, i)); + + put_unaligned_le64(0, + (u8 *)gsp_phys_chunk_page_va(pd0, 0) + + (size_t)pd0_idx * 16); + put_unaligned_le64(pde, + (u8 *)gsp_phys_chunk_page_va(pd0, 0) + + (size_t)pd0_idx * 16 + 8); + } + for (i = 0; i < GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES; i++) { + u32 win = i / 512u; + u32 slot = i % 512u; + + put_unaligned_le64(gsp_phys_rmcall_pte( + (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_gr_data_bounce, + i)), + (u8 *)gsp_phys_chunk_page_va(ptd, win) + + (size_t)slot * 8); + } + dma_wmb(); + pr_info("vgsp-boot: rm-call: data-bounce region planted: %u pages at VA 0x%llx..0x%llx (PD0 slots %u..%u, %u PT pages, before first fetch)\n", + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES, + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0, + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 + + (GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES - + 1) * GSP_PHYS_PAGE_SIZE, + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_SLOT0, + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_SLOT0 + + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PT_PAGES - + 1, + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PT_PAGES); + } + } /* * P6a guest windows: start from explicit zero (invalid PTE) for both @@ -16801,6 +16934,53 @@ static inline u64 gsp_phys_gr_fnv1a64(const u8 *p, size_t len) } /* + * B-2: the data chunk the active operand region lives in. With + * gsp_phys_gr_data_region=1 (default) that is the grown 16384-page region; + * with 0 it is the legacy 208-page bounce window (whose data pages sit below + * the mail slot). The copy path always asks this instead of naming a chunk so + * the two paths share one index space per mode. + */ +static struct gsp_phys_boot_chunk *gsp_phys_gr_data_chunk(void) +{ + if (gsp_phys_gr_data_region_value()) + return &boot_stage.rmcall_gr_data_bounce; + return &boot_stage.rmcall_gr_bounce; +} + +/* + * B-2: page index of a relocated operand VA inside the engine-visible data + * region, replacing the five inlined (reloc_va - 0x110000) >> 12 derivations + * and their MAIL_SLOT bounds. Grown region: VAs from + * GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0, bound + * GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES. Legacy window: VAs from + * GSP_PHYS_RMCALL_GR_BOUNCE_VA0, bound GSP_PHYS_RMCALL_GR_MAIL_SLOT so the + * mail (206) and marker (207) pages are never treated as data. Returns + * GSP_PHYS_GR_DATA_BI_NONE when the VA is outside the active region (the copy + * sites skip such records exactly as they skipped the old mail-slot bound). + */ +#define GSP_PHYS_GR_DATA_BI_NONE 0xffffffffu +static u32 gsp_phys_gr_data_bi(u64 reloc_va) +{ + u32 bi; + + if (gsp_phys_gr_data_region_value()) { + if (reloc_va < GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0) + return GSP_PHYS_GR_DATA_BI_NONE; + bi = (u32)((reloc_va - GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0) >> + 12); + if (bi >= GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES) + return GSP_PHYS_GR_DATA_BI_NONE; + return bi; + } + if (reloc_va < GSP_PHYS_RMCALL_GR_BOUNCE_VA0) + return GSP_PHYS_GR_DATA_BI_NONE; + bi = (u32)((reloc_va - GSP_PHYS_RMCALL_GR_BOUNCE_VA0) >> 12); + if (bi >= GSP_PHYS_RMCALL_GR_MAIL_SLOT) + return GSP_PHYS_GR_DATA_BI_NONE; + return bi; +} + +/* * m151: output retirement wait. qmd_rel is a schedule/fetch ack (the front * end latched and fetched the QMD), not a data-path retirement: the SM's * stores into the bounce pages can still be in flight when the ack lands. @@ -16825,7 +17005,15 @@ static noinline u32 gsp_phys_gr_output_retire_wait(const struct gsp_phys_gr_submit *req, const u32 *csum_in) { - u32 checked[GSP_PHYS_RMCALL_GR_MAIL_SLOT]; + /* + * B-2: one entry per page of the grown region (64 KB), static like the + * other submit-scope scratch: the only caller is the single-inflight + * ordered work item (see the static scratch note in + * gsp_phys_backend_guest_gr_submit). 64 KB must not sit on the kernel + * stack (this frame was already near the warning bound at 205 pages). + */ + static u32 checked[GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES]; + struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); u32 n_all = 0, di, iter; u32 changed = 0, prev = 0xffffffffu; const char *why = " (TIMEOUT)"; @@ -16834,12 +17022,11 @@ gsp_phys_gr_output_retire_wait(const struct gsp_phys_gr_submit *req, if (!req->n_data) return 0; for (di = 0; di < req->n_data; di++) { - u32 bi = (u32)((req->data[di].reloc_va - - GSP_PHYS_RMCALL_GR_BOUNCE_VA0) >> 12); + u32 bi = gsp_phys_gr_data_bi(req->data[di].reloc_va); u32 k; bool dup = false; - if (bi >= GSP_PHYS_RMCALL_GR_MAIL_SLOT) + if (bi == GSP_PHYS_GR_DATA_BI_NONE) continue; for (k = 0; k < n_all; k++) if (checked[k] == bi) { @@ -16857,7 +17044,7 @@ gsp_phys_gr_output_retire_wait(const struct gsp_phys_gr_submit *req, changed = 0; for (j = 0; j < n_all; j++) { const u8 *src = gsp_phys_chunk_page_va( - &boot_stage.rmcall_gr_bounce, checked[j]); + dchunk, checked[j]); u32 ci, csum = 0; if (!src) @@ -16897,20 +17084,19 @@ gsp_phys_gr_output_retire_wait(const struct gsp_phys_gr_submit *req, static void gsp_phys_gr_fill_bounce(const struct gsp_phys_gr_submit *req, u32 *csum_in) { + struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); u32 di; for (di = 0; di < req->n_data; di++) { - u32 bi = (u32)((req->data[di].reloc_va - - GSP_PHYS_RMCALL_GR_BOUNCE_VA0) >> 12); + u32 bi = gsp_phys_gr_data_bi(req->data[di].reloc_va); const u8 *src; u8 *dst; u32 ci, csum = 0; - if (bi >= GSP_PHYS_RMCALL_GR_MAIL_SLOT || - !req->data[di].hpa) + if (bi == GSP_PHYS_GR_DATA_BI_NONE || !req->data[di].hpa) continue; src = (const u8 *)__va(req->data[di].hpa); - dst = gsp_phys_chunk_page_va(&boot_stage.rmcall_gr_bounce, bi); + dst = gsp_phys_chunk_page_va(dchunk, bi); if (!dst) continue; memcpy(dst, src, GSP_PHYS_PAGE_SIZE); @@ -16958,14 +17144,16 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, static u32 sked_tick; static bool sked_first; /* - * m142: per-bounce-page checksums captured at copy-in. The copy-out + * m142: per-data-page checksums captured at copy-in. The copy-out * compares them to report how many pages the engine actually modified; * an all-unchanged window after a reported completion is the * "front-end ran, grid never dispatched / read zeros" signature that - * previously masqueraded as a successful launch. Safe as static: the - * only caller is the single-inflight ordered gsp-phys-work item. + * previously masqueraded as a successful launch. B-2: one entry per + * page of the grown region (64 KB, static - must not sit on the kernel + * stack). Safe as static: the only caller is the single-inflight + * ordered gsp-phys-work item. */ - static u32 bounce_csum_in[GSP_PHYS_RMCALL_GR_MAIL_SLOT]; + static u32 bounce_csum_in[GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES]; sked_tick = 0; sked_first = true; guest_param_pg = NULL; @@ -17775,20 +17963,18 @@ gr_page_unmapped: * removes the race entirely; per-page checksums are kept so the * copy-out can prove whether the engine modified the window at all. */ - if (req->n_data && boot_stage.rmcall_gr_bounce.pages) { + if (req->n_data && gsp_phys_gr_data_chunk()->pages) { + struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); u32 dcopied = 0, dfailed = 0, di; gsp_phys_gr_fill_bounce(req, bounce_csum_in); for (di = 0; di < req->n_data; di++) { - u32 bi = (u32)((req->data[di].reloc_va - - GSP_PHYS_RMCALL_GR_BOUNCE_VA0) >> - 12); + u32 bi = gsp_phys_gr_data_bi(req->data[di].reloc_va); - if (bi >= GSP_PHYS_RMCALL_GR_MAIL_SLOT || + if (bi == GSP_PHYS_GR_DATA_BI_NONE || !req->data[di].hpa) dfailed++; - else if (gsp_phys_chunk_page_va( - &boot_stage.rmcall_gr_bounce, bi)) + else if (gsp_phys_chunk_page_va(dchunk, bi)) dcopied++; } pr_info("vgsp-gr: bounce copy-in: %u pages, %u skipped (pre-publish)\n", @@ -18385,7 +18571,7 @@ gr_page_unmapped: * false-fired on correct launches and would re-execute * side-effecting kernels (atomics) up to 3x. */ - if (req->n_data && boot_stage.rmcall_gr_bounce.pages) { + if (req->n_data && gsp_phys_gr_data_chunk()->pages) { u32 wrote = gsp_phys_gr_output_retire_wait(req, bounce_csum_in); if (gsp_phys_rmcall_gr_nowrite_retry_value()) { @@ -18458,34 +18644,33 @@ gr_page_unmapped: * request). All mapped pages are written back; inputs are unchanged * by the kernels, so copying them back is a no-op. */ - if (req->n_data && boot_stage.rmcall_gr_bounce.pages) { + if (req->n_data && gsp_phys_gr_data_chunk()->pages) { + struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); u32 wbi, wrote = 0, changed = 0; u32 chg[8] = { }; for (wbi = 0; wbi < req->n_data; wbi++) { - u32 bi = (u32)((req->data[wbi].reloc_va - - GSP_PHYS_RMCALL_GR_BOUNCE_VA0) >> - 12); + u32 bi = gsp_phys_gr_data_bi(req->data[wbi].reloc_va); const u8 *src; u32 ci, csum = 0; - if (bi >= GSP_PHYS_RMCALL_GR_MAIL_SLOT || + if (bi == GSP_PHYS_GR_DATA_BI_NONE || !req->data[wbi].hpa) continue; - src = gsp_phys_chunk_page_va( - &boot_stage.rmcall_gr_bounce, bi); + src = gsp_phys_chunk_page_va(dchunk, bi); for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) csum += get_unaligned_le32(src + ci); if (csum != bounce_csum_in[bi]) { /* - * m142: which pages the engine touched. The - * indexes name the relocated VA (bi page of - * 0x110000); a c-range write is bi >= 0x40. - * Also dump the first words of the first two - * changed pages: float sums prove the kernel - * wrote them (the c-pointer landed on this - * buffer); pointers/QMD words prove something - * else wrote. + * m142: which pages the engine touched. bi is + * the page index inside the active data region + * (grown: VA 0x0a000000 + bi*4096; legacy: + * 0x110000 + bi*4096). Also dump the first + * words of the first two changed pages: float + * sums prove the kernel wrote them (the + * c-pointer landed on this buffer); + * pointers/QMD words prove something else + * wrote. */ if (changed < 2) pr_info("vgsp-gr: changed page bi=%u w0..3=%08x %08x %08x %08x\n", @@ -18503,23 +18688,21 @@ gr_page_unmapped: wrote++; } /* m143b: unconditional first words of the first page of each - * 32-page relocation window (a=0x110000, b=0x130000, c=0x150000) - * after the run, so the engine's read/write footprint per window - * is provable without relying on the changed-page counter. */ + * 32-page relocation window after the run, so the engine's + * read/write footprint per window is provable without relying + * on the changed-page counter. */ { u32 wi; for (wi = 0; wi + 1 < req->n_data; wi += 32) { - u32 bi = (u32)((req->data[wi].reloc_va - - GSP_PHYS_RMCALL_GR_BOUNCE_VA0) >> - 12); + u32 bi = gsp_phys_gr_data_bi( + req->data[wi].reloc_va); const u8 *wp; u32 wc = 0, k; - if (bi >= GSP_PHYS_RMCALL_GR_MAIL_SLOT) + if (bi == GSP_PHYS_GR_DATA_BI_NONE) continue; - wp = gsp_phys_chunk_page_va( - &boot_stage.rmcall_gr_bounce, bi); + wp = gsp_phys_chunk_page_va(dchunk, bi); for (k = 0; k < GSP_PHYS_PAGE_SIZE; k += 4) wc += get_unaligned_le32(wp + k); pr_info("vgsp-gr: window[%u] reloc=0x%llx first=%08x %08x %08x %08x changed=%u\n", @@ -19420,6 +19603,14 @@ static void gsp_phys_rmcall_fini(struct pci_dev *pdev) gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_pt[i]); gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_shadow); gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_qmdwin); + /* + * B-2: the old 208-page bounce window (previously leaked here) and the + * grown data-bounce region chunks. The region's PT chunk and data chunk + * are freed only when allocated. + */ + gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_bounce); + gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_data_bounce); + gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_data_bounce_pt); gsp_phys_chunk_free(pdev, &boot_stage.rmcall_patch); gsp_phys_chunk_free(pdev, &boot_stage.rmcall_pm); gsp_phys_chunk_free(pdev, &boot_stage.rmcall_fecs); diff --git a/kernel/gsp_phys_booter.h b/kernel/gsp_phys_booter.h index 896a436..fb7b2a4 100644 --- a/kernel/gsp_phys_booter.h +++ b/kernel/gsp_phys_booter.h @@ -349,6 +349,25 @@ void gsp_phys_backend_relay_counters(u64 *requested, u64 *ok, u64 *verify_fail, #define GSP_PHYS_GR_QMD_WORD_NONE 0xffffffffu #define GSP_PHYS_GR_RAW_WORDS 256u +/* + * B-2 (large-operand data path): the grown, boot-fixed GR data-bounce region. + * One PD0 dual entry per 2 MB window (PD0 slots 80..111, immediately above + * the m152 extension scratch pool's 32..79) backed by one fresh 4 KB small-PT + * page each, every leaf PTE pointing at one of the 16384 pages (64 MiB) of a + * single data chunk. The whole map is planted once at stage 3, before any + * engine fetch, so the GMMU's no-invalidate 128 B fetch-line rule is + * satisfied by construction (no PTE is ever rewritten at runtime). The SVC + * assigns operand reloc VAs from VA0 upward; gsp_sema_service.c derives its + * reloc base and page cap from these constants so the two sides cannot drift + * (the pre-B-2 205-page window below the mail page stays allocated for the + * mail injection and diagnostics, it just no longer carries operand data when + * gsp_phys_gr_data_region=1, the default). + */ +#define GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 0x0A000000ULL +#define GSP_PHYS_RMCALL_GR_DATA_BOUNCE_SLOT0 80u +#define GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PT_PAGES 32u +#define GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES (32u * 512u) /* 16384 */ + struct gsp_phys_gr_page { u64 guest_va; /* guest VA (page aligned) */ u64 hpa; /* host PA (page aligned; 0 = unresolved) */ diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c index 9b7235b..7932268 100644 --- a/kernel/gsp_phys_gpu.c +++ b/kernel/gsp_phys_gpu.c @@ -1075,6 +1075,28 @@ int gsp_phys_rmcall_gr_data_prewin_value(void) EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_data_prewin_value); /* + * B-2: grown boot-fixed data-bounce region (gsp_phys_gr_data_region; default + * 1). When set, gsp_phys_rmcall_pdb_build plants the 16384-page (64 MiB) + * region at GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 (PD0 80..111) and the SVC + * relocates operand pages into it; the booter's copy-in/copy-out index the + * grown chunk. When 0, everything falls back to the legacy 205-page / + * 68-pages-per-pointer window below the mail page, for one-flip A/B + * trialing. Boot-fixed: it is read by the stage-3 plant, so set it at + * insmod, not at runtime. + */ +static int gsp_phys_gr_data_region_param = 1; +module_param_named(gsp_phys_gr_data_region, + gsp_phys_gr_data_region_param, int, 0644); +MODULE_PARM_DESC(gsp_phys_gr_data_region, + "Use the grown 64 MiB data-bounce region for GR operands (default 1; 0 = legacy 205-page window)"); + +int gsp_phys_gr_data_region_value(void) +{ + return gsp_phys_gr_data_region_param; +} +EXPORT_SYMBOL_GPL(gsp_phys_gr_data_region_value); + +/* * m113: after planting the launch's data mappings, re-register the VAS page * directory (UNSET+SET) to force the server walker to re-snapshot the tree; * all evidence says the SM walker resolves only the commit-time tree, and diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h index 73b0822..84b56d6 100644 --- a/kernel/gsp_phys_gpu.h +++ b/kernel/gsp_phys_gpu.h @@ -352,6 +352,10 @@ int gsp_phys_rmcall_gr_mail_patch_value(void); /* m106: pre-install the data PT window pool in stage 3. */ int gsp_phys_rmcall_gr_data_prewin_value(void); +/* B-2: use the grown 64 MiB boot-fixed data-bounce region (default 1; 0 = + * legacy 205-page window). Read by the stage-3 plant and the SVC fill. */ +int gsp_phys_gr_data_region_value(void); + /* m113: re-register the VAS page directory after planting data. */ int gsp_phys_rmcall_gr_recommit_value(void); diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index f8925df..baac8d4 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -3924,11 +3924,11 @@ static struct svc_fwd_cand svc_fwd_cand; /* * m99: guest data identity-map capacity. The forwarded kernel's global data * pages (discovered by scanning the param/const-0 page for device VAs) are - * resolved on the service thread and identity-mapped into the engine tree. - * 16K pages = 64 MB of window coverage, comfortably above the demo buffers - * (2 x 16 MB observed) with headroom. + * resolved on the service thread and relocated into the engine-visible + * region. B-2: 32K records = 2x the grown region's 16384 pages of headroom, + * so the hand-off array is never the binding limit. */ -#define SVC_GR_DATA_MAX 16384 +#define SVC_GR_DATA_MAX 32768 #define SVC_GR_PUSH_WORDS 256 #define SVC_GR_QMD_WORD_NONE 0xffffffffu /* svc_gr_cand.rel_flags bits (decoded from qmd[24] and qmd[4]). */ @@ -12360,32 +12360,36 @@ static bool svc_gr_page_inline_covers(u64 page) * Resolution only happens on the service thread; the work item turns the * guest PAs into HPAs and the backend plants the identity PTEs. */ -#define SVC_GR_DATA_RELOC_BASE 0x110000ULL +/* + * B-2 (large-operand data path): the SVC assigns each resolved operand page + * a reloc VA in the grown boot-planted region (the layout constants live in + * gsp_phys_booter.h so the two sides cannot drift; gsp_phys_gr_data_region + * selects grown vs legacy, default grown). The legacy 0x110000 window below + * the mail page (205 data pages max, 68 per pointer) remains selectable with + * gsp_phys_gr_data_region=0 for A/B trialing. + */ +#define SVC_GR_DATA_RELOC_BASE GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 +#define SVC_GR_DATA_RELOC_BASE_OLD 0x110000ULL /* * Per candidate pointer, map at most this many pages from the pointer's - * page onward (the engine-visible main-PT window has ~220 free 4 KB slots; - * the demo buffers are small on the native path). - * m120: lowered 64 -> 32 so every pointer page fits under the 100-page - * bounce cap (mail 2x32=64, vadd 3x32=96); at 64/ptr the second/third - * candidate run was truncated by the cap and its pointer word never - * rewritten (m119: first LDG base c[0x0][0x168] possibly untouched). - * m140: 32 -> 33. A cudaMalloc'd 128 KB buffer whose pointer is 0x400 - * into its page spans 33 pages (32 full + the 1 KB tail), and the kernel - * accesses the tail: with 32 pages the tail read the NEXT candidate's - * relocated page (wrong values; vadd tails). 3x33=99 <= the 100-page cap. - */ -/* - * m150 (large buffers): the bounce window was grown to 208 pages (mail page - * at slot 206). 68 pages per candidate/pointer covers a 256 KB cudaMalloc'd - * operand whose pointer sits 0x400 into its page (64 full + offset + tail), - * and up to three such operands (3x68 = 204 <= 205) fit below the mail page. - * Larger-than-256 KB operands still hit the cap; they need a chunked data - * path (future work). - */ -#define SVC_GR_DATA_PAGES_PER_PTR 68u -/* Total pages the pre-commit bounce window can carry below the mail page - * (GSP_PHYS_RMCALL_GR_MAIL_SLOT = 206). */ -#define SVC_GR_DATA_BOUNCE_MAX 205u + * page onward. With the grown region this is a runaway guard, not a budget: + * a cold page still stops the run (stop-on-miss below), so a real operand's + * span is bounded by its extent. History: 64 -> 32 (m120) -> 33 (m140) under + * the old 100-page cap -> 68 (m150, 256 KB operands under the 205-page + * window); the old value is kept for the legacy path. + */ +#define SVC_GR_DATA_PAGES_PER_PTR 4096u +#define SVC_GR_DATA_PAGES_PER_PTR_OLD 68u +/* + * Total operand pages the engine-visible data region can carry; derived from + * the booter's shared constant (no drift). A cap hit is FAIL-CLOSED: the + * fill marks the launch stalled for the hold/retry path and returns instead + * of silently truncating later pointers - a truncated operand must never + * reach the engine. The legacy value is the pre-B-2 window below the mail + * page (GSP_PHYS_RMCALL_GR_MAIL_SLOT = 206). + */ +#define SVC_GR_DATA_BOUNCE_MAX GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES +#define SVC_GR_DATA_BOUNCE_MAX_OLD 205u static u32 svc_gr_data_log; /* m142: bounded per-page attribution log */ static u32 svc_gr_src_log; /* m150: bounded per-candidate source log */ @@ -12515,10 +12519,23 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) { u64 wins[16]; bool is_arg[16]; /* m150: candidate came from the argument scan */ - u32 n_win = 0, n_cand; + u32 n_cand; u32 off = 0, i, w; u8 kind; /* + * B-2: active data-region geometry. gsp_phys_gr_data_region=1 (the + * default) selects the grown 16384-page region planted by the booter + * at GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0; 0 keeps the legacy 205-page + * window below the mail page for A/B trialing. + */ + bool region_on = gsp_phys_gr_data_region_value() != 0; + u64 reloc_base = region_on ? SVC_GR_DATA_RELOC_BASE : + SVC_GR_DATA_RELOC_BASE_OLD; + u32 per_ptr = region_on ? SVC_GR_DATA_PAGES_PER_PTR : + SVC_GR_DATA_PAGES_PER_PTR_OLD; + u32 bounce_max = region_on ? SVC_GR_DATA_BOUNCE_MAX : + SVC_GR_DATA_BOUNCE_MAX_OLD; + /* * m150: high (user/device-space) argument candidates and how many of * them produced at least one resolved page. None resolving is the * second-client zeros signature: the engine would run the kernel @@ -12623,9 +12640,11 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) * inline-captured bytes as well (see svc_gr_scan_inline). */ svc_gr_scan_inline(wins, &n_cand, pp->va); } - /* m110: one contiguous run of pages per candidate pointer, relocated - * into the engine-walkable main-PT window (0x110000 onward, fresh - * PTE lines; the commit-time snapshot only preserves that window). */ + /* m110/B-2: one contiguous run of pages per candidate pointer, + * relocated into the active engine-visible data region (grown region + * 0x0a000000+ with gsp_phys_gr_data_region=1 - the default - or the + * legacy 0x110000 window; both are boot-planted, never-rewritten PTE + * ranges the commit-time snapshot preserves). */ for (w = 0; w < n_cand; w++) { u64 base = wins[w]; u32 p; @@ -12640,17 +12659,46 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) * operands (high, cudaMalloc-shaped VAs) get the full per-pointer * span. Low-VA candidates (the launch's own QMD/release/reloc * pointers) need a single page at most, and the launch's QMD VA - * is not a kernel operand at all - relocating 68 pages for it - * starved the operands that follow and blew the 205-page cap + * is not a kernel operand at all - relocating a full span for it + * starved the operands that follow and blew the old window cap * (r141: a 256 KB operand resolved 0 pages because the QMD and - * param candidates consumed the window). Excluding them lets - * three 256 KB operands (3x68 = 204) fit. + * param candidates consumed the window). */ if ((base & ~(u64)(SEMA_PAGE_SIZE - 1)) == (svc_fwd_req.gr_qmd_dev_va & ~(u64)(SEMA_PAGE_SIZE - 1))) continue; - span = (base >= 0x40000000000ULL) ? - SVC_GR_DATA_PAGES_PER_PTR : 1u; + /* + * R3 hygiene (large-buffer budget discipline): the launch's own + * param/bank pages (the qmd_word 32..47 gr_pages entries) and + * the param VA are driver bookkeeping, not kernel operands - the + * observed fills resolved a full 68-page span for the param page + * in every launch and starved the real operands under the old + * window. Skip them as data candidates; they are + * shadowed/rewritten through gr_pages/Tier 2, not this path. + */ + if (svc_fwd_req.gr_param_va && + (base & ~(u64)(SEMA_PAGE_SIZE - 1)) == + (svc_fwd_req.gr_param_va & ~(u64)(SEMA_PAGE_SIZE - 1))) + continue; + { + bool launch_internal = false; + + for (i = 0; i < svc_fwd_req.gr_n_pages; i++) { + const struct svc_gr_page *pp = + &svc_fwd_req.gr_pages[i]; + + if (pp->qmd_word >= 32u && + pp->qmd_word <= 47u && + (base & ~(u64)(SEMA_PAGE_SIZE - 1)) == + (pp->va & ~(u64)(SEMA_PAGE_SIZE - 1))) { + launch_internal = true; + break; + } + } + if (launch_internal) + continue; + } + span = (base >= 0x40000000000ULL) ? per_ptr : 1u; if (is_arg[w] && base >= 0x40000000000ULL) n_arg_hi++; @@ -12699,16 +12747,26 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) break; } } - if (svc_fwd_req.gr_n_data >= SVC_GR_DATA_BOUNCE_MAX) { - pr_info("vgsp-sema: svc-gr data fill: %u candidate ptrs, %u pages resolved (reloc 0x%llx+) CAP-HIT\n", + if (svc_fwd_req.gr_n_data >= bounce_max) { + /* + * B-2 fail-closed cap: the active region is + * full and later pointers would be silently + * dropped (the old CAP-HIT return forwarded + * a truncated operand set). Mark the launch + * stalled so svc_gr_request holds/retries it + * via the existing bounded hold instead of + * forwarding incomplete data. + */ + svc_gr_data_stalled = true; + pr_err("vgsp-sema: svc-gr data fill: %u candidate ptrs, %u pages resolved (reloc 0x%llx+) CAP-HIT: region full, launch held fail-closed (no silent truncation)\n", n_cand, svc_fwd_req.gr_n_data, - (u64)SVC_GR_DATA_RELOC_BASE); + reloc_base); return; } rec = &svc_fwd_req.gr_data[svc_fwd_req.gr_n_data]; rec->va = va; rec->pa = pa & ~(u64)(SEMA_PAGE_SIZE - 1); - rec->reloc_va = SVC_GR_DATA_RELOC_BASE + + rec->reloc_va = reloc_base + ((u64)svc_fwd_req.gr_n_data << 12); rec->ap = (u8)svc_ap_of(ap); svc_fwd_req.gr_n_data++; @@ -12773,8 +12831,7 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) n_cand, svc_fwd_req.gr_n_data, n_arg_hi); else pr_info("vgsp-sema: svc-gr data fill: %u candidate ptrs, %u pages resolved (reloc 0x%llx+)\n", - n_cand, svc_fwd_req.gr_n_data, - (u64)SVC_GR_DATA_RELOC_BASE); + n_cand, svc_fwd_req.gr_n_data, reloc_base); } static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, |