diff options
Diffstat (limited to 'kernel/gsp_phys_booter.c')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 852 |
1 files changed, 725 insertions, 127 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 72cda10..9eb1179 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -999,16 +999,34 @@ bool gsp_phys_boot_requested(void) #define GSP_PHYS_RMCALL_VA_DST (GSP_PHYS_RMCALL_VA_BASE + 3 * GSP_PHYS_PAGE_SIZE) #define GSP_PHYS_RMCALL_VA_SEM (GSP_PHYS_RMCALL_VA_BASE + 4 * GSP_PHYS_PAGE_SIZE) /* - * m69: window-0 shadow pool - 8 module pages at VAs 0x108000..0x10f000 - * (small-PT slots 0x108..0x10f, the same 128 B PTE fetch line as the - * GPFIFO/PB/QMDWIN slots, planted before the engine's first fetch). The - * Tier-2 copies of the QMD's program/param pages are placed here so the GR - * engine never has to read through the window-1 scratch VA range whose PT - * chain (DMA_UPDATE_PDE_2 pdeIndex=1) has never been proven engine-readable. + * m69: window-0 shadow pool - 16 module pages holding the Tier-2 copies of + * the QMD's program/param pages, so the GR engine never has to read through + * the window-1 scratch VA range whose PT chain (DMA_UPDATE_PDE_2 + * pdeIndex=1) has never been proven engine-readable. The pool is split + * into two 8-page banks; the forwarded launch alternates banks per launch + * (m151) so one launch's copy-in can never clobber the pages the previous + * launch's engine fetches may still be reading (late-fetch safety). + * + * The base MUST stay clear of the m114/m150 GR data bounce window + * (GSP_PHYS_RMCALL_GR_BOUNCE_VA0 .. +PAGES, main-PT slots 0x110..0x1df): + * this pool is re-planted on every gsp_phys_rmcall_pdb_build() while the + * bounce PTEs are a one-shot plant, so any overlap lets the later win0 + * plant steal the bounce slots and the engine writes results into the + * shadow pages, losing them. 0x1e0000 (slots 0x1e0..0x1ef) sits directly + * above the last bounce/mail/marker page (0x1df000) and below the + * mail-injection slot 0x1f0 and the guest scratch window (0x200000). Keep + * it that way (see the static_assert next to the bounce constants). */ #define GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE \ - (GSP_PHYS_RMCALL_VA_BASE + 0x90000ULL) -#define GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS 8u + (GSP_PHYS_RMCALL_VA_BASE + 0xE0000ULL) +#define GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS 16u +/* + * m151: one bank = one launch's worth of QMD shadow pages. A forwarded + * launch consumes at most eight pages (param via qmd_word 32, program 48, + * bank 0/1/2/3/5/7 at qmd_word 34..46 even); the per-launch bounds check + * uses this so a launch can never spill into the other bank. + */ +#define GSP_PHYS_RMCALL_GR_SHADOW0_BANK_SLOTS 8u /* * m99: engine-tree branches for identity-mapping the forwarded kernel's * guest data pages. The walker uses our root[0] -> PD2 -> PD1 -> PD0 dual -> @@ -1100,6 +1118,33 @@ bool gsp_phys_boot_requested(void) /* Full 128 B fetch lines per 4 KB small-PT page: (4096/8)/16 = 32. */ #define GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE \ ((GSP_PHYS_PAGE_SIZE / 8u) / GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES) +/* + * m152: extended scratch-PT pool. The original 16 x 2 MB PD0 windows + * (512 never-reused 128 B fetch lines per boot) exhausted under a normal + * launch mix (each distinct guest page consumed a line per launch) and the + * exhaustion was permanent for the boot: every later launch that needed a + * new mapping was refused ("launch refused ... no Tier-1 mapping and no + * captured inline data"), which is the deterministic once-broken-forever + * failure the longevity runs kept hitting. Two changes fix the class: + * (a) GR alias cache (below): a consumed fetch line is keyed by the HPA + * it maps, so a repeat launch of the same buffers (or the same chunk + * page / semaphore page) reuses the identical never-rewritten line and + * consumes nothing; + * (b) a 48-window extension pool right above the ladder's free gap + * (window VA 0x04000000.., PD0 indices 32..79) backed by one 48-page + * PT chunk, raising capacity to 64 windows x 32 lines = 2048. + * lines/boot. Both are pure capacity mechanics: no PTE and no mapping any + * engine may have fetched is ever rewritten. + */ +#define GSP_PHYS_RMCALL_VA_GUEST_PT_EXT_BASE 0x04000000ULL +#define GSP_PHYS_RMCALL_GUEST_PT_EXT_PAGES 48u +#define GSP_PHYS_RMCALL_GUEST_PT_TOTAL_PAGES \ + (GSP_PHYS_RMCALL_GUEST_PT_PAGES + GSP_PHYS_RMCALL_GUEST_PT_EXT_PAGES) +#define GSP_PHYS_RMCALL_GUEST_PT_TOTAL_LINES \ + (GSP_PHYS_RMCALL_GUEST_PT_TOTAL_PAGES * \ + GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE) +/* HPA -> shadow-VA reuse cache; hit = no new fetch line is consumed. */ +#define GSP_PHYS_RMCALL_GR_ALIAS_CACHE 768u /* * GR forwarding (gsp_phys_gr_forward=1; default off) scratch resources, @@ -1253,8 +1298,21 @@ bool gsp_phys_boot_requested(void) #define GSP_PHYS_RMCALL_CHANNEL_FLAG_USER_LANE \ (u32)(GSP_PHYS_RMCALL_CHANNEL_FLAG_PRIVILEGED | (4u << 8) | \ (1u << 21)) -/* Captured user-lane GPFIFO depth (#825: 1024 entries). */ -#define GSP_PHYS_RMCALL_USER_LANE_GPFIFO_ENTRIES 1024u +/* + * User-lane GPFIFO depth. MUST equal the host ring's push modulus + * (GSP_PHYS_RMCALL_GPFIFO_ENTRIES, 8): the host publishes into the + * one-page ladder ring modulo 8, so if the channel declares a different + * count the engine's GET/PUT wrap desyncs at the host's first wrap + * (GP_PUT 7 -> 0 while the engine's GET keeps counting to 8) and every + * later launch on the boot is never fetched (r138: launches 1..7 green, + * the 8th and all after it silent with `qmd_rel=5 ch_rel=0`). The captured + * #825 value (1024) describes the real client's own much larger ring, not + * this emulated one; the campaign's 2026-10-06 "fidelity" change to 1024 + * was a wrap-only difference that broke exactly here (M1's two-launch + * acceptance never wrapped). Keep the two ends equal. + */ +#define GSP_PHYS_RMCALL_USER_LANE_GPFIFO_ENTRIES \ + GSP_PHYS_RMCALL_GPFIFO_ENTRIES #define GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_USER_LANE 0x1du #define GSP_PHYS_RMCALL_TSG_FAULT_BUF_SIZE 0x2000u #define GSP_PHYS_RMCALL_USER_LANE_PM_RAW_SIZE 0xd100u @@ -1379,6 +1437,58 @@ enum gsp_phys_rmcall_buf { GSP_PHYS_RMCALL_BUF_GUEST_PT13, GSP_PHYS_RMCALL_BUF_GUEST_PT14, GSP_PHYS_RMCALL_BUF_GUEST_PT15, + /* m152: extension pool PT pages: 48 separate single-page chunks + * (GUEST_PT_EXT0..47). The server only accepts single-page memdescs + * for client buffers (a 48-page chunk was refused with rpc_result + * 0x1f + a firmware ASSERT in stage 2). */ + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT0, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT1, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT2, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT3, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT4, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT5, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT6, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT7, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT8, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT9, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT10, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT11, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT12, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT13, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT14, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT15, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT16, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT17, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT18, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT19, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT20, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT21, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT22, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT23, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT24, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT25, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT26, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT27, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT28, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT29, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT30, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT31, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT32, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT33, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT34, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT35, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT36, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT37, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT38, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT39, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT40, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT41, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT42, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT43, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT44, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT45, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT46, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT47, /* * Golden9g: the three client page-table levels the real golden VAS * hands to the server with VASPACE_COPY_SERVER_RESERVED_PDES @@ -1409,6 +1519,20 @@ enum gsp_phys_rmcall_buf { * generous and progress is logged every ~500 ms. */ #define GSP_PHYS_RMCALL_GR_POLL_MS 5000u +/* + * Bounded retry of a forwarded GR launch whose QMD did not run. A green + * launch completes in ~170-195 ms; when the engine drops it (cold context + * save/unload, or an entry the PBDMA never fetched) `qmd_rel` never reaches + * the wanted value and the client would fall back to the software release + * and read stale/zero results. Because the timeout path never copies the + * result out to the guest, a launch that timed out did not reach the + * client: re-publishing a FRESH ring entry (same PB/QMD) and re-issuing the + * schedule+doorbell gives the front-end another chance and the guest still + * sees only the success path. Bounded so the service thread is never held + * past the existing hold budget. + */ +#define GSP_PHYS_RMCALL_GR_RETRY_MS 2000u +#define GSP_PHYS_RMCALL_GR_ATTEMPTS 3u /* P6a guest-memory CE copies: same bounded poll cap as the P5 shadow path. */ #define GSP_PHYS_RMCALL_GUEST_POLL_MS 200u @@ -2136,17 +2260,31 @@ static struct gsp_phys_boot_stage { bool rmcall_gr_exec_ok; /* a real GR submission proved out */ u32 rmcall_gr_submit_epoch; /* per-submission mapper epoch */ u32 rmcall_gr_pt_used; /* Tier-1 window pool slots used */ - u32 rmcall_guest_pt_win_mask; /* Tier-2 scratch windows installed */ + u64 rmcall_guest_pt_win_mask; /* Tier-2 scratch windows installed */ + /* + * m152: shadow alias reuse cache: hpa -> never-rewritten shadow VA. + * Populated by gsp_phys_rmcall_gr_map_shadow_locked; a hit returns the + * existing line so repeat launches of the same physical pages consume + * no new fetch line. Fill-order/LRU-free by design: entries are dead + * weight once a client exits, but a boot has 2048 lines to give and + * the cache only ever returns identical PTE content. + */ + u64 rmcall_gr_alias_hpa[GSP_PHYS_RMCALL_GR_ALIAS_CACHE]; + u64 rmcall_gr_alias_va[GSP_PHYS_RMCALL_GR_ALIAS_CACHE]; + u32 rmcall_gr_alias_count; struct gsp_phys_boot_chunk rmcall_gr_pt[GSP_PHYS_RMCALL_GR_PT_PAGES]; struct gsp_phys_boot_chunk rmcall_gr_shadow; /* Tier-2 pages */ /* * m69: window-0 shadow pool (gsp_phys_rmcall_gr_shadow_win0=1). - * Eight module pages whose PTEs are planted in the proven window-0 - * PT line at slots 0x108..0x10f (VAs 0x108000..0x10f000) by - * gsp_phys_rmcall_pdb_build, before the engine's first fetch (same - * pattern as the qmdwin slot). The launch copies the QMD's - * program/param page content into them and rewrites the QMD pointers - * there, so the GR engine never reads the window-1 scratch range. + * Sixteen module pages (two 8-page banks; m151 alternates the bank + * per launch so a launch's copy-in cannot clobber pages the previous + * launch's engine fetches may still be reading) whose PTEs are + * planted in the proven window-0 PT (slots derive from + * GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE) by gsp_phys_rmcall_pdb_build, + * before the engine's first fetch (same pattern as the qmdwin slot). + * The launch copies the QMD's program/param page content into them + * and rewrites the QMD pointers there, so the GR engine never reads + * the window-1 scratch range. */ struct gsp_phys_boot_chunk rmcall_gr_shadow_win0; u32 rmcall_gr_shadow_win0_used; @@ -2161,19 +2299,52 @@ static struct gsp_phys_boot_stage { /* m104: module page holding the injected mail_kernel message. */ struct gsp_phys_boot_chunk rmcall_gr_mail; /* - * m114: pre-commit bounce window. 112 pages planted into the main - * small-PT at slots 0x110..0x17f during stage 3 (the proven win0 - * pattern: the SM walker resolves the commit-time tree only). The - * launch copies the kernel's data pages in, rewrites the pointers to - * these VAs (reloc base 0x110000), and copies the results back after - * the release; page 100 (VA 0x174000) carries the mail message. + * m114: pre-commit bounce window, planted into the main small-PT at + * slots 0x110..0x1df during stage 3 (the proven win0 pattern: the SM + * walker resolves the commit-time tree only). The launch copies the + * kernel's data pages in, rewrites the pointers to these VAs (reloc + * base 0x110000), and copies the results back after the release. + * + * The window is 208 pages so the data region below the mail page can + * carry the operand pages of kernels whose buffers exceed the old + * 128 KB (victory_matmul's 256 KB buffers need 64 pages each; three + * of them plus argument/bank candidates exceed the old 100-page / + * 33-pages-per-pointer caps and the engine then ran on truncated + * windows). The mail page is the LAST data-free page (slot 206) and + * the m132 marker sits at 207; the main PT page deliberately covers + * the whole 0x110000..0x1df000 range with nothing else planted there. */ struct gsp_phys_boot_chunk rmcall_gr_bounce; u32 rmcall_gr_bounce_used; -#define GSP_PHYS_RMCALL_GR_BOUNCE_PAGES 112u +#define GSP_PHYS_RMCALL_GR_BOUNCE_PAGES 208u #define GSP_PHYS_RMCALL_GR_BOUNCE_SLOT0 0x110u #define GSP_PHYS_RMCALL_GR_BOUNCE_VA0 0x110000ULL -#define GSP_PHYS_RMCALL_GR_MAIL_SLOT 100u +#define GSP_PHYS_RMCALL_GR_MAIL_SLOT 206u +/* + * m150b: the win0 QMD shadow pool must never overlap the bounce window. + * The win0 plant runs on every pdb_build() (unconditional) and the bounce + * plant is one-shot, so an overlap silently hijacks bounce PTEs and the + * engine's results land in the shadow pages (victory_matmul lost C rows + * 0..31 exactly this way before 0x1e0000 moved the pool clear). + */ +static_assert(GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE + + (u64)GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS * GSP_PHYS_PAGE_SIZE <= + GSP_PHYS_RMCALL_GR_BOUNCE_VA0 || + GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE >= + GSP_PHYS_RMCALL_GR_BOUNCE_VA0 + + (u64)GSP_PHYS_RMCALL_GR_BOUNCE_PAGES * GSP_PHYS_PAGE_SIZE, + "win0 shadow pool must not overlap the GR data bounce window"); +/* + * m151: and the two-bank pool must end below the reserved mail-injection + * slot 0x1f0 (VA 0x1f0000) so the extra slots never steal that PTE. + */ +static_assert(GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE + + (u64)GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS * GSP_PHYS_PAGE_SIZE <= + GSP_PHYS_RMCALL_VA_BASE + 0xF0000ULL, + "win0 shadow pool must end below the reserved mail-injection slot"); +static_assert(GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS == + 2u * GSP_PHYS_RMCALL_GR_SHADOW0_BANK_SLOTS, + "win0 shadow pool must be exactly two banks"); /* m99: guest data identity-map table branches (see the PD*_MAX block). */ struct gsp_phys_boot_chunk rmcall_gr_data_pd1[GSP_PHYS_RMCALL_GR_DATA_PD1_MAX]; @@ -2813,6 +2984,7 @@ static void gsp_phys_rmcall_reset(void) boot_stage.rmcall_gr_submit_epoch = 0; boot_stage.rmcall_gr_pt_used = 0; boot_stage.rmcall_guest_pt_win_mask = 0; + boot_stage.rmcall_gr_alias_count = 0; memset(boot_stage.rmcall_gr_pt, 0, sizeof(boot_stage.rmcall_gr_pt)); memset(&boot_stage.rmcall_gr_shadow, 0, sizeof(boot_stage.rmcall_gr_shadow)); @@ -7215,6 +7387,65 @@ static void gsp_phys_rmcall_ctx_diag_dump(const char *what, static void gsp_phys_rmcall_ctx_diag_decode(const char *tag); /* + * m152: scratch-PT pool geometry helpers. Pool local page P (0-based across + * the original 16-window region then the 48-window extension) maps to a PD0 + * window VA and to a page of either the per-window single-page PT chunks + * (P < 16) or the multi-page extension chunk (P >= 16). Keep every + * consumer on these two helpers so the two regions can never drift. + */ +static u64 gsp_phys_rmcall_guest_pt_win_va(u32 page) +{ + if (page < GSP_PHYS_RMCALL_GUEST_PT_PAGES) + return boot_stage.rmcall_va_guest_pt + ((u64)page << 21); + return GSP_PHYS_RMCALL_VA_GUEST_PT_EXT_BASE + + ((u64)(page - GSP_PHYS_RMCALL_GUEST_PT_PAGES) << 21); +} + +static u8 *gsp_phys_rmcall_guest_pt_page_va(u32 page) +{ + if (page < GSP_PHYS_RMCALL_GUEST_PT_PAGES) + return gsp_phys_chunk_page_va( + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GUEST_PT0 + + page], 0); + if (page < GSP_PHYS_RMCALL_GUEST_PT_TOTAL_PAGES) + return gsp_phys_chunk_page_va( + &boot_stage.rmcall_chunks[ + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT0 + + (page - GSP_PHYS_RMCALL_GUEST_PT_PAGES)], 0); + return NULL; +} + +static u64 gsp_phys_rmcall_guest_pt_page_dma(u32 page) +{ + if (page < GSP_PHYS_RMCALL_GUEST_PT_PAGES) + return (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GUEST_PT0 + + page], 0); + return (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_chunks[ + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT0 + + (page - GSP_PHYS_RMCALL_GUEST_PT_PAGES)], 0); +} + +/* + * PD0 index -> pool local page, or 0xffffffffu when the index is not a + * scratch-pool window. Consults both pool regions. + */ +static u32 gsp_phys_rmcall_guest_pt_page_of_pde(u32 pde_index) +{ + u32 base = (u32)(boot_stage.rmcall_va_guest_pt >> 21); + + if (pde_index >= base && + pde_index < base + GSP_PHYS_RMCALL_GUEST_PT_PAGES) + return pde_index - base; + base = (u32)(GSP_PHYS_RMCALL_VA_GUEST_PT_EXT_BASE >> 21); + if (pde_index >= base && + pde_index < base + GSP_PHYS_RMCALL_GUEST_PT_EXT_PAGES) + return pde_index - base + GSP_PHYS_RMCALL_GUEST_PT_PAGES; + return 0xffffffffu; +} + +/* * Shared-management VAS: install the ladder's page-table window through the * server's own MMU walker. NV0080_CTRL_CMD_DMA_UPDATE_PDE_2 (0x80180f, * gpu_vaspace.c gvaspaceUpdatePde2_IMPL) takes the system-physical base of a @@ -9364,7 +9595,7 @@ static void gsp_phys_rmcall_pdb_build(void) } } /* - * m69: plant the window-0 shadow pool PTEs (slots 0x108..0x10f) in the + * m69: plant the window-0 shadow pool PTEs (current win0 slot range) in the * same proven small-PT line, before the engine's first fetch. The * launch copies the QMD's program/param page content into these pages * and rewrites the QMD pointers to these VAs, so the GR engine never @@ -9410,12 +9641,18 @@ static void gsp_phys_rmcall_pdb_build(void) (size_t)idx * 8); } dma_wmb(); - pr_info("vgsp-boot: rm-call: shadow-win0 pool planted %u pages at VA 0x%llx..0x%llx (small-PT slots 0x108..0x10f, before first fetch)\n", + pr_info("vgsp-boot: rm-call: shadow-win0 pool planted %u pages at VA 0x%llx..0x%llx (small-PT slots 0x%x..0x%x, before first fetch)\n", GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS, GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE, GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE + (GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS - 1) * - GSP_PHYS_PAGE_SIZE); + GSP_PHYS_PAGE_SIZE, + (u32)((GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE >> 12) & + 0x1ffu), + (u32)(((GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE + + (GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS - 1) * + GSP_PHYS_PAGE_SIZE) >> 12) & + 0x1ffu)); } } /* @@ -9532,6 +9769,30 @@ static void gsp_phys_rmcall_pdb_build(void) (size_t)(1 + i) * 16 + 8); } /* + * m152: extension scratch pool - PD0 dual entries 32..79 back the + * GSP_PHYS_RMCALL_GUEST_PT_EXT_PAGES pages of the multi-page extension + * PT chunk at GSP_PHYS_RMCALL_VA_GUEST_PT_EXT_BASE (0x04000000; free + * space above the GR-context windows, which end well below 0x03000000). + * Same shape and same once-before-any-engine-access rule as the 16-window + * loop above. + */ + for (i = 0; i < GSP_PHYS_RMCALL_GUEST_PT_EXT_PAGES; i++) { + u64 guest_pt_pde = gsp_phys_rmcall_pde( + (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_chunks[ + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT0 + i], + 0)); + u32 pd0_idx = (u32)(GSP_PHYS_RMCALL_VA_GUEST_PT_EXT_BASE >> 21) + + i; + + put_unaligned_le64(0, + (u8 *)gsp_phys_chunk_page_va(pd0, 0) + + (size_t)pd0_idx * 16); + put_unaligned_le64(guest_pt_pde, + (u8 *)gsp_phys_chunk_page_va(pd0, 0) + + (size_t)pd0_idx * 16 + 8); + } + /* * H1 GR context window: when the GR ladder prepared the promoted MAIN * context buffer (gsp_phys_rmcall_gr_ctx_prepare ran before this * builder), append its page tables as fresh PD0 dual entries 17.. (the @@ -9698,11 +9959,12 @@ static void gsp_phys_rmcall_pdb_build(void) src_idx, pt_src, (u64)gsp_phys_chunk_page_dma(src, 0), dst_idx, pt_dst, (u64)gsp_phys_chunk_page_dma(dst, 0), sem_idx, pt_sem, (u64)gsp_phys_chunk_page_dma(sem, 0)); - pr_info("vgsp-boot: rm-call: guest scratch PT built: pages=%u vaBase=0x%llx capacity=%u\n", + pr_info("vgsp-boot: rm-call: guest scratch PT built: pages=%u (+%u ext) vaBase=0x%llx extBase=0x%llx capacity=%u\n", GSP_PHYS_RMCALL_GUEST_PT_PAGES, + GSP_PHYS_RMCALL_GUEST_PT_EXT_PAGES, GSP_PHYS_RMCALL_VA_GUEST_PT_BASE, - GSP_PHYS_RMCALL_GUEST_PT_PAGES * - GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE); + GSP_PHYS_RMCALL_VA_GUEST_PT_EXT_BASE, + GSP_PHYS_RMCALL_GUEST_PT_TOTAL_LINES); } /* @@ -15162,7 +15424,6 @@ static int gsp_phys_rmcall_guest_map_locked(u64 src_hpa, u64 dst_hpa, u64 sem_hpa, u64 *src_va, u64 *dst_va, u64 *sem_va) { - struct gsp_phys_boot_chunk *guest_pt_chunk; u32 line = boot_stage.rmcall_guest_pt_next_line; u32 page, line_in_page; u64 va; @@ -15170,19 +15431,14 @@ static int gsp_phys_rmcall_guest_map_locked(u64 src_hpa, u64 dst_hpa, if (!boot_stage.rmcall_va_guest_pt) return -EIO; - if (line >= GSP_PHYS_RMCALL_GUEST_PT_PAGES * - GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE) + if (line >= GSP_PHYS_RMCALL_GUEST_PT_TOTAL_LINES) return -ENOSPC; page = line / GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; line_in_page = line % GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; - guest_pt_chunk = &boot_stage.rmcall_chunks[ - GSP_PHYS_RMCALL_BUF_GUEST_PT0 + page]; - if (!guest_pt_chunk->pages) - return -EIO; /* 2 MB PD0 slot stride; a full line is 16 PTEs = 0x10000 of VA. */ - va = boot_stage.rmcall_va_guest_pt + ((u64)page << 21) + + va = gsp_phys_rmcall_guest_pt_win_va(page) + ((u64)line_in_page << 16); - pt = gsp_phys_chunk_page_va(guest_pt_chunk, 0); + pt = gsp_phys_rmcall_guest_pt_page_va(page); if (!pt) return -EIO; /* @@ -15403,15 +15659,13 @@ static int gsp_phys_guest_ce_copy_common(u64 src_hpa, u32 src_off, u64 dst_hpa, if ((gsp_phys_relay_pool_exhausted & 63) == 1) pr_err("vgsp-work: GUEST CE RELAY FAILED: scratch-PT pool exhausted (no line reuse) pool_exhausted=%llu capacity=%u; falling back to the software release\n", gsp_phys_relay_pool_exhausted, - GSP_PHYS_RMCALL_GUEST_PT_PAGES * - GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE); + GSP_PHYS_RMCALL_GUEST_PT_TOTAL_LINES); } else { gsp_phys_guest_pool_exhausted++; if ((gsp_phys_guest_pool_exhausted & 63) == 1) pr_err("vgsp-work: GUEST CE COPY FAILED: scratch-PT pool exhausted (no line reuse) pool_exhausted=%llu capacity=%u; falling back to the ladder path\n", gsp_phys_guest_pool_exhausted, - GSP_PHYS_RMCALL_GUEST_PT_PAGES * - GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE); + GSP_PHYS_RMCALL_GUEST_PT_TOTAL_LINES); } } else { if (relay) @@ -15738,6 +15992,7 @@ static u64 gsp_phys_gr_submitted; static u64 gsp_phys_gr_ok; static u64 gsp_phys_gr_refused; static u64 gsp_phys_gr_timed_out; +static u64 gsp_phys_gr_retries; /* fresh-entry launch retries published */ static u64 gsp_phys_gr_pool_exhausted; /* m86: program-page override blob (see gsp-phys-gr-override). */ static u8 gsp_phys_gr_override[GSP_PHYS_PAGE_SIZE]; @@ -16077,26 +16332,26 @@ static void __iomem *gsp_phys_gr_scratch_bar0; static int gsp_phys_rmcall_gr_scratch_win_install_locked(u32 page) { struct gsp_phys_rmcall_update_pde2_params p; - struct gsp_phys_boot_chunk *pt; u32 rpc_result = 0, status = 0, pde_index; + u64 pt_dma; int ret; - if (page >= GSP_PHYS_RMCALL_GUEST_PT_PAGES) + if (page >= GSP_PHYS_RMCALL_GUEST_PT_TOTAL_PAGES) return -EINVAL; if (!gsp_phys_rmcall_vas_shared_value()) return 0; - if (boot_stage.rmcall_guest_pt_win_mask & BIT(page)) + if (boot_stage.rmcall_guest_pt_win_mask & BIT_ULL(page)) return 0; - pt = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GUEST_PT0 + page]; - if (!pt->pages) + pt_dma = gsp_phys_rmcall_guest_pt_page_dma(page); + if (!pt_dma) return -EIO; if (!boot_stage.rmcall_va_guest_pt) return -EIO; - pde_index = (u32)((boot_stage.rmcall_va_guest_pt >> 21) + page); + pde_index = (u32)(gsp_phys_rmcall_guest_pt_win_va(page) >> 21); memset(&p, 0, sizeof(p)); p.pde_index = cpu_to_le32(pde_index); p.flags = cpu_to_le32(GSP_PHYS_RMCALL_PDE2_FLAG_FLUSH); - p.pt[0].phys_addr = cpu_to_le64((u64)gsp_phys_chunk_page_dma(pt, 0)); + p.pt[0].phys_addr = cpu_to_le64(pt_dma); p.pt[0].aperture = cpu_to_le32(GSP_PHYS_RMCALL_PDE2_APERTURE_SYS_COH); p.pt[1].aperture = cpu_to_le32(GSP_PHYS_RMCALL_PDE2_APERTURE_INVALID); p.h_va_space = cpu_to_le32(boot_stage.rmcall_vas); @@ -16113,10 +16368,9 @@ static int gsp_phys_rmcall_gr_scratch_win_install_locked(u32 page) ret); return ret; } - boot_stage.rmcall_guest_pt_win_mask |= BIT(page); + boot_stage.rmcall_guest_pt_win_mask |= BIT_ULL(page); pr_info("vgsp-gr: scratch window installed pdeIndex=%u (VA 0x%llx) ptPa=0x%llx status=0x%x (Tier-2 shadow legs now reachable through the server walker)\n", - pde_index, (u64)pde_index << 21, - (u64)gsp_phys_chunk_page_dma(pt, 0), status); + pde_index, (u64)pde_index << 21, pt_dma, status); return 0; } @@ -16129,13 +16383,11 @@ static int gsp_phys_rmcall_gr_scratch_win_install_locked(u32 page) static bool gsp_phys_rmcall_gr_win_is_installed_locked(u64 va) { u32 pde_index = (u32)(va >> 21); - u32 base = (u32)(boot_stage.rmcall_va_guest_pt >> 21); + u32 page = gsp_phys_rmcall_guest_pt_page_of_pde(pde_index); u32 i; - if (pde_index >= base && - pde_index < base + GSP_PHYS_RMCALL_GUEST_PT_PAGES) - return !!(boot_stage.rmcall_guest_pt_win_mask & - BIT(pde_index - base)); + if (page != 0xffffffffu) + return !!(boot_stage.rmcall_guest_pt_win_mask & BIT_ULL(page)); for (i = 0; i < boot_stage.rmcall_gr_pt_used; i++) if (boot_stage.rmcall_gr_win[i].used && boot_stage.rmcall_gr_win[i].pde_index == pde_index) @@ -16154,13 +16406,11 @@ static bool gsp_phys_rmcall_gr_win_is_installed_locked(u64 va) static int gsp_phys_rmcall_gr_win_ensure_locked(void __iomem *bar0, u64 va) { u32 pde_index = (u32)(va >> 21); - u32 base = (u32)(boot_stage.rmcall_va_guest_pt >> 21); + u32 page = gsp_phys_rmcall_guest_pt_page_of_pde(pde_index); u32 slot; - if (pde_index >= base && - pde_index < base + GSP_PHYS_RMCALL_GUEST_PT_PAGES) - return gsp_phys_rmcall_gr_scratch_win_install_locked( - pde_index - base); + if (page != 0xffffffffu) + return gsp_phys_rmcall_gr_scratch_win_install_locked(page); return gsp_phys_rmcall_gr_win_slot_locked(bar0, pde_index, &slot); } @@ -16171,16 +16421,35 @@ static int gsp_phys_rmcall_gr_win_ensure_locked(void __iomem *bar0, u64 va) * supplies the backing page. Called with gsp_phys_rm_lock held. */ static int gsp_phys_rmcall_gr_map_shadow_locked(u32 slot, u64 hpa, - u64 *va_out) + u64 *va_out, u32 *line_out) { struct gsp_phys_boot_chunk *sh = &boot_stage.rmcall_gr_shadow; - struct gsp_phys_boot_chunk *pt; u32 line = boot_stage.rmcall_guest_pt_next_line; - u32 page, line_in_page; + u32 page, line_in_page, i; u64 va; - u8 *pt_va; + u8 *pt; int ret; + /* + * m152: reuse before consume. A shadow VA is a never-rewritten PTE: + * once hpa X maps at VA V, every later request for X can be served by + * V with byte-identical engine-visible content, so the pool only + * needs one line per distinct HPA per boot instead of one per launch. + * This is safe against the GMMU's no-invalidate fetch-line cache by + * construction: the PTE value never differs from what the engine may + * have already fetched. A hit reports line_out = ~0 so a caller that + * records contiguity bookkeeping never contig-extends an old line + * (a new PTE in a line the engine may have already fetched would be + * invisible). + */ + for (i = 0; i < boot_stage.rmcall_gr_alias_count; i++) { + if (boot_stage.rmcall_gr_alias_hpa[i] == hpa) { + *va_out = boot_stage.rmcall_gr_alias_va[i]; + if (line_out) + *line_out = 0xffffffffu; + return 0; + } + } if (!sh->pages) { ret = gsp_phys_chunk_alloc(boot_stage.pdev, sh, GSP_PHYS_RMCALL_GR_SHADOW_PAGES); @@ -16192,28 +16461,34 @@ static int gsp_phys_rmcall_gr_map_shadow_locked(u32 slot, u64 hpa, return -ENOSPC; if (!boot_stage.rmcall_va_guest_pt) return -EIO; - if (line >= GSP_PHYS_RMCALL_GUEST_PT_PAGES * - GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE) + if (line >= GSP_PHYS_RMCALL_GUEST_PT_TOTAL_LINES) return -ENOSPC; page = line / GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; line_in_page = line % GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; ret = gsp_phys_rmcall_gr_scratch_win_install_locked(page); if (ret) return ret; - pt = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GUEST_PT0 + page]; - if (!pt->pages) - return -EIO; - pt_va = gsp_phys_chunk_page_va(pt, 0); - if (!pt_va) + pt = gsp_phys_rmcall_guest_pt_page_va(page); + if (!pt) return -EIO; put_unaligned_le64(gsp_phys_rmcall_pte(hpa), - pt_va + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES * + pt + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES * line_in_page) * 8); dma_wmb(); boot_stage.rmcall_guest_pt_next_line = line + 1; - va = boot_stage.rmcall_va_guest_pt + ((u64)page << 21) + + va = gsp_phys_rmcall_guest_pt_win_va(page) + ((u64)line_in_page << 16); *va_out = va; + if (line_out) + *line_out = line; + if (boot_stage.rmcall_gr_alias_count < + GSP_PHYS_RMCALL_GR_ALIAS_CACHE) { + u32 ci = boot_stage.rmcall_gr_alias_count; + + boot_stage.rmcall_gr_alias_hpa[ci] = hpa; + boot_stage.rmcall_gr_alias_va[ci] = va; + boot_stage.rmcall_gr_alias_count = ci + 1u; + } return 0; } @@ -16227,32 +16502,27 @@ static int gsp_phys_rmcall_gr_map_shadow_locked(u32 slot, u64 hpa, static int gsp_phys_rmcall_gr_map_shadow_contig_locked(u32 line, u32 pte, u64 hpa, u64 *va_out) { - struct gsp_phys_boot_chunk *pt; u32 page, line_in_page; - u8 *pt_va; + u8 *pt; int ret; if (pte >= GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES) return -EINVAL; - if (line >= GSP_PHYS_RMCALL_GUEST_PT_PAGES * - GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE) + if (line >= GSP_PHYS_RMCALL_GUEST_PT_TOTAL_LINES) return -ENOSPC; page = line / GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; line_in_page = line % GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; ret = gsp_phys_rmcall_gr_scratch_win_install_locked(page); if (ret) return ret; - pt = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GUEST_PT0 + page]; - if (!pt->pages) - return -EIO; - pt_va = gsp_phys_chunk_page_va(pt, 0); - if (!pt_va) + pt = gsp_phys_rmcall_guest_pt_page_va(page); + if (!pt) return -EIO; put_unaligned_le64(gsp_phys_rmcall_pte(hpa), - pt_va + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES * + pt + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES * line_in_page + pte) * 8); dma_wmb(); - *va_out = boot_stage.rmcall_va_guest_pt + ((u64)page << 21) + + *va_out = gsp_phys_rmcall_guest_pt_win_va(page) + ((u64)line_in_page << 16) + (u64)pte * GSP_PHYS_PAGE_SIZE; return 0; } @@ -16471,6 +16741,24 @@ static int gsp_phys_rmcall_gr_resched_locked(void __iomem *bar0) u32 runlist_id, ch_id, doorbell; int ret; + /* + * m155 (HF-3 extended re-arm): after the channel has been idle long + * enough for the FECS to save/unload the context, re-issuing only the + * two GPFIFO_SCHEDULE verbs does not revive dispatch - r154: matmul's + * post-smiley launches acked (qmd_rel/ch_rel) but SKED stayed idle + * (cwd0=0x2086/fecs_cur=0) for every attempt including outcome + * retries, while the same channel executed mail/smiley earlier in the + * boot. Replay the mode-35 lane-enable tail (CILP preempt mode, + * timeslice, device sched) that stage 4 ran when the lane first went + * live - it is proven accepted and is the design's extended re-arm. + * The poison variants (explicit PREEMPT/tail, BIND, PROMOTE) stay out. + */ + ret = gsp_phys_rmcall_user_lane_lane_enable(bar0); + if (ret) + pr_info("vgsp-gr: rearm lane-enable rc=%d (continuing to schedules)\n", + ret); + rpc_result = 0; + status = 0; ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_channel, GSP_PHYS_RMCALL_CTRL_GPFIFO_SCHEDULE, sched_params, sizeof(sched_params), 16, @@ -16495,6 +16783,144 @@ static int gsp_phys_rmcall_gr_resched_locked(void __iomem *bar0) return 0; } +/* + * m151: FNV-1a 64-bit over one page. Used to compare the win0 param page + * and the guest param page before and after the poll, so a launch that + * wrote (or clobbered) the page the engine read is provable from the log. + */ +static inline u64 gsp_phys_gr_fnv1a64(const u8 *p, size_t len) +{ + u64 h = 0xcbf29ce484222325ULL; + size_t i; + + for (i = 0; i < len; i++) { + h ^= p[i]; + h *= 0x100000001b3ULL; + } + return h; +} + +/* + * m151: output retirement wait. qmd_rel is a schedule/fetch ack (the front + * end latched and fetched the QMD), not a data-path retirement: the SM's + * stores into the bounce pages can still be in flight when the ack lands. + * Copying out - and the next launch's copy-in - before those stores land + * loses them: the guest observes zeros for a completed launch, or + * stale/mixed content when the next copy-in races the previous engine's + * write tail. + * + * The wait covers EVERY resolved data page (footprint-by-inference was + * tried twice and is unreliable: the fill's candidate list ends with the + * launch's own bank/param pages, so the max-reloc run is 3-4 pages, not + * the output operand). Adaptive exits make a whole-window count cheap: + * - all resolved pages changed -> done (normal case, one poll) + * - changed>0 and stable for 3 polls -> stores settled (masked or + * repeated-identical kernels) + * - no writes at all for 600 ms -> no output for this launch + * - hard cap 1.5 s -> genuinely late remainder + * noinline so this rarely-taken wait does not grow the submit frame, which + * is already near the warning bound. + */ +static noinline u32 +gsp_phys_gr_output_retire_wait(const struct gsp_phys_gr_submit *req, + const u32 *csum_in) +{ + u32 checked[GSP_PHYS_RMCALL_GR_MAIL_SLOT]; + u32 n_all = 0, di, iter; + u32 changed = 0, prev = 0xffffffffu; + const char *why = " (TIMEOUT)"; + unsigned long wait_start = jiffies; + + if (!req->n_data) + return 0; + for (di = 0; di < req->n_data; di++) { + u32 bi = (u32)((req->data[di].reloc_va - + GSP_PHYS_RMCALL_GR_BOUNCE_VA0) >> 12); + u32 k; + bool dup = false; + + if (bi >= GSP_PHYS_RMCALL_GR_MAIL_SLOT) + continue; + for (k = 0; k < n_all; k++) + if (checked[k] == bi) { + dup = true; + break; + } + if (!dup) + checked[n_all++] = bi; + } + if (!n_all) + return 0; + for (iter = 0; iter < 300u; iter++) { + u32 j; + + changed = 0; + for (j = 0; j < n_all; j++) { + const u8 *src = gsp_phys_chunk_page_va( + &boot_stage.rmcall_gr_bounce, checked[j]); + u32 ci, csum = 0; + + if (!src) + continue; + for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) + csum += get_unaligned_le32(src + ci); + if (csum != csum_in[checked[j]]) + changed++; + } + if (changed >= n_all) { + why = ""; + break; + } + if (changed > 0 && changed == prev && iter >= 3u) { + why = " (settled)"; + break; + } + if (changed == 0 && iter >= 120u) { + why = " (no writes)"; + break; + } + prev = changed; + usleep_range(4000, 6000); + } + pr_info("vgsp-gr: output retirement wait: %u/%u pages changed after %u ms%s\n", + changed, n_all, jiffies_to_msecs(jiffies - wait_start), why); + return changed; +} + +/* + * m153: fill the bounce window from the request's guest pages and capture a + * per-page checksum baseline. Used before the first publish and again before + * every outcome retry (a mis-executing first attempt can leave stale or + * wrong-window content behind, so a retry must start from the guest inputs + * again). + */ +static void gsp_phys_gr_fill_bounce(const struct gsp_phys_gr_submit *req, + u32 *csum_in) +{ + u32 di; + + for (di = 0; di < req->n_data; di++) { + u32 bi = (u32)((req->data[di].reloc_va - + GSP_PHYS_RMCALL_GR_BOUNCE_VA0) >> 12); + const u8 *src; + u8 *dst; + u32 ci, csum = 0; + + if (bi >= GSP_PHYS_RMCALL_GR_MAIL_SLOT || + !req->data[di].hpa) + continue; + src = (const u8 *)__va(req->data[di].hpa); + dst = gsp_phys_chunk_page_va(&boot_stage.rmcall_gr_bounce, bi); + if (!dst) + continue; + memcpy(dst, src, GSP_PHYS_PAGE_SIZE); + for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) + csum += get_unaligned_le32(dst + ci); + csum_in[bi] = csum; + } + dma_wmb(); +} + int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, struct gsp_phys_backend_copy_result *out) { @@ -16517,6 +16943,8 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, unsigned long deadline; u8 *sem_va; u32 epoch, words, i, n_shadow = 0; + u32 win0_bank_base = 0, shadow_slot_limit = 0; + const u8 *guest_param_pg; u32 want_qmd = 0, want_ch = 0; bool qmd_rel = false, ch_rel = false; u64 qmd_rel_hpa = 0, ch_rel_hpa = 0; @@ -16540,12 +16968,14 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, static u32 bounce_csum_in[GSP_PHYS_RMCALL_GR_MAIL_SLOT]; sked_tick = 0; sked_first = true; + guest_param_pg = NULL; u32 qmd_win_pde = 0, qmd_win_installed = 0; int qmd_win_ret = 0; bool qmd_win_diag = false; bool qmdwin_on = false; char qmdwin_note[32] = ""; bool rearmed = false; + u32 attempts = 0; int ret = 0; if (out) @@ -16588,8 +17018,13 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, * the program page (silent corruption; no launch can survive it). * The win0 PTEs are planted once at stage 3; reusing the same VAs * with fresh content is the proven pattern. + * + * m151: the cursor now selects one of two 8-page banks, alternating + * with the submission epoch (set below). The engine's index/data + * fetches of the previous launch's QMD shadow pages can still be in + * flight when this launch runs, so the bank alternation protects + * them from this launch's copy-in (late-fetch safety). */ - boot_stage.rmcall_gr_shadow_win0_used = 0; mutex_lock(&gsp_phys_rm_lock); if (!gsp_phys_backend_gr_ready()) { mutex_unlock(&gsp_phys_rm_lock); @@ -16600,6 +17035,11 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, epoch = ++boot_stage.rmcall_gr_submit_epoch; if (!epoch) epoch = ++boot_stage.rmcall_gr_submit_epoch; + boot_stage.rmcall_gr_shadow_win0_used = + (epoch & 1u) * GSP_PHYS_RMCALL_GR_SHADOW0_BANK_SLOTS; + win0_bank_base = boot_stage.rmcall_gr_shadow_win0_used; + shadow_slot_limit = win0_bank_base + + GSP_PHYS_RMCALL_GR_SHADOW0_BANK_SLOTS; memcpy(qmd, req->qmd, sizeof(qmd)); /* * Two-step PCAS (gsp_phys_rmcall_gr_pcas_two_step): set the QMD's @@ -16628,7 +17068,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, ret = gsp_phys_rmcall_gr_map_shadow_locked(n_shadow, req->rel_hpa, - &sh_va); + &sh_va, NULL); if (!ret) { n_shadow++; qmd_rel_hpa = req->rel_hpa; @@ -16719,6 +17159,8 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, for (k = 0; k < n_sh_rec; k++) { u64 va = 0; + if (sh_rec[k].line == 0xffffffffu) + continue; /* cached line: never extended */ if (pg_va != sh_rec[k].guest_va + (u64)sh_rec[k].npte * GSP_PHYS_PAGE_SIZE) @@ -16745,11 +17187,11 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, */ if (!mapped && pg->hpa && pg->qmd_word == GSP_PHYS_GR_QMD_WORD_NONE) { - u32 sh_line = boot_stage.rmcall_guest_pt_next_line; + u32 sh_line = 0xffffffffu; if (n_shadow < GSP_PHYS_RMCALL_GR_SHADOW_PAGES && !gsp_phys_rmcall_gr_map_shadow_locked( - n_shadow, pg->hpa, &sh_va)) { + n_shadow, pg->hpa, &sh_va, &sh_line)) { u32 k, slot = n_shadow; mapped = true; @@ -16975,7 +17417,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, gsp_phys_rmcall_gr_shadow_win0_value() && boot_stage.rmcall_gr_shadow_win0.pages && boot_stage.rmcall_gr_shadow_win0_used < - GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS; + shadow_slot_limit; /* * m86: SASS override. When gsp-phys-gr-override @@ -17036,7 +17478,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, n_shadow, (u64)gsp_phys_chunk_page_dma(sh, n_shadow), - &sh_va); + &sh_va, NULL); } if (!ret) { bool keep_guest = @@ -17197,7 +17639,7 @@ gr_page_unmapped: if (n_shadow < GSP_PHYS_RMCALL_GR_SHADOW_PAGES && !gsp_phys_rmcall_gr_map_shadow_locked( - n_shadow, req->qmd_hpa, &sh_va)) { + n_shadow, req->qmd_hpa, &sh_va, NULL)) { qmd_dst_mem = (u8 *)gsp_phys_chunk_page_va( &boot_stage.rmcall_gr_shadow, n_shadow) + qmd_off; @@ -17240,7 +17682,8 @@ gr_page_unmapped: u64 sh_va = 0; ret = gsp_phys_rmcall_gr_map_shadow_locked(n_shadow, - req->sem_hpa, &sh_va); + req->sem_hpa, &sh_va, + NULL); if (!ret) { n_shadow++; ch_rel_hpa = req->sem_hpa; @@ -17335,29 +17778,19 @@ gr_page_unmapped: if (req->n_data && boot_stage.rmcall_gr_bounce.pages) { u32 dcopied = 0, dfailed = 0, di; + gsp_phys_gr_fill_bounce(req, bounce_csum_in); for (di = 0; di < req->n_data; di++) { u32 bi = (u32)((req->data[di].reloc_va - GSP_PHYS_RMCALL_GR_BOUNCE_VA0) >> 12); - const u8 *src; - u8 *dst; - u32 ci = 0, csum = 0; if (bi >= GSP_PHYS_RMCALL_GR_MAIL_SLOT || - !req->data[di].hpa) { + !req->data[di].hpa) dfailed++; - continue; - } - src = (const u8 *)__va(req->data[di].hpa); - dst = gsp_phys_chunk_page_va( - &boot_stage.rmcall_gr_bounce, bi); - memcpy(dst, src, GSP_PHYS_PAGE_SIZE); - for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) - csum += get_unaligned_le32(dst + ci); - bounce_csum_in[bi] = csum; - dcopied++; + else if (gsp_phys_chunk_page_va( + &boot_stage.rmcall_gr_bounce, bi)) + dcopied++; } - dma_wmb(); pr_info("vgsp-gr: bounce copy-in: %u pages, %u skipped (pre-publish)\n", dcopied, dfailed); for (di = 0; di < req->n_data && di < 3; di++) @@ -17445,24 +17878,37 @@ gr_page_unmapped: gsp_phys_rmcall_gr_gpc_mmu_init(bar0); /* * m143 DIAGNOSTIC (second-client launch-state divergence): dump the - * exact param page the FE/SM will read (win0 slot 0, the QMD bank-0 - * target) and the guest param page at the three JIT parameter slots, - * right before the doorbell. The JIT reads a@0x160/b@0x168/c@0x170 - * and n@0x178 (verified with ptxas -arch=sm_75). If the win0 copy is - * correct here but the engine still stores to the wrong relocated - * window, the divergence is engine-side (a stale mapping), not the - * relaunched page content. + * exact param page the FE/SM will read (this launch's win0 bank base + * slot, the QMD bank-0 target) and the guest param page at the three + * JIT parameter slots, right before the doorbell. The JIT reads + * a@0x160/b@0x168/c@0x170 and n@0x178 (verified with ptxas + * -arch=sm_75). If the win0 copy is correct here but the engine still + * stores to the wrong relocated window, the divergence is engine-side + * (a stale mapping), not the relaunched page content. */ if (boot_stage.rmcall_gr_shadow_win0.pages) { const u8 *wp = gsp_phys_chunk_page_va( - &boot_stage.rmcall_gr_shadow_win0, 0); + &boot_stage.rmcall_gr_shadow_win0, win0_bank_base); - if (wp) - pr_info("vgsp-gr: DIAG win0[0] 160=%llx 168=%llx 170=%llx 178=%llx\n", + if (wp) { + u32 off; + + pr_info("vgsp-gr: DIAG win0[%u] 160=%llx 168=%llx 170=%llx 178=%llx\n", + win0_bank_base, (u64)get_unaligned_le64(wp + 0x160), (u64)get_unaligned_le64(wp + 0x168), (u64)get_unaligned_le64(wp + 0x170), (u64)get_unaligned_le64(wp + 0x178)); + pr_info("vgsp-gr: win0 param hash pre=0x%016llx\n", + gsp_phys_gr_fnv1a64(wp, GSP_PHYS_PAGE_SIZE)); + for (off = 0x100u; off < 0x200u; off += 0x20u) + pr_info("vgsp-gr: win0 page +0x%03x: %016llx %016llx %016llx %016llx\n", + off, + (u64)get_unaligned_le64(wp + off), + (u64)get_unaligned_le64(wp + off + 8), + (u64)get_unaligned_le64(wp + off + 16), + (u64)get_unaligned_le64(wp + off + 24)); + } } { u32 pi; @@ -17474,12 +17920,15 @@ gr_page_unmapped: !req->pages[pi].hpa) continue; gp = (const u8 *)__va(req->pages[pi].hpa); + guest_param_pg = gp; pr_info("vgsp-gr: DIAG guest-param pa=0x%llx 160=%llx 168=%llx 170=%llx 178=%llx\n", req->pages[pi].hpa, (u64)get_unaligned_le64(gp + 0x160), (u64)get_unaligned_le64(gp + 0x168), (u64)get_unaligned_le64(gp + 0x170), (u64)get_unaligned_le64(gp + 0x178)); + pr_info("vgsp-gr: guest-param hash pre=0x%016llx\n", + gsp_phys_gr_fnv1a64(gp, GSP_PHYS_PAGE_SIZE)); break; } } @@ -17491,9 +17940,10 @@ gr_page_unmapped: boot_stage.rmcall_token, qmd_rel, want_ch, GSP_PHYS_RMCALL_GR_POLL_MS, qmdwin_note); if (qmd_win_diag) - pr_info("vgsp-gr: qmd dest va=0x%llx pdeIndex=%u installed=%u win_mask=0x%x ensure_ret=%d\n", + pr_info("vgsp-gr: qmd dest va=0x%llx pdeIndex=%u installed=%u win_mask=0x%llx ensure_ret=%d\n", qmd_dst, qmd_win_pde, qmd_win_installed, - boot_stage.rmcall_guest_pt_win_mask, qmd_win_ret); + (unsigned long long)boot_stage.rmcall_guest_pt_win_mask, + qmd_win_ret); deadline = jiffies + msecs_to_jiffies(GSP_PHYS_RMCALL_GR_POLL_MS); for (;;) { @@ -17581,6 +18031,45 @@ gr_page_unmapped: pr_info("vgsp-gr: rearm after timeout failed rc=%d\n", rr); } + /* + * The re-arm above only replays the schedule verbs; if the + * entry was already fetched or the front-end never latched + * it, that cannot make the QMD run. Re-publish a FRESH ring + * entry (same PB content and QMD) with a re-issued schedule + * + doorbell, bounded. The timeout path never copied the + * result out to the guest, so a timed-out attempt did not + * reach the client; the guest therefore still sees only the + * successful attempt. + */ + if (attempts < GSP_PHYS_RMCALL_GR_ATTEMPTS) { + int pr_ret; + + attempts++; + mutex_lock(&gsp_phys_rm_lock); + pr_ret = gsp_phys_rmcall_publish_locked( + bar0, boot_stage.rmcall_va_pb, words, + &sub); + if (!pr_ret) + pr_ret = gsp_phys_rmcall_gr_resched_locked( + bar0); + mutex_unlock(&gsp_phys_rm_lock); + if (!pr_ret) { + gsp_phys_gr_retries++; + pr_info("vgsp-gr: retry %u/%u: fresh ring entry slot=%u published + rescheduled; re-polling %u ms\n", + attempts, + GSP_PHYS_RMCALL_GR_ATTEMPTS, + sub.slot, + GSP_PHYS_RMCALL_GR_RETRY_MS); + deadline = jiffies + + msecs_to_jiffies( + GSP_PHYS_RMCALL_GR_RETRY_MS); + rearmed = false; + grpoll_n = 0; + continue; + } + pr_info("vgsp-gr: retry %u publish failed rc=%d\n", + attempts, pr_ret); + } if (gsp_phys_gr_fault_active()) { struct gsp_phys_fault_info fi; u32 fep = 0; @@ -17871,6 +18360,99 @@ gr_page_unmapped: gsp_phys_gr_ok++; /* + * m151: retire the output footprint before the copy-out (and before + * any later launch's copy-in). qmd_rel above is a schedule/fetch ack, + * not a data-path retirement, so the engine's stores into the bounce + * pages can land after it and be lost by an early copy-out; the next + * launch's copy-in would also race that write tail. Success path + * only - the timeout/fault paths never copied out. See + * gsp_phys_gr_output_retire_wait(). + * + * m153: outcome-verifying retry. The intermittent per-boot + * no-execution class (qmd_rel/ch_rel satisfied but the engine wrote + * nothing at all) cannot be caught by the completion poll; the + * retirement wait is the only execution witness. On zero writes, + * refill the bounce window from the guest inputs, re-publish the + * same method block on a fresh ring entry, re-issue the schedule and + * re-poll, then re-wait - bounded by the same attempt budget as the + * timeout path. After the budget the launch proceeds to copy-out + * with whatever the engine produced: a genuinely read-only kernel is + * indistinguishable from a no-execution here and simply pays the + * bounded extra polls (safe to re-execute by construction - a kernel + * that writes nothing has no side effects to duplicate). A + * wrong-place write detector was tried and removed: the highest-reloc + * data entry is often a UVM-internal buffer, not the output, so it + * false-fired on correct launches and would re-execute + * side-effecting kernels (atomics) up to 3x. + */ + if (req->n_data && boot_stage.rmcall_gr_bounce.pages) { + u32 wrote = gsp_phys_gr_output_retire_wait(req, bounce_csum_in); + + if (gsp_phys_rmcall_gr_nowrite_retry_value()) { + while (attempts < GSP_PHYS_RMCALL_GR_ATTEMPTS) { + if (wrote > 0) + break; + attempts++; + if (attempts > GSP_PHYS_RMCALL_GR_ATTEMPTS) + break; + pr_info("vgsp-gr: outcome retry %u/%u (wrote=%u): refill + re-publish\n", + attempts, GSP_PHYS_RMCALL_GR_ATTEMPTS, + wrote); + gsp_phys_gr_fill_bounce(req, bounce_csum_in); + mutex_lock(&gsp_phys_rm_lock); + ret = gsp_phys_rmcall_publish_locked( + bar0, boot_stage.rmcall_va_pb, + words, &sub); + if (!ret) + ret = gsp_phys_rmcall_gr_resched_locked( + bar0); + mutex_unlock(&gsp_phys_rm_lock); + if (ret) { + pr_info("vgsp-gr: outcome retry %u publish/resched failed rc=%d (accepting current outcome)\n", + attempts, ret); + ret = 0; + break; + } + { + unsigned long dl = jiffies + + msecs_to_jiffies( + GSP_PHYS_RMCALL_GR_RETRY_MS); + u32 rw0 = 0, rw1 = 0; + + for (;;) { + if (qmd_rel_hpa) + rw0 = READ_ONCE( + *(u32 *)((u8 *)__va( + qmd_rel_hpa) + + qmd_rel_off)); + else if (qmd_rel) + rw0 = READ_ONCE( + *(u32 *)sem_va); + if (ch_rel_hpa) + rw1 = READ_ONCE( + *(u32 *)((u8 *)__va( + ch_rel_hpa) + + ch_rel_off)); + else + rw1 = READ_ONCE( + *(u32 *)(sem_va + 4)); + if ((!qmd_rel || + rw0 == want_qmd) && + rw1 == want_ch) + break; + if (time_after_eq(jiffies, dl)) + break; + msleep(GSP_PHYS_RMCALL_POLL_MS); + } + } + wrote = gsp_phys_gr_output_retire_wait( + req, bounce_csum_in); + pr_info("vgsp-gr: outcome retry %u done: engine wrote %u page(s)\n", + attempts, wrote); + } + } + } + /* * m114: the kernel ran against the pre-commit bounce window; copy the * results back to the guest's real pages (identity HPAs in the * request). All mapped pages are written back; inputs are unchanged @@ -17962,6 +18544,23 @@ gr_page_unmapped: ioread32(bar0 + 0x00407060u), ioread32(bar0 + 0x00407064u), ioread32(bar0 + 0x00407068u)); + /* m151: post-poll page hashes; compare with the pre-doorbell ones. */ + { + const u8 *wp = boot_stage.rmcall_gr_shadow_win0.pages ? + gsp_phys_chunk_page_va( + &boot_stage.rmcall_gr_shadow_win0, + win0_bank_base) : NULL; + + if (wp || guest_param_pg) + pr_info("vgsp-gr: win0 param hash post=0x%016llx guest-param hash post=0x%016llx\n", + wp ? gsp_phys_gr_fnv1a64(wp, + GSP_PHYS_PAGE_SIZE) : + 0ULL, + guest_param_pg ? + gsp_phys_gr_fnv1a64(guest_param_pg, + GSP_PHYS_PAGE_SIZE) : + 0ULL); + } pr_info("vgsp-gr: GUEST GR LAUNCH OK h=0x%x push=0x%llx qmd=0x%llx prog/pages=%u qmd_rel=0x%08x ch_rel=0x%08x elapsed_ms=%u ok=%llu\n", req->h_object, req->push_va, req->qmd_va, req->n_pages, w0, w1, jiffies_to_msecs(jiffies - start), gsp_phys_gr_ok); @@ -18992,8 +19591,7 @@ static int gsp_phys_state_show(struct seq_file *m, void *v) fwd_cand_none, fwd_cand_sys, fwd_cand_mixed, fwd_cand_vid, fwd_resolve_fail, boot_stage.rmcall_guest_pt_next_line, - GSP_PHYS_RMCALL_GUEST_PT_PAGES * - GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE); + GSP_PHYS_RMCALL_GUEST_PT_TOTAL_LINES); } { /* @@ -20625,8 +21223,8 @@ static_assert(offsetof(struct gsp_phys_rmcall_channel_params, process_id) == 296 "NV_CHANNEL_ALLOC_PARAMS.ProcessID must be at offset 296"); static_assert(offsetof(struct gsp_phys_rmcall_channel_params, tpc_config_id) == 360, "NV_CHANNEL_ALLOC_PARAMS.tpcConfigID must be at offset 360"); -static_assert(GSP_PHYS_RMCALL_BUF_COUNT == 33, - "the rm-call ladder registers exactly thirty-three DMA buffers (14 ladder + 16 single-page guest scratch PTs + 3 golden9g VAS levels)"); +static_assert(GSP_PHYS_RMCALL_BUF_COUNT == 81, + "the rm-call ladder registers exactly eighty-one DMA buffers (14 ladder + 16 single-page guest scratch PTs + 48 single-page extension scratch PTs + 3 golden9g VAS levels)"); static_assert((GSP_PHYS_RMCALL_HANDLE_MEM_BASE + (GSP_PHYS_RMCALL_BUF_COUNT - 1) * GSP_PHYS_RMCALL_HANDLE_MEM_STRIDE) < 0xcaf80000u, |