diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 852 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.c | 25 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.h | 4 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 392 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.h | 10 | ||||
| -rw-r--r-- | kernel/main.c | 31 |
6 files changed, 1173 insertions, 141 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 72cda10..9eb1179 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -999,16 +999,34 @@ bool gsp_phys_boot_requested(void) #define GSP_PHYS_RMCALL_VA_DST (GSP_PHYS_RMCALL_VA_BASE + 3 * GSP_PHYS_PAGE_SIZE) #define GSP_PHYS_RMCALL_VA_SEM (GSP_PHYS_RMCALL_VA_BASE + 4 * GSP_PHYS_PAGE_SIZE) /* - * m69: window-0 shadow pool - 8 module pages at VAs 0x108000..0x10f000 - * (small-PT slots 0x108..0x10f, the same 128 B PTE fetch line as the - * GPFIFO/PB/QMDWIN slots, planted before the engine's first fetch). The - * Tier-2 copies of the QMD's program/param pages are placed here so the GR - * engine never has to read through the window-1 scratch VA range whose PT - * chain (DMA_UPDATE_PDE_2 pdeIndex=1) has never been proven engine-readable. + * m69: window-0 shadow pool - 16 module pages holding the Tier-2 copies of + * the QMD's program/param pages, so the GR engine never has to read through + * the window-1 scratch VA range whose PT chain (DMA_UPDATE_PDE_2 + * pdeIndex=1) has never been proven engine-readable. The pool is split + * into two 8-page banks; the forwarded launch alternates banks per launch + * (m151) so one launch's copy-in can never clobber the pages the previous + * launch's engine fetches may still be reading (late-fetch safety). + * + * The base MUST stay clear of the m114/m150 GR data bounce window + * (GSP_PHYS_RMCALL_GR_BOUNCE_VA0 .. +PAGES, main-PT slots 0x110..0x1df): + * this pool is re-planted on every gsp_phys_rmcall_pdb_build() while the + * bounce PTEs are a one-shot plant, so any overlap lets the later win0 + * plant steal the bounce slots and the engine writes results into the + * shadow pages, losing them. 0x1e0000 (slots 0x1e0..0x1ef) sits directly + * above the last bounce/mail/marker page (0x1df000) and below the + * mail-injection slot 0x1f0 and the guest scratch window (0x200000). Keep + * it that way (see the static_assert next to the bounce constants). */ #define GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE \ - (GSP_PHYS_RMCALL_VA_BASE + 0x90000ULL) -#define GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS 8u + (GSP_PHYS_RMCALL_VA_BASE + 0xE0000ULL) +#define GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS 16u +/* + * m151: one bank = one launch's worth of QMD shadow pages. A forwarded + * launch consumes at most eight pages (param via qmd_word 32, program 48, + * bank 0/1/2/3/5/7 at qmd_word 34..46 even); the per-launch bounds check + * uses this so a launch can never spill into the other bank. + */ +#define GSP_PHYS_RMCALL_GR_SHADOW0_BANK_SLOTS 8u /* * m99: engine-tree branches for identity-mapping the forwarded kernel's * guest data pages. The walker uses our root[0] -> PD2 -> PD1 -> PD0 dual -> @@ -1100,6 +1118,33 @@ bool gsp_phys_boot_requested(void) /* Full 128 B fetch lines per 4 KB small-PT page: (4096/8)/16 = 32. */ #define GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE \ ((GSP_PHYS_PAGE_SIZE / 8u) / GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES) +/* + * m152: extended scratch-PT pool. The original 16 x 2 MB PD0 windows + * (512 never-reused 128 B fetch lines per boot) exhausted under a normal + * launch mix (each distinct guest page consumed a line per launch) and the + * exhaustion was permanent for the boot: every later launch that needed a + * new mapping was refused ("launch refused ... no Tier-1 mapping and no + * captured inline data"), which is the deterministic once-broken-forever + * failure the longevity runs kept hitting. Two changes fix the class: + * (a) GR alias cache (below): a consumed fetch line is keyed by the HPA + * it maps, so a repeat launch of the same buffers (or the same chunk + * page / semaphore page) reuses the identical never-rewritten line and + * consumes nothing; + * (b) a 48-window extension pool right above the ladder's free gap + * (window VA 0x04000000.., PD0 indices 32..79) backed by one 48-page + * PT chunk, raising capacity to 64 windows x 32 lines = 2048. + * lines/boot. Both are pure capacity mechanics: no PTE and no mapping any + * engine may have fetched is ever rewritten. + */ +#define GSP_PHYS_RMCALL_VA_GUEST_PT_EXT_BASE 0x04000000ULL +#define GSP_PHYS_RMCALL_GUEST_PT_EXT_PAGES 48u +#define GSP_PHYS_RMCALL_GUEST_PT_TOTAL_PAGES \ + (GSP_PHYS_RMCALL_GUEST_PT_PAGES + GSP_PHYS_RMCALL_GUEST_PT_EXT_PAGES) +#define GSP_PHYS_RMCALL_GUEST_PT_TOTAL_LINES \ + (GSP_PHYS_RMCALL_GUEST_PT_TOTAL_PAGES * \ + GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE) +/* HPA -> shadow-VA reuse cache; hit = no new fetch line is consumed. */ +#define GSP_PHYS_RMCALL_GR_ALIAS_CACHE 768u /* * GR forwarding (gsp_phys_gr_forward=1; default off) scratch resources, @@ -1253,8 +1298,21 @@ bool gsp_phys_boot_requested(void) #define GSP_PHYS_RMCALL_CHANNEL_FLAG_USER_LANE \ (u32)(GSP_PHYS_RMCALL_CHANNEL_FLAG_PRIVILEGED | (4u << 8) | \ (1u << 21)) -/* Captured user-lane GPFIFO depth (#825: 1024 entries). */ -#define GSP_PHYS_RMCALL_USER_LANE_GPFIFO_ENTRIES 1024u +/* + * User-lane GPFIFO depth. MUST equal the host ring's push modulus + * (GSP_PHYS_RMCALL_GPFIFO_ENTRIES, 8): the host publishes into the + * one-page ladder ring modulo 8, so if the channel declares a different + * count the engine's GET/PUT wrap desyncs at the host's first wrap + * (GP_PUT 7 -> 0 while the engine's GET keeps counting to 8) and every + * later launch on the boot is never fetched (r138: launches 1..7 green, + * the 8th and all after it silent with `qmd_rel=5 ch_rel=0`). The captured + * #825 value (1024) describes the real client's own much larger ring, not + * this emulated one; the campaign's 2026-10-06 "fidelity" change to 1024 + * was a wrap-only difference that broke exactly here (M1's two-launch + * acceptance never wrapped). Keep the two ends equal. + */ +#define GSP_PHYS_RMCALL_USER_LANE_GPFIFO_ENTRIES \ + GSP_PHYS_RMCALL_GPFIFO_ENTRIES #define GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_USER_LANE 0x1du #define GSP_PHYS_RMCALL_TSG_FAULT_BUF_SIZE 0x2000u #define GSP_PHYS_RMCALL_USER_LANE_PM_RAW_SIZE 0xd100u @@ -1379,6 +1437,58 @@ enum gsp_phys_rmcall_buf { GSP_PHYS_RMCALL_BUF_GUEST_PT13, GSP_PHYS_RMCALL_BUF_GUEST_PT14, GSP_PHYS_RMCALL_BUF_GUEST_PT15, + /* m152: extension pool PT pages: 48 separate single-page chunks + * (GUEST_PT_EXT0..47). The server only accepts single-page memdescs + * for client buffers (a 48-page chunk was refused with rpc_result + * 0x1f + a firmware ASSERT in stage 2). */ + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT0, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT1, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT2, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT3, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT4, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT5, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT6, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT7, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT8, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT9, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT10, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT11, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT12, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT13, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT14, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT15, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT16, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT17, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT18, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT19, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT20, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT21, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT22, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT23, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT24, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT25, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT26, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT27, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT28, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT29, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT30, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT31, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT32, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT33, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT34, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT35, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT36, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT37, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT38, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT39, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT40, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT41, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT42, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT43, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT44, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT45, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT46, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT47, /* * Golden9g: the three client page-table levels the real golden VAS * hands to the server with VASPACE_COPY_SERVER_RESERVED_PDES @@ -1409,6 +1519,20 @@ enum gsp_phys_rmcall_buf { * generous and progress is logged every ~500 ms. */ #define GSP_PHYS_RMCALL_GR_POLL_MS 5000u +/* + * Bounded retry of a forwarded GR launch whose QMD did not run. A green + * launch completes in ~170-195 ms; when the engine drops it (cold context + * save/unload, or an entry the PBDMA never fetched) `qmd_rel` never reaches + * the wanted value and the client would fall back to the software release + * and read stale/zero results. Because the timeout path never copies the + * result out to the guest, a launch that timed out did not reach the + * client: re-publishing a FRESH ring entry (same PB/QMD) and re-issuing the + * schedule+doorbell gives the front-end another chance and the guest still + * sees only the success path. Bounded so the service thread is never held + * past the existing hold budget. + */ +#define GSP_PHYS_RMCALL_GR_RETRY_MS 2000u +#define GSP_PHYS_RMCALL_GR_ATTEMPTS 3u /* P6a guest-memory CE copies: same bounded poll cap as the P5 shadow path. */ #define GSP_PHYS_RMCALL_GUEST_POLL_MS 200u @@ -2136,17 +2260,31 @@ static struct gsp_phys_boot_stage { bool rmcall_gr_exec_ok; /* a real GR submission proved out */ u32 rmcall_gr_submit_epoch; /* per-submission mapper epoch */ u32 rmcall_gr_pt_used; /* Tier-1 window pool slots used */ - u32 rmcall_guest_pt_win_mask; /* Tier-2 scratch windows installed */ + u64 rmcall_guest_pt_win_mask; /* Tier-2 scratch windows installed */ + /* + * m152: shadow alias reuse cache: hpa -> never-rewritten shadow VA. + * Populated by gsp_phys_rmcall_gr_map_shadow_locked; a hit returns the + * existing line so repeat launches of the same physical pages consume + * no new fetch line. Fill-order/LRU-free by design: entries are dead + * weight once a client exits, but a boot has 2048 lines to give and + * the cache only ever returns identical PTE content. + */ + u64 rmcall_gr_alias_hpa[GSP_PHYS_RMCALL_GR_ALIAS_CACHE]; + u64 rmcall_gr_alias_va[GSP_PHYS_RMCALL_GR_ALIAS_CACHE]; + u32 rmcall_gr_alias_count; struct gsp_phys_boot_chunk rmcall_gr_pt[GSP_PHYS_RMCALL_GR_PT_PAGES]; struct gsp_phys_boot_chunk rmcall_gr_shadow; /* Tier-2 pages */ /* * m69: window-0 shadow pool (gsp_phys_rmcall_gr_shadow_win0=1). - * Eight module pages whose PTEs are planted in the proven window-0 - * PT line at slots 0x108..0x10f (VAs 0x108000..0x10f000) by - * gsp_phys_rmcall_pdb_build, before the engine's first fetch (same - * pattern as the qmdwin slot). The launch copies the QMD's - * program/param page content into them and rewrites the QMD pointers - * there, so the GR engine never reads the window-1 scratch range. + * Sixteen module pages (two 8-page banks; m151 alternates the bank + * per launch so a launch's copy-in cannot clobber pages the previous + * launch's engine fetches may still be reading) whose PTEs are + * planted in the proven window-0 PT (slots derive from + * GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE) by gsp_phys_rmcall_pdb_build, + * before the engine's first fetch (same pattern as the qmdwin slot). + * The launch copies the QMD's program/param page content into them + * and rewrites the QMD pointers there, so the GR engine never reads + * the window-1 scratch range. */ struct gsp_phys_boot_chunk rmcall_gr_shadow_win0; u32 rmcall_gr_shadow_win0_used; @@ -2161,19 +2299,52 @@ static struct gsp_phys_boot_stage { /* m104: module page holding the injected mail_kernel message. */ struct gsp_phys_boot_chunk rmcall_gr_mail; /* - * m114: pre-commit bounce window. 112 pages planted into the main - * small-PT at slots 0x110..0x17f during stage 3 (the proven win0 - * pattern: the SM walker resolves the commit-time tree only). The - * launch copies the kernel's data pages in, rewrites the pointers to - * these VAs (reloc base 0x110000), and copies the results back after - * the release; page 100 (VA 0x174000) carries the mail message. + * m114: pre-commit bounce window, planted into the main small-PT at + * slots 0x110..0x1df during stage 3 (the proven win0 pattern: the SM + * walker resolves the commit-time tree only). The launch copies the + * kernel's data pages in, rewrites the pointers to these VAs (reloc + * base 0x110000), and copies the results back after the release. + * + * The window is 208 pages so the data region below the mail page can + * carry the operand pages of kernels whose buffers exceed the old + * 128 KB (victory_matmul's 256 KB buffers need 64 pages each; three + * of them plus argument/bank candidates exceed the old 100-page / + * 33-pages-per-pointer caps and the engine then ran on truncated + * windows). The mail page is the LAST data-free page (slot 206) and + * the m132 marker sits at 207; the main PT page deliberately covers + * the whole 0x110000..0x1df000 range with nothing else planted there. */ struct gsp_phys_boot_chunk rmcall_gr_bounce; u32 rmcall_gr_bounce_used; -#define GSP_PHYS_RMCALL_GR_BOUNCE_PAGES 112u +#define GSP_PHYS_RMCALL_GR_BOUNCE_PAGES 208u #define GSP_PHYS_RMCALL_GR_BOUNCE_SLOT0 0x110u #define GSP_PHYS_RMCALL_GR_BOUNCE_VA0 0x110000ULL -#define GSP_PHYS_RMCALL_GR_MAIL_SLOT 100u +#define GSP_PHYS_RMCALL_GR_MAIL_SLOT 206u +/* + * m150b: the win0 QMD shadow pool must never overlap the bounce window. + * The win0 plant runs on every pdb_build() (unconditional) and the bounce + * plant is one-shot, so an overlap silently hijacks bounce PTEs and the + * engine's results land in the shadow pages (victory_matmul lost C rows + * 0..31 exactly this way before 0x1e0000 moved the pool clear). + */ +static_assert(GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE + + (u64)GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS * GSP_PHYS_PAGE_SIZE <= + GSP_PHYS_RMCALL_GR_BOUNCE_VA0 || + GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE >= + GSP_PHYS_RMCALL_GR_BOUNCE_VA0 + + (u64)GSP_PHYS_RMCALL_GR_BOUNCE_PAGES * GSP_PHYS_PAGE_SIZE, + "win0 shadow pool must not overlap the GR data bounce window"); +/* + * m151: and the two-bank pool must end below the reserved mail-injection + * slot 0x1f0 (VA 0x1f0000) so the extra slots never steal that PTE. + */ +static_assert(GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE + + (u64)GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS * GSP_PHYS_PAGE_SIZE <= + GSP_PHYS_RMCALL_VA_BASE + 0xF0000ULL, + "win0 shadow pool must end below the reserved mail-injection slot"); +static_assert(GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS == + 2u * GSP_PHYS_RMCALL_GR_SHADOW0_BANK_SLOTS, + "win0 shadow pool must be exactly two banks"); /* m99: guest data identity-map table branches (see the PD*_MAX block). */ struct gsp_phys_boot_chunk rmcall_gr_data_pd1[GSP_PHYS_RMCALL_GR_DATA_PD1_MAX]; @@ -2813,6 +2984,7 @@ static void gsp_phys_rmcall_reset(void) boot_stage.rmcall_gr_submit_epoch = 0; boot_stage.rmcall_gr_pt_used = 0; boot_stage.rmcall_guest_pt_win_mask = 0; + boot_stage.rmcall_gr_alias_count = 0; memset(boot_stage.rmcall_gr_pt, 0, sizeof(boot_stage.rmcall_gr_pt)); memset(&boot_stage.rmcall_gr_shadow, 0, sizeof(boot_stage.rmcall_gr_shadow)); @@ -7215,6 +7387,65 @@ static void gsp_phys_rmcall_ctx_diag_dump(const char *what, static void gsp_phys_rmcall_ctx_diag_decode(const char *tag); /* + * m152: scratch-PT pool geometry helpers. Pool local page P (0-based across + * the original 16-window region then the 48-window extension) maps to a PD0 + * window VA and to a page of either the per-window single-page PT chunks + * (P < 16) or the multi-page extension chunk (P >= 16). Keep every + * consumer on these two helpers so the two regions can never drift. + */ +static u64 gsp_phys_rmcall_guest_pt_win_va(u32 page) +{ + if (page < GSP_PHYS_RMCALL_GUEST_PT_PAGES) + return boot_stage.rmcall_va_guest_pt + ((u64)page << 21); + return GSP_PHYS_RMCALL_VA_GUEST_PT_EXT_BASE + + ((u64)(page - GSP_PHYS_RMCALL_GUEST_PT_PAGES) << 21); +} + +static u8 *gsp_phys_rmcall_guest_pt_page_va(u32 page) +{ + if (page < GSP_PHYS_RMCALL_GUEST_PT_PAGES) + return gsp_phys_chunk_page_va( + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GUEST_PT0 + + page], 0); + if (page < GSP_PHYS_RMCALL_GUEST_PT_TOTAL_PAGES) + return gsp_phys_chunk_page_va( + &boot_stage.rmcall_chunks[ + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT0 + + (page - GSP_PHYS_RMCALL_GUEST_PT_PAGES)], 0); + return NULL; +} + +static u64 gsp_phys_rmcall_guest_pt_page_dma(u32 page) +{ + if (page < GSP_PHYS_RMCALL_GUEST_PT_PAGES) + return (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GUEST_PT0 + + page], 0); + return (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_chunks[ + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT0 + + (page - GSP_PHYS_RMCALL_GUEST_PT_PAGES)], 0); +} + +/* + * PD0 index -> pool local page, or 0xffffffffu when the index is not a + * scratch-pool window. Consults both pool regions. + */ +static u32 gsp_phys_rmcall_guest_pt_page_of_pde(u32 pde_index) +{ + u32 base = (u32)(boot_stage.rmcall_va_guest_pt >> 21); + + if (pde_index >= base && + pde_index < base + GSP_PHYS_RMCALL_GUEST_PT_PAGES) + return pde_index - base; + base = (u32)(GSP_PHYS_RMCALL_VA_GUEST_PT_EXT_BASE >> 21); + if (pde_index >= base && + pde_index < base + GSP_PHYS_RMCALL_GUEST_PT_EXT_PAGES) + return pde_index - base + GSP_PHYS_RMCALL_GUEST_PT_PAGES; + return 0xffffffffu; +} + +/* * Shared-management VAS: install the ladder's page-table window through the * server's own MMU walker. NV0080_CTRL_CMD_DMA_UPDATE_PDE_2 (0x80180f, * gpu_vaspace.c gvaspaceUpdatePde2_IMPL) takes the system-physical base of a @@ -9364,7 +9595,7 @@ static void gsp_phys_rmcall_pdb_build(void) } } /* - * m69: plant the window-0 shadow pool PTEs (slots 0x108..0x10f) in the + * m69: plant the window-0 shadow pool PTEs (current win0 slot range) in the * same proven small-PT line, before the engine's first fetch. The * launch copies the QMD's program/param page content into these pages * and rewrites the QMD pointers to these VAs, so the GR engine never @@ -9410,12 +9641,18 @@ static void gsp_phys_rmcall_pdb_build(void) (size_t)idx * 8); } dma_wmb(); - pr_info("vgsp-boot: rm-call: shadow-win0 pool planted %u pages at VA 0x%llx..0x%llx (small-PT slots 0x108..0x10f, before first fetch)\n", + pr_info("vgsp-boot: rm-call: shadow-win0 pool planted %u pages at VA 0x%llx..0x%llx (small-PT slots 0x%x..0x%x, before first fetch)\n", GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS, GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE, GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE + (GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS - 1) * - GSP_PHYS_PAGE_SIZE); + GSP_PHYS_PAGE_SIZE, + (u32)((GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE >> 12) & + 0x1ffu), + (u32)(((GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE + + (GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS - 1) * + GSP_PHYS_PAGE_SIZE) >> 12) & + 0x1ffu)); } } /* @@ -9532,6 +9769,30 @@ static void gsp_phys_rmcall_pdb_build(void) (size_t)(1 + i) * 16 + 8); } /* + * m152: extension scratch pool - PD0 dual entries 32..79 back the + * GSP_PHYS_RMCALL_GUEST_PT_EXT_PAGES pages of the multi-page extension + * PT chunk at GSP_PHYS_RMCALL_VA_GUEST_PT_EXT_BASE (0x04000000; free + * space above the GR-context windows, which end well below 0x03000000). + * Same shape and same once-before-any-engine-access rule as the 16-window + * loop above. + */ + for (i = 0; i < GSP_PHYS_RMCALL_GUEST_PT_EXT_PAGES; i++) { + u64 guest_pt_pde = gsp_phys_rmcall_pde( + (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_chunks[ + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT0 + i], + 0)); + u32 pd0_idx = (u32)(GSP_PHYS_RMCALL_VA_GUEST_PT_EXT_BASE >> 21) + + i; + + put_unaligned_le64(0, + (u8 *)gsp_phys_chunk_page_va(pd0, 0) + + (size_t)pd0_idx * 16); + put_unaligned_le64(guest_pt_pde, + (u8 *)gsp_phys_chunk_page_va(pd0, 0) + + (size_t)pd0_idx * 16 + 8); + } + /* * H1 GR context window: when the GR ladder prepared the promoted MAIN * context buffer (gsp_phys_rmcall_gr_ctx_prepare ran before this * builder), append its page tables as fresh PD0 dual entries 17.. (the @@ -9698,11 +9959,12 @@ static void gsp_phys_rmcall_pdb_build(void) src_idx, pt_src, (u64)gsp_phys_chunk_page_dma(src, 0), dst_idx, pt_dst, (u64)gsp_phys_chunk_page_dma(dst, 0), sem_idx, pt_sem, (u64)gsp_phys_chunk_page_dma(sem, 0)); - pr_info("vgsp-boot: rm-call: guest scratch PT built: pages=%u vaBase=0x%llx capacity=%u\n", + pr_info("vgsp-boot: rm-call: guest scratch PT built: pages=%u (+%u ext) vaBase=0x%llx extBase=0x%llx capacity=%u\n", GSP_PHYS_RMCALL_GUEST_PT_PAGES, + GSP_PHYS_RMCALL_GUEST_PT_EXT_PAGES, GSP_PHYS_RMCALL_VA_GUEST_PT_BASE, - GSP_PHYS_RMCALL_GUEST_PT_PAGES * - GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE); + GSP_PHYS_RMCALL_VA_GUEST_PT_EXT_BASE, + GSP_PHYS_RMCALL_GUEST_PT_TOTAL_LINES); } /* @@ -15162,7 +15424,6 @@ static int gsp_phys_rmcall_guest_map_locked(u64 src_hpa, u64 dst_hpa, u64 sem_hpa, u64 *src_va, u64 *dst_va, u64 *sem_va) { - struct gsp_phys_boot_chunk *guest_pt_chunk; u32 line = boot_stage.rmcall_guest_pt_next_line; u32 page, line_in_page; u64 va; @@ -15170,19 +15431,14 @@ static int gsp_phys_rmcall_guest_map_locked(u64 src_hpa, u64 dst_hpa, if (!boot_stage.rmcall_va_guest_pt) return -EIO; - if (line >= GSP_PHYS_RMCALL_GUEST_PT_PAGES * - GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE) + if (line >= GSP_PHYS_RMCALL_GUEST_PT_TOTAL_LINES) return -ENOSPC; page = line / GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; line_in_page = line % GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; - guest_pt_chunk = &boot_stage.rmcall_chunks[ - GSP_PHYS_RMCALL_BUF_GUEST_PT0 + page]; - if (!guest_pt_chunk->pages) - return -EIO; /* 2 MB PD0 slot stride; a full line is 16 PTEs = 0x10000 of VA. */ - va = boot_stage.rmcall_va_guest_pt + ((u64)page << 21) + + va = gsp_phys_rmcall_guest_pt_win_va(page) + ((u64)line_in_page << 16); - pt = gsp_phys_chunk_page_va(guest_pt_chunk, 0); + pt = gsp_phys_rmcall_guest_pt_page_va(page); if (!pt) return -EIO; /* @@ -15403,15 +15659,13 @@ static int gsp_phys_guest_ce_copy_common(u64 src_hpa, u32 src_off, u64 dst_hpa, if ((gsp_phys_relay_pool_exhausted & 63) == 1) pr_err("vgsp-work: GUEST CE RELAY FAILED: scratch-PT pool exhausted (no line reuse) pool_exhausted=%llu capacity=%u; falling back to the software release\n", gsp_phys_relay_pool_exhausted, - GSP_PHYS_RMCALL_GUEST_PT_PAGES * - GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE); + GSP_PHYS_RMCALL_GUEST_PT_TOTAL_LINES); } else { gsp_phys_guest_pool_exhausted++; if ((gsp_phys_guest_pool_exhausted & 63) == 1) pr_err("vgsp-work: GUEST CE COPY FAILED: scratch-PT pool exhausted (no line reuse) pool_exhausted=%llu capacity=%u; falling back to the ladder path\n", gsp_phys_guest_pool_exhausted, - GSP_PHYS_RMCALL_GUEST_PT_PAGES * - GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE); + GSP_PHYS_RMCALL_GUEST_PT_TOTAL_LINES); } } else { if (relay) @@ -15738,6 +15992,7 @@ static u64 gsp_phys_gr_submitted; static u64 gsp_phys_gr_ok; static u64 gsp_phys_gr_refused; static u64 gsp_phys_gr_timed_out; +static u64 gsp_phys_gr_retries; /* fresh-entry launch retries published */ static u64 gsp_phys_gr_pool_exhausted; /* m86: program-page override blob (see gsp-phys-gr-override). */ static u8 gsp_phys_gr_override[GSP_PHYS_PAGE_SIZE]; @@ -16077,26 +16332,26 @@ static void __iomem *gsp_phys_gr_scratch_bar0; static int gsp_phys_rmcall_gr_scratch_win_install_locked(u32 page) { struct gsp_phys_rmcall_update_pde2_params p; - struct gsp_phys_boot_chunk *pt; u32 rpc_result = 0, status = 0, pde_index; + u64 pt_dma; int ret; - if (page >= GSP_PHYS_RMCALL_GUEST_PT_PAGES) + if (page >= GSP_PHYS_RMCALL_GUEST_PT_TOTAL_PAGES) return -EINVAL; if (!gsp_phys_rmcall_vas_shared_value()) return 0; - if (boot_stage.rmcall_guest_pt_win_mask & BIT(page)) + if (boot_stage.rmcall_guest_pt_win_mask & BIT_ULL(page)) return 0; - pt = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GUEST_PT0 + page]; - if (!pt->pages) + pt_dma = gsp_phys_rmcall_guest_pt_page_dma(page); + if (!pt_dma) return -EIO; if (!boot_stage.rmcall_va_guest_pt) return -EIO; - pde_index = (u32)((boot_stage.rmcall_va_guest_pt >> 21) + page); + pde_index = (u32)(gsp_phys_rmcall_guest_pt_win_va(page) >> 21); memset(&p, 0, sizeof(p)); p.pde_index = cpu_to_le32(pde_index); p.flags = cpu_to_le32(GSP_PHYS_RMCALL_PDE2_FLAG_FLUSH); - p.pt[0].phys_addr = cpu_to_le64((u64)gsp_phys_chunk_page_dma(pt, 0)); + p.pt[0].phys_addr = cpu_to_le64(pt_dma); p.pt[0].aperture = cpu_to_le32(GSP_PHYS_RMCALL_PDE2_APERTURE_SYS_COH); p.pt[1].aperture = cpu_to_le32(GSP_PHYS_RMCALL_PDE2_APERTURE_INVALID); p.h_va_space = cpu_to_le32(boot_stage.rmcall_vas); @@ -16113,10 +16368,9 @@ static int gsp_phys_rmcall_gr_scratch_win_install_locked(u32 page) ret); return ret; } - boot_stage.rmcall_guest_pt_win_mask |= BIT(page); + boot_stage.rmcall_guest_pt_win_mask |= BIT_ULL(page); pr_info("vgsp-gr: scratch window installed pdeIndex=%u (VA 0x%llx) ptPa=0x%llx status=0x%x (Tier-2 shadow legs now reachable through the server walker)\n", - pde_index, (u64)pde_index << 21, - (u64)gsp_phys_chunk_page_dma(pt, 0), status); + pde_index, (u64)pde_index << 21, pt_dma, status); return 0; } @@ -16129,13 +16383,11 @@ static int gsp_phys_rmcall_gr_scratch_win_install_locked(u32 page) static bool gsp_phys_rmcall_gr_win_is_installed_locked(u64 va) { u32 pde_index = (u32)(va >> 21); - u32 base = (u32)(boot_stage.rmcall_va_guest_pt >> 21); + u32 page = gsp_phys_rmcall_guest_pt_page_of_pde(pde_index); u32 i; - if (pde_index >= base && - pde_index < base + GSP_PHYS_RMCALL_GUEST_PT_PAGES) - return !!(boot_stage.rmcall_guest_pt_win_mask & - BIT(pde_index - base)); + if (page != 0xffffffffu) + return !!(boot_stage.rmcall_guest_pt_win_mask & BIT_ULL(page)); for (i = 0; i < boot_stage.rmcall_gr_pt_used; i++) if (boot_stage.rmcall_gr_win[i].used && boot_stage.rmcall_gr_win[i].pde_index == pde_index) @@ -16154,13 +16406,11 @@ static bool gsp_phys_rmcall_gr_win_is_installed_locked(u64 va) static int gsp_phys_rmcall_gr_win_ensure_locked(void __iomem *bar0, u64 va) { u32 pde_index = (u32)(va >> 21); - u32 base = (u32)(boot_stage.rmcall_va_guest_pt >> 21); + u32 page = gsp_phys_rmcall_guest_pt_page_of_pde(pde_index); u32 slot; - if (pde_index >= base && - pde_index < base + GSP_PHYS_RMCALL_GUEST_PT_PAGES) - return gsp_phys_rmcall_gr_scratch_win_install_locked( - pde_index - base); + if (page != 0xffffffffu) + return gsp_phys_rmcall_gr_scratch_win_install_locked(page); return gsp_phys_rmcall_gr_win_slot_locked(bar0, pde_index, &slot); } @@ -16171,16 +16421,35 @@ static int gsp_phys_rmcall_gr_win_ensure_locked(void __iomem *bar0, u64 va) * supplies the backing page. Called with gsp_phys_rm_lock held. */ static int gsp_phys_rmcall_gr_map_shadow_locked(u32 slot, u64 hpa, - u64 *va_out) + u64 *va_out, u32 *line_out) { struct gsp_phys_boot_chunk *sh = &boot_stage.rmcall_gr_shadow; - struct gsp_phys_boot_chunk *pt; u32 line = boot_stage.rmcall_guest_pt_next_line; - u32 page, line_in_page; + u32 page, line_in_page, i; u64 va; - u8 *pt_va; + u8 *pt; int ret; + /* + * m152: reuse before consume. A shadow VA is a never-rewritten PTE: + * once hpa X maps at VA V, every later request for X can be served by + * V with byte-identical engine-visible content, so the pool only + * needs one line per distinct HPA per boot instead of one per launch. + * This is safe against the GMMU's no-invalidate fetch-line cache by + * construction: the PTE value never differs from what the engine may + * have already fetched. A hit reports line_out = ~0 so a caller that + * records contiguity bookkeeping never contig-extends an old line + * (a new PTE in a line the engine may have already fetched would be + * invisible). + */ + for (i = 0; i < boot_stage.rmcall_gr_alias_count; i++) { + if (boot_stage.rmcall_gr_alias_hpa[i] == hpa) { + *va_out = boot_stage.rmcall_gr_alias_va[i]; + if (line_out) + *line_out = 0xffffffffu; + return 0; + } + } if (!sh->pages) { ret = gsp_phys_chunk_alloc(boot_stage.pdev, sh, GSP_PHYS_RMCALL_GR_SHADOW_PAGES); @@ -16192,28 +16461,34 @@ static int gsp_phys_rmcall_gr_map_shadow_locked(u32 slot, u64 hpa, return -ENOSPC; if (!boot_stage.rmcall_va_guest_pt) return -EIO; - if (line >= GSP_PHYS_RMCALL_GUEST_PT_PAGES * - GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE) + if (line >= GSP_PHYS_RMCALL_GUEST_PT_TOTAL_LINES) return -ENOSPC; page = line / GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; line_in_page = line % GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; ret = gsp_phys_rmcall_gr_scratch_win_install_locked(page); if (ret) return ret; - pt = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GUEST_PT0 + page]; - if (!pt->pages) - return -EIO; - pt_va = gsp_phys_chunk_page_va(pt, 0); - if (!pt_va) + pt = gsp_phys_rmcall_guest_pt_page_va(page); + if (!pt) return -EIO; put_unaligned_le64(gsp_phys_rmcall_pte(hpa), - pt_va + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES * + pt + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES * line_in_page) * 8); dma_wmb(); boot_stage.rmcall_guest_pt_next_line = line + 1; - va = boot_stage.rmcall_va_guest_pt + ((u64)page << 21) + + va = gsp_phys_rmcall_guest_pt_win_va(page) + ((u64)line_in_page << 16); *va_out = va; + if (line_out) + *line_out = line; + if (boot_stage.rmcall_gr_alias_count < + GSP_PHYS_RMCALL_GR_ALIAS_CACHE) { + u32 ci = boot_stage.rmcall_gr_alias_count; + + boot_stage.rmcall_gr_alias_hpa[ci] = hpa; + boot_stage.rmcall_gr_alias_va[ci] = va; + boot_stage.rmcall_gr_alias_count = ci + 1u; + } return 0; } @@ -16227,32 +16502,27 @@ static int gsp_phys_rmcall_gr_map_shadow_locked(u32 slot, u64 hpa, static int gsp_phys_rmcall_gr_map_shadow_contig_locked(u32 line, u32 pte, u64 hpa, u64 *va_out) { - struct gsp_phys_boot_chunk *pt; u32 page, line_in_page; - u8 *pt_va; + u8 *pt; int ret; if (pte >= GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES) return -EINVAL; - if (line >= GSP_PHYS_RMCALL_GUEST_PT_PAGES * - GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE) + if (line >= GSP_PHYS_RMCALL_GUEST_PT_TOTAL_LINES) return -ENOSPC; page = line / GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; line_in_page = line % GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; ret = gsp_phys_rmcall_gr_scratch_win_install_locked(page); if (ret) return ret; - pt = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GUEST_PT0 + page]; - if (!pt->pages) - return -EIO; - pt_va = gsp_phys_chunk_page_va(pt, 0); - if (!pt_va) + pt = gsp_phys_rmcall_guest_pt_page_va(page); + if (!pt) return -EIO; put_unaligned_le64(gsp_phys_rmcall_pte(hpa), - pt_va + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES * + pt + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES * line_in_page + pte) * 8); dma_wmb(); - *va_out = boot_stage.rmcall_va_guest_pt + ((u64)page << 21) + + *va_out = gsp_phys_rmcall_guest_pt_win_va(page) + ((u64)line_in_page << 16) + (u64)pte * GSP_PHYS_PAGE_SIZE; return 0; } @@ -16471,6 +16741,24 @@ static int gsp_phys_rmcall_gr_resched_locked(void __iomem *bar0) u32 runlist_id, ch_id, doorbell; int ret; + /* + * m155 (HF-3 extended re-arm): after the channel has been idle long + * enough for the FECS to save/unload the context, re-issuing only the + * two GPFIFO_SCHEDULE verbs does not revive dispatch - r154: matmul's + * post-smiley launches acked (qmd_rel/ch_rel) but SKED stayed idle + * (cwd0=0x2086/fecs_cur=0) for every attempt including outcome + * retries, while the same channel executed mail/smiley earlier in the + * boot. Replay the mode-35 lane-enable tail (CILP preempt mode, + * timeslice, device sched) that stage 4 ran when the lane first went + * live - it is proven accepted and is the design's extended re-arm. + * The poison variants (explicit PREEMPT/tail, BIND, PROMOTE) stay out. + */ + ret = gsp_phys_rmcall_user_lane_lane_enable(bar0); + if (ret) + pr_info("vgsp-gr: rearm lane-enable rc=%d (continuing to schedules)\n", + ret); + rpc_result = 0; + status = 0; ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_channel, GSP_PHYS_RMCALL_CTRL_GPFIFO_SCHEDULE, sched_params, sizeof(sched_params), 16, @@ -16495,6 +16783,144 @@ static int gsp_phys_rmcall_gr_resched_locked(void __iomem *bar0) return 0; } +/* + * m151: FNV-1a 64-bit over one page. Used to compare the win0 param page + * and the guest param page before and after the poll, so a launch that + * wrote (or clobbered) the page the engine read is provable from the log. + */ +static inline u64 gsp_phys_gr_fnv1a64(const u8 *p, size_t len) +{ + u64 h = 0xcbf29ce484222325ULL; + size_t i; + + for (i = 0; i < len; i++) { + h ^= p[i]; + h *= 0x100000001b3ULL; + } + return h; +} + +/* + * m151: output retirement wait. qmd_rel is a schedule/fetch ack (the front + * end latched and fetched the QMD), not a data-path retirement: the SM's + * stores into the bounce pages can still be in flight when the ack lands. + * Copying out - and the next launch's copy-in - before those stores land + * loses them: the guest observes zeros for a completed launch, or + * stale/mixed content when the next copy-in races the previous engine's + * write tail. + * + * The wait covers EVERY resolved data page (footprint-by-inference was + * tried twice and is unreliable: the fill's candidate list ends with the + * launch's own bank/param pages, so the max-reloc run is 3-4 pages, not + * the output operand). Adaptive exits make a whole-window count cheap: + * - all resolved pages changed -> done (normal case, one poll) + * - changed>0 and stable for 3 polls -> stores settled (masked or + * repeated-identical kernels) + * - no writes at all for 600 ms -> no output for this launch + * - hard cap 1.5 s -> genuinely late remainder + * noinline so this rarely-taken wait does not grow the submit frame, which + * is already near the warning bound. + */ +static noinline u32 +gsp_phys_gr_output_retire_wait(const struct gsp_phys_gr_submit *req, + const u32 *csum_in) +{ + u32 checked[GSP_PHYS_RMCALL_GR_MAIL_SLOT]; + u32 n_all = 0, di, iter; + u32 changed = 0, prev = 0xffffffffu; + const char *why = " (TIMEOUT)"; + unsigned long wait_start = jiffies; + + if (!req->n_data) + return 0; + for (di = 0; di < req->n_data; di++) { + u32 bi = (u32)((req->data[di].reloc_va - + GSP_PHYS_RMCALL_GR_BOUNCE_VA0) >> 12); + u32 k; + bool dup = false; + + if (bi >= GSP_PHYS_RMCALL_GR_MAIL_SLOT) + continue; + for (k = 0; k < n_all; k++) + if (checked[k] == bi) { + dup = true; + break; + } + if (!dup) + checked[n_all++] = bi; + } + if (!n_all) + return 0; + for (iter = 0; iter < 300u; iter++) { + u32 j; + + changed = 0; + for (j = 0; j < n_all; j++) { + const u8 *src = gsp_phys_chunk_page_va( + &boot_stage.rmcall_gr_bounce, checked[j]); + u32 ci, csum = 0; + + if (!src) + continue; + for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) + csum += get_unaligned_le32(src + ci); + if (csum != csum_in[checked[j]]) + changed++; + } + if (changed >= n_all) { + why = ""; + break; + } + if (changed > 0 && changed == prev && iter >= 3u) { + why = " (settled)"; + break; + } + if (changed == 0 && iter >= 120u) { + why = " (no writes)"; + break; + } + prev = changed; + usleep_range(4000, 6000); + } + pr_info("vgsp-gr: output retirement wait: %u/%u pages changed after %u ms%s\n", + changed, n_all, jiffies_to_msecs(jiffies - wait_start), why); + return changed; +} + +/* + * m153: fill the bounce window from the request's guest pages and capture a + * per-page checksum baseline. Used before the first publish and again before + * every outcome retry (a mis-executing first attempt can leave stale or + * wrong-window content behind, so a retry must start from the guest inputs + * again). + */ +static void gsp_phys_gr_fill_bounce(const struct gsp_phys_gr_submit *req, + u32 *csum_in) +{ + u32 di; + + for (di = 0; di < req->n_data; di++) { + u32 bi = (u32)((req->data[di].reloc_va - + GSP_PHYS_RMCALL_GR_BOUNCE_VA0) >> 12); + const u8 *src; + u8 *dst; + u32 ci, csum = 0; + + if (bi >= GSP_PHYS_RMCALL_GR_MAIL_SLOT || + !req->data[di].hpa) + continue; + src = (const u8 *)__va(req->data[di].hpa); + dst = gsp_phys_chunk_page_va(&boot_stage.rmcall_gr_bounce, bi); + if (!dst) + continue; + memcpy(dst, src, GSP_PHYS_PAGE_SIZE); + for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) + csum += get_unaligned_le32(dst + ci); + csum_in[bi] = csum; + } + dma_wmb(); +} + int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, struct gsp_phys_backend_copy_result *out) { @@ -16517,6 +16943,8 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, unsigned long deadline; u8 *sem_va; u32 epoch, words, i, n_shadow = 0; + u32 win0_bank_base = 0, shadow_slot_limit = 0; + const u8 *guest_param_pg; u32 want_qmd = 0, want_ch = 0; bool qmd_rel = false, ch_rel = false; u64 qmd_rel_hpa = 0, ch_rel_hpa = 0; @@ -16540,12 +16968,14 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, static u32 bounce_csum_in[GSP_PHYS_RMCALL_GR_MAIL_SLOT]; sked_tick = 0; sked_first = true; + guest_param_pg = NULL; u32 qmd_win_pde = 0, qmd_win_installed = 0; int qmd_win_ret = 0; bool qmd_win_diag = false; bool qmdwin_on = false; char qmdwin_note[32] = ""; bool rearmed = false; + u32 attempts = 0; int ret = 0; if (out) @@ -16588,8 +17018,13 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, * the program page (silent corruption; no launch can survive it). * The win0 PTEs are planted once at stage 3; reusing the same VAs * with fresh content is the proven pattern. + * + * m151: the cursor now selects one of two 8-page banks, alternating + * with the submission epoch (set below). The engine's index/data + * fetches of the previous launch's QMD shadow pages can still be in + * flight when this launch runs, so the bank alternation protects + * them from this launch's copy-in (late-fetch safety). */ - boot_stage.rmcall_gr_shadow_win0_used = 0; mutex_lock(&gsp_phys_rm_lock); if (!gsp_phys_backend_gr_ready()) { mutex_unlock(&gsp_phys_rm_lock); @@ -16600,6 +17035,11 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, epoch = ++boot_stage.rmcall_gr_submit_epoch; if (!epoch) epoch = ++boot_stage.rmcall_gr_submit_epoch; + boot_stage.rmcall_gr_shadow_win0_used = + (epoch & 1u) * GSP_PHYS_RMCALL_GR_SHADOW0_BANK_SLOTS; + win0_bank_base = boot_stage.rmcall_gr_shadow_win0_used; + shadow_slot_limit = win0_bank_base + + GSP_PHYS_RMCALL_GR_SHADOW0_BANK_SLOTS; memcpy(qmd, req->qmd, sizeof(qmd)); /* * Two-step PCAS (gsp_phys_rmcall_gr_pcas_two_step): set the QMD's @@ -16628,7 +17068,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, ret = gsp_phys_rmcall_gr_map_shadow_locked(n_shadow, req->rel_hpa, - &sh_va); + &sh_va, NULL); if (!ret) { n_shadow++; qmd_rel_hpa = req->rel_hpa; @@ -16719,6 +17159,8 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, for (k = 0; k < n_sh_rec; k++) { u64 va = 0; + if (sh_rec[k].line == 0xffffffffu) + continue; /* cached line: never extended */ if (pg_va != sh_rec[k].guest_va + (u64)sh_rec[k].npte * GSP_PHYS_PAGE_SIZE) @@ -16745,11 +17187,11 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, */ if (!mapped && pg->hpa && pg->qmd_word == GSP_PHYS_GR_QMD_WORD_NONE) { - u32 sh_line = boot_stage.rmcall_guest_pt_next_line; + u32 sh_line = 0xffffffffu; if (n_shadow < GSP_PHYS_RMCALL_GR_SHADOW_PAGES && !gsp_phys_rmcall_gr_map_shadow_locked( - n_shadow, pg->hpa, &sh_va)) { + n_shadow, pg->hpa, &sh_va, &sh_line)) { u32 k, slot = n_shadow; mapped = true; @@ -16975,7 +17417,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, gsp_phys_rmcall_gr_shadow_win0_value() && boot_stage.rmcall_gr_shadow_win0.pages && boot_stage.rmcall_gr_shadow_win0_used < - GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS; + shadow_slot_limit; /* * m86: SASS override. When gsp-phys-gr-override @@ -17036,7 +17478,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, n_shadow, (u64)gsp_phys_chunk_page_dma(sh, n_shadow), - &sh_va); + &sh_va, NULL); } if (!ret) { bool keep_guest = @@ -17197,7 +17639,7 @@ gr_page_unmapped: if (n_shadow < GSP_PHYS_RMCALL_GR_SHADOW_PAGES && !gsp_phys_rmcall_gr_map_shadow_locked( - n_shadow, req->qmd_hpa, &sh_va)) { + n_shadow, req->qmd_hpa, &sh_va, NULL)) { qmd_dst_mem = (u8 *)gsp_phys_chunk_page_va( &boot_stage.rmcall_gr_shadow, n_shadow) + qmd_off; @@ -17240,7 +17682,8 @@ gr_page_unmapped: u64 sh_va = 0; ret = gsp_phys_rmcall_gr_map_shadow_locked(n_shadow, - req->sem_hpa, &sh_va); + req->sem_hpa, &sh_va, + NULL); if (!ret) { n_shadow++; ch_rel_hpa = req->sem_hpa; @@ -17335,29 +17778,19 @@ gr_page_unmapped: if (req->n_data && boot_stage.rmcall_gr_bounce.pages) { u32 dcopied = 0, dfailed = 0, di; + gsp_phys_gr_fill_bounce(req, bounce_csum_in); for (di = 0; di < req->n_data; di++) { u32 bi = (u32)((req->data[di].reloc_va - GSP_PHYS_RMCALL_GR_BOUNCE_VA0) >> 12); - const u8 *src; - u8 *dst; - u32 ci = 0, csum = 0; if (bi >= GSP_PHYS_RMCALL_GR_MAIL_SLOT || - !req->data[di].hpa) { + !req->data[di].hpa) dfailed++; - continue; - } - src = (const u8 *)__va(req->data[di].hpa); - dst = gsp_phys_chunk_page_va( - &boot_stage.rmcall_gr_bounce, bi); - memcpy(dst, src, GSP_PHYS_PAGE_SIZE); - for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) - csum += get_unaligned_le32(dst + ci); - bounce_csum_in[bi] = csum; - dcopied++; + else if (gsp_phys_chunk_page_va( + &boot_stage.rmcall_gr_bounce, bi)) + dcopied++; } - dma_wmb(); pr_info("vgsp-gr: bounce copy-in: %u pages, %u skipped (pre-publish)\n", dcopied, dfailed); for (di = 0; di < req->n_data && di < 3; di++) @@ -17445,24 +17878,37 @@ gr_page_unmapped: gsp_phys_rmcall_gr_gpc_mmu_init(bar0); /* * m143 DIAGNOSTIC (second-client launch-state divergence): dump the - * exact param page the FE/SM will read (win0 slot 0, the QMD bank-0 - * target) and the guest param page at the three JIT parameter slots, - * right before the doorbell. The JIT reads a@0x160/b@0x168/c@0x170 - * and n@0x178 (verified with ptxas -arch=sm_75). If the win0 copy is - * correct here but the engine still stores to the wrong relocated - * window, the divergence is engine-side (a stale mapping), not the - * relaunched page content. + * exact param page the FE/SM will read (this launch's win0 bank base + * slot, the QMD bank-0 target) and the guest param page at the three + * JIT parameter slots, right before the doorbell. The JIT reads + * a@0x160/b@0x168/c@0x170 and n@0x178 (verified with ptxas + * -arch=sm_75). If the win0 copy is correct here but the engine still + * stores to the wrong relocated window, the divergence is engine-side + * (a stale mapping), not the relaunched page content. */ if (boot_stage.rmcall_gr_shadow_win0.pages) { const u8 *wp = gsp_phys_chunk_page_va( - &boot_stage.rmcall_gr_shadow_win0, 0); + &boot_stage.rmcall_gr_shadow_win0, win0_bank_base); - if (wp) - pr_info("vgsp-gr: DIAG win0[0] 160=%llx 168=%llx 170=%llx 178=%llx\n", + if (wp) { + u32 off; + + pr_info("vgsp-gr: DIAG win0[%u] 160=%llx 168=%llx 170=%llx 178=%llx\n", + win0_bank_base, (u64)get_unaligned_le64(wp + 0x160), (u64)get_unaligned_le64(wp + 0x168), (u64)get_unaligned_le64(wp + 0x170), (u64)get_unaligned_le64(wp + 0x178)); + pr_info("vgsp-gr: win0 param hash pre=0x%016llx\n", + gsp_phys_gr_fnv1a64(wp, GSP_PHYS_PAGE_SIZE)); + for (off = 0x100u; off < 0x200u; off += 0x20u) + pr_info("vgsp-gr: win0 page +0x%03x: %016llx %016llx %016llx %016llx\n", + off, + (u64)get_unaligned_le64(wp + off), + (u64)get_unaligned_le64(wp + off + 8), + (u64)get_unaligned_le64(wp + off + 16), + (u64)get_unaligned_le64(wp + off + 24)); + } } { u32 pi; @@ -17474,12 +17920,15 @@ gr_page_unmapped: !req->pages[pi].hpa) continue; gp = (const u8 *)__va(req->pages[pi].hpa); + guest_param_pg = gp; pr_info("vgsp-gr: DIAG guest-param pa=0x%llx 160=%llx 168=%llx 170=%llx 178=%llx\n", req->pages[pi].hpa, (u64)get_unaligned_le64(gp + 0x160), (u64)get_unaligned_le64(gp + 0x168), (u64)get_unaligned_le64(gp + 0x170), (u64)get_unaligned_le64(gp + 0x178)); + pr_info("vgsp-gr: guest-param hash pre=0x%016llx\n", + gsp_phys_gr_fnv1a64(gp, GSP_PHYS_PAGE_SIZE)); break; } } @@ -17491,9 +17940,10 @@ gr_page_unmapped: boot_stage.rmcall_token, qmd_rel, want_ch, GSP_PHYS_RMCALL_GR_POLL_MS, qmdwin_note); if (qmd_win_diag) - pr_info("vgsp-gr: qmd dest va=0x%llx pdeIndex=%u installed=%u win_mask=0x%x ensure_ret=%d\n", + pr_info("vgsp-gr: qmd dest va=0x%llx pdeIndex=%u installed=%u win_mask=0x%llx ensure_ret=%d\n", qmd_dst, qmd_win_pde, qmd_win_installed, - boot_stage.rmcall_guest_pt_win_mask, qmd_win_ret); + (unsigned long long)boot_stage.rmcall_guest_pt_win_mask, + qmd_win_ret); deadline = jiffies + msecs_to_jiffies(GSP_PHYS_RMCALL_GR_POLL_MS); for (;;) { @@ -17581,6 +18031,45 @@ gr_page_unmapped: pr_info("vgsp-gr: rearm after timeout failed rc=%d\n", rr); } + /* + * The re-arm above only replays the schedule verbs; if the + * entry was already fetched or the front-end never latched + * it, that cannot make the QMD run. Re-publish a FRESH ring + * entry (same PB content and QMD) with a re-issued schedule + * + doorbell, bounded. The timeout path never copied the + * result out to the guest, so a timed-out attempt did not + * reach the client; the guest therefore still sees only the + * successful attempt. + */ + if (attempts < GSP_PHYS_RMCALL_GR_ATTEMPTS) { + int pr_ret; + + attempts++; + mutex_lock(&gsp_phys_rm_lock); + pr_ret = gsp_phys_rmcall_publish_locked( + bar0, boot_stage.rmcall_va_pb, words, + &sub); + if (!pr_ret) + pr_ret = gsp_phys_rmcall_gr_resched_locked( + bar0); + mutex_unlock(&gsp_phys_rm_lock); + if (!pr_ret) { + gsp_phys_gr_retries++; + pr_info("vgsp-gr: retry %u/%u: fresh ring entry slot=%u published + rescheduled; re-polling %u ms\n", + attempts, + GSP_PHYS_RMCALL_GR_ATTEMPTS, + sub.slot, + GSP_PHYS_RMCALL_GR_RETRY_MS); + deadline = jiffies + + msecs_to_jiffies( + GSP_PHYS_RMCALL_GR_RETRY_MS); + rearmed = false; + grpoll_n = 0; + continue; + } + pr_info("vgsp-gr: retry %u publish failed rc=%d\n", + attempts, pr_ret); + } if (gsp_phys_gr_fault_active()) { struct gsp_phys_fault_info fi; u32 fep = 0; @@ -17871,6 +18360,99 @@ gr_page_unmapped: gsp_phys_gr_ok++; /* + * m151: retire the output footprint before the copy-out (and before + * any later launch's copy-in). qmd_rel above is a schedule/fetch ack, + * not a data-path retirement, so the engine's stores into the bounce + * pages can land after it and be lost by an early copy-out; the next + * launch's copy-in would also race that write tail. Success path + * only - the timeout/fault paths never copied out. See + * gsp_phys_gr_output_retire_wait(). + * + * m153: outcome-verifying retry. The intermittent per-boot + * no-execution class (qmd_rel/ch_rel satisfied but the engine wrote + * nothing at all) cannot be caught by the completion poll; the + * retirement wait is the only execution witness. On zero writes, + * refill the bounce window from the guest inputs, re-publish the + * same method block on a fresh ring entry, re-issue the schedule and + * re-poll, then re-wait - bounded by the same attempt budget as the + * timeout path. After the budget the launch proceeds to copy-out + * with whatever the engine produced: a genuinely read-only kernel is + * indistinguishable from a no-execution here and simply pays the + * bounded extra polls (safe to re-execute by construction - a kernel + * that writes nothing has no side effects to duplicate). A + * wrong-place write detector was tried and removed: the highest-reloc + * data entry is often a UVM-internal buffer, not the output, so it + * false-fired on correct launches and would re-execute + * side-effecting kernels (atomics) up to 3x. + */ + if (req->n_data && boot_stage.rmcall_gr_bounce.pages) { + u32 wrote = gsp_phys_gr_output_retire_wait(req, bounce_csum_in); + + if (gsp_phys_rmcall_gr_nowrite_retry_value()) { + while (attempts < GSP_PHYS_RMCALL_GR_ATTEMPTS) { + if (wrote > 0) + break; + attempts++; + if (attempts > GSP_PHYS_RMCALL_GR_ATTEMPTS) + break; + pr_info("vgsp-gr: outcome retry %u/%u (wrote=%u): refill + re-publish\n", + attempts, GSP_PHYS_RMCALL_GR_ATTEMPTS, + wrote); + gsp_phys_gr_fill_bounce(req, bounce_csum_in); + mutex_lock(&gsp_phys_rm_lock); + ret = gsp_phys_rmcall_publish_locked( + bar0, boot_stage.rmcall_va_pb, + words, &sub); + if (!ret) + ret = gsp_phys_rmcall_gr_resched_locked( + bar0); + mutex_unlock(&gsp_phys_rm_lock); + if (ret) { + pr_info("vgsp-gr: outcome retry %u publish/resched failed rc=%d (accepting current outcome)\n", + attempts, ret); + ret = 0; + break; + } + { + unsigned long dl = jiffies + + msecs_to_jiffies( + GSP_PHYS_RMCALL_GR_RETRY_MS); + u32 rw0 = 0, rw1 = 0; + + for (;;) { + if (qmd_rel_hpa) + rw0 = READ_ONCE( + *(u32 *)((u8 *)__va( + qmd_rel_hpa) + + qmd_rel_off)); + else if (qmd_rel) + rw0 = READ_ONCE( + *(u32 *)sem_va); + if (ch_rel_hpa) + rw1 = READ_ONCE( + *(u32 *)((u8 *)__va( + ch_rel_hpa) + + ch_rel_off)); + else + rw1 = READ_ONCE( + *(u32 *)(sem_va + 4)); + if ((!qmd_rel || + rw0 == want_qmd) && + rw1 == want_ch) + break; + if (time_after_eq(jiffies, dl)) + break; + msleep(GSP_PHYS_RMCALL_POLL_MS); + } + } + wrote = gsp_phys_gr_output_retire_wait( + req, bounce_csum_in); + pr_info("vgsp-gr: outcome retry %u done: engine wrote %u page(s)\n", + attempts, wrote); + } + } + } + /* * m114: the kernel ran against the pre-commit bounce window; copy the * results back to the guest's real pages (identity HPAs in the * request). All mapped pages are written back; inputs are unchanged @@ -17962,6 +18544,23 @@ gr_page_unmapped: ioread32(bar0 + 0x00407060u), ioread32(bar0 + 0x00407064u), ioread32(bar0 + 0x00407068u)); + /* m151: post-poll page hashes; compare with the pre-doorbell ones. */ + { + const u8 *wp = boot_stage.rmcall_gr_shadow_win0.pages ? + gsp_phys_chunk_page_va( + &boot_stage.rmcall_gr_shadow_win0, + win0_bank_base) : NULL; + + if (wp || guest_param_pg) + pr_info("vgsp-gr: win0 param hash post=0x%016llx guest-param hash post=0x%016llx\n", + wp ? gsp_phys_gr_fnv1a64(wp, + GSP_PHYS_PAGE_SIZE) : + 0ULL, + guest_param_pg ? + gsp_phys_gr_fnv1a64(guest_param_pg, + GSP_PHYS_PAGE_SIZE) : + 0ULL); + } pr_info("vgsp-gr: GUEST GR LAUNCH OK h=0x%x push=0x%llx qmd=0x%llx prog/pages=%u qmd_rel=0x%08x ch_rel=0x%08x elapsed_ms=%u ok=%llu\n", req->h_object, req->push_va, req->qmd_va, req->n_pages, w0, w1, jiffies_to_msecs(jiffies - start), gsp_phys_gr_ok); @@ -18992,8 +19591,7 @@ static int gsp_phys_state_show(struct seq_file *m, void *v) fwd_cand_none, fwd_cand_sys, fwd_cand_mixed, fwd_cand_vid, fwd_resolve_fail, boot_stage.rmcall_guest_pt_next_line, - GSP_PHYS_RMCALL_GUEST_PT_PAGES * - GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE); + GSP_PHYS_RMCALL_GUEST_PT_TOTAL_LINES); } { /* @@ -20625,8 +21223,8 @@ static_assert(offsetof(struct gsp_phys_rmcall_channel_params, process_id) == 296 "NV_CHANNEL_ALLOC_PARAMS.ProcessID must be at offset 296"); static_assert(offsetof(struct gsp_phys_rmcall_channel_params, tpc_config_id) == 360, "NV_CHANNEL_ALLOC_PARAMS.tpcConfigID must be at offset 360"); -static_assert(GSP_PHYS_RMCALL_BUF_COUNT == 33, - "the rm-call ladder registers exactly thirty-three DMA buffers (14 ladder + 16 single-page guest scratch PTs + 3 golden9g VAS levels)"); +static_assert(GSP_PHYS_RMCALL_BUF_COUNT == 81, + "the rm-call ladder registers exactly eighty-one DMA buffers (14 ladder + 16 single-page guest scratch PTs + 48 single-page extension scratch PTs + 3 golden9g VAS levels)"); static_assert((GSP_PHYS_RMCALL_HANDLE_MEM_BASE + (GSP_PHYS_RMCALL_BUF_COUNT - 1) * GSP_PHYS_RMCALL_HANDLE_MEM_STRIDE) < 0xcaf80000u, diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c index c5419a5..9b7235b 100644 --- a/kernel/gsp_phys_gpu.c +++ b/kernel/gsp_phys_gpu.c @@ -949,6 +949,31 @@ int gsp_phys_rmcall_gr_skedrec_value(void) EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_skedrec_value); /* + * m153 no-write/wrong-place launch retry (gsp_phys_rmcall_gr_nowrite_retry; + * default 1). After a forwarded launch reports completion, the output + * retirement wait measures how many resolved data pages the engine actually + * changed. Two mis-execution signatures are known to be intermittent per + * boot: the engine writes nothing at all, or it writes into a lower operand + * window while the highest (output) operand stays untouched (a stale + * constant-source state). Both re-execute correctly on a fresh ring entry + * often enough to be worth a bounded retry: refill the bounce window from + * the guest inputs, re-publish the same method block, re-issue the schedule + * and re-poll, then re-wait. Set to 0 to disable the retry and accept the + * first outcome unconditionally. + */ +static int gsp_phys_rmcall_gr_nowrite_retry_param = 1; +module_param_named(gsp_phys_rmcall_gr_nowrite_retry, + gsp_phys_rmcall_gr_nowrite_retry_param, int, 0644); +MODULE_PARM_DESC(gsp_phys_rmcall_gr_nowrite_retry, + "Re-execute a forwarded GR launch whose retirement wait saw zero/wrong-place engine writes (default 1)"); + +int gsp_phys_rmcall_gr_nowrite_retry_value(void) +{ + return gsp_phys_rmcall_gr_nowrite_retry_param; +} +EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_nowrite_retry_value); + +/* * m69 window-0 shadow pool (gsp_phys_rmcall_gr_shadow_win0; default 0). When * set, the forwarded GR launch copies the QMD's program/param pages into the * window-0 shadow pool (module pages at VAs 0x108000..0x10f000, PTEs planted diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h index f93603c..73b0822 100644 --- a/kernel/gsp_phys_gpu.h +++ b/kernel/gsp_phys_gpu.h @@ -365,6 +365,10 @@ unsigned int gsp_phys_rmcall_gr_lmem_tpc_size_value(void); /* m72: log SKED CWD/HWW register transitions during the launch poll. */ int gsp_phys_rmcall_gr_skedrec_value(void); +/* m153: re-execute a forwarded launch whose retirement wait saw zero or + * wrong-place engine writes (default 1 = enabled). */ +int gsp_phys_rmcall_gr_nowrite_retry_value(void); + /* m73: use host-visible sysmem chunks for V7 instance/RAMFC (m62 shape). */ int gsp_phys_rmcall_gr_inst_sys_value(void); diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 9ab609a..f8925df 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -454,6 +454,45 @@ static u8 svc_sess_flush; static u8 svc_sess_active; /* a channel of a session has registered (until flush) */ static struct completion svc_sess_flush_done; /* a pending flush has executed */ static unsigned long svc_ring_done; /* CE ranks whose ring is pinned (session state) */ + +/* + * CeUtils scrubber channel (the guest PMA's internal CE channel created at + * driver init). When the PMA must hand out scrubbed pages it waits on the + * channel's finish payload (channel buffer offset finishPayloadOffset = + * channelPbSize + GPFIFO_SIZE + 4; for the regular scrubber = 0x6c004), + * which the physical CE would advance as the scrub work completes. In this + * emulation nothing authors it, so the third+ CUDA session's cudaMalloc + * freezes in scrubCheckAndWaitForSize. + * + * The scrubber channel is identified by its GPFIFO entry count + * (NUM_COPY_BLOCKS = 4096; client channels use 1024); its USERD PA is known + * from the channel ALLOC registration. The channel buffer's PA cannot be + * resolved through the global VA map (the kernel VA space collides + * numerically with the client's CE push-buffer window), so the guest + * scanner (vmallocinfo + kcore page-table walk) registers it through + * sema_scrubbuf_add. The service then authors, from the channel's own + * GP_PUT: USERD GP_GET = put, host semaphore = put (so the guest's + * channelWaitForFreeEntry never blocks), and finish payload = submitted + * block count (so the scrub wait completes). Idempotent, forward-only. + */ +struct svc_scrub { + bool userd_ok; /* a 4096-entry channel was registered */ + u32 h_object; /* handle of the registered scrubber channel */ + u64 userd_pa; + u32 userd_as; + u64 gpfifo_va; + u32 entries; + bool pa_ok; /* scanner registered fin/hsem PAs */ + u64 fin_pa; /* finish payload word (u32) */ + u64 hsem_pa; /* channel host semaphore word (u32) */ + bool seeded; + u32 put_last; + u64 submitted; /* blocks submitted since seeding */ + u64 authored; /* last authored finish value */ + u32 pa_gen; /* scanner registration generation (logged) */ + u32 log_budget; +}; +static struct svc_scrub svc_scrub; static u8 svc_pb_passes; static u64 svc_pb_last; /* @@ -602,6 +641,14 @@ struct sema_svc { u64 grce_log; /* GRCE inline-write budget */ u32 cur_ch; /* channel of the op being run */ u64 entry_log; /* per-push summary budget */ + /* + * m151: bounded log for CE pushes that were READABLE (nonzero first + * bytes) yet decoded no release. These are the entries the pump holds + * for the 8 s grace; naming their decode state (have_sem, rel) and + * head bytes is the Class-2 diagnostic (r128: correct content, right + * sem block at +0xC0 with RELEASE_ONE_WORD, yet rel_seen false). + */ + u64 entry_norel_log; u64 ops_denied; /* work refused: implausible stream */ u64 false_trans; /* resolvable VA mapped to non-push */ u64 sem_write_reresolve; /* writes re-resolved after read-back */ @@ -942,6 +989,13 @@ static bool svc_gr_content_rearm; static u32 svc_cepin_log; /* + * m154: injected pins dropped because they collide with a live channel's + * GPFIFO ring range (see svc_handle_map_add). Counted + logged so the + * defensive drop is visible in the boot log. + */ +static u32 svc_ring_pin_skips; + +/* * m148 (second-client wedge): re-arm the per-rank one-shot ring/push-buffer * latches for a (re-)registered channel. The guest driver rebuilds the GSP * session for every client context and RM reuses the same object handles and @@ -987,6 +1041,7 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) if (!desc->engine_type) { WRITE_ONCE(svc_gr_content_rearm, true); svc_cepin_log = 0; + svc_ring_pin_skips = 0; } /* @@ -1102,6 +1157,37 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) */ svc_chan_rearm_rank(ch); svc.last_chan_reg = jiffies; + /* + * CeUtils scrubber channel: NUM_COPY_BLOCKS = 4096 GPFIFO + * entries (client channels use 1024). Stash its USERD and ring + * geometry for the scrub pass; the channel itself is left + * entirely alone (no rank change, no pump touch changes) - the + * pass authors the finish payload from the USERD alone. A + * colliding numeric-handle FREE may still unregister the + * channel; the stashed geometry survives, so the scrub path + * does not depend on the channel-table identity at all. + */ + if (desc->gpfifo_entries >= 4096 && desc->gpfifo_va && + desc->userd_pa) { + /* Re-seed the counting on a fresh scrubber object. */ + if (!svc_scrub.userd_ok || + svc_scrub.h_object != desc->h_object || + svc_scrub.gpfifo_va != desc->gpfifo_va) { + svc_scrub.seeded = false; + svc_scrub.submitted = 0; + } + svc_scrub.userd_ok = true; + svc_scrub.h_object = desc->h_object; + svc_scrub.userd_pa = desc->userd_pa; + svc_scrub.userd_as = desc->userd_as ? desc->userd_as + : SVC_APERTURE_SYS; + svc_scrub.gpfifo_va = desc->gpfifo_va; + svc_scrub.entries = desc->gpfifo_entries; + pr_info("sema-svc: scrubber channel 0x%x detected (gpfifo=0x%llx entries=%u userd=0x%llx as=%u)\n", + desc->h_object, desc->gpfifo_va, + desc->gpfifo_entries, desc->userd_pa, + svc_scrub.userd_as); + } pr_info("sema-svc: channel 0x%x registered gpfifo=0x%llx entries=%u userd=0x%llx eng=%u cid=%u notif=0x%llx\n", desc->h_object, desc->gpfifo_va, desc->gpfifo_entries, desc->userd_pa, desc->engine_type, desc->cid, @@ -6936,6 +7022,36 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, tlen ? tail : NULL); } } + + /* + * m151 Class-2 diagnostic: a CE push with real content that decoded no + * release is the one that pays the 8 s grace. Log its decode state + * (bounded, content-gated so all-zero reads do not wash it out). + */ + if (gsp_sema_dbg && ch->engine_type && ch->h_object && !st.have_rel) { + u8 head[64]; + u32 hlen = len < sizeof(head) ? len : (u32)sizeof(head); + bool nz = false; + u32 i; + + for (i = 0; i + 4 <= hlen; i += 4) { + u32 w; + + memcpy(&w, buf + i, 4); + if (w) { + nz = true; + break; + } + } + if (nz && svc.entry_norel_log < 2048) { + svc.entry_norel_log++; + memcpy(head, buf, hlen); + pr_info("sema-norel: h=0x%x eng=%u va=0x%llx len=%u ops=%u have_sem=%u rel=0x%llx pay=%u head=%*phN\n", + ch->h_object, ch->engine_type, push_va, len, + ops_runs, st.have_sem, st.rel_va, + st.rel_payload, hlen, head); + } + } } /* ------------------------------------------------------------------ */ @@ -7942,6 +8058,8 @@ static bool svc_push_prefix_ok(const u8 *buf, size_t len) /* Fallback-line budget for the whole boot. */ static u32 svc_pushpg_fb_log; +/* Bounded split-read rejection lines (cross-page CE push, r129 class). */ +static u32 svc_pushpg_split_log; /* * Read a push the way the engine does: every 4 KB page of the range is @@ -7961,11 +8079,16 @@ static u32 svc_pushpg_fb_log; * method stream is rejected and the remaining candidates are tried * (observed: a wrong injected pin for the push page made every push * decode no release, starving the client's tracker); - * 2. validated self-pin (svc_pin_lookup_valid): read-back-validated pages + * 2. the displaced injected pair (svc_ipin_alt_lookup): what the page + * resolved to before a later injection overwrote the primary slot - + * the fall-back when a foreign/stale injection shadowed a good page + * (observed r129: a continuation page of a cross-page CE push under + * an all-FF pin); + * 3. validated self-pin (svc_pin_lookup_valid): read-back-validated pages * (content rescue, first-push pairing), window guesses excluded; - * 3. the channel's linear window mapping (svc_pb_win_lookup): last resort + * 4. the channel's linear window mapping (svc_pb_win_lookup): last resort * for a page no validated source knows; - * 4. outside any window: the remaining trusted sources + * 5. outside any window: the remaining trusted sources * (svc_lookup_trusted), exactly the old behaviour for GR/other pushes. * * For a contiguous allocation every path resolves to the same PA the old @@ -7993,8 +8116,9 @@ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, have_win = !svc_pb_win_lookup(ch, page_va, &win_pa); - for (src = 0; src < 4 && !done; src++) { + for (src = 0; src < 5 && !done; src++) { bool via_ipin = false; + bool untrusted = false; pa = 0; ap = SVC_APERTURE_SYS; @@ -8004,22 +8128,39 @@ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, svc_ipin_lookup(va, &pa, &ap)) continue; via_ipin = true; + untrusted = true; if (have_win) svc_pushpg_note(ch, page_va, pa, win_pa); break; case 1: + /* + * Displaced injected pin (svc_ipin_alt): + * the mapping this page had before a later + * injection overwrote it. The push decode + * is the consumer here, so a pin that no + * longer reads like the page's validated + * content (all-FF continuation chunks, + * see below) can fall back to the pair the + * current injection displaced. + */ + if (svc_ipin_alt_lookup(va, &pa, &ap)) + continue; + untrusted = true; + break; + case 2: if (svc_pin_lookup_valid(va, &pa, &ap)) continue; if (have_win) svc_pushpg_note(ch, page_va, pa, win_pa); break; - case 2: + case 3: if (!have_win) continue; pa = win_pa + (va & (SEMA_PAGE_SIZE - 1)); ap = SVC_APERTURE_SYS; + untrusted = true; break; default: /* learned cache / discovered map / page @@ -8045,18 +8186,45 @@ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, !svc_push_prefix_ok(p, chunk)) continue; /* content says: wrong pin */ /* + * A continuation chunk (not the push start) read from + * an unvalidated source can be a stale/foreign pin: + * observed r129, a 216-byte CE push whose last 24 + * bytes (the SET_SEMAPHORE_A + LAUNCH_DMA) fell in a + * page whose injected pin resolved to an all-0xFF + * page while the first page was correct. A full + * chunk of 0xFF is never a method stream - fresh + * emulation pages read 0xFF - so reject it and try + * the remaining sources (the displaced alt pair + * first). The bounded line names the rejected + * (va, pa, src) for the post-mortem. + */ + if (untrusted && !first && chunk >= 64 && + !memchr_inv(p, 0xff, chunk)) { + if (svc_pushpg_split_log < 8) { + svc_pushpg_split_log++; + pr_info("sema-pushpg: split reject va=0x%llx pa=0x%llx src=%s (all-FF) at page 0x%llx\n", + va, pa, + src == 0 ? "ipin" : + (src == 1 ? "ipin-alt" : + "win"), + page_va); + } + last_rc = -EIO; + continue; + } + /* * ISSUE-0002 (r51): a window guess that reads ALL * ZEROS cannot be right - the paired base page is * real, so a zero first chunk means the linear window * does not describe this page (r49: the second * session's CE push pages read zeros and, because the * read "succeeded", shadowed the per-page resolvers). - * Fall through to src=3 instead. Non-zero window + * Fall through to src=4 instead. Non-zero window * content stays trusted exactly as before (r50: * requiring method-stream plausibility rejected valid * non-method push prefixes and corrupted c1). */ - if (src == 2 && first && !memchr_inv(p, 0, chunk)) + if (src == 3 && first && !memchr_inv(p, 0, chunk)) continue; done = true; } @@ -12206,10 +12374,18 @@ static bool svc_gr_page_inline_covers(u64 page) * accesses the tail: with 32 pages the tail read the NEXT candidate's * relocated page (wrong values; vadd tails). 3x33=99 <= the 100-page cap. */ -#define SVC_GR_DATA_PAGES_PER_PTR 33u -/* m114: total pages the pre-commit bounce window can carry (slot 100 is - * the mail message). */ -#define SVC_GR_DATA_BOUNCE_MAX 100u +/* + * m150 (large buffers): the bounce window was grown to 208 pages (mail page + * at slot 206). 68 pages per candidate/pointer covers a 256 KB cudaMalloc'd + * operand whose pointer sits 0x400 into its page (64 full + offset + tail), + * and up to three such operands (3x68 = 204 <= 205) fit below the mail page. + * Larger-than-256 KB operands still hit the cap; they need a chunked data + * path (future work). + */ +#define SVC_GR_DATA_PAGES_PER_PTR 68u +/* Total pages the pre-commit bounce window can carry below the mail page + * (GSP_PHYS_RMCALL_GR_MAIL_SLOT = 206). */ +#define SVC_GR_DATA_BOUNCE_MAX 205u static u32 svc_gr_data_log; /* m142: bounded per-page attribution log */ static u32 svc_gr_src_log; /* m150: bounded per-candidate source log */ @@ -12456,8 +12632,26 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) u32 before = svc_fwd_req.gr_n_data; u64 pba = 0; u32 pba_ap = 0; + u32 span; const char *pba_src = "?"; + /* + * Window budget discipline (large buffers): only real device + * operands (high, cudaMalloc-shaped VAs) get the full per-pointer + * span. Low-VA candidates (the launch's own QMD/release/reloc + * pointers) need a single page at most, and the launch's QMD VA + * is not a kernel operand at all - relocating 68 pages for it + * starved the operands that follow and blew the 205-page cap + * (r141: a 256 KB operand resolved 0 pages because the QMD and + * param candidates consumed the window). Excluding them lets + * three 256 KB operands (3x68 = 204) fit. + */ + if ((base & ~(u64)(SEMA_PAGE_SIZE - 1)) == + (svc_fwd_req.gr_qmd_dev_va & ~(u64)(SEMA_PAGE_SIZE - 1))) + continue; + span = (base >= 0x40000000000ULL) ? + SVC_GR_DATA_PAGES_PER_PTR : 1u; + if (is_arg[w] && base >= 0x40000000000ULL) n_arg_hi++; @@ -12472,7 +12666,7 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) svc_pd_found, svc.ipin_n); } - for (p = 0; p < SVC_GR_DATA_PAGES_PER_PTR; p++) { + for (p = 0; p < span; p++) { u64 va = wins[w] + (u64)p * SEMA_PAGE_SIZE; u64 pa = 0; u32 ap = 0, x; @@ -15218,6 +15412,41 @@ static u32 svc_map_add_head; /* producer */ static u32 svc_map_add_tail; /* consumer (service thread) */ static DEFINE_SPINLOCK(svc_map_add_lock); +/* + * m154: does this injected-pin VA fall inside any live channel's GPFIFO ring + * range? The ring VA translation must come from the channel's own discovery + * path (channel object / push-buffer scan); a scanner run that happens to + * span the ring VA is enumerating a different mapping and its translation is + * always wrong for the ring (r153: an all-zero read consumed a ring entry + * and wedged the client for the boot). Called on the service thread; takes + * chan_lock like the rewind below. + */ +static bool svc_va_hits_chan_ring(u64 va) +{ + unsigned long flags; + u32 i; + bool hit = false; + + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + struct svc_channel *ch = &svc.chan[i]; + u64 lo, hi; + + if (!ch->valid || !ch->gpfifo_va || !ch->gpfifo_entries) + continue; + lo = ch->gpfifo_va & ~(u64)(SEMA_PAGE_SIZE - 1); + hi = (ch->gpfifo_va + + (u64)ch->gpfifo_entries * 8 + SEMA_PAGE_SIZE - 1) & + ~(u64)(SEMA_PAGE_SIZE - 1); + if (va >= lo && va < hi) { + hit = true; + break; + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + return hit; +} + static void svc_handle_map_add(void) { unsigned long flags; @@ -15242,8 +15471,33 @@ static void svc_handle_map_add(void) ap = req.ap ? req.ap : SVC_APERTURE_SYS; for (off = 0; off < (req.len ? req.len : 1); off += SEMA_PAGE_SIZE) { - svc_ipin_store(req.va + off, req.pa + off, ap); - svc_lat_store(req.va + off, req.pa + off, ap); + u64 va = req.va + off; + + /* + * m154: never let an injected pin shadow a live + * channel's GPFIFO ring. r153: the scanner's merged + * CE-window run (va=0x121000000 len=0x40000, the + * length varies per session) covered a CE channel's + * ring VA 0x121010000; the injected translation took + * precedence in svc_ipin_lookup/read, the ring then + * read the guest page at 0x102e02000 - an all-zero + * entry was silently consumed, the pool release was + * never authored, and the client spun in userspace + * for the rest of the boot (pushes frozen). The ring + * pages already have their own discovery path (the + * channel object / push-buffer scan); a colliding + * injection is always the wrong translation, so drop + * it and leave the ring to that path. + */ + if (svc_va_hits_chan_ring(va)) { + svc_ring_pin_skips++; + if (svc_ring_pin_skips <= 8) + pr_info("sema-svc: injected pin va=0x%llx DROPPED (live channel ring range; ring reads stay on the channel's own translation)\n", + va); + continue; + } + svc_ipin_store(va, req.pa + off, ap); + svc_lat_store(va, req.pa + off, ap); } /* * m149: bounded per-session evidence that the CE push-buffer @@ -15465,6 +15719,115 @@ int gsp_sema_service_counter_add(u32 h_object, u64 pa) return ret; } +/* + * Register the guest PMA scrubber's channel-buffer words. Called from the + * sema_scrubbuf_add debug parameter with the PAs the guest scanner resolved + * (vmallocinfo vmap area + kcore page-table walk). fin_pa is the finish + * payload the guest's scrub wait polls; hsem_pa is the channel's host + * semaphore (channelWaitForFreeEntry polls it). Plain state update. + */ +int gsp_sema_service_scrubbuf_add(u64 fin_pa, u64 hsem_pa) +{ + if (!svc.task) + return -ENODEV; + /* Both words are u32: require 4-byte alignment and a RAM-shaped PA. */ + if (!fin_pa || (fin_pa & 0x3) || fin_pa >= SEMA_PA_MAX) + return -EINVAL; + if (!hsem_pa || (hsem_pa & 0x3) || hsem_pa >= SEMA_PA_MAX) + return -EINVAL; + + svc_scrub.fin_pa = fin_pa; + svc_scrub.hsem_pa = hsem_pa; + svc_scrub.pa_ok = true; + svc_scrub.pa_gen++; + pr_info("sema-svc: scrubber buffer registered fin_pa=0x%llx hsem_pa=0x%llx (gen %u)\n", + fin_pa, hsem_pa, svc_scrub.pa_gen); + return 0; +} + +/* + * Author the CeUtils scrubber channel's progress once per service pass. + * + * The physical CE would advance, per completed scrub block, the channel's + * finish payload (channel buffer + 0x6c004) and write the channel's host + * semaphore as it consumes GPFIFO entries. Nothing in this emulation does, + * so the guest PMA's scrub wait (scrubCheckAndWaitForSize) freezes from the + * third CUDA session on. This pass closes that loop from the channel's own + * USERD GPPut (which the guest driver writes before every submit): + * + * submitted = number of scrub blocks submitted (GPPut is end-exclusive + * and counts submissions until the ring wraps; track the wrap ourselves), + * then author USERD GP_GET = put (consumed), host sema = put and finish + * payload = submitted. Forward-only, idempotent, retried every pass so a + * late scanner registration (or a post-flush reset) heals by itself. + * + * The scrub work itself is not executed (no shadow memsets): the guest + * only needs the progress to proceed. Executing the memsets requires + * decoding the channel's pushes, which the scanner can later provide pins + * for; correctness impact is limited to stale data in pages the guest + * believes scrubbed. + */ +static void svc_scrub_pass(struct nv_vgpu_device *vgpu) +{ + struct svc_scrub *s = &svc_scrub; + u32 put = 0; + u64 delta; + + if (!s->userd_ok || !s->pa_ok || !s->entries) + return; + if (svc_read_ap(vgpu, s->userd_pa + SVC_USERD_GP_PUT, s->userd_as, + &put, sizeof(put))) { + /* Retry through the alternate aperture (as registration did). */ + if (svc_read_ap(vgpu, s->userd_pa + SVC_USERD_GP_PUT, + s->userd_as == SVC_APERTURE_FB ? + SVC_APERTURE_SYS : SVC_APERTURE_FB, + &put, sizeof(put))) + return; + } + if (put >= s->entries) + return; /* junk translation; retry next pass */ + + if (!s->seeded) { + s->seeded = true; + s->put_last = put; + /* GPPut = (lastSubmittedEntry + 1) % entries: the submission + * count until the ring wraps. Seeding from it adopts any + * pre-host scrub work (the boot session's scrubbers). */ + s->submitted = put; + } else if (put != s->put_last) { + delta = (put + s->entries - s->put_last) % s->entries; + if (!delta) + delta = s->entries; /* exact wrap */ + s->submitted += delta; + s->put_last = put; + } else if (s->submitted == s->authored) { + return; /* nothing new; no guest writes */ + } + + if (s->log_budget < 64) { + s->log_budget++; + pr_info("sema-svc: scrub author put=%u submitted=%llu fin_pa=0x%llx\n", + put, s->submitted, s->fin_pa); + } + + /* + * Author (forward-only). The USERD GP_GET and host-sema writes mark + * the submissions consumed; the finish write releases the scrub + * wait. Written only when GP_PUT advanced (or on the seeding pass), + * so the per-pass cost is zero while the scrubber idles. + */ + svc_write_ap(vgpu, s->userd_pa + SVC_USERD_GP_GET, s->userd_as, + &put, sizeof(put)); + svc_write_ap(vgpu, s->hsem_pa, SVC_APERTURE_SYS, &put, sizeof(put)); + { + u32 fin = (u32)s->submitted; + + svc_write_ap(vgpu, s->fin_pa, SVC_APERTURE_SYS, &fin, + sizeof(fin)); + } + s->authored = s->submitted; +} + static int svc_thread(void *arg) { struct nv_vgpu_device *vgpu; @@ -15510,6 +15873,7 @@ static int svc_thread(void *arg) svc_handle_walk_probe(vgpu); svc_handle_map_add(); svc_handle_chan_dump(vgpu); + svc_scrub_pass(vgpu); if (svc_sess_flush || !svc.direct_base || time_after(jiffies, diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h index 4b04892..6822bd5 100644 --- a/kernel/gsp_sema_service.h +++ b/kernel/gsp_sema_service.h @@ -36,6 +36,16 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc); void gsp_sema_service_channel_del(u32 h_object); /* + * Register the guest PMA scrubber's channel-buffer words (the CeUtils + * finish payload the guest's scrub wait polls, and the channel's host + * semaphore). The PAs come from the guest scanner (vmallocinfo + kcore): + * the guest RM kernel channel lives in a GPU VA space that collides + * numerically with the client's CE window in the global translation map, + * so it cannot be resolved through that map. + */ +int gsp_sema_service_scrubbuf_add(u64 fin_pa, u64 hsem_pa); + +/* * A GPU VA space whose top-level page directory the driver has published (the * driver builds its own page tables for the address spaces it owns, then hands * the directory's physical address to the GPU through SET_PAGE_DIRECTORY). diff --git a/kernel/main.c b/kernel/main.c index ac970f1..d0308c4 100644 --- a/kernel/main.c +++ b/kernel/main.c @@ -2544,6 +2544,37 @@ module_param_cb(sema_probe, &nv_vgpu_sema_probe_ops, NULL, 0200); MODULE_PARM_DESC(sema_probe, "Dump sema-service map + slot values for a GPU VA"); /* + * Register the guest PMA scrubber channel buffer words (finish payload PA, + * host-semaphore PA) the guest scanner resolved: + * echo 0x11a8ff004,0x11a8ff000 > .../parameters/sema_scrubbuf_add + */ +static int nv_vgpu_sema_scrubbuf_set(const char *val, + const struct kernel_param *kp) +{ + char buf[64]; + char *comma; + u64 fin_pa = 0, hsem_pa = 0; + + strscpy(buf, val, sizeof(buf)); + comma = strchr(buf, ','); + if (!comma) + return -EINVAL; + *comma = '\0'; + if (kstrtoull(buf, 0, &fin_pa) || kstrtoull(comma + 1, 0, &hsem_pa)) + return -EINVAL; + + return gsp_sema_service_scrubbuf_add(fin_pa, hsem_pa); +} + +static const struct kernel_param_ops nv_vgpu_sema_scrubbuf_ops = { + .set = nv_vgpu_sema_scrubbuf_set, +}; + +module_param_cb(sema_scrubbuf_add, &nv_vgpu_sema_scrubbuf_ops, NULL, 0200); +MODULE_PARM_DESC(sema_scrubbuf_add, + "Register the PMA scrubber finish/host-sema PAs (hex fin,hex hsem)"); + +/* * Debug: run the page-table walks for a GPU VA with per-level tracing, * bypassing the pin/lat/pair map (see gsp_sema_service_walk_probe): * echo 0x20400f000 > .../parameters/walk_probe |