diff options
Diffstat (limited to 'kernel/gsp_phys_booter.c')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 210 |
1 files changed, 202 insertions, 8 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 60b57f2..4cbbd40 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -650,6 +650,16 @@ bool gsp_phys_boot_requested(void) */ #define GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST 29u /* + * Mode 30 (2026-10-06, experiment): the TSG-sibling split. A clean UNWIRED + * golden channel A (captured shape, no ring/USERD/HOBJ wiring) is parented to + * the ladder TSG and gets the 9-entry promote plus its own TURING_A (0xc597) + * construct, so the first GR init + forced golden save + TSG-wide commit run + * on A. V6 stays the execution channel (no swap) and its FWCLIENT promote is + * suppressed; the group commit (or the new-channel setup inheritance) must set + * V6's instance ENG_CTX pointer and RAMFC ENG_CTX_VALID. + */ +#define GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING 30u +/* * Mode-28 golden execution channel shape bits (gsp_phys_rmcall_golden_exec_shape, * default 7). Every combination keeps the captured flags 0x00200320 and the * FBMEM instance/RAMFC/mthd slices; the bits only choose the client-visible @@ -763,7 +773,7 @@ bool gsp_phys_boot_requested(void) */ #define GSP_PHYS_RMCALL_FWCLIENT_MASK_FECS_SYSMEM (1u << 16) #define GSP_PHYS_RMCALL_GR_PROMOTE_MAX \ - (GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST) + (GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING) /* * Work-submit token notifier slot. The default index is @@ -6889,6 +6899,9 @@ static int gsp_phys_rmcall_control(void __iomem *bar0, u32 h_object, u32 cmd, const char *what); static void gsp_phys_rmcall_pdb_build(void); static u64 gsp_phys_rmcall_pde(u64 pa); +static void gsp_phys_rmcall_ctx_diag_dump(const char *what, + struct gsp_phys_boot_chunk *c); +static void gsp_phys_rmcall_ctx_diag_decode(const char *tag); /* * Shared-management VAS: install the ladder's page-table window through the @@ -7614,6 +7627,8 @@ static const char *gsp_phys_rmcall_gr_promote_name(void) return "mode 28: golden execution lane (golden channel with client GPFIFO/USERD, 9-entry promote on it, execution switched to it)"; case GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST: return "mode 29: construct-last golden execution lane (mode 28 minus the TURING_COMPUTE_A object construct on the wired golden channel)"; + case GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING: + return "mode 30: clean unwired TSG sibling A (promote + TURING_A first-init) + V6 execution"; default: return "off (PROMOTE_CTX never sent)"; } @@ -7800,7 +7815,9 @@ static int gsp_phys_rmcall_alloc_channel(void __iomem *bar0) boot_stage.rmcall_gr_promote == GSP_PHYS_RMCALL_GR_PROMOTE_FWCLIENT || boot_stage.rmcall_gr_promote == - GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9)) { + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 || + boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING)) { vars = &gsp_phys_rmcall_channel_variant_skipctx; attempts = 1; } @@ -8259,7 +8276,9 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0) boot_stage.rmcall_gr_promote == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC || boot_stage.rmcall_gr_promote == - GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST) && + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST || + boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING) && gsp_phys_rmcall_engine_is_gr()) { if (gsp_phys_rmcall_fwclient_mask_value() & GSP_PHYS_RMCALL_GOLDEN9_MASK_LADDER_VAS) { @@ -8367,7 +8386,9 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0) boot_stage.rmcall_gr_promote == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC || boot_stage.rmcall_gr_promote == - GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST) && + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST || + boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING) && gsp_phys_rmcall_engine_is_gr()) { ch.flags = cpu_to_le32(GSP_PHYS_RMCALL_CHANNEL_FLAG_GOLDEN9); ch.internal_flags = cpu_to_le32( @@ -8467,6 +8488,20 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0) ch.h_va_space = cpu_to_le32(0); } } + /* + * Mode 30: clean unwired sibling. Captured flags/memdescs stay; + * the submission wiring is deliberately absent (gpFifoOffset 0, + * captured userd slice, hObjectBuffer 0) and the channel joins + * the ladder TSG so the promote's group commit and A's + * first-init construct are visible to V6. hVASpace=0 per the TSG + * channel rule (kernel_channel.c:429-435). + */ + if (boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING) { + ch_parent = boot_stage.rmcall_tsg; + ch.h_va_space = cpu_to_le32(0); + pr_info("vgsp-boot: rm-call: mode 30 GOLDEN SIBLING A: TSG-parented unwired captured shape (TURING_A will construct here for the first GR init + commit)\n"); + } pr_info("vgsp-boot: rm-call: GOLDEN9 channel shape: flags=0x%x (PRIVILEGED|USERD_INDEX_VALUE=3|USERD_INDEX_PAGE_FIXED, no SKIP_SCRUBBER/SKIP_CTXBUFFER_ALLOC) hObjectBuffer=0 gpFifoEntries=0x%x instance/userd/ramfc/mthd=FBMEM memdescs block=0x%llx (capture-matched apertures)\n", GSP_PHYS_RMCALL_CHANNEL_FLAG_GOLDEN9, GSP_PHYS_RMCALL_GOLDEN_GPFIFO_ENTRIES, golden_fb); @@ -8555,14 +8590,31 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0) boot_stage.rmcall_gr_promote == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC || boot_stage.rmcall_gr_promote == - GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST) && + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST || + boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING) && gsp_phys_rmcall_engine_is_gr()) { ret = gsp_phys_rmcall_golden9_promote(bar0); if (ret) return ret; } - /* 6. TURING_A (0xc597) 3D object on the golden channel (kernel_graphics.c:2409-2433). */ + /* + * 6. TURING_A (0xc597) 3D object on the golden channel + * (kernel_graphics.c:2409-2433). + * + * Mode 30 deliberately falls through the skip gate below (it only + * matches modes 28/29 with shape bit5): sibling A keeps its own + * TURING_A construct, and that first GR object construct on A drives + * the first-init + forced golden save (grAlloc_GK104 -> first-init -> + * grInitGoldenCtxImage) whose image and TSG-group commit V6 later + * inherits. + */ + if (boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING) { + pr_info("vgsp-boot: rm-call: mode 30: GOLDEN TURING_A on sibling A skipped by design (m30a: any GR object construct on A blocks RM -110); A is a pure carrier for the 9-entry promote's group commit, the ladder's TURING_COMPUTE_A on V6 provides the instance state\n"); + return 0; + } if ((boot_stage.rmcall_gr_promote == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC || boot_stage.rmcall_gr_promote == @@ -10045,13 +10097,27 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0) if ((mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 || mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6_ONLY || mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC || - mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST) && + mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST || + mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING) && (!gsp_phys_rmcall_golden_enabled() || !gsp_phys_gpu_enabled())) { pr_err("vgsp-boot: rm-call: mode %u requires gsp_phys_rmcall_golden=1 and gsp_phys_gpu=1 (the golden replay did not run); no PROMOTE_CTX is sent and the ladder aborts with no retry\n", mode); return -EINVAL; } /* + * Mode 30: the sibling A's 9-entry promote carries the group context + * commit; nothing is sent on the V6 channel. A multi-entry promote + * containing MAIN is refused for a SKIP-ctx channel + * (graphics_context.c: bKMDVirtualContext -> NV_ERR_INVALID_ARGUMENT) + * and the single-MAIN exemption needs the pCtxPhysMemDesc the FWCLIENT + * shapes never establish, so a V6 promote would abort stage 3 on 0x1f. + */ + if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING) { + pr_info("vgsp-boot: rm-call: mode 30: no PROMOTE_CTX on V6; the sibling A carries the group context commit (config_group ppEngCtxDesc), V6 inherits via the TSG commit/channel setup\n"); + boot_stage.rmcall_gr_promote_step = 1; + return 0; + } + /* * Mode 28: the 9-entry promote already ran on the golden execution * channel inside the golden replay (it owns the committed context and * the forwarded launches); nothing is sent on the V6 channel at all. @@ -11881,7 +11947,9 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0) boot_stage.rmcall_gr_promote == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC || boot_stage.rmcall_gr_promote == - GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST) && + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST || + boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING) && gsp_phys_rmcall_engine_is_gr()) { pr_err("vgsp-boot: rm-call: stage 3 aborted: mode-%u golden replay failed (%d); the golden construct is atomic and is not retried\n", boot_stage.rmcall_gr_promote, ret); @@ -11926,6 +11994,25 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0) return ret; /* + * Mode 30 readback point 1: V6's instance/RAMFC right after the channel + * alloc (A's replay contains the 9-entry promote and A's TURING_A + * first-init by now), so A's commit effect can be separated from the + * later BIND/GPFIFO_SCHEDULE. The decode line prints the two decisive + * fields (instance WFI pointer pair + RAMFC ENG_CTX_VALID). + */ + if (gsp_phys_rmcall_engine_is_gr() && + boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING) { + gsp_phys_rmcall_ctx_diag_dump( + "INSTANCE (post-A-commit pre-BIND)", + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_INSTANCE]); + gsp_phys_rmcall_ctx_diag_dump( + "RAMFC (post-A-commit pre-BIND)", + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_RAMFC]); + gsp_phys_rmcall_ctx_diag_decode("post-A-commit pre-BIND"); + } + + /* * Mode 28: the golden execution lane. The golden replay allocated the * golden channel with a client GPFIFO/USERD and the accepted 9-entry * context commit; switch the ladder's execution channel to it BEFORE @@ -13758,6 +13845,50 @@ static int gsp_phys_rmcall_gr_scratch_win_install_locked(u32 page) } /* + * Is the 2 MiB window containing `va` marked installed in the engine's table + * bookkeeping? Scratch-pool VAs consult rmcall_guest_pt_win_mask (bit = pool + * page); every other VA consults the Tier-1 window pool by pdeIndex. Called + * with gsp_phys_rm_lock held. + */ +static bool gsp_phys_rmcall_gr_win_is_installed_locked(u64 va) +{ + u32 pde_index = (u32)(va >> 21); + u32 base = (u32)(boot_stage.rmcall_va_guest_pt >> 21); + u32 i; + + if (pde_index >= base && + pde_index < base + GSP_PHYS_RMCALL_GUEST_PT_PAGES) + return !!(boot_stage.rmcall_guest_pt_win_mask & + BIT(pde_index - base)); + for (i = 0; i < boot_stage.rmcall_gr_pt_used; i++) + if (boot_stage.rmcall_gr_win[i].used && + boot_stage.rmcall_gr_win[i].pde_index == pde_index) + return true; + return false; +} + +/* + * Make the 2 MiB window containing `va` reachable by the engine: scratch-pool + * VAs install their pool window through the scratch installer (shared VAS + * only; an external VAS keeps the client PDB spine), every other VA installs + * its Tier-1 window at its own pdeIndex. Both paths are idempotent; a window + * already installed is a bookkeeping hit and sends no RPC. Called with + * gsp_phys_rm_lock held. + */ +static int gsp_phys_rmcall_gr_win_ensure_locked(void __iomem *bar0, u64 va) +{ + u32 pde_index = (u32)(va >> 21); + u32 base = (u32)(boot_stage.rmcall_va_guest_pt >> 21); + u32 slot; + + if (pde_index >= base && + pde_index < base + GSP_PHYS_RMCALL_GUEST_PT_PAGES) + return gsp_phys_rmcall_gr_scratch_win_install_locked( + pde_index - base); + return gsp_phys_rmcall_gr_win_slot_locked(bar0, pde_index, &slot); +} + +/* * Tier-2 map: consume one never-reused 128 B fetch line from the guest * scratch PT pool (PTE0 only) for a ladder-owned shadow page and return the * line's base VA. The shadow chunk is allocated lazily; page slot `slot` @@ -13899,6 +14030,9 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, u8 *qmd_dst_mem = NULL; u32 qmd_rel_off = 0, ch_rel_off = 0; u32 poll = 0, w0 = 0, w1 = 0, grpoll_n = 0; + u32 qmd_win_pde = 0, qmd_win_installed = 0; + int qmd_win_ret = 0; + bool qmd_win_diag = false; int ret = 0; if (out) @@ -14236,12 +14370,14 @@ gr_page_unmapped: if (req->qmd_hpa) { u64 qmd_page = req->qmd_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1); u32 qmd_off = (u32)(req->qmd_va & (GSP_PHYS_PAGE_SIZE - 1)); + bool dst_ok = false; if (qmd_page && !gsp_phys_rmcall_gr_map_va_locked(bar0, qmd_page, req->qmd_hpa, epoch)) { qmd_dst = qmd_page; qmd_dst_mem = (u8 *)__va(req->qmd_hpa) + qmd_off; + dst_ok = true; } else { u64 sh_va = 0; @@ -14253,8 +14389,27 @@ gr_page_unmapped: n_shadow) + qmd_off; n_shadow++; qmd_dst = sh_va + qmd_off; + dst_ok = true; } } + /* + * The engine must reach the rewritten destination: make sure + * the 2 MiB window containing it is installed in the engine's + * table view (a Tier-1 guest window, or the Tier-2 scratch + * pool window the shadow line lives in). Idempotent; only a + * window not yet installed costs an RPC. + */ + if (dst_ok) { + qmd_win_diag = true; + qmd_win_pde = (u32)(qmd_dst >> 21); + qmd_win_installed = + gsp_phys_rmcall_gr_win_is_installed_locked( + qmd_dst) ? 1u : 0u; + qmd_win_ret = gsp_phys_rmcall_gr_win_ensure_locked( + bar0, qmd_dst); + if (!qmd_win_ret) + qmd_win_installed = 1u; + } pr_info("vgsp-gr: inline-QMD dest va=0x%llx hpa=0x%llx -> 0x%llx (shadow=%u)\n", req->qmd_va, req->qmd_hpa, qmd_dst, n_shadow); } @@ -14324,6 +14479,10 @@ gr_page_unmapped: req->h_object, req->push_va, req->push_len, req->n_pages, n_shadow, req->qmd_va, sub.slot, words, boot_stage.rmcall_token, qmd_rel, want_ch, GSP_PHYS_RMCALL_GR_POLL_MS); + if (qmd_win_diag) + pr_info("vgsp-gr: qmd dest va=0x%llx pdeIndex=%u installed=%u win_mask=0x%x ensure_ret=%d\n", + qmd_dst, qmd_win_pde, qmd_win_installed, + boot_stage.rmcall_guest_pt_win_mask, qmd_win_ret); deadline = jiffies + msecs_to_jiffies(GSP_PHYS_RMCALL_GR_POLL_MS); for (;;) { @@ -14506,6 +14665,40 @@ static void gsp_phys_rmcall_ctx_diag_dump(const char *what, } } +/* + * One-line decode of the two fields that decide whether V6's engine context is + * live: the instance block's NV_RAMIN_ENGINE_WFI_PTR pair (dword 132 bits + * 31:12 = PA[31:12], dword 133 bits 7:0 = PA[39:32]; fifo_gv100.c) and RAMFC + * NV_RAMFC_TARGET (dword 43, NV_PPBDMA_TARGET_ENG_CTX_VALID bit 16). Host-view + * LE chunks, same as the dump above; used at the mode-30 post-replay and the + * stage-4 post-BIND points so A's commit can be told apart from BIND. + */ +static void gsp_phys_rmcall_ctx_diag_decode(const char *tag) +{ + struct gsp_phys_boot_chunk *inst = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_INSTANCE]; + struct gsp_phys_boot_chunk *ramfc = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_RAMFC]; + const __le32 *iv, *rv; + u64 ctx_pa; + u32 target; + bool valid; + + if (!inst->pages || !ramfc->pages) { + pr_info("vgsp-boot: rm-call: V6 ctx (%s): instance/RAMFC chunk absent\n", + tag); + return; + } + iv = gsp_phys_chunk_page_va(inst, 0); + rv = gsp_phys_chunk_page_va(ramfc, 0); + ctx_pa = ((u64)(le32_to_cpu(iv[133]) & 0xffu) << 32) | + (le32_to_cpu(iv[132]) & 0xfffff000u); + target = le32_to_cpu(rv[43]); + valid = !!(target & (1u << 16)); + pr_info("vgsp-boot: rm-call: V6 ctx (%s): inst_wfi=0x%llx RAMFC_TARGET=0x%08x ENG_CTX_VALID=%u\n", + tag, ctx_pa, target, valid); +} + static int gsp_phys_rmcall_stage4(void __iomem *bar0) { struct gsp_phys_boot_chunk *userd_c = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_USERD]; @@ -14616,6 +14809,7 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0) &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_INSTANCE]); gsp_phys_rmcall_ctx_diag_dump("RAMFC", &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_RAMFC]); + gsp_phys_rmcall_ctx_diag_decode("post-BIND"); } /* |