diff options
Diffstat (limited to 'kernel/gsp_phys_booter.c')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 172 |
1 files changed, 139 insertions, 33 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 11f205c..dc8e641 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -660,6 +660,12 @@ bool gsp_phys_boot_requested(void) */ #define GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING 30u /* + * Mode 31 (2026-10-06): mode-25 D first-init + a clean TSG sibling A whose + * 9-entry promote runs after V6 exists, so fifoCommitRamfcEngCtxForTsg + * commits V6's instance pointer + RAMFC while V6's own promote is suppressed. + */ +#define GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING_COMMIT 31u +/* * Mode-28 golden execution channel shape bits (gsp_phys_rmcall_golden_exec_shape, * default 7). Every combination keeps the captured flags 0x00200320 and the * FBMEM instance/RAMFC/mthd slices; the bits only choose the client-visible @@ -773,7 +779,7 @@ bool gsp_phys_boot_requested(void) */ #define GSP_PHYS_RMCALL_FWCLIENT_MASK_FECS_SYSMEM (1u << 16) #define GSP_PHYS_RMCALL_GR_PROMOTE_MAX \ - (GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING) + (GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING_COMMIT) /* * Work-submit token notifier slot. The default index is @@ -1027,6 +1033,12 @@ bool gsp_phys_boot_requested(void) #define GSP_PHYS_RMCALL_GOLDEN_FB_USERD 0x1000u #define GSP_PHYS_RMCALL_GOLDEN_FB_RAMFC 0x2000u #define GSP_PHYS_RMCALL_GOLDEN_FB_MTHD 0x3000u +/* + * Mode 31: the TSG-parented sibling A (the explicit TSG-commit carrier) and + * its FBMEM instance/RAMFC/mthd memdesc block. + */ +#define GSP_PHYS_RMCALL_HANDLE_GOLDEN_SIBLING 0xcaf00084u +#define GSP_PHYS_RMCALL_HANDLE_GOLDEN_SIBLING_FB 0xcaf00088u /* * Golden-image replay sizes (kernel_graphics.c:2058-2060, 2273-2276): the @@ -7629,6 +7641,8 @@ static const char *gsp_phys_rmcall_gr_promote_name(void) return "mode 29: construct-last golden execution lane (mode 28 minus the TURING_COMPUTE_A object construct on the wired golden channel)"; case GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING: return "mode 30: clean unwired TSG sibling A (promote + TURING_A first-init) + V6 execution"; + case GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING_COMMIT: + return "mode 31: mode-25 D first-init + TSG sibling A 9-entry promote after V6 (explicit TSG commit) + V6 execution"; default: return "off (PROMOTE_CTX never sent)"; } @@ -7817,7 +7831,9 @@ static int gsp_phys_rmcall_alloc_channel(void __iomem *bar0) boot_stage.rmcall_gr_promote == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 || boot_stage.rmcall_gr_promote == - GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING)) { + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING || + boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING_COMMIT)) { vars = &gsp_phys_rmcall_channel_variant_skipctx; attempts = 1; } @@ -8278,7 +8294,9 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0) boot_stage.rmcall_gr_promote == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST || boot_stage.rmcall_gr_promote == - GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING) && + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING || + boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING_COMMIT) && gsp_phys_rmcall_engine_is_gr()) { if (gsp_phys_rmcall_fwclient_mask_value() & GSP_PHYS_RMCALL_GOLDEN9_MASK_LADDER_VAS) { @@ -8388,7 +8406,9 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0) boot_stage.rmcall_gr_promote == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST || boot_stage.rmcall_gr_promote == - GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING) && + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING || + boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING_COMMIT) && gsp_phys_rmcall_engine_is_gr()) { ch.flags = cpu_to_le32(GSP_PHYS_RMCALL_CHANNEL_FLAG_GOLDEN9); ch.internal_flags = cpu_to_le32( @@ -8592,7 +8612,9 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0) boot_stage.rmcall_gr_promote == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST || boot_stage.rmcall_gr_promote == - GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING) && + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING || + boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING_COMMIT) && gsp_phys_rmcall_engine_is_gr()) { ret = gsp_phys_rmcall_golden9_promote(bar0); if (ret) @@ -9914,6 +9936,59 @@ static int gsp_phys_rmcall_golden9_promote(void __iomem *bar0) } /* + * Mode 31: allocate the clean TSG-parented sibling A (captured shape, no + * ring/USERD/HOBJ wiring) and run its 9-entry golden9 promote after V6's own + * constructs, so the server's TSG-wide engine-context commit walk + * (fifo_gv100.c:1480-1494) commits V6's instance pointer + RAMFC while V6's + * own promote is suppressed. Two 9-entry promotes total (D and A); V6 gets + * none. + */ +static int gsp_phys_rmcall_golden_sibling_alloc(void __iomem *bar0) +{ + struct gsp_phys_rmcall_channel_params ch; + u32 rpc = 0, status = 0; + u64 fb = 0; + int ret; + + memset(&ch, 0, sizeof(ch)); + ch.h_va_space = cpu_to_le32(0); + ch.gp_fifo_entries = + cpu_to_le32(GSP_PHYS_RMCALL_GOLDEN_GPFIFO_ENTRIES); + ch.engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_GR0); + ch.flags = cpu_to_le32(GSP_PHYS_RMCALL_CHANNEL_FLAG_GOLDEN9); + ch.internal_flags = cpu_to_le32( + GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_PRIV_KERNEL); + ret = gsp_phys_rmcall_alloc_fb_mem(bar0, + GSP_PHYS_RMCALL_HANDLE_GOLDEN_SIBLING_FB, 0x4000, &fb, + "GOLDEN SIBLING A memdescs"); + if (ret) + return ret; + ch.instance_mem = gsp_phys_rmcall_mem_desc_fb(fb + 0x0000, 0x1000); + ch.ramfc_mem = gsp_phys_rmcall_mem_desc_fb(fb + 0x2000, 0x1000); + ch.mthdbuf_mem = gsp_phys_rmcall_mem_desc_fb(fb + 0x3000, 0x1000); + ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client, + boot_stage.rmcall_tsg, + GSP_PHYS_RMCALL_HANDLE_GOLDEN_SIBLING, + GSP_PHYS_RMCALL_CLASS_CHANNEL, &ch, sizeof(ch), + false, NULL, + "mode 31 GOLDEN SIBLING A (TSG commit carrier)", + true, &rpc, &status); + if (ret) + return ret; + gsp_phys_rmcall_ctx_diag_decode("post-V6-constructs pre-A-commit"); + ret = gsp_phys_rmcall_golden9_promote_on(bar0, + GSP_PHYS_RMCALL_HANDLE_GOLDEN_SIBLING); + if (ret) + return ret; + gsp_phys_rmcall_ctx_diag_dump("INSTANCE (post-A-commit pre-BIND)", + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_INSTANCE]); + gsp_phys_rmcall_ctx_diag_dump("RAMFC (post-A-commit pre-BIND)", + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_RAMFC]); + gsp_phys_rmcall_ctx_diag_decode("post-A-commit pre-BIND"); + return 0; +} + +/* * Fill one MAIN promote entry with the prepared PA/VA and the snapshot's * entry size. initialize/nonmapped carry the bisection shape: modes 1/3/5 * entry 0 and mode 2 step 1 use a full field set; mode 2 step 2 repeats the @@ -10098,22 +10173,27 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0) mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6_ONLY || mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC || mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST || - mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING) && + mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING || + mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING_COMMIT) && (!gsp_phys_rmcall_golden_enabled() || !gsp_phys_gpu_enabled())) { pr_err("vgsp-boot: rm-call: mode %u requires gsp_phys_rmcall_golden=1 and gsp_phys_gpu=1 (the golden replay did not run); no PROMOTE_CTX is sent and the ladder aborts with no retry\n", mode); return -EINVAL; } /* - * Mode 30: the sibling A's 9-entry promote carries the group context - * commit; nothing is sent on the V6 channel. A multi-entry promote - * containing MAIN is refused for a SKIP-ctx channel + * Mode 30/31: the sibling A's 9-entry promote carries the group + * context commit; nothing is sent on the V6 channel. A multi-entry + * promote containing MAIN is refused for a SKIP-ctx channel * (graphics_context.c: bKMDVirtualContext -> NV_ERR_INVALID_ARGUMENT) * and the single-MAIN exemption needs the pCtxPhysMemDesc the FWCLIENT * shapes never establish, so a V6 promote would abort stage 3 on 0x1f. */ - if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING) { - pr_info("vgsp-boot: rm-call: mode 30: no PROMOTE_CTX on V6; the sibling A carries the group context commit (config_group ppEngCtxDesc), V6 inherits via the TSG commit/channel setup\n"); + if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING || + mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING_COMMIT) { + if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING) + pr_info("vgsp-boot: rm-call: mode 30: no PROMOTE_CTX on V6; the sibling A carries the group context commit (config_group ppEngCtxDesc), V6 inherits via the TSG commit/channel setup\n"); + else + pr_info("vgsp-boot: rm-call: mode 31: no PROMOTE_CTX on V6; sibling A's 9-entry promote ran after V6's alloc and the TSG loop (fifo_gv100.c:1480-1494) already committed V6\n"); boot_stage.rmcall_gr_promote_step = 1; return 0; } @@ -11949,7 +12029,9 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0) boot_stage.rmcall_gr_promote == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST || boot_stage.rmcall_gr_promote == - GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING) && + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING || + boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING_COMMIT) && gsp_phys_rmcall_engine_is_gr()) { pr_err("vgsp-boot: rm-call: stage 3 aborted: mode-%u golden replay failed (%d); the golden construct is atomic and is not retried\n", boot_stage.rmcall_gr_promote, ret); @@ -11994,22 +12076,26 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0) return ret; /* - * Mode 30 readback point 1: V6's instance/RAMFC right after the channel - * alloc (A's replay contains the 9-entry promote and A's TURING_A - * first-init by now), so A's commit effect can be separated from the - * later BIND/GPFIFO_SCHEDULE. The decode line prints the two decisive - * fields (instance WFI pointer pair + RAMFC ENG_CTX_VALID). + * Mode 30/31 readback point 1: V6's instance/RAMFC right after the + * channel alloc (mode 30: A's replay contains the 9-entry promote and + * A's TURING_A first-init by now; mode 31: D's replay ran and the + * sibling A commit plus V6's constructs are still ahead), so the + * commit effect can be separated from the later BIND/GPFIFO_SCHEDULE. + * The decode line prints the decisive fields (instance WFI pointer + * pair + both ENG_CTX_VALID target words). */ if (gsp_phys_rmcall_engine_is_gr() && - boot_stage.rmcall_gr_promote == - GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING) { + (boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING || + boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING_COMMIT)) { gsp_phys_rmcall_ctx_diag_dump( - "INSTANCE (post-A-commit pre-BIND)", + "INSTANCE (post-V6-alloc pre-A-commit)", &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_INSTANCE]); gsp_phys_rmcall_ctx_diag_dump( - "RAMFC (post-A-commit pre-BIND)", + "RAMFC (post-V6-alloc pre-A-commit)", &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_RAMFC]); - gsp_phys_rmcall_ctx_diag_decode("post-A-commit pre-BIND"); + gsp_phys_rmcall_ctx_diag_decode("post-V6-alloc pre-A-commit"); } /* @@ -12079,6 +12165,21 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0) boot_stage.rmcall_ctx_size); return -EOPNOTSUPP; } + /* + * Mode 31: allocate the TSG-parented sibling A and run its + * 9-entry promote here, after V6's own constructs, so the + * server's TSG-wide commit walk sees V6 in its group; V6's + * own promote is suppressed in gsp_phys_rmcall_gr_ctx_promote. + */ + if (boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING_COMMIT) { + ret = gsp_phys_rmcall_golden_sibling_alloc(bar0); + if (ret) { + pr_err("vgsp-boot: rm-call: mode 31 sibling A commit failed (%d); no V6 promote follows; the ladder aborts with no retry\n", + ret); + return ret; + } + } ret = gsp_phys_rmcall_gr_ctx_promote(bar0); if (ret) { pr_err("vgsp-boot: rm-call: stage 3 aborted: GPU_PROMOTE_CTX failed (%d); the GR context is not committed and BIND/SCHEDULE/submit are not attempted (no retry; UNLOAD + rmmod remains the fail-safe)\n", @@ -14694,12 +14795,14 @@ static void gsp_phys_rmcall_ctx_diag_dump(const char *what, } /* - * One-line decode of the two fields that decide whether V6's engine context is + * One-line decode of the fields that decide whether V6's engine context is * live: the instance block's NV_RAMIN_ENGINE_WFI_PTR pair (dword 132 bits - * 31:12 = PA[31:12], dword 133 bits 7:0 = PA[39:32]; fifo_gv100.c) and RAMFC - * NV_RAMFC_TARGET (dword 43, NV_PPBDMA_TARGET_ENG_CTX_VALID bit 16). Host-view - * LE chunks, same as the dump above; used at the mode-30 post-replay and the - * stage-4 post-BIND points so A's commit can be told apart from BIND. + * 31:12 = PA[31:12], dword 133 bits 7:0 = PA[39:32]; fifo_gv100.c) and the + * NV_RAMFC_TARGET dword (43, NV_PPBDMA_TARGET_ENG_CTX_VALID bit 16) in BOTH + * the instance and RAMFC chunks - which of the two the PBDMA actually reads + * is an open question (m30b saw the bit only in the instance view). Host-view + * LE chunks, same as the dump above; used at the mode-30/31 pre- and + * post-commit points and the stage-4 post-BIND point. */ static void gsp_phys_rmcall_ctx_diag_decode(const char *tag) { @@ -14709,8 +14812,8 @@ static void gsp_phys_rmcall_ctx_diag_decode(const char *tag) &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_RAMFC]; const __le32 *iv, *rv; u64 ctx_pa; - u32 target; - bool valid; + u32 inst_target, ramfc_target; + bool inst_valid, ramfc_valid; if (!inst->pages || !ramfc->pages) { pr_info("vgsp-boot: rm-call: V6 ctx (%s): instance/RAMFC chunk absent\n", @@ -14721,10 +14824,13 @@ static void gsp_phys_rmcall_ctx_diag_decode(const char *tag) rv = gsp_phys_chunk_page_va(ramfc, 0); ctx_pa = ((u64)(le32_to_cpu(iv[133]) & 0xffu) << 32) | (le32_to_cpu(iv[132]) & 0xfffff000u); - target = le32_to_cpu(rv[43]); - valid = !!(target & (1u << 16)); - pr_info("vgsp-boot: rm-call: V6 ctx (%s): inst_wfi=0x%llx RAMFC_TARGET=0x%08x ENG_CTX_VALID=%u\n", - tag, ctx_pa, target, valid); + inst_target = le32_to_cpu(iv[43]); + ramfc_target = le32_to_cpu(rv[43]); + inst_valid = !!(inst_target & (1u << 16)); + ramfc_valid = !!(ramfc_target & (1u << 16)); + pr_info("vgsp-boot: rm-call: V6 ctx (%s): inst_wfi=0x%llx INST_TARGET=0x%08x RAMFC_TARGET=0x%08x ENG_CTX_VALID(inst=%u,ramfc=%u)\n", + tag, ctx_pa, inst_target, ramfc_target, inst_valid, + ramfc_valid); } static int gsp_phys_rmcall_stage4(void __iomem *bar0) |