diff options
Diffstat (limited to 'kernel/gsp_phys_booter.c')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 407 |
1 files changed, 384 insertions, 23 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 040428c..3383b35 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -618,6 +618,38 @@ bool gsp_phys_boot_requested(void) */ #define GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6_ONLY 27u /* + * Mode 28 (2026-10-06, experiment): the golden execution lane. Mode 25's + * captured golden construct (VAS binding, FBMEM instance/RAMFC/mthd slices, + * single atomic 9-entry GPU_PROMOTE_CTX on the golden channel), but the + * golden channel itself gets a client-visible submission path: the ladder's + * GPFIFO ring VA (gpFifoOffset) and the ladder's sysmem USERD control page + * (host GP_PUT writes reach the hardware word, the V6 shape). The channel + * stays device-parented like the captured golden construct (a golden9 + * promote on a TSG channel poisons the next GR object construct: modes 27/28 + * first run both died -110 there), and the real nv-push path schedules + * device-parented channels with BIND + channel-level GPFIFO_SCHEDULE alone. + * The ladder swaps boot_stage.rmcall_channel to the golden handle before the + * 0xc5c0 engine-object alloc, so stage 4's BIND/GPFIFO_SCHEDULE/token + * sequence, the GR sem check and every forwarded publish run on the golden + * channel; V6 stays allocated but is never bound or scheduled, and its + * FWCLIENT promote is skipped. The hypothesis: the golden channel owns the + * complete committed context, so the engine will finally dispatch the + * forwarded QMD. + */ +#define GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC 28u +/* + * Mode-28 golden execution channel shape bits (gsp_phys_rmcall_golden_exec_shape, + * default 7). Every combination keeps the captured flags 0x00200320 and the + * FBMEM instance/RAMFC/mthd slices; the bits only choose the client-visible + * submission path, so a deadlock can be bisected without a rebuild. + */ +#define GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_RING BIT(0) +#define GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_ENTRIES8 BIT(1) +#define GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_USERD_SYSMEM BIT(2) +#define GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_USERD_SIZE200 BIT(3) +#define GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_HOBJ_VAS BIT(4) +#define GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_SKIP_TURING_A BIT(5) +/* * Existing fwclient mask bit 8: mode-25 gate for the captured VAS * COPY_SERVER_RESERVED_PDES control. The captured control copies three * server-RM-internal page-table levels the ladder cannot source safely, so @@ -643,8 +675,34 @@ bool gsp_phys_boot_requested(void) * for comparison. */ #define GSP_PHYS_RMCALL_GOLDEN9_MASK_SIZED_BUFFERS (1u << 10) +/* + * Mode-25 bisection bit 11: send each built non-MAIN FWCLIENT entry as its + * own one-entry GPU_PROMOTE_CTX control instead of one batched control, so a + * single refused entry cannot refuse the whole set (m2b/m2f: the batched + * five-entry control returned 0x1f as a whole; individual acceptance was + * never tested). + */ +#define GSP_PHYS_RMCALL_FWCLIENT_MASK_INDIVIDUAL (1u << 11) +/* + * Mode-25 bisection bit 12: additionally promote the golden9 VA-only global + * bookings (bufferIds 3/4/5/6 = BUNDLE_CB, PAGEPOOL, ATTRIBUTE_CB, + * RTV_CB_GLOBAL) on the ladder channel as one-entry controls. The golden9 + * context carries them; the FWCLIENT set does not, so the V6 context may be + * missing the GR engine's code/attribute data entirely. + */ +#define GSP_PHYS_RMCALL_FWCLIENT_MASK_GOLDEN9_GLOBALS (1u << 12) +/* + * Mode-25 bisection bits 13/14: the FECS_EVENT entry is refused 0x1f on the + * V6 channel in its combined PA+VA form (the same form the golden channel + * accepts). Try its halves separately: bit 13 = PA-only init entry + * (bInitialize=1 bNonmapped=1, like the accepted PM entry), bit 14 = VA-only + * booking entry (bInitialize=0, like the accepted globals). Both booked + * together may complete the FECS state the combined control cannot. + */ +#define GSP_PHYS_RMCALL_FWCLIENT_MASK_FECS_PA_ONLY (1u << 13) +#define GSP_PHYS_RMCALL_FWCLIENT_MASK_FECS_VA_ONLY (1u << 14) #define GSP_PHYS_RMCALL_GR_PROMOTE_MAX \ - (GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6_ONLY) + (GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC) /* * Work-submit token notifier slot. The default index is @@ -946,6 +1004,15 @@ bool gsp_phys_boot_requested(void) (1u << 21)) /* + * Mode-28 alternative golden flags (unused after the m3b first run kept the + * captured word): the captured golden word with the USERD placement bits + * clear, for experiments where the hardware control block must sit at + * userd+0 (publish offset 0x8c). + */ +#define GSP_PHYS_RMCALL_CHANNEL_FLAG_GOLDEN_EXEC \ + (u32)(GSP_PHYS_RMCALL_CHANNEL_FLAG_PRIVILEGED) + +/* * BANNED - do not use: internalFlags bit 6 = 0x40 marks the channel as * GSP-owned. A live run with this bit set wedged the GSP-RM task: the alloc * RPC never replied, every later RPC timed out, and the fail-safe held the @@ -1014,6 +1081,9 @@ enum gsp_phys_rmcall_buf { #define GSP_PHYS_RMCALL_GPFIFO_ENTRIES 8u #define GSP_PHYS_RMCALL_USERD_GPPUT_OFF 0x8cu /* Nvc46fControl GPPut (clc46f.h:57) */ #define GSP_PHYS_RMCALL_USERD_GPGET_OFF 0x88u /* Nvc46fControl GPGet (clc46f.h:56) */ +/* Captured golden flags pin USERD_INDEX_VALUE=3, so the control block (and + * the GP_PUT/GP_GET words inside it) sits 0x600 into the USERD chunk. */ +#define GSP_PHYS_RMCALL_USERD_CTL_OFF_CAPTURED 0x600u #define GSP_PHYS_RMCALL_SEM_PAYLOAD 0xc0ffee01u #define GSP_PHYS_RMCALL_TIMEOUT_MS 2000u #define GSP_PHYS_RMCALL_POLL_MS 5u @@ -7477,6 +7547,8 @@ static const char *gsp_phys_rmcall_gr_promote_name(void) return "mode 26: golden9 9-entry promote on the V6 execution channel (plus the golden-channel promote)"; case GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6_ONLY: return "mode 27: golden9 9-entry promote on the V6 execution channel only (no golden-channel promote)"; + case GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC: + return "mode 28: golden execution lane (golden channel with client GPFIFO/USERD, 9-entry promote on it, execution switched to it)"; default: return "off (PROMOTE_CTX never sent)"; } @@ -7851,6 +7923,14 @@ static void gsp_phys_rmcall_golden_3d_on_ladder(void __iomem *bar0) if (!gsp_phys_rmcall_golden_enabled() || !gsp_phys_gpu_enabled() || !boot_stage.rmcall_channel) return; + /* + * Mode 28: the golden execution channel already carries its TURING_A + * object from the replay; the ladder's extra 3D object (a mode-25 + * GR-instance by-product) must not attach a second one to it. + */ + if (boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC) + return; ret = gsp_phys_rmcall_alloc_gr_object_class(bar0, GSP_PHYS_RMCALL_CLASS_GR_3D, GSP_PHYS_RMCALL_HANDLE_GOLDEN_GR3D_LADDER, @@ -8068,6 +8148,7 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0) struct gsp_phys_rmcall_channel_params ch; struct gsp_phys_rmcall_mem_obj_shape shape; u32 rpc = 0, status = 0; + u32 ch_parent = boot_stage.rmcall_device; u64 golden_fb = 0; bool pb_ok = true; int ret; @@ -8107,7 +8188,9 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0) boot_stage.rmcall_gr_promote == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6 || boot_stage.rmcall_gr_promote == - GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6_ONLY) && + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6_ONLY || + boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC) && gsp_phys_rmcall_engine_is_gr()) { if (gsp_phys_rmcall_fwclient_mask_value() & GSP_PHYS_RMCALL_GOLDEN9_MASK_LADDER_VAS) { @@ -8211,7 +8294,9 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0) boot_stage.rmcall_gr_promote == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6 || boot_stage.rmcall_gr_promote == - GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6_ONLY) && + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6_ONLY || + boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC) && gsp_phys_rmcall_engine_is_gr()) { ch.flags = cpu_to_le32(GSP_PHYS_RMCALL_CHANNEL_FLAG_GOLDEN9); ch.internal_flags = cpu_to_le32( @@ -8260,12 +8345,69 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0) golden_fb + GSP_PHYS_RMCALL_GOLDEN_FB_RAMFC, 0x1000); ch.mthdbuf_mem = gsp_phys_rmcall_mem_desc_fb( golden_fb + GSP_PHYS_RMCALL_GOLDEN_FB_MTHD, 0x1000); + /* + * Mode 28: the golden execution lane. Keep the captured shape + * bit-for-bit (same captured flags 0x00200320, same FBMEM + * instance/RAMFC/mthd slices) and only make the submission + * path client-visible: the ladder GPFIFO ring at the proven VA + * and the ladder sysmem USERD page (host GP_PUT writes land in + * the hardware word; the captured USERD index 3 puts the + * control block at userd+0x600, which + * gsp_phys_rmcall_userd_ctl_off() applies). The channel stays + * DEVICE-parented like the captured golden construct (a + * golden9 promote on a TSG channel poisoned the next GR object + * construct in modes 27/28 first run). Stage 3 swaps + * boot_stage.rmcall_channel to this handle before the compute + * object alloc. + */ + if (boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC) { + u32 exec_shape = + gsp_phys_rmcall_golden_exec_shape_value(); + + if (exec_shape & + GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_RING) + ch.gp_fifo_offset = + cpu_to_le64(GSP_PHYS_RMCALL_VA_GPFIFO); + if (exec_shape & + GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_ENTRIES8) + ch.gp_fifo_entries = cpu_to_le32( + GSP_PHYS_RMCALL_GPFIFO_ENTRIES); + if (exec_shape & + GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_USERD_SYSMEM) { + struct gsp_phys_rmcall_mem_desc ud = + gsp_phys_rmcall_mem_desc( + GSP_PHYS_RMCALL_BUF_USERD); + + if (exec_shape & + GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_USERD_SIZE200) + ud.size = cpu_to_le64(0x200); + ch.userd_mem = ud; + } + if (exec_shape & + GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_HOBJ_VAS) + ch.h_object_buffer = + cpu_to_le32(boot_stage.rmcall_vas); + } pr_info("vgsp-boot: rm-call: GOLDEN9 channel shape: flags=0x%x (PRIVILEGED|USERD_INDEX_VALUE=3|USERD_INDEX_PAGE_FIXED, no SKIP_SCRUBBER/SKIP_CTXBUFFER_ALLOC) hObjectBuffer=0 gpFifoEntries=0x%x instance/userd/ramfc/mthd=FBMEM memdescs block=0x%llx (capture-matched apertures)\n", GSP_PHYS_RMCALL_CHANNEL_FLAG_GOLDEN9, GSP_PHYS_RMCALL_GOLDEN_GPFIFO_ENTRIES, golden_fb); + if (boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC) + pr_info("vgsp-boot: rm-call: GOLDEN EXEC channel shape (mode 28): flags=0x%x (captured) shape=0x%x gpFifoOffset=0x%llx gpFifoEntries=%u hObjectBuffer=0x%x userd=%s size=0x%llx hParent=device hVASpace=0x%x; instance/ramfc/mthd stay FBMEM\n", + GSP_PHYS_RMCALL_CHANNEL_FLAG_GOLDEN9, + gsp_phys_rmcall_golden_exec_shape_value(), + le64_to_cpu(ch.gp_fifo_offset), + le32_to_cpu(ch.gp_fifo_entries), + le32_to_cpu(ch.h_object_buffer), + (ch.userd_mem.address_space == + cpu_to_le32(GSP_PHYS_RMCALL_ADDR_SYSMEM)) ? + "sysmem" : "FBMEM", + le64_to_cpu(ch.userd_mem.size), + boot_stage.rmcall_vas); } ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client, - boot_stage.rmcall_device, + ch_parent, GSP_PHYS_RMCALL_HANDLE_GOLDEN_CHANNEL, GSP_PHYS_RMCALL_CLASS_CHANNEL, &ch, sizeof(ch), false, NULL, @@ -8295,7 +8437,9 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0) if ((boot_stage.rmcall_gr_promote == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 || boot_stage.rmcall_gr_promote == - GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6) && + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6 || + boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC) && gsp_phys_rmcall_engine_is_gr()) { ret = gsp_phys_rmcall_golden9_promote(bar0); if (ret) @@ -8303,6 +8447,13 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0) } /* 6. TURING_A (0xc597) 3D object on the golden channel (kernel_graphics.c:2409-2433). */ + if (boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC && + (gsp_phys_rmcall_golden_exec_shape_value() & + GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_SKIP_TURING_A)) { + pr_info("vgsp-boot: rm-call: mode 28: GOLDEN TURING_A 3D object skipped (shape bit5); the ladder's TURING_COMPUTE_A on the same channel provides the GR instance state\n"); + return 0; + } ret = gsp_phys_rmcall_alloc_gr_object_class(bar0, GSP_PHYS_RMCALL_CLASS_GR_3D, GSP_PHYS_RMCALL_HANDLE_GOLDEN_GR_OBJ, @@ -9769,13 +9920,25 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0) * GR-instance state to consume. Abort instead of sending it blind. */ if ((mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 || - mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6_ONLY) && + mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6_ONLY || + mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC) && (!gsp_phys_rmcall_golden_enabled() || !gsp_phys_gpu_enabled())) { pr_err("vgsp-boot: rm-call: mode %u requires gsp_phys_rmcall_golden=1 and gsp_phys_gpu=1 (the golden replay did not run); no PROMOTE_CTX is sent and the ladder aborts with no retry\n", mode); return -EINVAL; } /* + * Mode 28: the 9-entry promote already ran on the golden execution + * channel inside the golden replay (it owns the committed context and + * the forwarded launches); nothing is sent on the V6 channel at all. + */ + if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC) { + pr_info("vgsp-boot: rm-call: mode-28 golden execution lane: no PROMOTE_CTX on V6 (the golden channel 0x%x carries the committed context); continuing to stage 4 on the golden channel\n", + GSP_PHYS_RMCALL_HANDLE_GOLDEN_CHANNEL); + boot_stage.rmcall_gr_promote_step = 1; + return 0; + } + /* * Modes 8/9/10 must run on the V6 channel: without the * SKIP_CTXBUFFER_ALLOC bit the promote takes the VA/bookkeeping * route that asserted GSP-RM in modes 1/2/4 (or rejects the @@ -10734,7 +10897,64 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0) pr_warn("vgsp-boot: rm-call: PROMOTE_CTX fwclient: UNRESTRICTED_PRIV_ACCESS_MAP omitted (allocation unavailable)\n"); } - if (n > 0) { + if (n > 0 && (mask & GSP_PHYS_RMCALL_FWCLIENT_MASK_INDIVIDUAL)) { + u32 k; + u32 n_single_ok = 0; + + /* + * Bisection: the batched non-MAIN control was refused + * 0x1f as a whole (m2b/m2f); send each built entry as + * its own one-entry control so one refused entry cannot + * refuse the rest, and the accepted subset is visible + * entry by entry. + */ + for (k = 0; k < n; k++) { + struct gsp_phys_rmcall_promote_entry e = + p->entry[k]; + + /* + * Overwrite only slot 0: the entries k+1..n-1 + * must survive for their own iterations (a full + * memset here destroyed the not-yet-sent ones). + */ + p->entry[0] = e; + p->entry_count = cpu_to_le32(1); + rpc_result = 0; + status = 0; + ret = gsp_phys_rmcall_control(bar0, + boot_stage.rmcall_subdevice, + GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX, + p, sizeof(*p), 24, NULL, 0, + &rpc_result, &status, + "GPU_PROMOTE_CTX (FWCLIENT non-MAIN single)"); + pr_info("vgsp-boot: rm-call: PROMOTE_CTX fwclient non-MAIN single[%u/%u]: bufferId=%u status=0x%x rpc_result=0x%x (hObject=0x%x)\n", + k + 1, n, le16_to_cpu(e.buffer_id), + status, rpc_result, + boot_stage.rmcall_channel); + if (ret) { + /* + * Bisection: keep going so the whole + * acceptance matrix is visible in one + * run; the ladder aborts only if no + * entry was accepted at all. + */ + pr_warn("vgsp-boot: rm-call: PROMOTE_CTX fwclient non-MAIN single[%u] bufferId=%u refused (status=0x%x rpc_result=0x%x, %s, ret=%d); continuing the bisection\n", + k, le16_to_cpu(e.buffer_id), + status, rpc_result, + gsp_phys_rmcall_status_str(status), + ret); + continue; + } + n_single_ok++; + } + if (!n_single_ok) { + pr_err("vgsp-boot: rm-call: PROMOTE_CTX fwclient non-MAIN single: no entry accepted; the GR context commit is incomplete and the ladder aborts before BIND with no retry\n"); + kvfree(p); + return -EIO; + } + boot_stage.rmcall_gr_promote_step = + (mask & BIT(0)) ? 2 : 1; + } else if (n > 0) { p->entry_count = cpu_to_le32(n); rpc_result = 0; status = 0; @@ -10762,6 +10982,103 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0) } else if (!(mask & BIT(0))) { pr_warn("vgsp-boot: rm-call: PROMOTE_CTX fwclient: MAIN omitted by mask\n"); } + + /* + * Bit 12: the golden9 VA-only global bookings, one control per + * entry. Accepted entries add the GR engine's bundle/pagepool/ + * attribute/RTV reservations to the ladder channel's context; + * a refused one is logged and skipped (bisection style). + */ + if (mask & GSP_PHYS_RMCALL_FWCLIENT_MASK_GOLDEN9_GLOBALS) { + static const struct { + u16 id; + u64 va; + const char *name; + } gl[4] = { + { 3, GSP_PHYS_RMCALL_VA_GOLDEN9_BUNDLE_CB, + "BUNDLE_CB" }, + { 4, GSP_PHYS_RMCALL_VA_GOLDEN9_PAGEPOOL, + "PAGEPOOL" }, + { 5, GSP_PHYS_RMCALL_VA_GOLDEN9_ATTRIBUTE_CB, + "ATTRIBUTE_CB" }, + { 6, GSP_PHYS_RMCALL_VA_GOLDEN9_RTV_CB_GLOBAL, + "RTV_CB_GLOBAL" }, + }; + u32 g; + + for (g = 0; g < 4; g++) { + gsp_phys_rmcall_promote_entry_fill(p, 0, + gl[g].id, 0, gl[g].va, 0, 0, + 0, 0); + p->entry_count = cpu_to_le32(1); + rpc_result = 0; + status = 0; + ret = gsp_phys_rmcall_control(bar0, + boot_stage.rmcall_subdevice, + GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX, + p, sizeof(*p), 24, NULL, 0, + &rpc_result, &status, + "GPU_PROMOTE_CTX (golden9 global)"); + pr_info("vgsp-boot: rm-call: PROMOTE_CTX golden9-global[%u/4] %s bufferId=%u va=0x%llx status=0x%x rpc_result=0x%x (hObject=0x%x)\n", + g + 1, gl[g].name, gl[g].id, gl[g].va, + status, rpc_result, + boot_stage.rmcall_channel); + if (ret) + pr_warn("vgsp-boot: rm-call: PROMOTE_CTX golden9-global[%u] %s refused (status=0x%x rpc_result=0x%x, %s, ret=%d); continuing the bisection\n", + g, gl[g].name, status, + rpc_result, + gsp_phys_rmcall_status_str(status), + ret); + } + } + + /* + * Bits 13/14: the FECS_EVENT halves. The combined form is + * refused 0x1f on V6; a PA-only init entry and a VA-only + * booking entry are each individually acceptable shapes on + * this channel, and together they may carry the FECS state. + */ + if (mask & GSP_PHYS_RMCALL_FWCLIENT_MASK_FECS_PA_ONLY) { + if (boot_stage.rmcall_fecs_pa && + boot_stage.rmcall_fecs_size) { + gsp_phys_rmcall_promote_entry_fill(p, 0, 9, + boot_stage.rmcall_fecs_pa, 0, + boot_stage.rmcall_fecs_size, fattr, + 1, 1); + p->entry_count = cpu_to_le32(1); + rpc_result = 0; + status = 0; + ret = gsp_phys_rmcall_control(bar0, + boot_stage.rmcall_subdevice, + GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX, + p, sizeof(*p), 24, NULL, 0, + &rpc_result, &status, + "GPU_PROMOTE_CTX (FECS PA-only)"); + pr_info("vgsp-boot: rm-call: PROMOTE_CTX FECS PA-only: bufferId=9 pa=0x%llx size=0x%x physAttr=0x%x status=0x%x rpc_result=0x%x (hObject=0x%x)\n", + boot_stage.rmcall_fecs_pa, + boot_stage.rmcall_fecs_size, fattr, + status, rpc_result, + boot_stage.rmcall_channel); + } else { + pr_warn("vgsp-boot: rm-call: PROMOTE_CTX FECS PA-only skipped: no FECS buffer prepared\n"); + } + } + if (mask & GSP_PHYS_RMCALL_FWCLIENT_MASK_FECS_VA_ONLY) { + gsp_phys_rmcall_promote_entry_fill(p, 0, 9, 0, + GSP_PHYS_RMCALL_VA_GOLDEN9_FECS, 0, 0, 0, 0); + p->entry_count = cpu_to_le32(1); + rpc_result = 0; + status = 0; + ret = gsp_phys_rmcall_control(bar0, + boot_stage.rmcall_subdevice, + GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX, + p, sizeof(*p), 24, NULL, 0, + &rpc_result, &status, + "GPU_PROMOTE_CTX (FECS VA-only)"); + pr_info("vgsp-boot: rm-call: PROMOTE_CTX FECS VA-only: bufferId=9 va=0x%llx status=0x%x rpc_result=0x%x (hObject=0x%x)\n", + GSP_PHYS_RMCALL_VA_GOLDEN9_FECS, status, + rpc_result, boot_stage.rmcall_channel); + } } else { pr_err("vgsp-boot: rm-call: unknown gr_promote_param=%u reached the promote step; no PROMOTE_CTX is sent and the ladder aborts with no retry\n", mode); @@ -10883,7 +11200,8 @@ static int gsp_phys_rmcall_srv_ctx_commit(void __iomem *bar0) u64 main_pa, main_size; int ret; - if (boot_stage.rmcall_srv_main_valid) { + if (boot_stage.rmcall_srv_main_valid && + !gsp_phys_srv_main_ignore_value()) { main_pa = boot_stage.rmcall_srv_main_pa; main_size = boot_stage.rmcall_srv_main_size; /* @@ -11367,11 +11685,13 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0) if (gsp_phys_rmcall_golden_enabled() && gsp_phys_gpu_enabled()) { ret = gsp_phys_rmcall_golden_replay(bar0); if (ret) { - if (boot_stage.rmcall_gr_promote == - GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 && + if ((boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 || + boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC) && gsp_phys_rmcall_engine_is_gr()) { - pr_err("vgsp-boot: rm-call: stage 3 aborted: mode-25 golden replay failed (%d); the golden construct is atomic and is not retried\n", - ret); + pr_err("vgsp-boot: rm-call: stage 3 aborted: mode-%u golden replay failed (%d); the golden construct is atomic and is not retried\n", + boot_stage.rmcall_gr_promote, ret); return ret; } pr_warn("vgsp-boot: rm-call: golden-image replay failed (%d); continuing the ladder (the golden replay is best-effort and the on-channel TURING_A init still runs); no retry\n", @@ -11413,6 +11733,29 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0) return ret; /* + * Mode 28: the golden execution lane. The golden replay allocated the + * golden channel with a client GPFIFO/USERD and the accepted 9-entry + * context commit; switch the ladder's execution channel to it BEFORE + * the compute-object alloc so the 0xc5c0 object (the forwarded + * launch's SET_OBJECT target), stage 4's BIND/SCHEDULE/token sequence, + * the GR sem check and every forwarded publish all run on the golden + * channel. The V6 channel stays allocated but is never bound or + * scheduled, and its FWCLIENT promote is suppressed (mode 28 returns + * early in gsp_phys_rmcall_gr_ctx_promote). + */ + if (gsp_phys_rmcall_engine_is_gr() && + boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC) { + pr_info("vgsp-boot: rm-call: mode 28: execution channel switched from V6 0x%x to golden execution channel 0x%x (client ring gpFifoVa=0x%llx userd=sysmem, device-parented); the V6 channel stays unscheduled\n", + boot_stage.rmcall_channel, + GSP_PHYS_RMCALL_HANDLE_GOLDEN_CHANNEL, + GSP_PHYS_RMCALL_VA_GPFIFO); + boot_stage.rmcall_channel = + GSP_PHYS_RMCALL_HANDLE_GOLDEN_CHANNEL; + boot_stage.rmcall_channel_server_mem = false; + } + + /* * H1: bind the compute engine object to the just-allocated GR channel * and commit the engine context before stage 3 returns. The TURING_ * COMPUTE_A alloc rejection aborts the ladder (the GR method block's @@ -11636,6 +11979,24 @@ struct gsp_phys_rmcall_ce_submit { }; /* + * The hardware USERD control block (Nvc46fControl GP_PUT/GP_GET) sits at + * userd base + USERD_INDEX_VALUE * 0x200. The exec-shape channels (V6 and + * every ladder variant) use index 0, so their GP_PUT/GP_GET live at offset + * 0x8c/0x88 of the chunk. Mode 28 keeps the captured golden channel flags + * (USERD_INDEX_VALUE=3 within a fixed USERD page), so the hardware control + * block sits at userd+0x600 and every host USERD touch must use the same + * offset or host and hardware would poll different words. + */ +static u32 gsp_phys_rmcall_userd_ctl_off(void) +{ + if (boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC && + gsp_phys_rmcall_engine_is_gr()) + return GSP_PHYS_RMCALL_USERD_CTL_OFF_CAPTURED; + return 0; +} + +/* * Shared publish tail of the ladder submissions: read USERD GP_PUT/GP_GET, * encode and write the GPFIFO entry for the module pushbuffer VA, advance * GP_PUT, snapshot the C3 fault epoch and ring the PF doorbell. Extracted @@ -11669,8 +12030,8 @@ static int gsp_phys_rmcall_publish_locked(void __iomem *bar0, u64 pb_gpu_va, if (!ring_va || !userd) return -EIO; - gpput = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF); - gpget = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPGET_OFF); + gpput = get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPPUT_OFF); + gpget = get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPGET_OFF); if (gpput >= GSP_PHYS_RMCALL_GPFIFO_ENTRIES) { pr_err("vgsp-boot: rm-call: USERD GP_PUT=%u is outside the %u-entry ring; refusing to submit\n", gpput, GSP_PHYS_RMCALL_GPFIFO_ENTRIES); @@ -11705,7 +12066,7 @@ static int gsp_phys_rmcall_publish_locked(void __iomem *bar0, u64 pb_gpu_va, * the host write is therefore what the engine polls (cycle 21). */ put_unaligned_le32((gpput + 1) % GSP_PHYS_RMCALL_GPFIFO_ENTRIES, - (u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF); + (u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPPUT_OFF); /* * Doorbell value: workSubmitToken = ((runlistId << 16) & 0x7f00000) | * (chId & 0xfff) (kernel_fifo_tu102.c:118-121). The server-generated @@ -11879,9 +12240,9 @@ static int gsp_phys_rmcall_gr_sem_push_locked(void __iomem *bar0) if (poll == GSP_PHYS_RMCALL_SEM_PAYLOAD) break; if (time_after_eq(jiffies, deadline)) { - gpget = get_unaligned_le32((u8 *)userd + + gpget = get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPGET_OFF); - gpput = get_unaligned_le32((u8 *)userd + + gpput = get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPPUT_OFF); pr_err("vgsp-boot: rm-call: REAL GR SEM FAILED: no release within %u ms (sem=0x%08x want=0x%08x GPGet=%u GPPut=%u engine=%s); the channel did not execute the GR block\n", GSP_PHYS_RMCALL_TIMEOUT_MS, poll, @@ -11893,10 +12254,10 @@ static int gsp_phys_rmcall_gr_sem_push_locked(void __iomem *bar0) msleep(GSP_PHYS_RMCALL_POLL_MS); } - gpget = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPGET_OFF); + gpget = get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPGET_OFF); pr_info("vgsp-boot: rm-call: REAL GR SEM OK sem=0x%08x want=0x%08x GPGet=%u GPPut=%u words=%u token=0x%08x\n", poll, GSP_PHYS_RMCALL_SEM_PAYLOAD, gpget, - get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF), + get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPPUT_OFF), sub.words, boot_stage.rmcall_token); /* * Latch the GR execution gate: this is the only place a real GR @@ -14059,9 +14420,9 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0) if (poll == GSP_PHYS_RMCALL_SEM_PAYLOAD) break; if (time_after_eq(jiffies, deadline)) { - gpget = get_unaligned_le32((u8 *)userd + + gpget = get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPGET_OFF); - gpput = get_unaligned_le32((u8 *)userd + + gpput = get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPPUT_OFF); pr_err("vgsp-boot: rm-call: no semaphore release within %u ms (sem=0x%08x want=0x%08x GPGet=%u GPPut=%u); the channel did not execute the CE block\n", GSP_PHYS_RMCALL_TIMEOUT_MS, poll, @@ -14087,10 +14448,10 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0) return -EIO; } - gpget = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPGET_OFF); + gpget = get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPGET_OFF); pr_info("vgsp-boot: rm-call: REAL COPY OK sem=0x%08x src=0x%llx dst=0x%llx bytes=%u GPGet=%u GPPut=%u words=%u token=0x%08x\n", poll, sub.src_dma, sub.dst_dma, GSP_PHYS_RMCALL_COPY_SIZE, gpget, - get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF), + get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPPUT_OFF), sub.words, boot_stage.rmcall_token); return 0; } |