diff options
| -rw-r--r-- | kernel/gsp_phys_booter.c | 407 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.c | 46 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.h | 15 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 65 |
4 files changed, 503 insertions, 30 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 040428c..3383b35 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -618,6 +618,38 @@ bool gsp_phys_boot_requested(void) */ #define GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6_ONLY 27u /* + * Mode 28 (2026-10-06, experiment): the golden execution lane. Mode 25's + * captured golden construct (VAS binding, FBMEM instance/RAMFC/mthd slices, + * single atomic 9-entry GPU_PROMOTE_CTX on the golden channel), but the + * golden channel itself gets a client-visible submission path: the ladder's + * GPFIFO ring VA (gpFifoOffset) and the ladder's sysmem USERD control page + * (host GP_PUT writes reach the hardware word, the V6 shape). The channel + * stays device-parented like the captured golden construct (a golden9 + * promote on a TSG channel poisons the next GR object construct: modes 27/28 + * first run both died -110 there), and the real nv-push path schedules + * device-parented channels with BIND + channel-level GPFIFO_SCHEDULE alone. + * The ladder swaps boot_stage.rmcall_channel to the golden handle before the + * 0xc5c0 engine-object alloc, so stage 4's BIND/GPFIFO_SCHEDULE/token + * sequence, the GR sem check and every forwarded publish run on the golden + * channel; V6 stays allocated but is never bound or scheduled, and its + * FWCLIENT promote is skipped. The hypothesis: the golden channel owns the + * complete committed context, so the engine will finally dispatch the + * forwarded QMD. + */ +#define GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC 28u +/* + * Mode-28 golden execution channel shape bits (gsp_phys_rmcall_golden_exec_shape, + * default 7). Every combination keeps the captured flags 0x00200320 and the + * FBMEM instance/RAMFC/mthd slices; the bits only choose the client-visible + * submission path, so a deadlock can be bisected without a rebuild. + */ +#define GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_RING BIT(0) +#define GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_ENTRIES8 BIT(1) +#define GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_USERD_SYSMEM BIT(2) +#define GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_USERD_SIZE200 BIT(3) +#define GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_HOBJ_VAS BIT(4) +#define GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_SKIP_TURING_A BIT(5) +/* * Existing fwclient mask bit 8: mode-25 gate for the captured VAS * COPY_SERVER_RESERVED_PDES control. The captured control copies three * server-RM-internal page-table levels the ladder cannot source safely, so @@ -643,8 +675,34 @@ bool gsp_phys_boot_requested(void) * for comparison. */ #define GSP_PHYS_RMCALL_GOLDEN9_MASK_SIZED_BUFFERS (1u << 10) +/* + * Mode-25 bisection bit 11: send each built non-MAIN FWCLIENT entry as its + * own one-entry GPU_PROMOTE_CTX control instead of one batched control, so a + * single refused entry cannot refuse the whole set (m2b/m2f: the batched + * five-entry control returned 0x1f as a whole; individual acceptance was + * never tested). + */ +#define GSP_PHYS_RMCALL_FWCLIENT_MASK_INDIVIDUAL (1u << 11) +/* + * Mode-25 bisection bit 12: additionally promote the golden9 VA-only global + * bookings (bufferIds 3/4/5/6 = BUNDLE_CB, PAGEPOOL, ATTRIBUTE_CB, + * RTV_CB_GLOBAL) on the ladder channel as one-entry controls. The golden9 + * context carries them; the FWCLIENT set does not, so the V6 context may be + * missing the GR engine's code/attribute data entirely. + */ +#define GSP_PHYS_RMCALL_FWCLIENT_MASK_GOLDEN9_GLOBALS (1u << 12) +/* + * Mode-25 bisection bits 13/14: the FECS_EVENT entry is refused 0x1f on the + * V6 channel in its combined PA+VA form (the same form the golden channel + * accepts). Try its halves separately: bit 13 = PA-only init entry + * (bInitialize=1 bNonmapped=1, like the accepted PM entry), bit 14 = VA-only + * booking entry (bInitialize=0, like the accepted globals). Both booked + * together may complete the FECS state the combined control cannot. + */ +#define GSP_PHYS_RMCALL_FWCLIENT_MASK_FECS_PA_ONLY (1u << 13) +#define GSP_PHYS_RMCALL_FWCLIENT_MASK_FECS_VA_ONLY (1u << 14) #define GSP_PHYS_RMCALL_GR_PROMOTE_MAX \ - (GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6_ONLY) + (GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC) /* * Work-submit token notifier slot. The default index is @@ -946,6 +1004,15 @@ bool gsp_phys_boot_requested(void) (1u << 21)) /* + * Mode-28 alternative golden flags (unused after the m3b first run kept the + * captured word): the captured golden word with the USERD placement bits + * clear, for experiments where the hardware control block must sit at + * userd+0 (publish offset 0x8c). + */ +#define GSP_PHYS_RMCALL_CHANNEL_FLAG_GOLDEN_EXEC \ + (u32)(GSP_PHYS_RMCALL_CHANNEL_FLAG_PRIVILEGED) + +/* * BANNED - do not use: internalFlags bit 6 = 0x40 marks the channel as * GSP-owned. A live run with this bit set wedged the GSP-RM task: the alloc * RPC never replied, every later RPC timed out, and the fail-safe held the @@ -1014,6 +1081,9 @@ enum gsp_phys_rmcall_buf { #define GSP_PHYS_RMCALL_GPFIFO_ENTRIES 8u #define GSP_PHYS_RMCALL_USERD_GPPUT_OFF 0x8cu /* Nvc46fControl GPPut (clc46f.h:57) */ #define GSP_PHYS_RMCALL_USERD_GPGET_OFF 0x88u /* Nvc46fControl GPGet (clc46f.h:56) */ +/* Captured golden flags pin USERD_INDEX_VALUE=3, so the control block (and + * the GP_PUT/GP_GET words inside it) sits 0x600 into the USERD chunk. */ +#define GSP_PHYS_RMCALL_USERD_CTL_OFF_CAPTURED 0x600u #define GSP_PHYS_RMCALL_SEM_PAYLOAD 0xc0ffee01u #define GSP_PHYS_RMCALL_TIMEOUT_MS 2000u #define GSP_PHYS_RMCALL_POLL_MS 5u @@ -7477,6 +7547,8 @@ static const char *gsp_phys_rmcall_gr_promote_name(void) return "mode 26: golden9 9-entry promote on the V6 execution channel (plus the golden-channel promote)"; case GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6_ONLY: return "mode 27: golden9 9-entry promote on the V6 execution channel only (no golden-channel promote)"; + case GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC: + return "mode 28: golden execution lane (golden channel with client GPFIFO/USERD, 9-entry promote on it, execution switched to it)"; default: return "off (PROMOTE_CTX never sent)"; } @@ -7851,6 +7923,14 @@ static void gsp_phys_rmcall_golden_3d_on_ladder(void __iomem *bar0) if (!gsp_phys_rmcall_golden_enabled() || !gsp_phys_gpu_enabled() || !boot_stage.rmcall_channel) return; + /* + * Mode 28: the golden execution channel already carries its TURING_A + * object from the replay; the ladder's extra 3D object (a mode-25 + * GR-instance by-product) must not attach a second one to it. + */ + if (boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC) + return; ret = gsp_phys_rmcall_alloc_gr_object_class(bar0, GSP_PHYS_RMCALL_CLASS_GR_3D, GSP_PHYS_RMCALL_HANDLE_GOLDEN_GR3D_LADDER, @@ -8068,6 +8148,7 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0) struct gsp_phys_rmcall_channel_params ch; struct gsp_phys_rmcall_mem_obj_shape shape; u32 rpc = 0, status = 0; + u32 ch_parent = boot_stage.rmcall_device; u64 golden_fb = 0; bool pb_ok = true; int ret; @@ -8107,7 +8188,9 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0) boot_stage.rmcall_gr_promote == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6 || boot_stage.rmcall_gr_promote == - GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6_ONLY) && + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6_ONLY || + boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC) && gsp_phys_rmcall_engine_is_gr()) { if (gsp_phys_rmcall_fwclient_mask_value() & GSP_PHYS_RMCALL_GOLDEN9_MASK_LADDER_VAS) { @@ -8211,7 +8294,9 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0) boot_stage.rmcall_gr_promote == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6 || boot_stage.rmcall_gr_promote == - GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6_ONLY) && + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6_ONLY || + boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC) && gsp_phys_rmcall_engine_is_gr()) { ch.flags = cpu_to_le32(GSP_PHYS_RMCALL_CHANNEL_FLAG_GOLDEN9); ch.internal_flags = cpu_to_le32( @@ -8260,12 +8345,69 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0) golden_fb + GSP_PHYS_RMCALL_GOLDEN_FB_RAMFC, 0x1000); ch.mthdbuf_mem = gsp_phys_rmcall_mem_desc_fb( golden_fb + GSP_PHYS_RMCALL_GOLDEN_FB_MTHD, 0x1000); + /* + * Mode 28: the golden execution lane. Keep the captured shape + * bit-for-bit (same captured flags 0x00200320, same FBMEM + * instance/RAMFC/mthd slices) and only make the submission + * path client-visible: the ladder GPFIFO ring at the proven VA + * and the ladder sysmem USERD page (host GP_PUT writes land in + * the hardware word; the captured USERD index 3 puts the + * control block at userd+0x600, which + * gsp_phys_rmcall_userd_ctl_off() applies). The channel stays + * DEVICE-parented like the captured golden construct (a + * golden9 promote on a TSG channel poisoned the next GR object + * construct in modes 27/28 first run). Stage 3 swaps + * boot_stage.rmcall_channel to this handle before the compute + * object alloc. + */ + if (boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC) { + u32 exec_shape = + gsp_phys_rmcall_golden_exec_shape_value(); + + if (exec_shape & + GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_RING) + ch.gp_fifo_offset = + cpu_to_le64(GSP_PHYS_RMCALL_VA_GPFIFO); + if (exec_shape & + GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_ENTRIES8) + ch.gp_fifo_entries = cpu_to_le32( + GSP_PHYS_RMCALL_GPFIFO_ENTRIES); + if (exec_shape & + GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_USERD_SYSMEM) { + struct gsp_phys_rmcall_mem_desc ud = + gsp_phys_rmcall_mem_desc( + GSP_PHYS_RMCALL_BUF_USERD); + + if (exec_shape & + GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_USERD_SIZE200) + ud.size = cpu_to_le64(0x200); + ch.userd_mem = ud; + } + if (exec_shape & + GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_HOBJ_VAS) + ch.h_object_buffer = + cpu_to_le32(boot_stage.rmcall_vas); + } pr_info("vgsp-boot: rm-call: GOLDEN9 channel shape: flags=0x%x (PRIVILEGED|USERD_INDEX_VALUE=3|USERD_INDEX_PAGE_FIXED, no SKIP_SCRUBBER/SKIP_CTXBUFFER_ALLOC) hObjectBuffer=0 gpFifoEntries=0x%x instance/userd/ramfc/mthd=FBMEM memdescs block=0x%llx (capture-matched apertures)\n", GSP_PHYS_RMCALL_CHANNEL_FLAG_GOLDEN9, GSP_PHYS_RMCALL_GOLDEN_GPFIFO_ENTRIES, golden_fb); + if (boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC) + pr_info("vgsp-boot: rm-call: GOLDEN EXEC channel shape (mode 28): flags=0x%x (captured) shape=0x%x gpFifoOffset=0x%llx gpFifoEntries=%u hObjectBuffer=0x%x userd=%s size=0x%llx hParent=device hVASpace=0x%x; instance/ramfc/mthd stay FBMEM\n", + GSP_PHYS_RMCALL_CHANNEL_FLAG_GOLDEN9, + gsp_phys_rmcall_golden_exec_shape_value(), + le64_to_cpu(ch.gp_fifo_offset), + le32_to_cpu(ch.gp_fifo_entries), + le32_to_cpu(ch.h_object_buffer), + (ch.userd_mem.address_space == + cpu_to_le32(GSP_PHYS_RMCALL_ADDR_SYSMEM)) ? + "sysmem" : "FBMEM", + le64_to_cpu(ch.userd_mem.size), + boot_stage.rmcall_vas); } ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client, - boot_stage.rmcall_device, + ch_parent, GSP_PHYS_RMCALL_HANDLE_GOLDEN_CHANNEL, GSP_PHYS_RMCALL_CLASS_CHANNEL, &ch, sizeof(ch), false, NULL, @@ -8295,7 +8437,9 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0) if ((boot_stage.rmcall_gr_promote == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 || boot_stage.rmcall_gr_promote == - GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6) && + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6 || + boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC) && gsp_phys_rmcall_engine_is_gr()) { ret = gsp_phys_rmcall_golden9_promote(bar0); if (ret) @@ -8303,6 +8447,13 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0) } /* 6. TURING_A (0xc597) 3D object on the golden channel (kernel_graphics.c:2409-2433). */ + if (boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC && + (gsp_phys_rmcall_golden_exec_shape_value() & + GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_SKIP_TURING_A)) { + pr_info("vgsp-boot: rm-call: mode 28: GOLDEN TURING_A 3D object skipped (shape bit5); the ladder's TURING_COMPUTE_A on the same channel provides the GR instance state\n"); + return 0; + } ret = gsp_phys_rmcall_alloc_gr_object_class(bar0, GSP_PHYS_RMCALL_CLASS_GR_3D, GSP_PHYS_RMCALL_HANDLE_GOLDEN_GR_OBJ, @@ -9769,13 +9920,25 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0) * GR-instance state to consume. Abort instead of sending it blind. */ if ((mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 || - mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6_ONLY) && + mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6_ONLY || + mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC) && (!gsp_phys_rmcall_golden_enabled() || !gsp_phys_gpu_enabled())) { pr_err("vgsp-boot: rm-call: mode %u requires gsp_phys_rmcall_golden=1 and gsp_phys_gpu=1 (the golden replay did not run); no PROMOTE_CTX is sent and the ladder aborts with no retry\n", mode); return -EINVAL; } /* + * Mode 28: the 9-entry promote already ran on the golden execution + * channel inside the golden replay (it owns the committed context and + * the forwarded launches); nothing is sent on the V6 channel at all. + */ + if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC) { + pr_info("vgsp-boot: rm-call: mode-28 golden execution lane: no PROMOTE_CTX on V6 (the golden channel 0x%x carries the committed context); continuing to stage 4 on the golden channel\n", + GSP_PHYS_RMCALL_HANDLE_GOLDEN_CHANNEL); + boot_stage.rmcall_gr_promote_step = 1; + return 0; + } + /* * Modes 8/9/10 must run on the V6 channel: without the * SKIP_CTXBUFFER_ALLOC bit the promote takes the VA/bookkeeping * route that asserted GSP-RM in modes 1/2/4 (or rejects the @@ -10734,7 +10897,64 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0) pr_warn("vgsp-boot: rm-call: PROMOTE_CTX fwclient: UNRESTRICTED_PRIV_ACCESS_MAP omitted (allocation unavailable)\n"); } - if (n > 0) { + if (n > 0 && (mask & GSP_PHYS_RMCALL_FWCLIENT_MASK_INDIVIDUAL)) { + u32 k; + u32 n_single_ok = 0; + + /* + * Bisection: the batched non-MAIN control was refused + * 0x1f as a whole (m2b/m2f); send each built entry as + * its own one-entry control so one refused entry cannot + * refuse the rest, and the accepted subset is visible + * entry by entry. + */ + for (k = 0; k < n; k++) { + struct gsp_phys_rmcall_promote_entry e = + p->entry[k]; + + /* + * Overwrite only slot 0: the entries k+1..n-1 + * must survive for their own iterations (a full + * memset here destroyed the not-yet-sent ones). + */ + p->entry[0] = e; + p->entry_count = cpu_to_le32(1); + rpc_result = 0; + status = 0; + ret = gsp_phys_rmcall_control(bar0, + boot_stage.rmcall_subdevice, + GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX, + p, sizeof(*p), 24, NULL, 0, + &rpc_result, &status, + "GPU_PROMOTE_CTX (FWCLIENT non-MAIN single)"); + pr_info("vgsp-boot: rm-call: PROMOTE_CTX fwclient non-MAIN single[%u/%u]: bufferId=%u status=0x%x rpc_result=0x%x (hObject=0x%x)\n", + k + 1, n, le16_to_cpu(e.buffer_id), + status, rpc_result, + boot_stage.rmcall_channel); + if (ret) { + /* + * Bisection: keep going so the whole + * acceptance matrix is visible in one + * run; the ladder aborts only if no + * entry was accepted at all. + */ + pr_warn("vgsp-boot: rm-call: PROMOTE_CTX fwclient non-MAIN single[%u] bufferId=%u refused (status=0x%x rpc_result=0x%x, %s, ret=%d); continuing the bisection\n", + k, le16_to_cpu(e.buffer_id), + status, rpc_result, + gsp_phys_rmcall_status_str(status), + ret); + continue; + } + n_single_ok++; + } + if (!n_single_ok) { + pr_err("vgsp-boot: rm-call: PROMOTE_CTX fwclient non-MAIN single: no entry accepted; the GR context commit is incomplete and the ladder aborts before BIND with no retry\n"); + kvfree(p); + return -EIO; + } + boot_stage.rmcall_gr_promote_step = + (mask & BIT(0)) ? 2 : 1; + } else if (n > 0) { p->entry_count = cpu_to_le32(n); rpc_result = 0; status = 0; @@ -10762,6 +10982,103 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0) } else if (!(mask & BIT(0))) { pr_warn("vgsp-boot: rm-call: PROMOTE_CTX fwclient: MAIN omitted by mask\n"); } + + /* + * Bit 12: the golden9 VA-only global bookings, one control per + * entry. Accepted entries add the GR engine's bundle/pagepool/ + * attribute/RTV reservations to the ladder channel's context; + * a refused one is logged and skipped (bisection style). + */ + if (mask & GSP_PHYS_RMCALL_FWCLIENT_MASK_GOLDEN9_GLOBALS) { + static const struct { + u16 id; + u64 va; + const char *name; + } gl[4] = { + { 3, GSP_PHYS_RMCALL_VA_GOLDEN9_BUNDLE_CB, + "BUNDLE_CB" }, + { 4, GSP_PHYS_RMCALL_VA_GOLDEN9_PAGEPOOL, + "PAGEPOOL" }, + { 5, GSP_PHYS_RMCALL_VA_GOLDEN9_ATTRIBUTE_CB, + "ATTRIBUTE_CB" }, + { 6, GSP_PHYS_RMCALL_VA_GOLDEN9_RTV_CB_GLOBAL, + "RTV_CB_GLOBAL" }, + }; + u32 g; + + for (g = 0; g < 4; g++) { + gsp_phys_rmcall_promote_entry_fill(p, 0, + gl[g].id, 0, gl[g].va, 0, 0, + 0, 0); + p->entry_count = cpu_to_le32(1); + rpc_result = 0; + status = 0; + ret = gsp_phys_rmcall_control(bar0, + boot_stage.rmcall_subdevice, + GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX, + p, sizeof(*p), 24, NULL, 0, + &rpc_result, &status, + "GPU_PROMOTE_CTX (golden9 global)"); + pr_info("vgsp-boot: rm-call: PROMOTE_CTX golden9-global[%u/4] %s bufferId=%u va=0x%llx status=0x%x rpc_result=0x%x (hObject=0x%x)\n", + g + 1, gl[g].name, gl[g].id, gl[g].va, + status, rpc_result, + boot_stage.rmcall_channel); + if (ret) + pr_warn("vgsp-boot: rm-call: PROMOTE_CTX golden9-global[%u] %s refused (status=0x%x rpc_result=0x%x, %s, ret=%d); continuing the bisection\n", + g, gl[g].name, status, + rpc_result, + gsp_phys_rmcall_status_str(status), + ret); + } + } + + /* + * Bits 13/14: the FECS_EVENT halves. The combined form is + * refused 0x1f on V6; a PA-only init entry and a VA-only + * booking entry are each individually acceptable shapes on + * this channel, and together they may carry the FECS state. + */ + if (mask & GSP_PHYS_RMCALL_FWCLIENT_MASK_FECS_PA_ONLY) { + if (boot_stage.rmcall_fecs_pa && + boot_stage.rmcall_fecs_size) { + gsp_phys_rmcall_promote_entry_fill(p, 0, 9, + boot_stage.rmcall_fecs_pa, 0, + boot_stage.rmcall_fecs_size, fattr, + 1, 1); + p->entry_count = cpu_to_le32(1); + rpc_result = 0; + status = 0; + ret = gsp_phys_rmcall_control(bar0, + boot_stage.rmcall_subdevice, + GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX, + p, sizeof(*p), 24, NULL, 0, + &rpc_result, &status, + "GPU_PROMOTE_CTX (FECS PA-only)"); + pr_info("vgsp-boot: rm-call: PROMOTE_CTX FECS PA-only: bufferId=9 pa=0x%llx size=0x%x physAttr=0x%x status=0x%x rpc_result=0x%x (hObject=0x%x)\n", + boot_stage.rmcall_fecs_pa, + boot_stage.rmcall_fecs_size, fattr, + status, rpc_result, + boot_stage.rmcall_channel); + } else { + pr_warn("vgsp-boot: rm-call: PROMOTE_CTX FECS PA-only skipped: no FECS buffer prepared\n"); + } + } + if (mask & GSP_PHYS_RMCALL_FWCLIENT_MASK_FECS_VA_ONLY) { + gsp_phys_rmcall_promote_entry_fill(p, 0, 9, 0, + GSP_PHYS_RMCALL_VA_GOLDEN9_FECS, 0, 0, 0, 0); + p->entry_count = cpu_to_le32(1); + rpc_result = 0; + status = 0; + ret = gsp_phys_rmcall_control(bar0, + boot_stage.rmcall_subdevice, + GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX, + p, sizeof(*p), 24, NULL, 0, + &rpc_result, &status, + "GPU_PROMOTE_CTX (FECS VA-only)"); + pr_info("vgsp-boot: rm-call: PROMOTE_CTX FECS VA-only: bufferId=9 va=0x%llx status=0x%x rpc_result=0x%x (hObject=0x%x)\n", + GSP_PHYS_RMCALL_VA_GOLDEN9_FECS, status, + rpc_result, boot_stage.rmcall_channel); + } } else { pr_err("vgsp-boot: rm-call: unknown gr_promote_param=%u reached the promote step; no PROMOTE_CTX is sent and the ladder aborts with no retry\n", mode); @@ -10883,7 +11200,8 @@ static int gsp_phys_rmcall_srv_ctx_commit(void __iomem *bar0) u64 main_pa, main_size; int ret; - if (boot_stage.rmcall_srv_main_valid) { + if (boot_stage.rmcall_srv_main_valid && + !gsp_phys_srv_main_ignore_value()) { main_pa = boot_stage.rmcall_srv_main_pa; main_size = boot_stage.rmcall_srv_main_size; /* @@ -11367,11 +11685,13 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0) if (gsp_phys_rmcall_golden_enabled() && gsp_phys_gpu_enabled()) { ret = gsp_phys_rmcall_golden_replay(bar0); if (ret) { - if (boot_stage.rmcall_gr_promote == - GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 && + if ((boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 || + boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC) && gsp_phys_rmcall_engine_is_gr()) { - pr_err("vgsp-boot: rm-call: stage 3 aborted: mode-25 golden replay failed (%d); the golden construct is atomic and is not retried\n", - ret); + pr_err("vgsp-boot: rm-call: stage 3 aborted: mode-%u golden replay failed (%d); the golden construct is atomic and is not retried\n", + boot_stage.rmcall_gr_promote, ret); return ret; } pr_warn("vgsp-boot: rm-call: golden-image replay failed (%d); continuing the ladder (the golden replay is best-effort and the on-channel TURING_A init still runs); no retry\n", @@ -11413,6 +11733,29 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0) return ret; /* + * Mode 28: the golden execution lane. The golden replay allocated the + * golden channel with a client GPFIFO/USERD and the accepted 9-entry + * context commit; switch the ladder's execution channel to it BEFORE + * the compute-object alloc so the 0xc5c0 object (the forwarded + * launch's SET_OBJECT target), stage 4's BIND/SCHEDULE/token sequence, + * the GR sem check and every forwarded publish all run on the golden + * channel. The V6 channel stays allocated but is never bound or + * scheduled, and its FWCLIENT promote is suppressed (mode 28 returns + * early in gsp_phys_rmcall_gr_ctx_promote). + */ + if (gsp_phys_rmcall_engine_is_gr() && + boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC) { + pr_info("vgsp-boot: rm-call: mode 28: execution channel switched from V6 0x%x to golden execution channel 0x%x (client ring gpFifoVa=0x%llx userd=sysmem, device-parented); the V6 channel stays unscheduled\n", + boot_stage.rmcall_channel, + GSP_PHYS_RMCALL_HANDLE_GOLDEN_CHANNEL, + GSP_PHYS_RMCALL_VA_GPFIFO); + boot_stage.rmcall_channel = + GSP_PHYS_RMCALL_HANDLE_GOLDEN_CHANNEL; + boot_stage.rmcall_channel_server_mem = false; + } + + /* * H1: bind the compute engine object to the just-allocated GR channel * and commit the engine context before stage 3 returns. The TURING_ * COMPUTE_A alloc rejection aborts the ladder (the GR method block's @@ -11636,6 +11979,24 @@ struct gsp_phys_rmcall_ce_submit { }; /* + * The hardware USERD control block (Nvc46fControl GP_PUT/GP_GET) sits at + * userd base + USERD_INDEX_VALUE * 0x200. The exec-shape channels (V6 and + * every ladder variant) use index 0, so their GP_PUT/GP_GET live at offset + * 0x8c/0x88 of the chunk. Mode 28 keeps the captured golden channel flags + * (USERD_INDEX_VALUE=3 within a fixed USERD page), so the hardware control + * block sits at userd+0x600 and every host USERD touch must use the same + * offset or host and hardware would poll different words. + */ +static u32 gsp_phys_rmcall_userd_ctl_off(void) +{ + if (boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC && + gsp_phys_rmcall_engine_is_gr()) + return GSP_PHYS_RMCALL_USERD_CTL_OFF_CAPTURED; + return 0; +} + +/* * Shared publish tail of the ladder submissions: read USERD GP_PUT/GP_GET, * encode and write the GPFIFO entry for the module pushbuffer VA, advance * GP_PUT, snapshot the C3 fault epoch and ring the PF doorbell. Extracted @@ -11669,8 +12030,8 @@ static int gsp_phys_rmcall_publish_locked(void __iomem *bar0, u64 pb_gpu_va, if (!ring_va || !userd) return -EIO; - gpput = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF); - gpget = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPGET_OFF); + gpput = get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPPUT_OFF); + gpget = get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPGET_OFF); if (gpput >= GSP_PHYS_RMCALL_GPFIFO_ENTRIES) { pr_err("vgsp-boot: rm-call: USERD GP_PUT=%u is outside the %u-entry ring; refusing to submit\n", gpput, GSP_PHYS_RMCALL_GPFIFO_ENTRIES); @@ -11705,7 +12066,7 @@ static int gsp_phys_rmcall_publish_locked(void __iomem *bar0, u64 pb_gpu_va, * the host write is therefore what the engine polls (cycle 21). */ put_unaligned_le32((gpput + 1) % GSP_PHYS_RMCALL_GPFIFO_ENTRIES, - (u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF); + (u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPPUT_OFF); /* * Doorbell value: workSubmitToken = ((runlistId << 16) & 0x7f00000) | * (chId & 0xfff) (kernel_fifo_tu102.c:118-121). The server-generated @@ -11879,9 +12240,9 @@ static int gsp_phys_rmcall_gr_sem_push_locked(void __iomem *bar0) if (poll == GSP_PHYS_RMCALL_SEM_PAYLOAD) break; if (time_after_eq(jiffies, deadline)) { - gpget = get_unaligned_le32((u8 *)userd + + gpget = get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPGET_OFF); - gpput = get_unaligned_le32((u8 *)userd + + gpput = get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPPUT_OFF); pr_err("vgsp-boot: rm-call: REAL GR SEM FAILED: no release within %u ms (sem=0x%08x want=0x%08x GPGet=%u GPPut=%u engine=%s); the channel did not execute the GR block\n", GSP_PHYS_RMCALL_TIMEOUT_MS, poll, @@ -11893,10 +12254,10 @@ static int gsp_phys_rmcall_gr_sem_push_locked(void __iomem *bar0) msleep(GSP_PHYS_RMCALL_POLL_MS); } - gpget = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPGET_OFF); + gpget = get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPGET_OFF); pr_info("vgsp-boot: rm-call: REAL GR SEM OK sem=0x%08x want=0x%08x GPGet=%u GPPut=%u words=%u token=0x%08x\n", poll, GSP_PHYS_RMCALL_SEM_PAYLOAD, gpget, - get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF), + get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPPUT_OFF), sub.words, boot_stage.rmcall_token); /* * Latch the GR execution gate: this is the only place a real GR @@ -14059,9 +14420,9 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0) if (poll == GSP_PHYS_RMCALL_SEM_PAYLOAD) break; if (time_after_eq(jiffies, deadline)) { - gpget = get_unaligned_le32((u8 *)userd + + gpget = get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPGET_OFF); - gpput = get_unaligned_le32((u8 *)userd + + gpput = get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPPUT_OFF); pr_err("vgsp-boot: rm-call: no semaphore release within %u ms (sem=0x%08x want=0x%08x GPGet=%u GPPut=%u); the channel did not execute the CE block\n", GSP_PHYS_RMCALL_TIMEOUT_MS, poll, @@ -14087,10 +14448,10 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0) return -EIO; } - gpget = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPGET_OFF); + gpget = get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPGET_OFF); pr_info("vgsp-boot: rm-call: REAL COPY OK sem=0x%08x src=0x%llx dst=0x%llx bytes=%u GPGet=%u GPPut=%u words=%u token=0x%08x\n", poll, sub.src_dma, sub.dst_dma, GSP_PHYS_RMCALL_COPY_SIZE, gpget, - get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF), + get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPPUT_OFF), sub.words, boot_stage.rmcall_token); return 0; } diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c index fa008a6..61ec22e 100644 --- a/kernel/gsp_phys_gpu.c +++ b/kernel/gsp_phys_gpu.c @@ -202,6 +202,25 @@ bool gsp_phys_srv_ctxcommit_value(void) } EXPORT_SYMBOL_GPL(gsp_phys_srv_ctxcommit_value); +/* + * Force the srv-ctx-commit CLIENT fallback (default 0): when set, a + * server-reported MAIN context buffer is ignored and the client MAIN + * INITIALIZE_CTX + VA-only promote path runs even after an initialized aux + * entry made the server materialize a MAIN (the server-MAIN PA-only promote + * then returns 0x1f and aborts the ladder). + */ +static bool gsp_phys_srv_main_ignore_param; +module_param_named(gsp_phys_rmcall_srv_main_ignore, + gsp_phys_srv_main_ignore_param, bool, 0644); +MODULE_PARM_DESC(gsp_phys_rmcall_srv_main_ignore, + "1 = ignore the server-reported MAIN ctx buffer in the srv-ctx commit and force the client INITIALIZE_CTX+VA promote fallback"); + +bool gsp_phys_srv_main_ignore_value(void) +{ + return gsp_phys_srv_main_ignore_param; +} +EXPORT_SYMBOL_GPL(gsp_phys_srv_main_ignore_value); + int gsp_phys_srv_ctxcommit_mode(void) { return gsp_phys_srv_ctxcommit_param; @@ -247,6 +266,27 @@ u32 gsp_phys_rmcall_fwclient_mask_value(void) EXPORT_SYMBOL_GPL(gsp_phys_rmcall_fwclient_mask_value); /* + * Mode-28 golden execution channel shape bisection (default 7 = the full + * client-visible target). Bits: 0 GPFIFO ring (gpFifoOffset=VA_GPFIFO), 1 + * gpFifoEntries=8 (else the captured 32), 2 sysmem USERD chunk memdesc + * (else the captured FBMEM slice). Every shape keeps the captured channel + * flags and the FBMEM instance/RAMFC/mthd slices; the bitmask exists because + * the golden9-promote + GR-object-construct sequence deadlocks with the full + * shape attached, so the poison must be bisected without a rebuild per step. + */ +static u32 gsp_phys_rmcall_golden_exec_shape_param = 7u; +module_param_named(gsp_phys_rmcall_golden_exec_shape, + gsp_phys_rmcall_golden_exec_shape_param, uint, 0644); +MODULE_PARM_DESC(gsp_phys_rmcall_golden_exec_shape, + "Mode-28 golden execution channel shape bitmask (default 7): bit0 GPFIFO ring at VA_GPFIFO, bit1 gpFifoEntries=8 (else 32), bit2 sysmem USERD chunk (else FBMEM slice)"); + +u32 gsp_phys_rmcall_golden_exec_shape_value(void) +{ + return gsp_phys_rmcall_golden_exec_shape_param; +} +EXPORT_SYMBOL_GPL(gsp_phys_rmcall_golden_exec_shape_value); + +/* * Golden-image replay (default 0). When set (and gsp_phys_gpu=1), the rm-call * ladder's stage 3 replays the stock client's post-load golden-image channel * allocations (kgraphicsCreateGoldenImageChannel: a separate @@ -511,8 +551,8 @@ static int gsp_phys_rmcall_gr_promote_set(const char *val, return ret; if (gsp_phys_rmcall_gr_promote_param < 0 || - gsp_phys_rmcall_gr_promote_param > 27) { - pr_warn("vgsp: gsp_phys_rmcall_gr_promote=%d is outside 0..27; clamping to 0 (no PROMOTE_CTX)\n", + gsp_phys_rmcall_gr_promote_param > 28) { + pr_warn("vgsp: gsp_phys_rmcall_gr_promote=%d is outside 0..28; clamping to 0 (no PROMOTE_CTX)\n", gsp_phys_rmcall_gr_promote_param); gsp_phys_rmcall_gr_promote_param = 0; } @@ -531,7 +571,7 @@ module_param_cb(gsp_phys_rmcall_gr_promote, &gsp_phys_rmcall_gr_promote_ops, &gsp_phys_rmcall_gr_promote_param, 0644); MODULE_PARM_DESC(gsp_phys_rmcall_gr_promote, - "GR context-promote bisection (default 0: never send PROMOTE_CTX; 1: single entry init+nonmapped=0 size=raw+subctx, the p6h1b shape; 2: two-phase PA-only nonmapped then VA-only with the same fields; 3: single entry size=raw; 4: two-phase with a zero-field VA-only step; 5: batched entryCount=2 PA-only init + zero-field VA-only; 6: no-entry hVirtMemory, NV01_MEMORY_VIRTUAL (class 0x70) on the VAS at the context window + PROMOTE_CTX entryCount=0; 7: no-entry bare virtAddress+size, entryCount=0; 8: single MAIN entry like 1 on the KMD-managed channel (V6 SKIP_CTXBUFFER_ALLOC), server creates the virtual context memdesc itself; 9: same V6 channel with the stock client's zero-field VA-only MAIN entry, bInitialize=0; 10: same V6 channel, GPU_INITIALIZE_CTX (0x2080012d) installs the physical context memdesc then PROMOTE_CTX entryCount=0 bare virtAddress+size; 11: mode 10 with a server FBMEM context buffer, VIDMEM physAttr 0x4; 12: V5 UVM/vGPU-faithful PA-only MAIN entry (bInitialize=1 bNonmapped=1, no VA) and continue to BIND/SCHEDULE/submit; 13: mode 12 plus a PATCH PA-only promote first, sized from GET_ENGINE_CONTEXT_PROPERTIES engineId 0x10; 14: KMD channel (V6) + PATCH PA-only then the single-entry MAIN init+VA (the flow that books the context VA and commits ENG_CTX_VALID); 15: mode 14 with a server FBMEM MAIN context buffer and VIDMEM physAttr 0x4; 16: mode 14 plus a PM ctxsw PA-only promote first; 17: no client PROMOTE_CTX, paired with gsp_phys_srv_ctxbuf=1 so the GSP-RM server-manages and commits the context; 18: mode 8 shape plus a client FERMI_CONTEXT_SHARE_A on the TSG (hContextShare) whose construct allocates the GraphicsSubcontext; 19: the stock guest's exact 9-entry promote set (MAIN 2 MB PA+VA, PATCH, global buffers, FECS event, private access maps) on V5; 20: mode 6's no-entry hVirtMemory promote on the KMD-managed V6 channel (SKIP_CTXBUFFER_ALLOC), untested - the combination the V5 refusal (0x1f) points at; 21: mode 7's no-entry bare (virtAddress, size) promote on the V6 KMD-managed channel; 22: GPU_INITIALIZE_CTX then mode 20's no-entry hVirtMemory promote; 23: the hVirtMemory no-entry promote backed by the server-owned FBMEM context object (GSP_RM_ALLOC class 0x40) on the server-managed shared V5 channel - a physical memdesc reaches the per-buffer commit without the ADDR_VIRTUAL vaList gate; 24: the reference FWCLIENT non-3D set {MAIN, PM, PATCH, FECS_EVENT, PRIV_ACCESS_MAP, UNRESTRICTED_PRIV_ACCESS_MAP} as one control on the V6 KMD-managed channel, combined PA+VA entries where the ladder has a VA (MAIN PDB-mapped; PATCH/FECS/UNRESTRICTED booking-only), PM and PRIV_ACCESS_MAP init-only (no VA source), unsourceable buffers omitted and logged); 25: the real golden-client construct on the golden replay handles (captured channel flags 0x00200320 + single atomic 9-entry PROMOTE_CTX on the golden channel), then the V6 FWCLIENT follow-on promote on the golden-initialized GR-instance state; requires gsp_phys_rmcall_golden=1 and gsp_phys_rmcall_engine=1; values outside 0..25 clamp to 0 with a warning"); + "GR context-promote bisection (default 0: never send PROMOTE_CTX; 1: single entry init+nonmapped=0 size=raw+subctx, the p6h1b shape; 2: two-phase PA-only nonmapped then VA-only with the same fields; 3: single entry size=raw; 4: two-phase with a zero-field VA-only step; 5: batched entryCount=2 PA-only init + zero-field VA-only; 6: no-entry hVirtMemory, NV01_MEMORY_VIRTUAL (class 0x70) on the VAS at the context window + PROMOTE_CTX entryCount=0; 7: no-entry bare virtAddress+size, entryCount=0; 8: single MAIN entry like 1 on the KMD-managed channel (V6 SKIP_CTXBUFFER_ALLOC), server creates the virtual context memdesc itself; 9: same V6 channel with the stock client's zero-field VA-only MAIN entry, bInitialize=0; 10: same V6 channel, GPU_INITIALIZE_CTX (0x2080012d) installs the physical context memdesc then PROMOTE_CTX entryCount=0 bare virtAddress+size; 11: mode 10 with a server FBMEM context buffer, VIDMEM physAttr 0x4; 12: V5 UVM/vGPU-faithful PA-only MAIN entry (bInitialize=1 bNonmapped=1, no VA) and continue to BIND/SCHEDULE/submit; 13: mode 12 plus a PATCH PA-only promote first, sized from GET_ENGINE_CONTEXT_PROPERTIES engineId 0x10; 14: KMD channel (V6) + PATCH PA-only then the single-entry MAIN init+VA (the flow that books the context VA and commits ENG_CTX_VALID); 15: mode 14 with a server FBMEM MAIN context buffer and VIDMEM physAttr 0x4; 16: mode 14 plus a PM ctxsw PA-only promote first; 17: no client PROMOTE_CTX, paired with gsp_phys_srv_ctxbuf=1 so the GSP-RM server-manages and commits the context; 18: mode 8 shape plus a client FERMI_CONTEXT_SHARE_A on the TSG (hContextShare) whose construct allocates the GraphicsSubcontext; 19: the stock guest's exact 9-entry promote set (MAIN 2 MB PA+VA, PATCH, global buffers, FECS event, private access maps) on V5; 20: mode 6's no-entry hVirtMemory promote on the KMD-managed V6 channel (SKIP_CTXBUFFER_ALLOC), untested - the combination the V5 refusal (0x1f) points at; 21: mode 7's no-entry bare (virtAddress, size) promote on the V6 KMD-managed channel; 22: GPU_INITIALIZE_CTX then mode 20's no-entry hVirtMemory promote; 23: the hVirtMemory no-entry promote backed by the server-owned FBMEM context object (GSP_RM_ALLOC class 0x40) on the server-managed shared V5 channel - a physical memdesc reaches the per-buffer commit without the ADDR_VIRTUAL vaList gate; 24: the reference FWCLIENT non-3D set {MAIN, PM, PATCH, FECS_EVENT, PRIV_ACCESS_MAP, UNRESTRICTED_PRIV_ACCESS_MAP} as one control on the V6 KMD-managed channel, combined PA+VA entries where the ladder has a VA (MAIN PDB-mapped; PATCH/FECS/UNRESTRICTED booking-only), PM and PRIV_ACCESS_MAP init-only (no VA source), unsourceable buffers omitted and logged); 25: the real golden-client construct on the golden replay handles (captured channel flags 0x00200320 + single atomic 9-entry PROMOTE_CTX on the golden channel), then the V6 FWCLIENT follow-on promote on the golden-initialized GR-instance state; requires gsp_phys_rmcall_golden=1 and gsp_phys_rmcall_engine=1; 26/27: golden9 promote on the V6 execution channel (dead ends); 28: the golden execution lane - the golden channel gets a client GPFIFO ring (VA_GPFIFO), a sysmem host-writable USERD and the ladder TSG as parent, keeps the 9-entry promote, and the ladder switches execution to it (V6 stays unscheduled); values outside 0..28 clamp to 0 with a warning"); int gsp_phys_rmcall_gr_promote_value(void) { diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h index a38067a..65e1e1f 100644 --- a/kernel/gsp_phys_gpu.h +++ b/kernel/gsp_phys_gpu.h @@ -89,6 +89,13 @@ bool gsp_phys_srv_ctxbuf_value(void); * controls. gsp_phys_srv_ctxcommit_value() is true for any nonzero mode. */ bool gsp_phys_srv_ctxcommit_value(void); + +/* + * gsp_phys_rmcall_srv_main_ignore (default 0): when set, the srv-ctx commit + * ignores a server-reported MAIN ctx buffer and forces the client + * INITIALIZE_CTX + VA-only promote fallback. + */ +bool gsp_phys_srv_main_ignore_value(void); int gsp_phys_srv_ctxcommit_mode(void); /* @@ -106,6 +113,14 @@ int gsp_phys_rmcall_channel_value(void); u32 gsp_phys_rmcall_fwclient_mask_value(void); /* + * gsp_phys_rmcall_golden_exec_shape (default 7): mode-28 golden execution + * channel shape bitmask. bit0 GPFIFO ring at VA_GPFIFO, bit1 + * gpFifoEntries=8 (else the captured 32), bit2 sysmem USERD chunk (else the + * captured FBMEM slice). + */ +u32 gsp_phys_rmcall_golden_exec_shape_value(void); + +/* * gsp_phys_rmcall_golden (default 0): replay the stock client's golden-image * channel allocations in stage 3 before the ladder's own channel when set and * gsp_phys_gpu=1. With 0, the ladder is byte-identical to before. diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index a5f58db..2c6836f 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -2316,6 +2316,8 @@ static int svc_sem_read(struct nv_vgpu_device *vgpu, u64 va, u32 *val) return svc_read_ap(vgpu, addr, svc_ap_of(ap), val, sizeof(*val)); } +static void svc_pending_add(u64 va, u32 payload, u32 op); + static int svc_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, u32 val) { u64 addr = 0, old_addr; @@ -2335,10 +2337,25 @@ static int svc_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, u32 val) * historical order below. */ if (is_pool) { - if (!svc_pool_pa_resolve(vgpu, h, va, &addr, &ap, &src)) + if (!svc_pool_pa_resolve(vgpu, h, va, &addr, &ap, &src)) { verified = true; - else + } else { + /* + * A pool semaphore must never be written through the + * learned/content-scanned map: a stale PA there is + * silently lost (observed PA==VA fallback writes) + * while the client polls the real slot forever. The + * exact slot is known from the decoded release, so + * queue the absolute target for the retry pass until + * the verified resolver (pin -> read-back cache -> + * walk -> targeted rm_mem with read-back) can + * translate it. svc_apply_pending() retries every + * service pass and drops the entry once it lands. + */ svc_pool_pa_miss_note(va); + svc_pending_add(va, val, 0); + return -ENOENT; + } } if (!verified && svc_resolve_va(vgpu, va, &addr, &ap)) { /* @@ -5579,7 +5596,27 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, if (st.have_sem && st.sem_payload && sem_va >= SEMA_VA_LO && sem_va < SEMA_VA_HI) { - if (svc_sem_read(vgpu, sem_va, &cur)) { + int rrc2; + + if (svc_pool_sem_va(sem_va)) { + u64 ppa = 0; + int pap = SVC_APERTURE_SYS; + + if (svc_pool_pa_resolve(vgpu, + ch->h_object, + sem_va, &ppa, + &pap, NULL)) + rrc2 = -ENOENT; + else + rrc2 = svc_read_ap(vgpu, ppa, + svc_ap_of(pap), + &cur, + sizeof(cur)); + } else { + rrc2 = svc_sem_read(vgpu, sem_va, + &cur); + } + if (rrc2) { svc.pushes_nomap++; /* * The semaphore page has no @@ -5826,7 +5863,27 @@ static void svc_apply_pending(struct nv_vgpu_device *vgpu) struct svc_pending_release *p = &svc_pending[i]; u32 cur = 0; - if (svc_sem_read(vgpu, p->va, &cur)) { + if (svc_pool_sem_va(p->va)) { + u64 ppa = 0; + int pap = SVC_APERTURE_SYS; + + /* + * Judge a pool semaphore only through the verified + * resolver: the learned map can answer with a stale + * page whose value makes this release look already + * delivered and drops it (see svc_sem_write). + */ + if (svc_pool_pa_resolve(vgpu, 0, p->va, &ppa, &pap, + NULL)) { + i++; /* not translatable yet: keep */ + continue; + } + if (svc_read_ap(vgpu, ppa, svc_ap_of(pap), &cur, + sizeof(cur))) { + i++; + continue; + } + } else if (svc_sem_read(vgpu, p->va, &cur)) { i++; /* translation still missing: keep waiting */ continue; } |