diff options
Diffstat (limited to 'kernel/gsp_phys_booter.c')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 130 |
1 files changed, 125 insertions, 5 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index e8df827..67ade76 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -255,6 +255,17 @@ bool gsp_phys_boot_requested(void) #define GSP_PHYS_RMCALL_CTRL_BIND_CHANNEL 0xa06f0104u #define GSP_PHYS_RMCALL_CTRL_TSG_GPFIFO_SCHEDULE 0xa06c0101u /* + * Mode 35 / E2 lane-enable tail (R2 capture #1059/#1061/#1063): the real + * CUDA client sends these after the TSG/channel schedules and before the + * first launch. Our lane never did; the FECS readback shows the V7 engine + * context never becomes the engine's valid current context + * (NV_PGRAPH_PRI_FECS_CURRENT_CTX VALID=0) while a live real CUDA channel + * runs with VALID=1, so the SKED schedule leg has no valid context. + */ +#define GSP_PHYS_RMCALL_CTRL_SET_TIMESLICE 0xa06c0103u +#define GSP_PHYS_RMCALL_CTRL_GR_SET_CTXSW_PREEMPTION_MODE 0x20801210u +#define GSP_PHYS_RMCALL_CTRL_DEVICE_SCHED 0x00802009u +/* * NVA06C_CTRL_CMD_INTERNAL_PROMOTE_FAULT_METHOD_BUFFERS (0xa06c010a, * ctrla06c.h:354-364): set the per-runqueue fault method buffer descriptors * on the channel group (TSG). Mode 35 sends it immediately after the ladder @@ -12764,8 +12775,16 @@ static int gsp_phys_rmcall_alloc_user_lane_channel(void __iomem *bar0) ch.sub_device_id = cpu_to_le32(0); ch.internal_flags = cpu_to_le32( GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_USER_LANE); - ch.instance_mem = gsp_phys_rmcall_mem_desc_fb(fb, 0x1000); - ch.ramfc_mem = gsp_phys_rmcall_mem_desc_fb(fb, 0x200); + /* + * Mode-35 diagnostic (m62): instance/ramfc on the host-visible sysmem + * chunks so the server's instance/RAMFC writes (WFI pointer, PBDMA + * TARGET bits) can be decoded from the driver; the captured lane uses + * FBMEM slices, which the host cannot read. One-variable diagnostic; + * revert to mem_desc_fb() to restore the capture-exact shape. + */ + ch.instance_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_INSTANCE); + ch.ramfc_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_RAMFC); + ch.ramfc_mem.size = cpu_to_le64(0x200); ch.userd_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_USERD); ch.h_userd_memory[0] = cpu_to_le32(GSP_PHYS_RMCALL_HANDLE_USER_LANE_USERD); @@ -12786,13 +12805,15 @@ static int gsp_phys_rmcall_alloc_user_lane_channel(void __iomem *bar0) false, NULL, "mode 35 V7 real-client user lane channel", true, &rpc, &status); - pr_info("vgsp-boot: rm-call: mode 35 V7 channel alloc: hObject=0x%x hParent=hTsg=0x%x flags=0x%x internalFlags=0x%x engineType=0 hContextShare=0x%x hVASpace=0 gpFifoOffset=0x%llx gpFifoEntries=%u hUserdMemory[0]=0 userdMem=sysmem instance=0x%llx/0x1000 FBMEM ramfc=0x%llx/0x200 FBMEM hObjectError=0x%x errorNotifier=sysmem status=0x%x rpc_result=0x%x ret=%d (deviations: entries=%u vs captured 1024; sysmem USERD vs captured hUserdMemory object)\n", + pr_info("vgsp-boot: rm-call: mode 35 V7 channel alloc: hObject=0x%x hParent=hTsg=0x%x flags=0x%x internalFlags=0x%x engineType=0 hContextShare=0x%x hVASpace=0 gpFifoOffset=0x%llx gpFifoEntries=%u hUserdMemory[0]=0 userdMem=sysmem instance=0x%llx/0x1000 sysmem ramfc=0x%llx/0x200 sysmem (m62 diagnostic readback) hObjectError=0x%x errorNotifier=sysmem status=0x%x rpc_result=0x%x ret=%d (deviations: entries=%u vs captured 1024; sysmem USERD vs captured hUserdMemory object)\n", GSP_PHYS_RMCALL_HANDLE_USER_LANE, boot_stage.rmcall_tsg, GSP_PHYS_RMCALL_CHANNEL_FLAG_USER_LANE, GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_USER_LANE, le32_to_cpu(ch.h_context_share), le64_to_cpu(ch.gp_fifo_offset), - le32_to_cpu(ch.gp_fifo_entries), fb, fb, + le32_to_cpu(ch.gp_fifo_entries), + le64_to_cpu(ch.instance_mem.base), + le64_to_cpu(ch.ramfc_mem.base), GSP_PHYS_RMCALL_HANDLE_USER_LANE_NOTIFIER, status, rpc, ret, GSP_PHYS_RMCALL_GPFIFO_ENTRIES); return ret; @@ -13029,6 +13050,89 @@ static int gsp_phys_rmcall_user_lane_va_promote(void __iomem *bar0) return ret; } +/* + * Mode 35 / E2: the real lane-enable tail (R2 #1059/#1061/#1063) sent after + * the TSG/channel schedules and before the first launch: + * 1. NV2080_CTRL_CMD_GR_SET_CTXSW_PREEMPTION_MODE on the subdevice with + * flags=CILP_SET, hChannelGroup=TSG, cilpMode=CILP (value 2) - the same + * call CUDA makes in channelPoolSetPreemptionModeRM (CU_COMPUTE_ + * PREEMPTION_MODE_CILP); + * 2. NVA06C_CTRL_CMD_SET_TIMESLICE (0xa06c0103) on the TSG with 0x800; + * 3. the NV01 device scheduling control 0x00802009 with enable=1. + * The FECS readback (m61) showed V7's engine context never becomes the + * engine's valid current context (FECS_CURRENT_CTX VALID=0), which leaves + * the SKED schedule leg with no valid context; this tail is the real + * client-visible step our lane never sent. Best-effort; each step logged. + */ +struct gsp_phys_rmcall_preemption_params { + __le32 flags; + __le32 h_channel_group; + __le32 gfxp_ctxsw_preempt_mode; + __le32 cilp_ctxsw_preempt_mode; + __le32 pad[4]; +}; + +struct gsp_phys_rmcall_timeslice_params { + __le32 timeslice; + __le32 pad; +}; + +struct gsp_phys_rmcall_device_sched_params { + __le32 enable; + __le32 pad; +}; + +#define GSP_PHYS_RMCALL_PREEMPT_FLAG_CILP_SET 0x1u +#define GSP_PHYS_RMCALL_PREEMPT_MODE_CILP 0x2u +#define GSP_PHYS_RMCALL_LANE_TIMESLICE 0x800u + +static int gsp_phys_rmcall_user_lane_lane_enable(void __iomem *bar0) +{ + struct gsp_phys_rmcall_preemption_params pp; + struct gsp_phys_rmcall_timeslice_params ts; + struct gsp_phys_rmcall_device_sched_params ds; + u32 rpc_result = 0, status = 0; + int ret; + + memset(&pp, 0, sizeof(pp)); + pp.flags = cpu_to_le32(GSP_PHYS_RMCALL_PREEMPT_FLAG_CILP_SET); + pp.h_channel_group = cpu_to_le32(boot_stage.rmcall_tsg); + pp.cilp_ctxsw_preempt_mode = + cpu_to_le32(GSP_PHYS_RMCALL_PREEMPT_MODE_CILP); + ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_subdevice, + GSP_PHYS_RMCALL_CTRL_GR_SET_CTXSW_PREEMPTION_MODE, + &pp, sizeof(pp), 16, NULL, 0, &rpc_result, &status, + "GR_SET_CTXSW_PREEMPTION_MODE (mode 35 lane enable, R2 #1059)"); + pr_info("vgsp-boot: rm-call: mode 35 lane enable #1059 GR_SET_CTXSW_PREEMPTION_MODE: flags=0x%x hChannelGroup=0x%x cilp=%u status=0x%x rpc_result=0x%x ret=%d\n", + le32_to_cpu(pp.flags), le32_to_cpu(pp.h_channel_group), + le32_to_cpu(pp.cilp_ctxsw_preempt_mode), status, rpc_result, ret); + if (ret) + return ret; + + memset(&ts, 0, sizeof(ts)); + ts.timeslice = cpu_to_le32(GSP_PHYS_RMCALL_LANE_TIMESLICE); + ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_tsg, + GSP_PHYS_RMCALL_CTRL_SET_TIMESLICE, + &ts, sizeof(ts), 16, NULL, 0, &rpc_result, &status, + "SET_TIMESLICE (mode 35 lane enable, R2 #1061)"); + pr_info("vgsp-boot: rm-call: mode 35 lane enable #1061 SET_TIMESLICE: timeslice=0x%x hTsg=0x%x status=0x%x rpc_result=0x%x ret=%d\n", + le32_to_cpu(ts.timeslice), boot_stage.rmcall_tsg, + status, rpc_result, ret); + if (ret) + return ret; + + memset(&ds, 0, sizeof(ds)); + ds.enable = cpu_to_le32(1); + ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_device, + GSP_PHYS_RMCALL_CTRL_DEVICE_SCHED, + &ds, sizeof(ds), 16, NULL, 0, &rpc_result, &status, + "NV01 device sched (mode 35 lane enable, R2 #1063)"); + pr_info("vgsp-boot: rm-call: mode 35 lane enable #1063 device sched: enable=%u hDevice=0x%x status=0x%x rpc_result=0x%x ret=%d\n", + le32_to_cpu(ds.enable), boot_stage.rmcall_device, + status, rpc_result, ret); + return ret; +} + static int gsp_phys_rmcall_stage3(void __iomem *bar0) { struct gsp_phys_rmcall_vas_params vas; @@ -13464,6 +13568,7 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0) if (ret) pr_err("vgsp-boot: rm-call: mode 35 VA-only booking promote (R1) failed (%d); continuing best-effort (the construct result shows whether the per-VAS ctx-buffer maps were the gate)\n", ret); + gsp_phys_rmcall_ctx_diag_decode("mode35 V7 post-promotes (m62 readback)"); ret = gsp_phys_rmcall_alloc_gr_object_class(bar0, GSP_PHYS_RMCALL_CLASS_COMPUTE, GSP_PHYS_RMCALL_HANDLE_USER_LANE_GR_OBJ, @@ -13477,7 +13582,7 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0) boot_stage.rmcall_gr_object = GSP_PHYS_RMCALL_HANDLE_USER_LANE_GR_OBJ; } - pr_info("vgsp-boot: rm-call: mode 35: execution channel switched from V6 0x%x to V7 0x%x (captured flags=0x%x internalFlags=0x%x hContextShare=0x%x hVASpace=0, gpFifoVa=0x%llx entries=%u, sysmem USERD, FBMEM instance/ramfc, TURING_COMPUTE_A 0x%x); V6 stays allocated and unscheduled, stage 4 + forwarding run on V7\n", + pr_info("vgsp-boot: rm-call: mode 35: execution channel switched from V6 0x%x to V7 0x%x (captured flags=0x%x internalFlags=0x%x hContextShare=0x%x hVASpace=0, gpFifoVa=0x%llx entries=%u, sysmem USERD, sysmem instance/ramfc (m62 readback), TURING_COMPUTE_A 0x%x); V6 stays allocated and unscheduled, stage 4 + forwarding run on V7\n", boot_stage.rmcall_channel, GSP_PHYS_RMCALL_HANDLE_USER_LANE, GSP_PHYS_RMCALL_CHANNEL_FLAG_USER_LANE, @@ -13489,6 +13594,7 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0) boot_stage.rmcall_channel = GSP_PHYS_RMCALL_HANDLE_USER_LANE; boot_stage.rmcall_channel_server_mem = false; + gsp_phys_rmcall_ctx_diag_decode("mode35 V7 post-construct (m62 readback)"); } else { ret = gsp_phys_rmcall_alloc_gr_object(bar0); if (ret) { @@ -16402,6 +16508,20 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0) boot_stage.rmcall_tsg, rpc_result, status); /* + * Mode 35 / E2: the real lane-enable tail (R2 #1059/#1061/#1063) after + * the schedules and before the forwarded launch. Best-effort: the + * launch result and the FECS readback show whether the lane enable + * was the gate. + */ + if (boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_USER_LANE) { + ret = gsp_phys_rmcall_user_lane_lane_enable(bar0); + if (ret) + pr_err("vgsp-boot: rm-call: mode 35 lane enable tail failed (%d); continuing best-effort (the launch result shows whether the lane enable was the gate)\n", + ret); + } + + /* * Shared-management GR diagnostic: the server owns the engine context; * dump the instance/RAMFC words here (post-BIND/SCHEDULE, pre-submit) * so the committed engine-context pointer/VA can be correlated with |