vgsp: mode 35 m62 readback + m63 E2 lane-enable tail

m62 diagnostic: V7 instance/ramfc on host-visible sysmem chunks so the
server's WFI-pointer/TARGET-bit writes are decodable (captured lane uses
unreadable FBMEM slices); decode calls post-promotes and post-construct.

m63 E2: send the real CUDA lane-enable tail after stage-4 schedules
(R2 #1059 NV2080_CTRL_CMD_GR_SET_CTXSW_PREEMPTION_MODE flags=CILP_SET
hChannelGroup=TSG cilp=CILP on the subdevice, #1061
NVA06C_CTRL_CMD_SET_TIMESLICE 0x800 on the TSG, #1063 NV01 device
0x00802009 enable=1). Ground truth: live real CUDA runs with
FECS_CURRENT_CTX VALID=1; our m61 snapshots show V7's context never valid,
and the SKED CWD table never leaves the cold value. Best-effort, mode-35
only.

1 files changed, 125 insertions, 5 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index e8df827..67ade76 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -255,6 +255,17 @@ bool gsp_phys_boot_requested(void)
 #define GSP_PHYS_RMCALL_CTRL_BIND_CHANNEL	0xa06f0104u
 #define GSP_PHYS_RMCALL_CTRL_TSG_GPFIFO_SCHEDULE	0xa06c0101u
 /*
+ * Mode 35 / E2 lane-enable tail (R2 capture #1059/#1061/#1063): the real
+ * CUDA client sends these after the TSG/channel schedules and before the
+ * first launch. Our lane never did; the FECS readback shows the V7 engine
+ * context never becomes the engine's valid current context
+ * (NV_PGRAPH_PRI_FECS_CURRENT_CTX VALID=0) while a live real CUDA channel
+ * runs with VALID=1, so the SKED schedule leg has no valid context.
+ */
+#define GSP_PHYS_RMCALL_CTRL_SET_TIMESLICE	0xa06c0103u
+#define GSP_PHYS_RMCALL_CTRL_GR_SET_CTXSW_PREEMPTION_MODE	0x20801210u
+#define GSP_PHYS_RMCALL_CTRL_DEVICE_SCHED	0x00802009u
+/*
  * NVA06C_CTRL_CMD_INTERNAL_PROMOTE_FAULT_METHOD_BUFFERS (0xa06c010a,
  * ctrla06c.h:354-364): set the per-runqueue fault method buffer descriptors
  * on the channel group (TSG). Mode 35 sends it immediately after the ladder
@@ -12764,8 +12775,16 @@ static int gsp_phys_rmcall_alloc_user_lane_channel(void __iomem *bar0)
 	ch.sub_device_id = cpu_to_le32(0);
 	ch.internal_flags = cpu_to_le32(
 		GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_USER_LANE);
-	ch.instance_mem = gsp_phys_rmcall_mem_desc_fb(fb, 0x1000);
-	ch.ramfc_mem = gsp_phys_rmcall_mem_desc_fb(fb, 0x200);
+	/*
+	 * Mode-35 diagnostic (m62): instance/ramfc on the host-visible sysmem
+	 * chunks so the server's instance/RAMFC writes (WFI pointer, PBDMA
+	 * TARGET bits) can be decoded from the driver; the captured lane uses
+	 * FBMEM slices, which the host cannot read. One-variable diagnostic;
+	 * revert to mem_desc_fb() to restore the capture-exact shape.
+	 */
+	ch.instance_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_INSTANCE);
+	ch.ramfc_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_RAMFC);
+	ch.ramfc_mem.size = cpu_to_le64(0x200);
 	ch.userd_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_USERD);
 	ch.h_userd_memory[0] =
 		cpu_to_le32(GSP_PHYS_RMCALL_HANDLE_USER_LANE_USERD);
@@ -12786,13 +12805,15 @@ static int gsp_phys_rmcall_alloc_user_lane_channel(void __iomem *bar0)
 			false, NULL,
 			"mode 35 V7 real-client user lane channel", true,
 			&rpc, &status);
-	pr_info("vgsp-boot: rm-call: mode 35 V7 channel alloc: hObject=0x%x hParent=hTsg=0x%x flags=0x%x internalFlags=0x%x engineType=0 hContextShare=0x%x hVASpace=0 gpFifoOffset=0x%llx gpFifoEntries=%u hUserdMemory[0]=0 userdMem=sysmem instance=0x%llx/0x1000 FBMEM ramfc=0x%llx/0x200 FBMEM hObjectError=0x%x errorNotifier=sysmem status=0x%x rpc_result=0x%x ret=%d (deviations: entries=%u vs captured 1024; sysmem USERD vs captured hUserdMemory object)\n",
+	pr_info("vgsp-boot: rm-call: mode 35 V7 channel alloc: hObject=0x%x hParent=hTsg=0x%x flags=0x%x internalFlags=0x%x engineType=0 hContextShare=0x%x hVASpace=0 gpFifoOffset=0x%llx gpFifoEntries=%u hUserdMemory[0]=0 userdMem=sysmem instance=0x%llx/0x1000 sysmem ramfc=0x%llx/0x200 sysmem (m62 diagnostic readback) hObjectError=0x%x errorNotifier=sysmem status=0x%x rpc_result=0x%x ret=%d (deviations: entries=%u vs captured 1024; sysmem USERD vs captured hUserdMemory object)\n",
 		GSP_PHYS_RMCALL_HANDLE_USER_LANE, boot_stage.rmcall_tsg,
 		GSP_PHYS_RMCALL_CHANNEL_FLAG_USER_LANE,
 		GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_USER_LANE,
 		le32_to_cpu(ch.h_context_share),
 		le64_to_cpu(ch.gp_fifo_offset),
-		le32_to_cpu(ch.gp_fifo_entries), fb, fb,
+		le32_to_cpu(ch.gp_fifo_entries),
+		le64_to_cpu(ch.instance_mem.base),
+		le64_to_cpu(ch.ramfc_mem.base),
 		GSP_PHYS_RMCALL_HANDLE_USER_LANE_NOTIFIER,
 		status, rpc, ret, GSP_PHYS_RMCALL_GPFIFO_ENTRIES);
 	return ret;
@@ -13029,6 +13050,89 @@ static int gsp_phys_rmcall_user_lane_va_promote(void __iomem *bar0)
 	return ret;
 }
 
+/*
+ * Mode 35 / E2: the real lane-enable tail (R2 #1059/#1061/#1063) sent after
+ * the TSG/channel schedules and before the first launch:
+ *   1. NV2080_CTRL_CMD_GR_SET_CTXSW_PREEMPTION_MODE on the subdevice with
+ *      flags=CILP_SET, hChannelGroup=TSG, cilpMode=CILP (value 2) - the same
+ *      call CUDA makes in channelPoolSetPreemptionModeRM (CU_COMPUTE_
+ *      PREEMPTION_MODE_CILP);
+ *   2. NVA06C_CTRL_CMD_SET_TIMESLICE (0xa06c0103) on the TSG with 0x800;
+ *   3. the NV01 device scheduling control 0x00802009 with enable=1.
+ * The FECS readback (m61) showed V7's engine context never becomes the
+ * engine's valid current context (FECS_CURRENT_CTX VALID=0), which leaves
+ * the SKED schedule leg with no valid context; this tail is the real
+ * client-visible step our lane never sent. Best-effort; each step logged.
+ */
+struct gsp_phys_rmcall_preemption_params {
+	__le32 flags;
+	__le32 h_channel_group;
+	__le32 gfxp_ctxsw_preempt_mode;
+	__le32 cilp_ctxsw_preempt_mode;
+	__le32 pad[4];
+};
+
+struct gsp_phys_rmcall_timeslice_params {
+	__le32 timeslice;
+	__le32 pad;
+};
+
+struct gsp_phys_rmcall_device_sched_params {
+	__le32 enable;
+	__le32 pad;
+};
+
+#define GSP_PHYS_RMCALL_PREEMPT_FLAG_CILP_SET	0x1u
+#define GSP_PHYS_RMCALL_PREEMPT_MODE_CILP	0x2u
+#define GSP_PHYS_RMCALL_LANE_TIMESLICE		0x800u
+
+static int gsp_phys_rmcall_user_lane_lane_enable(void __iomem *bar0)
+{
+	struct gsp_phys_rmcall_preemption_params pp;
+	struct gsp_phys_rmcall_timeslice_params ts;
+	struct gsp_phys_rmcall_device_sched_params ds;
+	u32 rpc_result = 0, status = 0;
+	int ret;
+
+	memset(&pp, 0, sizeof(pp));
+	pp.flags = cpu_to_le32(GSP_PHYS_RMCALL_PREEMPT_FLAG_CILP_SET);
+	pp.h_channel_group = cpu_to_le32(boot_stage.rmcall_tsg);
+	pp.cilp_ctxsw_preempt_mode =
+		cpu_to_le32(GSP_PHYS_RMCALL_PREEMPT_MODE_CILP);
+	ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_subdevice,
+			GSP_PHYS_RMCALL_CTRL_GR_SET_CTXSW_PREEMPTION_MODE,
+			&pp, sizeof(pp), 16, NULL, 0, &rpc_result, &status,
+			"GR_SET_CTXSW_PREEMPTION_MODE (mode 35 lane enable, R2 #1059)");
+	pr_info("vgsp-boot: rm-call: mode 35 lane enable #1059 GR_SET_CTXSW_PREEMPTION_MODE: flags=0x%x hChannelGroup=0x%x cilp=%u status=0x%x rpc_result=0x%x ret=%d\n",
+		le32_to_cpu(pp.flags), le32_to_cpu(pp.h_channel_group),
+		le32_to_cpu(pp.cilp_ctxsw_preempt_mode), status, rpc_result, ret);
+	if (ret)
+		return ret;
+
+	memset(&ts, 0, sizeof(ts));
+	ts.timeslice = cpu_to_le32(GSP_PHYS_RMCALL_LANE_TIMESLICE);
+	ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_tsg,
+			GSP_PHYS_RMCALL_CTRL_SET_TIMESLICE,
+			&ts, sizeof(ts), 16, NULL, 0, &rpc_result, &status,
+			"SET_TIMESLICE (mode 35 lane enable, R2 #1061)");
+	pr_info("vgsp-boot: rm-call: mode 35 lane enable #1061 SET_TIMESLICE: timeslice=0x%x hTsg=0x%x status=0x%x rpc_result=0x%x ret=%d\n",
+		le32_to_cpu(ts.timeslice), boot_stage.rmcall_tsg,
+		status, rpc_result, ret);
+	if (ret)
+		return ret;
+
+	memset(&ds, 0, sizeof(ds));
+	ds.enable = cpu_to_le32(1);
+	ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_device,
+			GSP_PHYS_RMCALL_CTRL_DEVICE_SCHED,
+			&ds, sizeof(ds), 16, NULL, 0, &rpc_result, &status,
+			"NV01 device sched (mode 35 lane enable, R2 #1063)");
+	pr_info("vgsp-boot: rm-call: mode 35 lane enable #1063 device sched: enable=%u hDevice=0x%x status=0x%x rpc_result=0x%x ret=%d\n",
+		le32_to_cpu(ds.enable), boot_stage.rmcall_device,
+		status, rpc_result, ret);
+	return ret;
+}
+
 static int gsp_phys_rmcall_stage3(void __iomem *bar0)
 {
 	struct gsp_phys_rmcall_vas_params vas;
@@ -13464,6 +13568,7 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0)
 			if (ret)
 				pr_err("vgsp-boot: rm-call: mode 35 VA-only booking promote (R1) failed (%d); continuing best-effort (the construct result shows whether the per-VAS ctx-buffer maps were the gate)\n",
 				       ret);
+			gsp_phys_rmcall_ctx_diag_decode("mode35 V7 post-promotes (m62 readback)");
 			ret = gsp_phys_rmcall_alloc_gr_object_class(bar0,
 					GSP_PHYS_RMCALL_CLASS_COMPUTE,
 					GSP_PHYS_RMCALL_HANDLE_USER_LANE_GR_OBJ,
@@ -13477,7 +13582,7 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0)
 				boot_stage.rmcall_gr_object =
 					GSP_PHYS_RMCALL_HANDLE_USER_LANE_GR_OBJ;
 			}
-			pr_info("vgsp-boot: rm-call: mode 35: execution channel switched from V6 0x%x to V7 0x%x (captured flags=0x%x internalFlags=0x%x hContextShare=0x%x hVASpace=0, gpFifoVa=0x%llx entries=%u, sysmem USERD, FBMEM instance/ramfc, TURING_COMPUTE_A 0x%x); V6 stays allocated and unscheduled, stage 4 + forwarding run on V7\n",
+			pr_info("vgsp-boot: rm-call: mode 35: execution channel switched from V6 0x%x to V7 0x%x (captured flags=0x%x internalFlags=0x%x hContextShare=0x%x hVASpace=0, gpFifoVa=0x%llx entries=%u, sysmem USERD, sysmem instance/ramfc (m62 readback), TURING_COMPUTE_A 0x%x); V6 stays allocated and unscheduled, stage 4 + forwarding run on V7\n",
 				boot_stage.rmcall_channel,
 				GSP_PHYS_RMCALL_HANDLE_USER_LANE,
 				GSP_PHYS_RMCALL_CHANNEL_FLAG_USER_LANE,
@@ -13489,6 +13594,7 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0)
 			boot_stage.rmcall_channel =
 				GSP_PHYS_RMCALL_HANDLE_USER_LANE;
 			boot_stage.rmcall_channel_server_mem = false;
+			gsp_phys_rmcall_ctx_diag_decode("mode35 V7 post-construct (m62 readback)");
 		} else {
 			ret = gsp_phys_rmcall_alloc_gr_object(bar0);
 			if (ret) {
@@ -16402,6 +16508,20 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0)
 			boot_stage.rmcall_tsg, rpc_result, status);
 
 	/*
+	 * Mode 35 / E2: the real lane-enable tail (R2 #1059/#1061/#1063) after
+	 * the schedules and before the forwarded launch. Best-effort: the
+	 * launch result and the FECS readback show whether the lane enable
+	 * was the gate.
+	 */
+	if (boot_stage.rmcall_gr_promote ==
+			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_USER_LANE) {
+		ret = gsp_phys_rmcall_user_lane_lane_enable(bar0);
+		if (ret)
+			pr_err("vgsp-boot: rm-call: mode 35 lane enable tail failed (%d); continuing best-effort (the launch result shows whether the lane enable was the gate)\n",
+			       ret);
+	}
+
+	/*
 	 * Shared-management GR diagnostic: the server owns the engine context;
 	 * dump the instance/RAMFC words here (post-BIND/SCHEDULE, pre-submit)
 	 * so the committed engine-context pointer/VA can be correlated with