vgsp: SKED/FE readback + QMD w50..53 + two-step PCAS gate (m41 instrumentation)

3 files changed, 61 insertions, 2 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index 8b78d9a..dc8e961 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -13827,6 +13827,7 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb,
 	u32 *p = pb;
 	int prelude = gsp_phys_rmcall_gr_prelude_value();
 	int qmdvirt = gsp_phys_rmcall_gr_qmdvirt_value();
+	int two_step = gsp_phys_rmcall_gr_pcas_two_step_value();
 
 	p = gsp_phys_rmcall_build_gr_init(p);
 	if (qmdvirt)
@@ -13882,7 +13883,16 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb,
 					    (u32)(qmd_va >> 8));
 		p = gsp_phys_rmcall_push_gr(p,
 				GSP_PHYS_RMCALL_GR_SEND_SIGNALING_PCAS_B,
+				two_step ? 1u :
 				GSP_PHYS_RMCALL_GR_PCAS_INVALIDATE_SCHEDULE);
+		if (two_step) {
+			p = gsp_phys_rmcall_push_gr(p,
+					GSP_PHYS_RMCALL_GR_SEND_PCAS_A,
+					(u32)(qmd_va >> 8));
+			p = gsp_phys_rmcall_push_gr(p,
+					GSP_PHYS_RMCALL_GR_SEND_SIGNALING_PCAS_B,
+					2u);
+		}
 	}
 	p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_A,
 				    (u32)(sem_va >> 32));
@@ -14305,6 +14315,13 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
 	if (!epoch)
 		epoch = ++boot_stage.rmcall_gr_submit_epoch;
 	memcpy(qmd, req->qmd, sizeof(qmd));
+	/*
+	 * Two-step PCAS (gsp_phys_rmcall_gr_pcas_two_step): set the QMD's
+	 * REQUIRE_SCHEDULING_PCAS (w4 bit12) so the copy PCAS does not
+	 * schedule and the trailing schedule-only PCAS does.
+	 */
+	if (gsp_phys_rmcall_gr_pcas_two_step_value())
+		qmd[4] |= 1u << 12;
 
 	/*
 	 * QMD release0: when the launch has a one-word absolute release and
@@ -14814,6 +14831,7 @@ gr_page_unmapped:
 					: 0;
 				u32 q0 = 0, q1 = 0, q23 = 0, q24 = 0, q25 = 0;
 				u32 q48 = 0, q49 = 0, q50 = 0;
+				u32 q51 = 0, q52 = 0, q53 = 0;
 				u32 q63 = 0;
 				u32 relw = 0;
 
@@ -14832,6 +14850,9 @@ gr_page_unmapped:
 						q48 = get_unaligned_le32(qp + 192);
 						q49 = get_unaligned_le32(qp + 196);
 						q50 = get_unaligned_le32(qp + 200);
+						q51 = get_unaligned_le32(qp + 204);
+						q52 = get_unaligned_le32(qp + 208);
+						q53 = get_unaligned_le32(qp + 212);
 						q63 = get_unaligned_le32(
 							qp +
 							GSP_PHYS_GR_QMD_SENTINEL_WORD *
@@ -14842,13 +14863,26 @@ gr_page_unmapped:
 					relw = READ_ONCE(
 						*(u32 *)((u8 *)__va(qmd_rel_hpa) +
 							 qmd_rel_off));
-				pr_info("vgsp-gr: post-mortem gpget=%u->%u qmd_page w0=%08x w1=%08x w23..25=%08x %08x %08x w48..50=%08x %08x %08x w63=%08x relw=%08x iq2m=%s%s\n",
+				pr_info("vgsp-gr: post-mortem gpget=%u->%u qmd_page w0=%08x w1=%08x w23..25=%08x %08x %08x w48..53=%08x %08x %08x %08x %08x %08x w63=%08x relw=%08x iq2m=%s%s\n",
 					sub.gpget, gpget_post, q0, q1, q23, q24,
-					q25, q48, q49, q50, q63, relw,
+					q25, q48, q49, q50, q51, q52, q53, q63,
+					relw,
 					q63 == GSP_PHYS_GR_QMD_SENTINEL ?
 						"not-run (sentinel intact)" :
 						"ran (page overwritten)",
 					qmdwin_note);
+				if (bar0)
+					pr_info("vgsp-gr: FE readback raw 0x40a500..0x40a50c = %08x %08x %08x %08x (subctx-1 control at 0x40a504)\n",
+						ioread32(bar0 + 0x0040a500u),
+						ioread32(bar0 + 0x0040a504u),
+						ioread32(bar0 + 0x0040a508u),
+						ioread32(bar0 + 0x0040a50cu));
+				if (bar0)
+					pr_info("vgsp-gr: SKED readback: HWW_ESR=0x%08x CWD_T0=0x%08x CWD_T1=0x%08x FECS_CTX=0x%08x\n",
+						ioread32(bar0 + 0x00407020u),
+						ioread32(bar0 + 0x00407060u),
+						ioread32(bar0 + 0x00407064u),
+						ioread32(bar0 + 0x00409b00u));
 			}
 			poll = w1;
 			ret = -ETIMEDOUT;
diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c
index 9100959..5ec97e2 100644
--- a/kernel/gsp_phys_gpu.c
+++ b/kernel/gsp_phys_gpu.c
@@ -638,6 +638,24 @@ int gsp_phys_rmcall_gr_pcas_value(void)
 EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_pcas_value);
 
 /*
+ * Two-step PCAS gate (default 0). With the explicit PCAS pair on, setting
+ * this marks the QMD's REQUIRE_SCHEDULING_PCAS (w4 bit12) and emits the copy
+ * PCAS (INVALIDATE only) followed by a schedule-only PCAS (SCHEDULE), the
+ * dependent-QMD flow (cuda_device.h:1108-1113).
+ */
+static int gsp_phys_rmcall_gr_pcas_two_step_param;
+module_param_named(gsp_phys_rmcall_gr_pcas_two_step,
+		   gsp_phys_rmcall_gr_pcas_two_step_param, int, 0644);
+MODULE_PARM_DESC(gsp_phys_rmcall_gr_pcas_two_step,
+		 "Two-step PCAS: set QMD REQUIRE_SCHEDULING_PCAS and emit copy PCAS then schedule-only PCAS (default 0; requires gsp_phys_rmcall_gr_pcas=1)");
+
+int gsp_phys_rmcall_gr_pcas_two_step_value(void)
+{
+	return gsp_phys_rmcall_gr_pcas_two_step_param;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_pcas_two_step_value);
+
+/*
  * Rank-2 qmdwin diagnostic (gsp_phys_rmcall_gr_qmdwin; default 0). When set,
  * a forwarded GR launch relocates its inline-QMD destination from the guest
  * QMD VA to a dedicated module page mapped at the ladder's own window-0 VA
diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h
index d37f5d5..8cfb5e9 100644
--- a/kernel/gsp_phys_gpu.h
+++ b/kernel/gsp_phys_gpu.h
@@ -301,6 +301,13 @@ int gsp_phys_rmcall_gr_qmdvirt_value(void);
 int gsp_phys_rmcall_gr_pcas_value(void);
 
 /*
+ * gsp_phys_rmcall_gr_pcas_two_step (default 0): set the QMD
+ * REQUIRE_SCHEDULING_PCAS bit and emit copy PCAS followed by a schedule-only
+ * PCAS; requires gsp_phys_rmcall_gr_pcas=1.
+ */
+int gsp_phys_rmcall_gr_pcas_two_step_value(void);
+
+/*
  * gsp_phys_rmcall_gr_qmdwin (default 0): 1 relocates the forwarded GR
  * inline-QMD destination from the guest QMD VA to a dedicated module page
  * mapped at the proven engine-reachable window-0 VA 0x107000 (the PTE is