vgsp: m26 GR launch discriminator - rebuilt window (prelude off), QMD-virt 0x29c, w63 sentinel

4 files changed, 94 insertions, 14 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index 4cbbd40..11f205c 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -13526,12 +13526,21 @@ EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_ce_copy);
 #define GSP_PHYS_RMCALL_GR_INIT_VSPAN_C			0x208u
 #define GSP_PHYS_RMCALL_GR_INIT_NON_THROTTLED_C		0x2ecu
 #define GSP_PHYS_RMCALL_GR_INIT_SM_DISP_CTRL		0x250u
+/*
+ * NVC5C0_SET_QMD_VIRTUALIZATION_CONTROL (class/clc5c0.h): emitted between the
+ * compute init and the inline-QMD window when gsp_phys_rmcall_gr_qmdvirt=1
+ * (the default). Value enables IQ2M_ADDR_ENABLE (bit 16) and
+ * SEND_PCAS_ENABLE (bit 24).
+ */
+#define GSP_PHYS_RMCALL_GR_SET_QMD_VIRT_CONTROL		0x29cu
+#define GSP_PHYS_RMCALL_GR_QMD_VIRT_IQ2M_SEND_PCAS	0x01010000u
 #define GSP_PHYS_RMCALL_GR_SHARED_WINDOW_BASE		0x1000000u
 #define GSP_PHYS_RMCALL_GR_SPA_VERSION			0x0705u
 #define GSP_PHYS_RMCALL_GR_CWD_REFCOUNT			1536u
 #define GSP_PHYS_RMCALL_GR_CWD_LEVELS			64u
 #define GSP_PHYS_RMCALL_GR_MAX_SM_COUNT			24u
-/* Method block bound: init(152) + inline run(67) + PCAS(4) + REPORT_SEM(8). */
+/* Method block bound: init(152) + inline run(67) + PCAS(4) + REPORT_SEM(8) +
+ * the optional QMD-virt pair(2). */
 #define GSP_PHYS_RMCALL_GR_METHOD_WORDS		512u
 
 static u64 gsp_phys_gr_submitted;
@@ -13608,16 +13617,24 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb,
 					   u64 sem_va, u32 sem_payload)
 {
 	u32 *p = pb;
+	int prelude = gsp_phys_rmcall_gr_prelude_value();
+	int qmdvirt = gsp_phys_rmcall_gr_qmdvirt_value();
 
 	p = gsp_phys_rmcall_build_gr_init(p);
+	if (qmdvirt)
+		p = gsp_phys_rmcall_push_gr(p,
+				GSP_PHYS_RMCALL_GR_SET_QMD_VIRT_CONTROL,
+				GSP_PHYS_RMCALL_GR_QMD_VIRT_IQ2M_SEND_PCAS);
 	/*
 	 * Verbatim replay: copy the captured GRCE prelude and the guest's
 	 * own inline-QMD run, keeping the method headers, and relocate only
 	 * the ADDRESS_A/B data words and the 64 QMD words exactly like the
 	 * rebuilt window. Only a contiguous 0x318 run is replayed; anything
-	 * else falls back to the rebuilt window.
+	 * else falls back to the rebuilt window. With
+	 * gsp_phys_rmcall_gr_prelude=0 (default) the rebuilt window is always
+	 * used even when a verbatim prelude is available.
 	 */
-	if (req && req->raw &&
+	if (prelude && req && req->raw &&
 	    req->raw_words >=
 		    (u32)req->raw_qmd_hdr + GSP_PHYS_RMCALL_GR_QMD_WINDOW_WORDS &&
 	    req->raw_qmd_end ==
@@ -13639,7 +13656,7 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb,
 		pr_info("vgsp-gr: verbatim replay prelude+window words=%u qmd_hdr=%u\n",
 			pre, req->raw_qmd_hdr);
 	} else {
-		if (req && req->raw)
+		if (prelude && req && req->raw)
 			pr_info("vgsp-gr: raw prelude unusable n=%u hdr=%u end=%u; rebuilt window\n",
 				req->raw_words, req->raw_qmd_hdr,
 				req->raw_qmd_end);
@@ -14450,10 +14467,13 @@ gr_page_unmapped:
 	 */
 	if (qmd_dst_mem) {
 		memcpy(qmd_dst_mem, qmd, GSP_PHYS_GR_QMD_WORDS * 4);
-		put_unaligned_le32(GSP_PHYS_GR_QMD_SENTINEL, qmd_dst_mem);
+		put_unaligned_le32(GSP_PHYS_GR_QMD_SENTINEL,
+				   qmd_dst_mem +
+				   GSP_PHYS_GR_QMD_SENTINEL_WORD * 4);
 		wmb();
-		pr_info("vgsp-gr: QMD prewritten to dest 0x%llx (%u words, w0 sentinel=0x%08x)\n",
+		pr_info("vgsp-gr: QMD prewritten to dest 0x%llx (%u words, w%u sentinel=0x%08x; w0 left authentic)\n",
 			qmd_dst, GSP_PHYS_GR_QMD_WORDS,
+			GSP_PHYS_GR_QMD_SENTINEL_WORD,
 			GSP_PHYS_GR_QMD_SENTINEL);
 	}
 
@@ -14475,9 +14495,12 @@ gr_page_unmapped:
 		goto out;
 	}
 
-	pr_info("vgsp-gr: GR LAUNCH submitted h=0x%x push=0x%llx len=%u pages=%u shadow=%u qmd_va=0x%llx slot=%u words=%u token=0x%x; polling qmd_rel=%u ch_rel=0x%08x (<=%u ms)\n",
+	pr_info("vgsp-gr: GR LAUNCH submitted h=0x%x push=0x%llx len=%u pages=%u shadow=%u qmd_va=0x%llx slot=%u words=%u prelude=%d qmdvirt=%d token=0x%x; polling qmd_rel=%u ch_rel=0x%08x (<=%u ms)\n",
 		req->h_object, req->push_va, req->push_len, req->n_pages,
-		n_shadow, req->qmd_va, sub.slot, words, boot_stage.rmcall_token,
+		n_shadow, req->qmd_va, sub.slot, words,
+		gsp_phys_rmcall_gr_prelude_value(),
+		gsp_phys_rmcall_gr_qmdvirt_value(),
+		boot_stage.rmcall_token,
 		qmd_rel, want_ch, GSP_PHYS_RMCALL_GR_POLL_MS);
 	if (qmd_win_diag)
 		pr_info("vgsp-gr: qmd dest va=0x%llx pdeIndex=%u installed=%u win_mask=0x%x ensure_ret=%d\n",
@@ -14540,6 +14563,7 @@ gr_page_unmapped:
 						ud + GSP_PHYS_RMCALL_USERD_GPGET_OFF)
 					: 0;
 				u32 q0 = 0, q1 = 0, q23 = 0, q24 = 0, q25 = 0;
+				u32 q63 = 0;
 				u32 relw = 0;
 
 				{
@@ -14554,16 +14578,20 @@ gr_page_unmapped:
 						q23 = get_unaligned_le32(qp + 92);
 						q24 = get_unaligned_le32(qp + 96);
 						q25 = get_unaligned_le32(qp + 100);
+						q63 = get_unaligned_le32(
+							qp +
+							GSP_PHYS_GR_QMD_SENTINEL_WORD *
+								4);
 					}
 				}
 				if (qmd_rel_hpa)
 					relw = READ_ONCE(
 						*(u32 *)((u8 *)__va(qmd_rel_hpa) +
 							 qmd_rel_off));
-				pr_info("vgsp-gr: post-mortem gpget=%u->%u qmd_page w0=%08x w1=%08x w23..25=%08x %08x %08x relw=%08x iq2m=%s\n",
+				pr_info("vgsp-gr: post-mortem gpget=%u->%u qmd_page w0=%08x w1=%08x w23..25=%08x %08x %08x w63=%08x relw=%08x iq2m=%s\n",
 					sub.gpget, gpget_post, q0, q1, q23, q24,
-					q25, relw,
-					q0 == GSP_PHYS_GR_QMD_SENTINEL ?
+					q25, q63, relw,
+					q63 == GSP_PHYS_GR_QMD_SENTINEL ?
 						"not-run (sentinel intact)" :
 						"ran (page overwritten)");
 			}
diff --git a/kernel/gsp_phys_booter.h b/kernel/gsp_phys_booter.h
index 67d5d5c..e46fb86 100644
--- a/kernel/gsp_phys_booter.h
+++ b/kernel/gsp_phys_booter.h
@@ -336,14 +336,16 @@ void gsp_phys_backend_relay_counters(u64 *requested, u64 *ok, u64 *verify_fail,
 #define GSP_PHYS_GR_INLINE_BYTES	64u
 #define GSP_PHYS_GR_QMD_WORDS		64u
 /*
- * IQ2M sentinel: written to the inline-QMD destination's word 0 after the
- * prewrite. If the engine processes the inline window, the copy overwrites
- * the page (word 0 -> the inline value, 0) and the post-mortem sees the page
+ * IQ2M sentinel: written to the inline-QMD destination's word 63 after the
+ * prewrite (a word SKED does not consume; word 0 stays the guest's authentic
+ * OUTER_PUT). If the engine processes the inline window, the copy overwrites
+ * the page (word 63 -> the inline value) and the post-mortem sees the page
  * changed; if the sentinel survives, the engine stalled before the inline
  * window and the page was never touched. Distinguishes "IQ2M never ran" from
  * "QMD deposited but not executed" without a server-side capture.
  */
 #define GSP_PHYS_GR_QMD_SENTINEL	0xdeadbeefu
+#define GSP_PHYS_GR_QMD_SENTINEL_WORD	63u
 #define GSP_PHYS_GR_QMD_WORD_NONE	0xffffffffu
 #define GSP_PHYS_GR_RAW_WORDS		256u
 
diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c
index 2f441a0..ad2244e 100644
--- a/kernel/gsp_phys_gpu.c
+++ b/kernel/gsp_phys_gpu.c
@@ -580,6 +580,42 @@ int gsp_phys_rmcall_gr_promote_value(void)
 EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_promote_value);
 
 /*
+ * GR launch verbatim-prelude gate (default 0). 0 = skip the captured
+ * 136-word verbatim guest prelude and emit the rebuilt inline-QMD window,
+ * i.e. the proven init(152)+window(67)+PCAS(4)+sem(8) shape; 1 = replay the
+ * verbatim prelude exactly as before. Read at each launch.
+ */
+static int gsp_phys_rmcall_gr_prelude_param;
+module_param_named(gsp_phys_rmcall_gr_prelude,
+		   gsp_phys_rmcall_gr_prelude_param, int, 0644);
+MODULE_PARM_DESC(gsp_phys_rmcall_gr_prelude,
+		 "GR launch verbatim-prelude gate (default 0: skip the captured guest prelude and emit the rebuilt inline-QMD window; 1: replay the verbatim prelude)");
+
+int gsp_phys_rmcall_gr_prelude_value(void)
+{
+	return gsp_phys_rmcall_gr_prelude_param;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_prelude_value);
+
+/*
+ * QMD virtualization control gate (default 1). When set, the GR launch block
+ * emits NVC5C0_SET_QMD_VIRTUALIZATION_CONTROL (0x29c) = IQ2M_ADDR_ENABLE
+ * (bit 16) | SEND_PCAS_ENABLE (bit 24) after the compute init and before the
+ * inline-QMD window; 0 omits the method pair. Read at each launch.
+ */
+static int gsp_phys_rmcall_gr_qmdvirt_param = 1;
+module_param_named(gsp_phys_rmcall_gr_qmdvirt,
+		   gsp_phys_rmcall_gr_qmdvirt_param, int, 0644);
+MODULE_PARM_DESC(gsp_phys_rmcall_gr_qmdvirt,
+		 "Emit NVC5C0_SET_QMD_VIRTUALIZATION_CONTROL (0x29c = IQ2M_ADDR_ENABLE|SEND_PCAS_ENABLE) in the GR launch block (default 1; 0 omits the method pair)");
+
+int gsp_phys_rmcall_gr_qmdvirt_value(void)
+{
+	return gsp_phys_rmcall_gr_qmdvirt_param;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_qmdvirt_value);
+
+/*
  * Ladder VAS ownership selector (gsp_phys_rmcall_vas_shared; default 0).
  * The H1 GR promote wall is a direct consequence of the ladder's
  * externally-owned FERMI_VASPACE_A: with IS_EXTERNALLY_OWNED the RM treats
diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h
index 65e1e1f..3dce1a6 100644
--- a/kernel/gsp_phys_gpu.h
+++ b/kernel/gsp_phys_gpu.h
@@ -280,6 +280,20 @@ int gsp_phys_rmcall_engine_value(void);
 int gsp_phys_rmcall_gr_promote_value(void);
 
 /*
+ * gsp_phys_rmcall_gr_prelude (default 0): 0 skips the captured verbatim
+ * guest prelude and emits the rebuilt inline-QMD window (proven
+ * init+window+PCAS+sem shape); 1 replays the verbatim prelude.
+ */
+int gsp_phys_rmcall_gr_prelude_value(void);
+
+/*
+ * gsp_phys_rmcall_gr_qmdvirt (default 1): emit
+ * NVC5C0_SET_QMD_VIRTUALIZATION_CONTROL (0x29c = IQ2M_ADDR_ENABLE |
+ * SEND_PCAS_ENABLE) in the GR launch block; 0 omits the method pair.
+ */
+int gsp_phys_rmcall_gr_qmdvirt_value(void);
+
+/*
  * Ladder VAS ownership selector (gsp_phys_rmcall_vas_shared; default 0).
  * 0 = FERMI_VASPACE_A is allocated IS_EXTERNALLY_OWNED (historical ladder
  * behaviour: the client owns page tables, VA bookings and engine-context