vgsp: verbatim replay of the captured GR launch prelude+window

1 files changed, 57 insertions, 12 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index 2c0fdc0..badf14f 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -12765,6 +12765,12 @@ EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_ce_copy);
 #define GSP_PHYS_RMCALL_GR_LOAD_INLINE_QMD_DATA	0x320u
 /* Header count: ADDRESS_A + ADDRESS_B + 64 QMD words. */
 #define GSP_PHYS_RMCALL_GR_QMD_INLINE_WORDS	66u
+/* Whole run: one method header + the inline-QMD data words. */
+#define GSP_PHYS_RMCALL_GR_QMD_WINDOW_WORDS	67u
+/* Dwords the per-channel compute init emits (see build_gr_init). */
+#define GSP_PHYS_RMCALL_GR_INIT_WORDS		152u
+/* PCAS(4) + REPORT_SEM(8). */
+#define GSP_PHYS_RMCALL_GR_TAIL_WORDS		12u
 #define GSP_PHYS_RMCALL_GR_QMD_REL_PAYLOAD	GSP_PHYS_RMCALL_SEM_PAYLOAD
 #define GSP_PHYS_RMCALL_GR_SEM_PAYLOAD		0xc0ffee02u
 /*
@@ -12874,21 +12880,56 @@ static u32 *gsp_phys_rmcall_build_gr_init(u32 *p)
  * channel long before the launch), so the replay must not either.
  * Returns the block size in dwords.
  */
-static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, const u32 *qmd,
-					   u64 qmd_va, u64 sem_va,
-					   u32 sem_payload)
+static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb,
+					   const struct gsp_phys_gr_submit *req,
+					   const u32 *qmd, u64 qmd_va,
+					   u64 sem_va, u32 sem_payload)
 {
 	u32 *p = pb;
 
 	p = gsp_phys_rmcall_build_gr_init(p);
-	*p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) |
-	       (GSP_PHYS_RMCALL_GR_QMD_INLINE_WORDS << 16) |
-	       ((u32)GSP_PHYS_RMCALL_GR_SUBCHANNEL << 13) |
-	       (GSP_PHYS_RMCALL_GR_INLINE_QMD_ADDRESS_A >> 2);
-	*p++ = (u32)(qmd_va >> 40);
-	*p++ = (u32)((qmd_va >> 8) & 0xffffffffu);
-	memcpy(p, qmd, GSP_PHYS_GR_QMD_WORDS * 4);
-	p += GSP_PHYS_GR_QMD_WORDS;
+	/*
+	 * Verbatim replay: copy the captured GRCE prelude and the guest's
+	 * own inline-QMD run, keeping the method headers, and relocate only
+	 * the ADDRESS_A/B data words and the 64 QMD words exactly like the
+	 * rebuilt window. Only a contiguous 0x318 run is replayed; anything
+	 * else falls back to the rebuilt window.
+	 */
+	if (req && req->raw &&
+	    req->raw_words >=
+		    (u32)req->raw_qmd_hdr + GSP_PHYS_RMCALL_GR_QMD_WINDOW_WORDS &&
+	    req->raw_qmd_end ==
+		    (u32)req->raw_qmd_hdr + GSP_PHYS_RMCALL_GR_QMD_WINDOW_WORDS &&
+	    GSP_PHYS_RMCALL_GR_INIT_WORDS +
+		    (u32)req->raw_qmd_hdr + GSP_PHYS_RMCALL_GR_QMD_WINDOW_WORDS +
+		    GSP_PHYS_RMCALL_GR_TAIL_WORDS <=
+		    GSP_PHYS_RMCALL_GR_METHOD_WORDS) {
+		u32 pre = (u32)req->raw_qmd_hdr +
+			  GSP_PHYS_RMCALL_GR_QMD_WINDOW_WORDS;
+
+		memcpy(p, req->raw, pre * 4);
+		p[req->raw_qmd_hdr + 1] = (u32)(qmd_va >> 40);
+		p[req->raw_qmd_hdr + 2] =
+			(u32)((qmd_va >> 8) & 0xffffffffu);
+		memcpy(p + req->raw_qmd_hdr + 3, qmd,
+		       GSP_PHYS_GR_QMD_WORDS * 4);
+		p += pre;
+		pr_info("vgsp-gr: verbatim replay prelude+window words=%u qmd_hdr=%u\n",
+			pre, req->raw_qmd_hdr);
+	} else {
+		if (req && req->raw)
+			pr_info("vgsp-gr: raw prelude unusable n=%u hdr=%u end=%u; rebuilt window\n",
+				req->raw_words, req->raw_qmd_hdr,
+				req->raw_qmd_end);
+		*p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) |
+		       (GSP_PHYS_RMCALL_GR_QMD_INLINE_WORDS << 16) |
+		       ((u32)GSP_PHYS_RMCALL_GR_SUBCHANNEL << 13) |
+		       (GSP_PHYS_RMCALL_GR_INLINE_QMD_ADDRESS_A >> 2);
+		*p++ = (u32)(qmd_va >> 40);
+		*p++ = (u32)((qmd_va >> 8) & 0xffffffffu);
+		memcpy(p, qmd, GSP_PHYS_GR_QMD_WORDS * 4);
+		p += GSP_PHYS_GR_QMD_WORDS;
+	}
 	p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SEND_PCAS_A,
 				    (u32)(qmd_va >> 8));
 	p = gsp_phys_rmcall_push_gr(p,
@@ -13554,7 +13595,7 @@ gr_page_unmapped:
 			qmd_dst, GSP_PHYS_GR_QMD_WORDS);
 	}
 
-	words = gsp_phys_rmcall_build_gr_launch(methods, qmd,
+	words = gsp_phys_rmcall_build_gr_launch(methods, req, qmd,
 						qmd_dst,
 						ch_sem_va, want_ch);
 	memcpy(gsp_phys_chunk_page_va(pb, 0), methods, words * 4);
@@ -16029,6 +16070,10 @@ static_assert(offsetof(struct gsp_phys_gr_inline, data) == 12 &&
 static_assert((2u + 1u + GSP_PHYS_RMCALL_GR_QMD_INLINE_WORDS + 8u) <=
 	      GSP_PHYS_RMCALL_GR_METHOD_WORDS,
 	      "the GR launch method block (compute init + SET_OBJECT + inline-QMD run + PCAS + REPORT_SEMAPHORE) must fit the bound");
+static_assert(GSP_PHYS_RMCALL_GR_INIT_WORDS + GSP_PHYS_GR_RAW_WORDS +
+	      GSP_PHYS_RMCALL_GR_TAIL_WORDS <=
+	      GSP_PHYS_RMCALL_GR_METHOD_WORDS,
+	      "the verbatim GR launch block (compute init + raw tail + PCAS + REPORT_SEMAPHORE) must fit the bound");
 static_assert(GSP_PHYS_RMCALL_GR_METHOD_WORDS * 4u <=
 	      GSP_PHYS_RMCALL_COPY_SIZE,
 	      "the GR launch method block must fit one module pushbuffer page");