vgsp: verbatim replay of the captured GR launch prelude+window

3 files changed, 112 insertions, 13 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index 2c0fdc0..badf14f 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -12765,6 +12765,12 @@ EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_ce_copy);
 #define GSP_PHYS_RMCALL_GR_LOAD_INLINE_QMD_DATA	0x320u
 /* Header count: ADDRESS_A + ADDRESS_B + 64 QMD words. */
 #define GSP_PHYS_RMCALL_GR_QMD_INLINE_WORDS	66u
+/* Whole run: one method header + the inline-QMD data words. */
+#define GSP_PHYS_RMCALL_GR_QMD_WINDOW_WORDS	67u
+/* Dwords the per-channel compute init emits (see build_gr_init). */
+#define GSP_PHYS_RMCALL_GR_INIT_WORDS		152u
+/* PCAS(4) + REPORT_SEM(8). */
+#define GSP_PHYS_RMCALL_GR_TAIL_WORDS		12u
 #define GSP_PHYS_RMCALL_GR_QMD_REL_PAYLOAD	GSP_PHYS_RMCALL_SEM_PAYLOAD
 #define GSP_PHYS_RMCALL_GR_SEM_PAYLOAD		0xc0ffee02u
 /*
@@ -12874,21 +12880,56 @@ static u32 *gsp_phys_rmcall_build_gr_init(u32 *p)
  * channel long before the launch), so the replay must not either.
  * Returns the block size in dwords.
  */
-static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, const u32 *qmd,
-					   u64 qmd_va, u64 sem_va,
-					   u32 sem_payload)
+static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb,
+					   const struct gsp_phys_gr_submit *req,
+					   const u32 *qmd, u64 qmd_va,
+					   u64 sem_va, u32 sem_payload)
 {
 	u32 *p = pb;
 
 	p = gsp_phys_rmcall_build_gr_init(p);
-	*p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) |
-	       (GSP_PHYS_RMCALL_GR_QMD_INLINE_WORDS << 16) |
-	       ((u32)GSP_PHYS_RMCALL_GR_SUBCHANNEL << 13) |
-	       (GSP_PHYS_RMCALL_GR_INLINE_QMD_ADDRESS_A >> 2);
-	*p++ = (u32)(qmd_va >> 40);
-	*p++ = (u32)((qmd_va >> 8) & 0xffffffffu);
-	memcpy(p, qmd, GSP_PHYS_GR_QMD_WORDS * 4);
-	p += GSP_PHYS_GR_QMD_WORDS;
+	/*
+	 * Verbatim replay: copy the captured GRCE prelude and the guest's
+	 * own inline-QMD run, keeping the method headers, and relocate only
+	 * the ADDRESS_A/B data words and the 64 QMD words exactly like the
+	 * rebuilt window. Only a contiguous 0x318 run is replayed; anything
+	 * else falls back to the rebuilt window.
+	 */
+	if (req && req->raw &&
+	    req->raw_words >=
+		    (u32)req->raw_qmd_hdr + GSP_PHYS_RMCALL_GR_QMD_WINDOW_WORDS &&
+	    req->raw_qmd_end ==
+		    (u32)req->raw_qmd_hdr + GSP_PHYS_RMCALL_GR_QMD_WINDOW_WORDS &&
+	    GSP_PHYS_RMCALL_GR_INIT_WORDS +
+		    (u32)req->raw_qmd_hdr + GSP_PHYS_RMCALL_GR_QMD_WINDOW_WORDS +
+		    GSP_PHYS_RMCALL_GR_TAIL_WORDS <=
+		    GSP_PHYS_RMCALL_GR_METHOD_WORDS) {
+		u32 pre = (u32)req->raw_qmd_hdr +
+			  GSP_PHYS_RMCALL_GR_QMD_WINDOW_WORDS;
+
+		memcpy(p, req->raw, pre * 4);
+		p[req->raw_qmd_hdr + 1] = (u32)(qmd_va >> 40);
+		p[req->raw_qmd_hdr + 2] =
+			(u32)((qmd_va >> 8) & 0xffffffffu);
+		memcpy(p + req->raw_qmd_hdr + 3, qmd,
+		       GSP_PHYS_GR_QMD_WORDS * 4);
+		p += pre;
+		pr_info("vgsp-gr: verbatim replay prelude+window words=%u qmd_hdr=%u\n",
+			pre, req->raw_qmd_hdr);
+	} else {
+		if (req && req->raw)
+			pr_info("vgsp-gr: raw prelude unusable n=%u hdr=%u end=%u; rebuilt window\n",
+				req->raw_words, req->raw_qmd_hdr,
+				req->raw_qmd_end);
+		*p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) |
+		       (GSP_PHYS_RMCALL_GR_QMD_INLINE_WORDS << 16) |
+		       ((u32)GSP_PHYS_RMCALL_GR_SUBCHANNEL << 13) |
+		       (GSP_PHYS_RMCALL_GR_INLINE_QMD_ADDRESS_A >> 2);
+		*p++ = (u32)(qmd_va >> 40);
+		*p++ = (u32)((qmd_va >> 8) & 0xffffffffu);
+		memcpy(p, qmd, GSP_PHYS_GR_QMD_WORDS * 4);
+		p += GSP_PHYS_GR_QMD_WORDS;
+	}
 	p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SEND_PCAS_A,
 				    (u32)(qmd_va >> 8));
 	p = gsp_phys_rmcall_push_gr(p,
@@ -13554,7 +13595,7 @@ gr_page_unmapped:
 			qmd_dst, GSP_PHYS_GR_QMD_WORDS);
 	}
 
-	words = gsp_phys_rmcall_build_gr_launch(methods, qmd,
+	words = gsp_phys_rmcall_build_gr_launch(methods, req, qmd,
 						qmd_dst,
 						ch_sem_va, want_ch);
 	memcpy(gsp_phys_chunk_page_va(pb, 0), methods, words * 4);
@@ -16029,6 +16070,10 @@ static_assert(offsetof(struct gsp_phys_gr_inline, data) == 12 &&
 static_assert((2u + 1u + GSP_PHYS_RMCALL_GR_QMD_INLINE_WORDS + 8u) <=
 	      GSP_PHYS_RMCALL_GR_METHOD_WORDS,
 	      "the GR launch method block (compute init + SET_OBJECT + inline-QMD run + PCAS + REPORT_SEMAPHORE) must fit the bound");
+static_assert(GSP_PHYS_RMCALL_GR_INIT_WORDS + GSP_PHYS_GR_RAW_WORDS +
+	      GSP_PHYS_RMCALL_GR_TAIL_WORDS <=
+	      GSP_PHYS_RMCALL_GR_METHOD_WORDS,
+	      "the verbatim GR launch block (compute init + raw tail + PCAS + REPORT_SEMAPHORE) must fit the bound");
 static_assert(GSP_PHYS_RMCALL_GR_METHOD_WORDS * 4u <=
 	      GSP_PHYS_RMCALL_COPY_SIZE,
 	      "the GR launch method block must fit one module pushbuffer page");
diff --git a/kernel/gsp_phys_booter.h b/kernel/gsp_phys_booter.h
index 42bf1ff..9a0331f 100644
--- a/kernel/gsp_phys_booter.h
+++ b/kernel/gsp_phys_booter.h
@@ -336,6 +336,7 @@ void gsp_phys_backend_relay_counters(u64 *requested, u64 *ok, u64 *verify_fail,
 #define GSP_PHYS_GR_INLINE_BYTES	64u
 #define GSP_PHYS_GR_QMD_WORDS		64u
 #define GSP_PHYS_GR_QMD_WORD_NONE	0xffffffffu
+#define GSP_PHYS_GR_RAW_WORDS		256u
 
 struct gsp_phys_gr_page {
 	u64 guest_va;		/* guest VA (page aligned) */
@@ -353,6 +354,17 @@ struct gsp_phys_gr_inline {
 struct gsp_phys_gr_submit {
 	const u32 *qmd;		/* 64 rewritten-QMD source words */
 	u32 qmd_words;		/* must be GSP_PHYS_GR_QMD_WORDS */
+	/*
+	 * Verbatim replay: the captured tail of the guest push (up to
+	 * GSP_PHYS_GR_RAW_WORDS dwords) and the dword offsets of its
+	 * inline-QMD run inside that tail. A contiguous 0x318 run
+	 * (header + A/B + 64 words) is replayed with the same data-field
+	 * relocations; anything else leaves the rebuilt window in use.
+	 */
+	const u32 *raw;
+	u32 raw_words;
+	u32 raw_qmd_hdr;
+	u32 raw_qmd_end;
 	u64 qmd_va;		/* guest VA of the QMD buffer */
 	u64 qmd_hpa;		/* guest QMD buffer page HPA, or 0 */
 	const struct gsp_phys_gr_page *pages;
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 663e1ba..a5f58db 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -2925,6 +2925,8 @@ struct svc_push_state {
 	u32	qmd_have;	/* A/B seen bitmap */
 	u32	qmd_n;		/* QMD words collected so far */
 	u64	qmd_bits;	/* received-word bitmap */
+	u32	qmd_hdr;	/* push offset of the 0x318 run header */
+	u32	qmd_end;	/* push offset just past that run */
 	u64	push_va;	/* push VA, for launch logging */
 	u32	qmd[SVC_QMD_WORDS];
 };
@@ -3189,6 +3191,7 @@ static struct svc_fwd_cand svc_fwd_cand;
 #define SVC_GR_INLINE_BYTES	64
 #define SVC_GR_POINTER_MAX	8
 #define SVC_GR_PAGES_MAX	16
+#define SVC_GR_PUSH_WORDS	256
 #define SVC_GR_QMD_WORD_NONE	0xffffffffu
 /* svc_gr_cand.rel_flags bits (decoded from qmd[24] and qmd[4]). */
 #define SVC_GR_REL_REDUCTION	0x1u
@@ -3216,6 +3219,10 @@ struct svc_gr_cand {
 	u64 push_va;
 	u32 push_len;
 	u64 push_hash;
+	u32 push_n;		/* raw tail words kept in raw[] */
+	u32 push_qmd_hdr;	/* raw[] offset of the 0x318 run header */
+	u32 push_qmd_end;	/* raw[] offset just past that run */
+	u32 raw[SVC_GR_PUSH_WORDS];
 	u32 qmd[SVC_GR_QMD_WORDS];
 	u64 qmd_dev_va;
 	u32 n_inline;
@@ -3236,6 +3243,8 @@ static struct svc_gr_cand svc_gr_cand;
 
 static_assert(SVC_GR_QMD_WORDS == GSP_PHYS_GR_QMD_WORDS,
 	      "the sema GR QMD word count must match the backend ABI");
+static_assert(SVC_GR_PUSH_WORDS == GSP_PHYS_GR_RAW_WORDS,
+	      "the sema raw push cap must match the backend ABI");
 static_assert(SVC_GR_INLINE_MAX <= GSP_PHYS_GR_MAX_INLINE,
 	      "the sema GR inline capture must fit the backend ABI");
 static_assert(sizeof(struct svc_gr_inline) == 80 &&
@@ -3312,6 +3321,20 @@ static void svc_gr_cand_capture(struct svc_channel *ch,
 	svc_gr_cand.push_len = push_len;
 	if (push && push_len)
 		svc_gr_cand.push_hash = svc_push_hash(push, push_len);
+	if (push && push_len >= 4) {
+		u32 total = push_len / 4;
+		u32 n = total < SVC_GR_PUSH_WORDS ? total : SVC_GR_PUSH_WORDS;
+		u32 base = total - n;
+		u32 hdr = st->qmd_hdr / 4;
+		u32 end = st->qmd_end / 4;
+
+		memcpy(svc_gr_cand.raw, push + base * 4, n * 4);
+		svc_gr_cand.push_n = n;
+		if (end > hdr && hdr >= base && end - base <= n) {
+			svc_gr_cand.push_qmd_hdr = hdr - base;
+			svc_gr_cand.push_qmd_end = end - base;
+		}
+	}
 	memcpy(svc_gr_cand.qmd, st->qmd, sizeof(svc_gr_cand.qmd));
 	svc_gr_cand.qmd_dev_va = st->qmd_dev_va;
 	n = n_recs < SVC_GR_INLINE_MAX ? n_recs : SVC_GR_INLINE_MAX;
@@ -4804,6 +4827,8 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 		st->qmd_n = 0;
 		st->qmd_bits = 0;
 		st->qmd_have = 0;
+		st->qmd_hdr = 0;
+		st->qmd_end = 0;
 	}
 }
 
@@ -5007,9 +5032,13 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 		    (gsp_cuda_bridge_enabled() ||
 		     gsp_phys_gr_forward_enabled()) &&
 		    addr >= SVC_QMD_ADDR_A &&
-		    addr <= SVC_QMD_DATA + (SVC_QMD_WORDS - 1) * 4 &&
+		     addr <= SVC_QMD_DATA + (SVC_QMD_WORDS - 1) * 4 &&
 		    (st.qmd_have ||
 		     (addr <= SVC_QMD_ADDR_B && addr + count * 4 > SVC_QMD_ADDR_A))) {
+			if (!st.qmd_end) {
+				st.qmd_hdr = off;
+				st.qmd_end = off + 4 + count * 4;
+			}
 			svc_qmd_accum(vgpu, ch, &st, addr, count,
 				      buf + off + 4, buf, len, trusted, replay);
 			off += 4 + count * 4;
@@ -8004,6 +8033,10 @@ struct svc_fwd_req {
 	u32 gr_h_object;
 	u64 gr_push_va;
 	u32 gr_push_len;
+	u32 gr_push_n;
+	u32 gr_push_qmd_hdr;
+	u32 gr_push_qmd_end;
+	u32 gr_raw[SVC_GR_PUSH_WORDS];
 	u64 gr_qmd_dev_va;
 	/*
 	 * The QMD buffer page (the inline-QMD destination the CUDA driver
@@ -8926,6 +8959,10 @@ static void svc_fwd_gr_work(void)
 	gs.sem_payload = 0;
 	gs.push_va = svc_fwd_req.gr_push_va;
 	gs.push_len = svc_fwd_req.gr_push_len;
+	gs.raw = svc_fwd_req.gr_raw;
+	gs.raw_words = svc_fwd_req.gr_push_n;
+	gs.raw_qmd_hdr = svc_fwd_req.gr_push_qmd_hdr;
+	gs.raw_qmd_end = svc_fwd_req.gr_push_qmd_end;
 	gs.h_object = svc_fwd_req.gr_h_object;
 
 	hw_rc = gsp_phys_backend_guest_gr_submit(&gs, &cp);
@@ -9267,6 +9304,11 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu,
 	svc_fwd_req.gr_h_object = ch->h_object;
 	svc_fwd_req.gr_push_va = svc_gr_cand.push_va;
 	svc_fwd_req.gr_push_len = svc_gr_cand.push_len;
+	svc_fwd_req.gr_push_n = svc_gr_cand.push_n;
+	svc_fwd_req.gr_push_qmd_hdr = svc_gr_cand.push_qmd_hdr;
+	svc_fwd_req.gr_push_qmd_end = svc_gr_cand.push_qmd_end;
+	memcpy(svc_fwd_req.gr_raw, svc_gr_cand.raw,
+	       sizeof(svc_fwd_req.gr_raw));
 	svc_fwd_req.gr_qmd_dev_va = svc_gr_cand.qmd_dev_va;
 	/*
 	 * Resolve the QMD buffer page now, on the service thread. A miss