vgsp: 3D method-stream replay lane (verbatim C597 window + shadow relocation + release tail; engine retires real 3D methods, no QMD) (SESSION 153 r932)

1 files changed, 205 insertions, 10 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index 4e6acc5..f7b43fb 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -16865,6 +16865,22 @@ static u32 gsp_phys_rmcall_gr_emit_acquires(u32 *p,
 }
 
 /*
+ * r929 (GL 3D execution, Path A): replay a captured raw 3D method window.
+ * The sema service marks stream captures (the X server's C597 sections that
+ * carried their own operands) with a raw window (req->stream); when this is
+ * on, gsp_phys_rmcall_build_gr_launch emits SET_OBJECT TURING_A + the window
+ * verbatim + the channel release tail instead of the compute prelude/QMD, so
+ * the engine executes the REAL 3D methods (state binds, LINE_LENGTH_IN +
+ * LAUNCH_DMA uploads, program/tex binds) and, once draws are in the window,
+ * rasterizes. No effect on any submit without a stream window (all CUDA
+ * launches). Default on; A/B with gsp_phys_rmcall_gr_3d=0.
+ */
+static bool gsp_phys_rmcall_gr_3d = true;
+module_param_named(gsp_phys_rmcall_gr_3d, gsp_phys_rmcall_gr_3d, bool, 0644);
+MODULE_PARM_DESC(gsp_phys_rmcall_gr_3d,
+	"Replay a captured raw C597 method window verbatim instead of the compute prelude (default 1)");
+
+/*
  * GR launch method block: the inline QMD run at its original shape and the
  * channel REPORT_SEMAPHORE release. The stock guest push does NOT emit
  * SET_OBJECT before the inline run (the compute class is bound on the
@@ -16874,7 +16890,8 @@ static u32 gsp_phys_rmcall_gr_emit_acquires(u32 *p,
 static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb,
 					   const struct gsp_phys_gr_submit *req,
 					   const u32 *qmd, u64 qmd_va,
-					   u64 sem_va, u32 sem_payload)
+					   u64 sem_va, u32 sem_payload,
+					   const u32 *win3d, u32 win3d_n)
 {
 	u32 *p = pb;
 	int prelude = gsp_phys_rmcall_gr_prelude_value();
@@ -16882,6 +16899,51 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb,
 	int two_step = gsp_phys_rmcall_gr_pcas_two_step_value();
 
 	/*
+	 * r929 (Path A): a captured 3D method window replays verbatim. Bind
+	 * TURING_A (0xC597) on the channel subchannel (the same bind the
+	 * window itself carries - belt and braces, a rebind is harmless),
+	 * then the window words (address operands relocated to shadow-pool
+	 * VAs by gsp_phys_rmcall_gr_3d_patch_window), then the channel
+	 * release tail. No compute init, no QMD, no PCAS: the engine walks
+	 * real 3D methods (SET_OBJECT C597, state binds, LINE_LENGTH_IN +
+	 * LAUNCH_DMA inline uploads, program/tex binds). Completion is the
+	 * trailing REPORT_SEMAPHORE alone (the submit path leaves qmd_rel
+	 * off for this mode).
+	 */
+	if (gsp_phys_rmcall_gr_3d && req && win3d && win3d_n &&
+	    win3d_n + 2u + 8u <= GSP_PHYS_RMCALL_GR_METHOD_WORDS) {
+		*p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | (1u << 16) |
+		       (0u << 13) |
+		       (GSP_PHYS_RMCALL_GR_SET_OBJECT >> 2);
+		*p++ = GSP_PHYS_RMCALL_CLASS_GR_3D;
+		memcpy(p, win3d, (size_t)win3d_n * 4);
+		p += win3d_n;
+		p = gsp_phys_rmcall_push_gr(p,
+					    GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_A,
+					    (u32)(sem_va >> 32));
+		p = gsp_phys_rmcall_push_gr(p,
+					    GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_B,
+					    (u32)sem_va);
+		p = gsp_phys_rmcall_push_gr(p,
+					    GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_C,
+					    sem_payload);
+		p = gsp_phys_rmcall_push_gr(p,
+					    GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_D,
+					    GSP_PHYS_RMCALL_GR_SEM_RELEASE_D);
+		{
+			static u32 gr3d_log;
+
+			if (gr3d_log < 16) {
+				gr3d_log++;
+				pr_info("vgsp-gr: 3D STREAM replay h=0x%x push=0x%llx class=0xc597 window=%u words=%u verbatim (no prelude/QMD)\n",
+					req->h_object, req->push_va,
+					win3d_n, (u32)(p - pb));
+			}
+		}
+		return (u32)(p - pb);
+	}
+
+	/*
 	 * r425 A/B: with gsp_phys_rmcall_gr_init_once the compute init preamble
 	 * is emitted only on the first forwarded launch (the stock driver's
 	 * per-channel shape); subsequent launches start at the inline-QMD
@@ -19228,6 +19290,104 @@ static void gsp_phys_gr_stall_log(const char *reason, u64 push_va,
 		reason, push_va, jiffies_to_msecs(jiffies - start), action);
 }
 
+/*
+ * r929 v3 (Path A): relocate one 3D-window address operand to a shadow-pool
+ * VA. The guest VAs the 3D methods name (uploads 0x1_214f8f00.., vertex,
+ * program, tex pools) are installed in the ladder page tree by the Tier-1
+ * mapper yet the engine's FE/PD walks still fault FAULT_PDE on them (r929
+ * v2 evidence: "ENGINE GRAPHICS HUBCLIENT_FE faulted @ 0x1_214f8000 ...
+ * FAULT_PDE"); the shadow pool lines (map_shadow_locked) ARE the proven
+ * engine-visible region (every CUDA data/page relocation lives there). The
+ * page offset is preserved; a VA whose 4 KB page has no resolved hpa stays
+ * verbatim (it faults later and is named in the drain). Called with
+ * gsp_phys_rm_lock held.
+ */
+static bool gsp_phys_rmcall_gr_3d_reloc(u32 *up_w, u32 *lo_w,
+					const struct gsp_phys_gr_submit *req,
+					u32 *n_shadow)
+{
+	u64 va = ((u64)(*up_w & 0xffu) << 32) | (u64)(*lo_w & 0xfffffffcu);
+	u64 page = va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1);
+	u64 sh = 0;
+	u32 line = 0, i;
+
+	if (!va || !page)
+		return false;
+	for (i = 0; i < req->n_pages; i++) {
+		if ((req->pages[i].guest_va &
+		     ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) != page ||
+		    !req->pages[i].hpa)
+			continue;
+		if (*n_shadow >= GSP_PHYS_RMCALL_GR_SHADOW_PAGES)
+			return false;
+		if (gsp_phys_rmcall_gr_map_shadow_locked(*n_shadow,
+							 req->pages[i].hpa,
+							 &sh, &line))
+			return false;
+		if (line != 0xffffffffu)
+			(*n_shadow)++;	/* alias hit consumes no slot */
+		sh |= va & (GSP_PHYS_PAGE_SIZE - 1);
+		*lo_w = (*lo_w & 0x3u) | ((u32)sh & 0xfffffffcu);
+		*up_w = (*up_w & ~0xffu) | ((u32)(sh >> 32) & 0xffu);
+		return true;
+	}
+	return false;
+}
+
+/*
+ * r929 v3: copy the 3D window and relocate every address operand a 3D-class
+ * method run carries (same recognizers as the sema service's svc_gl_field:
+ * LINE_LENGTH_IN 0x180 OFFSET_OUT pair, SET_VERTEX_STREAM_SUBSTITUTE 0xf84,
+ * SET_TEX_SAMPLER_POOL 0x155c, SET_TEX_HEADER_POOL 0x1574 and the
+ * SET_PIPELINE_PROGRAM_ADDRESS 0x2014/0x2018+j*0x40 pairs). Returns the
+ * number of relocated pair(s). Run widths follow the service's decoder:
+ * op 1/3/5 are 1+count dwords, everything else resyncs one dword.
+ */
+static u32 gsp_phys_rmcall_gr_3d_patch_window(const u32 *in, u32 n, u32 *out,
+					      const struct gsp_phys_gr_submit *req,
+					      u32 *n_shadow)
+{
+	u32 off = 0, reloc = 0;
+
+	memcpy(out, in, (size_t)n * 4);
+	while (off < n) {
+		u32 hdr = out[off];
+		u32 op = (hdr >> 29) & 0x7u;
+		u32 cnt = (hdr >> 16) & 0xfffu;
+		u32 addr = (hdr & 0x1fffu) << 2;
+
+		if ((op == 1 || op == 3 || op == 5) && cnt &&
+		    off + 1u + cnt <= n) {
+			u32 *d = &out[off + 1];
+			u32 lo_i = 0;
+			bool pair = false;
+
+			if (op == 1 && addr == 0x0180 && cnt >= 4) {
+				lo_i = 2;	/* OFFSET_OUT is v2/v3 */
+				pair = true;
+			} else if (op == 1 && addr == 0x0f84 && cnt >= 2) {
+				pair = true;
+			} else if (op == 1 && addr == 0x155c && cnt >= 3) {
+				pair = true;
+			} else if (op == 1 && addr == 0x1574 && cnt >= 3) {
+				pair = true;
+			} else if (op == 1 && addr >= 0x2014 &&
+				   addr < 0x4000 && (addr & 0x3fu) == 0x14 &&
+				   cnt >= 2) {
+				pair = true;
+			}
+			if (pair && gsp_phys_rmcall_gr_3d_reloc(&d[lo_i],
+								&d[lo_i + 1],
+								req, n_shadow))
+				reloc++;
+			off += 1u + cnt;
+			continue;
+		}
+		off++;	/* header-only / resync, like the service decoder */
+	}
+	return reloc;
+}
+
 int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
 				     struct gsp_phys_backend_copy_result *out)
 {
@@ -19245,6 +19405,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
 	 * under the warning bound.
 	 */
 	static u32 methods[GSP_PHYS_RMCALL_GR_METHOD_WORDS];
+	static u32 stream_patch[GSP_PHYS_RMCALL_GR_METHOD_WORDS];
 	static u32 qmd[GSP_PHYS_GR_QMD_WORDS];
 	static struct gsp_phys_fault_info fault;
 	unsigned long start = jiffies;
@@ -19255,6 +19416,17 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
 	const u8 *guest_param_pg;
 	u32 want_qmd = 0, want_ch = 0;
 	bool qmd_rel = false, ch_rel = false;
+	/*
+	 * r929 (Path A): this submission is a raw 3D method window. The QMD
+	 * machinery (qmdwin relocation, QMD prewrite, release0 redirect,
+	 * PCAS) does not apply: the block is SET_OBJECT C597 + the window +
+	 * the channel release tail, and completion is the channel release
+	 * alone (qmd_rel stays off so the poll does not wait for a QMD
+	 * release word no engine will write).
+	 */
+	bool stream3d = gsp_phys_rmcall_gr_3d && req && req->stream_n;
+	const u32 *win3d = NULL;
+	u32 win3d_n = 0;
 	bool qmd_rel_redirected = false;	/* r420b: release0 -> module sem */
 	u64 qmd_rel_hpa = 0, ch_rel_hpa = 0;
 	u64 ch_sem_va = GSP_PHYS_RMCALL_VA_SEM + 4;
@@ -19534,9 +19706,10 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
 	 * a fresh shadow line so the guest observes its own QMD release; else
 	 * point release0 at the module semaphore so the host observes engine
 	 * completion. A reduction release is never redirected (its arithmetic
-	 * would change).
+	 * would change). r929: skipped entirely for a 3D stream replay -
+	 * there is no QMD, so there is no release0 to point anywhere.
 	 */
-	if (((qmd[4] >> 10) & 1)) {
+	if (!stream3d && ((qmd[4] >> 10) & 1)) {
 		u32 red_en = (qmd[24] >> 26) & 1;
 		u32 one_word = (qmd[24] >> 31) & 1;
 		bool rel_mod = gsp_phys_rmcall_gr_rel_mod_value() != 0;
@@ -20270,8 +20443,9 @@ gr_page_unmapped:
 	 * GPFIFO/PB/REPORT_SEM legs already use. The 2 MiB window is installed
 	 * by the ladder itself, so no window ensure is sent for it (a
 	 * pdeIndex-0 Tier-1 install would touch the ladder's own PDE).
+	 * r929: a 3D stream replay has no inline QMD at all (skipped).
 	 */
-	if (gsp_phys_rmcall_gr_qmdwin_value()) {
+	if (!stream3d && gsp_phys_rmcall_gr_qmdwin_value()) {
 		u8 *qw = boot_stage.rmcall_gr_qmdwin.pages ?
 			gsp_phys_chunk_page_va(&boot_stage.rmcall_gr_qmdwin, 0) :
 			NULL;
@@ -20298,9 +20472,9 @@ gr_page_unmapped:
 	 * use it as the destination when the caller resolved it; a Tier-1
 	 * install keeps the guest VA (the guest sees its own QMD), a Tier-2
 	 * alias keeps the launch runnable, and an unresolved page keeps the
-	 * module scratch VA.
+	 * module scratch VA. r929: skipped for a 3D stream replay.
 	 */
-	if (!qmdwin_on && req->qmd_hpa) {
+	if (!stream3d && !qmdwin_on && req->qmd_hpa) {
 		u64 qmd_page = req->qmd_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1);
 		u32 qmd_off = (u32)(req->qmd_va & (GSP_PHYS_PAGE_SIZE - 1));
 		bool dst_ok = false;
@@ -20418,9 +20592,28 @@ gr_page_unmapped:
 			GSP_PHYS_GR_QMD_SENTINEL);
 	}
 
+	/*
+	 * r929 v3 (Path A): relocate the 3D window's address operands to
+	 * shadow-pool VAs the engine walkers can reach (see
+	 * gsp_phys_rmcall_gr_3d_patch_window; the r929 v2 FE faulted
+	 * FAULT_PDE on a Tier-1-mapped guest VA). Runs with the rm lock
+	 * held, before the method block is built.
+	 */
+	if (stream3d) {
+		u32 nrel = gsp_phys_rmcall_gr_3d_patch_window(
+			req->stream, req->stream_n, stream_patch, req,
+			&n_shadow);
+
+		win3d = stream_patch;
+		win3d_n = req->stream_n;
+		pr_info("vgsp-gr: 3D window relocated %u operand pair(s) to shadow VAs (window=%u shadow=%u)\n",
+			nrel, win3d_n, n_shadow);
+	}
+
 	words = gsp_phys_rmcall_build_gr_launch(methods, req, qmd,
 						qmd_dst,
-						ch_sem_va, want_ch);
+						ch_sem_va, want_ch,
+						win3d, win3d_n);
 	memcpy(gsp_phys_chunk_page_va(pb, 0), methods, words * 4);
 	pr_info("vgsp-gr: qmd w0..3=%08x %08x %08x %08x w4..7=%08x %08x %08x %08x w23..25=%08x %08x %08x w32/33=%08x %08x w48/49=%08x %08x\n",
 		qmd[0], qmd[1], qmd[2], qmd[3], qmd[4], qmd[5], qmd[6], qmd[7],
@@ -21636,7 +21829,8 @@ gr_page_unmapped:
 						words = gsp_phys_rmcall_build_gr_launch(
 							methods, req, qmd,
 							qmd_dst, ch_sem_va,
-							want_ch);
+							want_ch, win3d,
+							win3d_n);
 						memcpy(gsp_phys_chunk_page_va(pb, 0),
 						       methods, words * 4);
 					}
@@ -25193,10 +25387,11 @@ static_assert(offsetof(struct gsp_phys_gr_inline, data) == 12 &&
 static_assert((2u + 1u + GSP_PHYS_RMCALL_GR_QMD_INLINE_WORDS + 8u) <=
 	      GSP_PHYS_RMCALL_GR_METHOD_WORDS,
 	      "the GR launch method block (compute init + SET_OBJECT + inline-QMD run + PCAS + REPORT_SEMAPHORE) must fit the bound");
-static_assert(GSP_PHYS_RMCALL_GR_INIT_WORDS + GSP_PHYS_GR_RAW_WORDS +
+static_assert(GSP_PHYS_RMCALL_GR_INIT_WORDS +
+	      GSP_PHYS_RMCALL_GR_QMD_WINDOW_WORDS +
 	      GSP_PHYS_RMCALL_GR_TAIL_WORDS <=
 	      GSP_PHYS_RMCALL_GR_METHOD_WORDS,
-	      "the verbatim GR launch block (compute init + raw tail + PCAS + REPORT_SEMAPHORE) must fit the bound");
+	      "the verbatim GR launch block minimum (compute init + window + PCAS + REPORT_SEMAPHORE) must fit the bound; the per-launch runtime check bounds raw_qmd_hdr for larger preludes");
 static_assert(GSP_PHYS_RMCALL_GR_METHOD_WORDS * 4u <=
 	      GSP_PHYS_RMCALL_COPY_SIZE,
 	      "the GR launch method block must fit one module pushbuffer page");