vgsp-gr: capture the full 4K-word GL draw window, emit SET_SHADER_EXCEPTIONS as immd13, and base the SPH region on the bound program's shadow page

- SVC_GR_PUSH_WORDS / GSP_PHYS_GR_RAW_WORDS raised 2048 -> 4096: the beef
  full-frame push is 12288 B / 3072 words, so the 2048-word capture tail
  failed the 3D mark (raw_ok=0 fit_ok=0) and the launch fell back to a
  239-word stub, losing the real 2115-word draw. Now '3D stream chunked
  window=1429 words -> 4 chunk(s)'.
- SET_SHADER_EXCEPTIONS (0x1528) is emitted as the client's immd13
  (0x8000054a) and skipped when the window already carries it; the
  remaining 0x1528 in the log is the client's own benign write.
- sema_gl_sph_region_bind (default 1): the SPH region base is the first
  bound program's relocated shadow page, not the forced 0x1500000 pool
  base; r1484 latched the fetched SPHs (0,019a0f00,019a0200,0,0,019a0000)
  with ds=86040646 (region mismatch), r1485 uses base 0x19b0000 bind
  0x19b0900 and the DS accepts (ds=80000000).
- r1484/r1485: the screen now shows gear-class pixels (xwd bbox spanning
  the 800x600 window, multicolor); GUEST-WIT shows RT0 page 0 written. The
  remaining wall is the MME draw macro 0xf1 / pc 513 poison on the first
  calls+window launch after the arming hold is released.

3 files changed, 235 insertions, 21 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index 1cfa11f..657a418 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -18504,6 +18504,87 @@ static bool gsp_phys_gr_pair_present(const u32 *w, u32 n)
 }
 
 /*
+ * SESSION 154-1: true when the method run in [w, w+n) already writes
+ * `method` (byte offset). Used to keep the channel-init re-emission from
+ * duplicating a write the captured window itself carries: the X channel's
+ * per-frame window contains the client's own SET_SHADER_EXCEPTIONS
+ * (0x1528) run (sema-pushraw `[79] off=0x5a4 op=4 addr=0x1528`), so the
+ * booter's extra INC-form write is the first method the FE sees and the
+ * one its trapped-method register reports (r1483 `trap=80001528`).
+ *
+ * The run grammar is the same as every window walk in this file: op 1/3/5
+ * carry 1+count data words, op 4 is the client's one-word immd13 header
+ * (the 13-bit method field is in the header, no data word), everything
+ * else is a single header word. The low 13 bits are the method >> 2 for
+ * both the 0x2000 and 0x8000 header forms.
+ */
+static bool gsp_phys_gr_method_present(const u32 *w, u32 n, u32 method)
+{
+	u32 r = 0;
+
+	while (r < n) {
+		u32 h = w[r];
+		u32 op = (h >> 29) & 7u;
+		u32 cnt = (h >> 16) & 0x1fffu;
+		u32 span = 1u;
+
+		if ((op == 1u || op == 3u || op == 5u) && cnt &&
+		    r + 1u + cnt <= n)
+			span = 1u + cnt;
+		if (((h & 0x1fffu) << 2) == method)
+			return true;
+		r += span;
+	}
+	return false;
+}
+
+/*
+ * SESSION 154-1: first SET_PIPELINE_PROGRAM_ADDRESS pair in an already
+ * relocated window. Returns the pair's engine-visible (shadow) VA. The
+ * DS validates each fetched SPH against the channel's PROGRAM REGION;
+ * the reference driver programs that region with the program's own GPU
+ * base (nv3dGetProgramGpuAddress -> NV9097_SET_PROGRAM_REGION_A/B), not
+ * with the whole shader heap. The replay's engine fetch goes through the
+ * relocated shadow VA, so the matching region base is the bound VA's 4K
+ * page (e.g. beef bind 0x19a0f00 -> region 0x19a0000), not the shadow
+ * pool's base (the r1483/r1484 forced 0x1500000, which does not contain
+ * the beef program's 0x19a0000 page and makes the DS reject the fetched
+ * SPH: r1484 `GR-STATE sph=0,019a0f00,019a0200,...,019a0000` together
+ * with `ds=86040646`).
+ *
+ * Only op 1 (INC) can carry the two-word A/B pair; the run grammar is the
+ * same as the window walkers.
+ */
+static bool gsp_phys_gr_first_prog_bind(const u32 *w, u32 n, u64 *va_out)
+{
+	u32 r = 0;
+
+	while (r < n) {
+		u32 h = w[r];
+		u32 op = (h >> 29) & 7u;
+		u32 cnt = (h >> 16) & 0x1fffu;
+		u32 addr = (h & 0x1fffu) << 2;
+		u32 span = 1u;
+
+		if ((op == 1u || op == 3u || op == 5u) && cnt &&
+		    r + 1u + cnt <= n)
+			span = 1u + cnt;
+		if (op == 1u && cnt >= 2u && addr >= 0x2014u &&
+		    addr < 0x4000u && (addr & 0x3fu) == 0x14u) {
+			u64 pva = ((u64)(w[r + 1u] & 0xffu) << 32) |
+				  (u64)(w[r + 2u] & 0xfffffffcu);
+
+			if (pva) {
+				*va_out = pva;
+				return true;
+			}
+		}
+		r += span;
+	}
+	return false;
+}
+
+/*
  * Apply the pair-chain rule to the final emitted window (prepends
  * included). Returns the number of inserted drains; *np is updated.
  * The scan is method-run aligned and insertion copies whole runs, so a
@@ -19089,6 +19170,42 @@ MODULE_PARM_DESC(sema_gl_shader_exc,
 	"Emit SET_SHADER_EXCEPTIONS (0x1528) <- 0 after SET_OBJECT like _nv3dInitChannelTuring (default 1)");
 
 /*
+ * SESSION 154-1: encoding of the emitted SET_SHADER_EXCEPTIONS write.
+ *
+ * The client (and NVIDIA's _nv3dInitChannelTuring, which uses
+ * nvPushImmedVal) writes this method in the one-word immd13 form: header
+ * 0x80000000 | (value << 16) | (method >> 2), value 0, no data word - the
+ * captured init pushraw line is `off=0x5a4 op=4 addr=0x1528 cnt=0 v0=0`.
+ * The booter historically emitted the two-word INC form
+ * `0x2001054a 0x00000000`. Both encode the same register write, but the
+ * INC form is the method the FE's trapped-method register names
+ * (r1483 `trap=80001528`, mthd=0x1528) on every replay, while the
+ * client's own immd13 write is accepted (the X window's later 0x22c is
+ * what the trap advances to). Default 1 emits the client's exact form;
+ * A/B with 0 restores the legacy INC pair.
+ */
+static bool gsp_phys_rmcall_gr_shader_exc_immd = true;
+module_param_named(sema_gl_shader_exc_immd,
+		   gsp_phys_rmcall_gr_shader_exc_immd, bool, 0644);
+MODULE_PARM_DESC(sema_gl_shader_exc_immd,
+	"Emit SET_SHADER_EXCEPTIONS (0x1528) in the client's immd13 form (0x8000054a, one word) instead of the legacy INC pair (default 1)");
+
+/*
+ * SESSION 154-1: do not duplicate a window-carried SET_SHADER_EXCEPTIONS.
+ * The X channel's captured per-frame window carries the client's own
+ * 0x1528 run (the full channel-init block is inside it), so emitting the
+ * booter's copy again just adds an extra method ahead of the client's
+ * init. When this is on and the replayed window already contains a 0x1528
+ * write, the emission is skipped (logged). Default 1; A/B with 0 keeps
+ * the unconditional emission.
+ */
+static bool gsp_phys_rmcall_gr_shader_exc_skip = true;
+module_param_named(sema_gl_shader_exc_skip,
+		   gsp_phys_rmcall_gr_shader_exc_skip, bool, 0644);
+MODULE_PARM_DESC(sema_gl_shader_exc_skip,
+	"Skip the emitted SET_SHADER_EXCEPTIONS when the captured window already carries a 0x1528 write (default 1)");
+
+/*
  * SESSION 153-CONTINUED-91: clear DS_HWW_REPORT_MASK/_2 (0x405844/0x40584c)
  * ahead of every replayed 3D window (host write + in-stream MME PRI R/W
  * macro 5) so a DS SPH validation miss latches without poisoning the PRI
@@ -19130,6 +19247,28 @@ MODULE_PARM_DESC(sema_sph_region_base,
 	"Program-region base override (0 = launch program family page; nonzero = explicit engine-side base)");
 
 /*
+ * SESSION 154-1: prefer the bound program's own engine-visible VA page
+ * as the program region base. The reference driver sets the region to
+ * the program's GPU address (nv3dGetProgramGpuAddress); the replay's
+ * program is fetched at the relocated shadow VA, so the region must be
+ * that VA's 4K page. The forced sema_sph_region_base (0x1500000 in the
+ * campaign args) is the shadow pool's base: the X program's shadow
+ * 0x15c0000 lands inside it, but the beef program's 0x19a0000 does not,
+ * and the DS rejects the fetched SPH (r1484 `GR-STATE sph` stages 1/2/5
+ * = 0x19a0f00/0x19a0200/0x19a0000 together with `ds=86040646`). With
+ * this on, the builder scans the already-relocated window for the first
+ * SET_PIPELINE_PROGRAM_ADDRESS pair and uses that VA's page as the base
+ * (beef: 0x19a0000), falling back to the override/auto only when the
+ * window carries no program bind. Default 1; A/B with 0 uses the old
+ * forced/auto base.
+ */
+static bool gsp_phys_rmcall_gr_sph_region_bind = true;
+module_param_named(sema_gl_sph_region_bind,
+		   gsp_phys_rmcall_gr_sph_region_bind, bool, 0644);
+MODULE_PARM_DESC(sema_gl_sph_region_bind,
+	"Use the first bound program's relocated shadow page as the program-region base (default 1; 0 = forced/auto base)");
+
+/*
  * SESSION 153-CONTINUED-101: MME fixed-constant-buffer scratch seed
  * (SET_MME_SHADOW_SCRATCH(0x18), method 0x3460).
  *
@@ -20602,15 +20741,46 @@ if (gsp_phys_gr_slot_kept(j, sa_mask,
 		 */
 		if (READ_ONCE(gsp_phys_rmcall_gr_shader_exc)) {
 			static u32 shader_exc_log;
-
-			*p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) |
-			       (1u << 16) | (0x1528u >> 2);
-			*p++ = 0u;
-			if (shader_exc_log < 8u) {
-				shader_exc_log++;
-				pr_info("vgsp-gr: SHADER-EXC emitted 0x1528 <- 0 (2 words; pos=%u h=0x%x push=0x%llx)\n",
-					(u32)(p - pb), req->h_object,
-					req->push_va);
+			bool in_window =
+				READ_ONCE(gsp_phys_rmcall_gr_shader_exc_skip) &&
+				gsp_phys_gr_method_present(win3d, win3d_n,
+							   0x1528u);
+
+			if (in_window) {
+				if (shader_exc_log < 8u) {
+					shader_exc_log++;
+					pr_info("vgsp-gr: SHADER-EXC skipped (window carries 0x1528) h=0x%x push=0x%llx\n",
+						req->h_object,
+						req->push_va);
+				}
+			} else if (READ_ONCE(
+					   gsp_phys_rmcall_gr_shader_exc_immd)) {
+				/*
+				 * SESSION 154-1: the client's own
+				 * encoding (nvPushImmedVal): immd13
+				 * header with the 13-bit value in bits
+				 * 16..28 and NO following data word.
+				 * Value 0 = ENABLE FALSE.
+				 */
+				*p++ = 0x80000000u | (0x1528u >> 2);
+				if (shader_exc_log < 8u) {
+					shader_exc_log++;
+					pr_info("vgsp-gr: SHADER-EXC emitted 0x1528 <- 0 (1 word immd13 %08x; pos=%u h=0x%x push=0x%llx)\n",
+						*(p - 1), (u32)(p - pb),
+						req->h_object, req->push_va);
+				}
+			} else {
+				*p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM
+					<< 29) |
+				       (1u << 16) | (0x1528u >> 2);
+				*p++ = 0u;
+				if (shader_exc_log < 8u) {
+					shader_exc_log++;
+					pr_info("vgsp-gr: SHADER-EXC emitted 0x1528 <- 0 (2 words INC; pos=%u h=0x%x push=0x%llx)\n",
+						(u32)(p - pb),
+						req->h_object,
+						req->push_va);
+				}
 			}
 		}
 		/*
@@ -20745,8 +20915,24 @@ if (gsp_phys_gr_slot_kept(j, sa_mask,
 		if (READ_ONCE(gsp_phys_rmcall_gr_sph_region) && req->prog_va) {
 			static u32 sph_reg_log;
 			u64 base = READ_ONCE(gsp_phys_rmcall_gr_sph_region_base);
+			u64 bind_va = 0;
+			bool bind_base =
+				READ_ONCE(gsp_phys_rmcall_gr_sph_region_bind) &&
+				gsp_phys_gr_first_prog_bind(win3d, win3d_n,
+							    &bind_va);
 
-			if (!base)
+			/*
+			 * SESSION 154-1: the DS compares the fetched SPH
+			 * address against this region. The engine fetches
+			 * the program through its relocated shadow VA, so
+			 * the correct base is the bound pair's own shadow
+			 * page; the forced override/shadow-pool base is
+			 * only the fallback when the window binds nothing.
+			 */
+			if (bind_base)
+				base = bind_va &
+				       ~(u64)(GSP_PHYS_PAGE_SIZE - 1);
+			else if (!base)
 				base = req->prog_va &
 				       ~(u64)(GSP_PHYS_PAGE_SIZE - 1);
 
@@ -20758,8 +20944,10 @@ if (gsp_phys_gr_slot_kept(j, sa_mask,
 			*p++ = (u32)base;
 			if (sph_reg_log < 16u) {
 				sph_reg_log++;
-				pr_info("vgsp-gr: SPH-REGION emitted base=0x%llx (scratch 0x78/0x79) h=0x%x push=0x%llx\n",
-					base, req->h_object, req->push_va);
+				pr_info("vgsp-gr: SPH-REGION emitted base=0x%llx (scratch 0x78/0x79; src=%s bind=0x%llx) h=0x%x push=0x%llx\n",
+					base, bind_base ? "bind-shadow" :
+					"override/auto", bind_va,
+					req->h_object, req->push_va);
 			}
 		}
 		/*
@@ -21043,8 +21231,16 @@ if (gsp_phys_gr_slot_kept(j, sa_mask,
 		    READ_ONCE(gsp_phys_rmcall_gr_sph_region_legacy)) {
 			static u32 sph_reg_leg_log;
 			u64 base = READ_ONCE(gsp_phys_rmcall_gr_sph_region_base);
-
-			if (!base)
+			u64 bind_va = 0;
+			bool bind_base =
+				READ_ONCE(gsp_phys_rmcall_gr_sph_region_bind) &&
+				gsp_phys_gr_first_prog_bind(win3d, win3d_n,
+							    &bind_va);
+
+			if (bind_base)
+				base = bind_va &
+				       ~(u64)(GSP_PHYS_PAGE_SIZE - 1);
+			else if (!base)
 				base = req->prog_va &
 				       ~(u64)(GSP_PHYS_PAGE_SIZE - 1);
 			*p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) |
@@ -21055,8 +21251,10 @@ if (gsp_phys_gr_slot_kept(j, sa_mask,
 			*p++ = (u32)base;
 			if (sph_reg_leg_log < 16u) {
 				sph_reg_leg_log++;
-				pr_info("vgsp-gr: SPH-REGION-LEGACY emitted base=0x%llx (0x1608/0x160c) h=0x%x push=0x%llx\n",
-					base, req->h_object, req->push_va);
+				pr_info("vgsp-gr: SPH-REGION-LEGACY emitted base=0x%llx (0x1608/0x160c; src=%s bind=0x%llx) h=0x%x push=0x%llx\n",
+					base, bind_base ? "bind-shadow" :
+					"override/auto", bind_va,
+					req->h_object, req->push_va);
 			}
 		}
 		/*
diff --git a/kernel/gsp_phys_booter.h b/kernel/gsp_phys_booter.h
index 5ae5ae6..6689771 100644
--- a/kernel/gsp_phys_booter.h
+++ b/kernel/gsp_phys_booter.h
@@ -390,10 +390,14 @@ void gsp_phys_backend_relay_counters(u64 *requested, u64 *ok, u64 *verify_fail,
  * tail; the raw window is the source of the 3D method-stream replay.
  * SESSION 153-CONTINUED-45: 512 -> 2048 dwords (8 KB) for the DIRECT GL
  * client's 5016-7144 B render pushes (glcore); the raw window must carry
- * the whole push for the section capture's `len == push_n*4` gate. Must
- * equal SVC_GR_PUSH_WORDS (static_assert in gsp_sema_service.c).
+ * the whole push for the section capture's `len == push_n*4` gate.
+ * SESSION 154-1: 2048 -> 4096 dwords (16 KB) for the client's full-frame
+ * 12288-B pushes (r1483 `3D mark failed ... raw_ok=0 fit_ok=0
+ * rawlen=3072 pushn=2048`), so the chunked window replay
+ * (svc_gl_stream_chunks) can run instead of falling back. Must equal
+ * SVC_GR_PUSH_WORDS (static_assert in gsp_sema_service.c).
  */
-#define GSP_PHYS_GR_RAW_WORDS		2048u
+#define GSP_PHYS_GR_RAW_WORDS		4096u
 
 /*
  * B-2 (large-operand data path): the grown, boot-fixed GR data-bounce region.
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 18d5e78..327a932 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -7420,8 +7420,20 @@ static struct svc_fwd_cand svc_fwd_cand;
  * (CONT-44: zero src=stream captures on the client channel). The window
  * slice still has to fit the backend's method block; only the raw[]
  * retention is widened here.
- */
-#define SVC_GR_PUSH_WORDS	2048
+ *
+ * SESSION 154-1: 2048 -> 4096 dwords (16 KB). The direct client's
+ * occasional full-frame push is 12288 B (3072 dwords; r1483
+ * `section open (draw) push=0x4310110 len=12288`, window 2115 words at
+ * off 0xef4). With the 2048-word tail the window start fell before the
+ * retained tail, `len == push_n*4` failed and the whole capture was
+ * dropped: `3D mark failed ... raw_ok=0 fit_ok=0 (rawlen=3072
+ * pushn=2048 len=12288)`. The launch then replayed the previous
+ * window's fallback words and the client's real draw never reached the
+ * engine. 4096 words carries the whole 12288-B push so the existing
+ * method-aligned svc_gl_stream_chunks() split (<=448-word chunks, six
+ * per frame) can run. Must stay equal to GSP_PHYS_GR_RAW_WORDS.
+ */
+#define SVC_GR_PUSH_WORDS	4096
 #define SVC_GR_QMD_WORD_NONE	0xffffffffu
 /* svc_gr_cand.rel_flags bits (decoded from qmd[24] and qmd[4]). */
 #define SVC_GR_REL_REDUCTION	0x1u