vgsp-gr: deliver the real GL shader images and the client fixed constant buffer to the engine-visible pages

- inline-apply writes the image into the guest HPA whenever the page is kept
  at the guest VA (code_keep_guest), not only for stream3d windows; a shadow
  copy is useless when the engine fetches the guest page (r1470: guest=0 VS
  then lost to the window-0 shadow copy).
- the standalone 0x180/0x1b0 context-init upload pair (outside any C597
  section) is also recorded in the GRCE destination ring, so a later draw
  window can find the client's real VTG image at its bind VA.
- sema_gl_cb_copy_to / sema_gl_cb_src_va: copy the client fixed-CB page into
  the forced, mapped MME seed page before the launch, so the vertex shader
  reads the client's transform constants instead of screen/stale data. Falls
  back to a synthetic identity MVP when the client VA cannot be walked.
- cache any push chunk whose first word is a real SPH (020a0481 VTG /
  000a1482 PS) and use it as the program-record fallback when the window's
  own bind-VA upload was never captured (the init push's continuation chunks
  are otherwise judged implausible and never reach the ring).
- r1481/r1482: the real VS SPH+code and PS are applied guest=1 at the bind
  VA, the client CB page resolves and is copied, poison=0 over 9-10 launches;
  render-target writes still zero - the remaining wall is the draw's
  render-target/colour-target binding (the beef window's output retirement
  set is 10 pages of shader-like data, not the 900-page screen surface).

2 files changed, 301 insertions, 7 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index 62ab6eb..33e1b35 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -27933,7 +27933,21 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
 				 * into the guest page too, emulating the
 				 * upload that never executed.
 				 */
-				if (stream3d && pg->hpa)
+				/*
+				 * SESSION 154: a program page whose CODESRC
+				 * verdict keeps it at the guest VA is read by
+				 * the engine directly (code_keep_guest sets
+				 * mapped without a QMD rewrite). The inline
+				 * image MUST land in the guest HPA there even
+				 * when the window was not classified
+				 * stream3d: the shadow copy is then unused and
+				 * the engine fetches the empty guest page.
+				 * r1470: the complete 0x41c0000 window's VS
+				 * applied guest=0 into the shadow while the
+				 * guest page stayed zero (the win0 shadow copy
+				 * then restored the zero page as well).
+				 */
+				if ((stream3d || code_keep_guest) && pg->hpa)
 					memcpy((u8 *)__va(pg->hpa) +
 						       (inl->dst_va &
 							(GSP_PHYS_PAGE_SIZE - 1)),
@@ -27951,8 +27965,8 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
 						inl_apply_log++;
 						pr_info("vgsp-gr: INLINE-APPLY dst=0x%llx len=%u pg_va=0x%llx guest=%u first=%08x %08x\n",
 							inl->dst_va, len, pg_va,
-							stream3d && pg->hpa ?
-								1u : 0u,
+							(stream3d || code_keep_guest) &&
+								pg->hpa ? 1u : 0u,
 							len >= 4 ? get_unaligned_le32(inl->data) : 0u,
 							len >= 8 ? get_unaligned_le32(inl->data + 4) : 0u);
 					}
@@ -27967,7 +27981,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
 				 * still rejects the header, the bytes are known
 				 * present at the VA it fetched.
 				 */
-				if (stream3d && pg->hpa) {
+				if ((stream3d || code_keep_guest) && pg->hpa) {
 					static u32 inl_rb_log;
 					const u32 *gw = (const u32 *)
 						((u8 *)__va(pg->hpa) +
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 498198d..bfed3b7 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -7690,6 +7690,9 @@ static bool svc_gr_cand_push_seen(u64 push_va, u64 hash);
 /* SESSION 149 R16 (M3): trusted-walk VA read, defined below the decode. */
 static int svc_read_va_trusted(struct nv_vgpu_device *vgpu, u64 va, void *buf,
 			       size_t len);
+/* r1475: general VA read (trusted sources + vidmem window), defined below. */
+static int svc_read_va(struct nv_vgpu_device *vgpu, u64 va, void *buf,
+		       size_t len);
 /*
  * SESSION 149 R17 (M4): the section's memory-writing upload commands are
  * applied through the shared inline-write path (defined below the section
@@ -7709,6 +7712,10 @@ static void svc_inline_once_store(u64 push_va, u64 hash, u64 dst, u32 bytes);
  */
 static bool svc_grce_dst_newest(u64 dst_va, u32 min_len, u32 *len, u8 *buf,
 				u32 buf_sz);
+/* r1476: newest captured upload whose first word is the given SPH magic
+ * (used when a window's own bind-VA upload was never captured). */
+static bool svc_grce_dst_sph(u32 sph_w0, u32 min_len, u32 *len, u8 *buf,
+			     u32 buf_sz);
 
 /* QMD field decode: the exact shape the host CUDA bridge uses. */
 static void svc_gr_qmd_decode(struct svc_gr_cand_rec *c)
@@ -7875,6 +7882,37 @@ MODULE_PARM_DESC(sema_gl_xlate,
 	"GL/3D (C597) direct stream -> synthesized GR candidate (default 0 = off)");
 
 /*
+ * r1475: fixed-CB page copy into the forced MME seed page.
+ *
+ * The client's VS reads its transform constants from the fixed constant
+ * buffer whose base is captured by SET_MME_SHADOW_SCRATCH(0x18) (method
+ * 0x3460) as (VA >> 8). The GR launch builder forces the MME seed to a
+ * mapped page (sema_gl_mme_seed_val) so the engine always has a base, but
+ * that page holds screen/stale data, not the client's uniforms. When this
+ * is set, the service copies the client's real fixed-CB page into that
+ * forced seed page at decode time, before the launch, so the VS reads the
+ * client's constants; the MME's later constant writes (offsets
+ * 0x20/0x30/0x1480/0x1e0/0xc8) overwrite only their own offsets and the
+ * VS's 0x80.. region survives. 0 = off.
+ */
+static unsigned long long sema_gl_cb_copy_to;
+module_param_named(sema_gl_cb_copy_to, sema_gl_cb_copy_to, ullong, 0644);
+MODULE_PARM_DESC(sema_gl_cb_copy_to,
+	"VA of the forced MME seed page to receive the client's fixed-CB page (0 = off)");
+
+/*
+ * r1476: explicit fixed-CB source VA. The client's 0x3460 write is not always
+ * decoded before the first draw (the GL client's channel init can land after
+ * the first replayed window), so allow the caller to name the CB base
+ * directly (observed 0x0cd70000 for the glxgears beef channel). 0 = use the
+ * captured scratch18 value.
+ */
+static unsigned long long sema_gl_cb_src_va;
+module_param_named(sema_gl_cb_src_va, sema_gl_cb_src_va, ullong, 0644);
+MODULE_PARM_DESC(sema_gl_cb_src_va,
+	"explicit fixed-CB source VA for sema_gl_cb_copy_to (0 = captured scratch18)");
+
+/*
  * SESSION 153-CONTINUED-4: native publish-counter advance (pace-wall fix).
  *
  * The GLX client's swap-completion ladder waits for the u32 at +0x8 of the
@@ -8312,6 +8350,17 @@ MODULE_PARM_DESC(sema_gl_ps_hex,
 static u8 svc_gl_prog[GSP_BRIDGE_INLINE_BYTES];
 static u32 svc_gl_prog_len;
 static u8 svc_gl_vs[GSP_BRIDGE_INLINE_BYTES];
+/*
+ * r1481: shader-image cache filled from any push chunk that starts with a
+ * real SPH magic. The client's init push carries its VS/PS images as raw
+ * continuation chunks (first word 020a0481 / 000a1482) whose method
+ * headers sit in a zero first page; the normal upload decoder calls them
+ * implausible (unreadable=1), so the ring may never hold them. Cache the
+ * bytes here and let the program-record fallback author them at the bind
+ * VA (VS cache index 1, PS index 0).
+ */
+static u8 svc_gl_sph_cache[2][GSP_BRIDGE_INLINE_BYTES];
+static u32 svc_gl_sph_cache_len[2];
 static u32 svc_gl_vs_len;
 static u8 svc_gl_ps[GSP_BRIDGE_INLINE_BYTES];
 static u32 svc_gl_ps_len;
@@ -10033,9 +10082,42 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu,
 				u32 ri0 = n_recs;
 
 				memset(&prs[n_recs], 0, sizeof(prs[n_recs]));
-				if (svc_grce_dst_newest(svc_gl.progs[pi], 128u,
-							&rlen, prs[n_recs].data,
-							(u32)sizeof(prs[n_recs].data))) {
+				/*
+				 * r1476: when this window's own bind-VA
+				 * upload was never captured (the client's
+				 * per-window 0x1b0 pair can be lost), fall
+				 * back to the newest captured VTG SPH image
+				 * (020a0481) and deliver it at the bind VA.
+				 * The init push's real VS image is in the
+				 * ring from the standalone 0x180/0x1b0 arm;
+				 * without this a window whose VTG page is
+				 * empty executes an empty VS (r1477:
+				 * 0x41c0900 all zero, no pixels).
+				 */
+				{
+					bool got = svc_grce_dst_newest(
+						svc_gl.progs[pi], 128u, &rlen,
+						prs[n_recs].data,
+						(u32)sizeof(prs[n_recs].data));
+
+					if (!got && t != 5u && t != 0xffu) {
+						got = svc_grce_dst_sph(
+							0x020a0481u, 128u,
+							&rlen,
+							prs[n_recs].data,
+							(u32)sizeof(prs[n_recs].data));
+						if (!got &&
+						    svc_gl_sph_cache_len[1]) {
+							rlen = min_t(u32,
+								svc_gl_sph_cache_len[1],
+								(u32)sizeof(prs[n_recs].data));
+							memcpy(prs[n_recs].data,
+							       svc_gl_sph_cache[1],
+							       rlen);
+							got = true;
+						}
+					}
+					if (got) {
 					u64 d = svc_gl.progs[pi];
 					u32 head = 4096u - (u32)(d & 0xfff);
 
@@ -10101,6 +10183,7 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu,
 							get_unaligned_le32(prs[ri0].data));
 					}
 					continue;
+					}
 				}
 			}
 			/*
@@ -13309,6 +13392,27 @@ static bool svc_grce_dst_newest(u64 dst_va, u32 min_len, u32 *len, u8 *buf,
 	return false;
 }
 
+static bool svc_grce_dst_sph(u32 sph_w0, u32 min_len, u32 *len, u8 *buf,
+			     u32 buf_sz)
+{
+	u32 ri;
+
+	for (ri = 0; ri < SVC_GRCE_DST_MAX; ri++) {
+		u32 idx = (svc_grce_dst_ring_next + SVC_GRCE_DST_MAX - 1u -
+			   ri) % SVC_GRCE_DST_MAX;
+		const struct svc_grce_rec *r = &svc_grce_dst_ring[idx];
+
+		if (r->len < min_len || !r->dst_va)
+			continue;
+		if (get_unaligned_le32(r->data) != sph_w0)
+			continue;
+		*len = min_t(u32, r->len, buf_sz);
+		memcpy(buf, r->data, *len);
+		return true;
+	}
+	return false;
+}
+
 static void svc_counter_learn(struct nv_vgpu_device *vgpu, u32 h, u64 sem_va);
 static void svc_pending_add(u64 va, u32 payload, u32 op);
 static int svc_rescue_push_page(struct nv_vgpu_device *vgpu,
@@ -14974,6 +15078,137 @@ static void svc_scratch18_watch(const u8 *buf, u32 len, u64 push_va,
 }
 
 /*
+ * r1475: copy the client's fixed-CB page into the forced MME seed page.
+ *
+ * The VS reads its transform constants from the fixed constant buffer whose
+ * base the client publishes through SET_MME_SHADOW_SCRATCH(0x18) (method
+ * 0x3460) as (VA >> 8). The GR launch builder forces the MME seed to a
+ * mapped page (sema_gl_mme_seed_val) so the engine always has a base, but
+ * that page holds screen/stale data, not the client's uniforms. Copy the
+ * client's real fixed-CB page into the forced seed page at decode time,
+ * before the launch, so the VS reads the client's constants; the MME's
+ * later constant writes (offsets 0x20/0x30/0x1480/0x1e0/0xc8) overwrite
+ * only their own offsets and the VS's 0x80.. region survives.
+ *
+ * Trigger: once per decode pass for a beef GL channel, gated by a content
+ * hash of the source page. The 0x3460 write itself is a once-per-channel
+ * event, so keying the copy on that write alone would copy stale init-time
+ * constants forever; the client rewrites the CB page every frame, and the
+ * hash gate picks that up while skipping an unchanged page (and any
+ * re-decode of the same push). The copy is idempotent and runs before
+ * svc_gl_flush's launch in the same pass.
+ */
+static u32 svc_gl_cb_copy_h;
+static u32 svc_gl_cb_copy_val;
+static u32 svc_gl_cb_copy_hash;
+static u32 svc_gl_cb_copy_log;
+
+static u32 svc_gl_cb_hash(const u8 *p, u32 n)
+{
+	u32 h = 2166136261u;
+	u32 i;
+
+	for (i = 0; i < n; i++) {
+		h ^= p[i];
+		h *= 16777619u;
+	}
+	return h;
+}
+
+static void svc_gl_cb_copy(struct nv_vgpu_device *vgpu, u32 h_object)
+{
+	/* Static: 4 KB is too much for the kernel stack; this runs on the
+	 * single service thread (same reason as the static arrays in
+	 * svc_gl_flush and svc_decode_stream). */
+	static u8 buf[SEMA_PAGE_SIZE];
+	u64 src_va, pa = 0;
+	u32 ap = 0, val, hash;
+	const char *src = "resolve";
+	int rc;
+
+	if (!sema_gl_cb_copy_to)
+		return;
+	if ((h_object & 0xfffff000u) != 0xbeef0000u)
+		return;
+	if (sema_gl_cb_src_va) {
+		src_va = sema_gl_cb_src_va;
+		val = (u32)(src_va >> 8);
+	} else {
+		if (!svc_scratch18_seen)
+			return;
+		val = READ_ONCE(svc_scratch18_val);
+		if (!val)
+			return;
+		src_va = (u64)val << 8;
+	}
+	rc = svc_resolve_va(vgpu, src_va, &pa, &ap);
+	if (rc) {
+		/* Fallback 1: the trusted walk svc_gl_flush uses for the
+		 * program page. */
+		rc = svc_read_va_trusted(vgpu, src_va, buf, sizeof(buf));
+		if (!rc) {
+			src = "trusted";
+			goto have;
+		}
+		/* Fallback 2: the general lookup (adds the vidmem window
+		 * convention). */
+		rc = svc_read_va(vgpu, src_va, buf, sizeof(buf));
+		if (!rc) {
+			src = "lookup";
+			goto have;
+		}
+		/*
+		 * r1476: the client's fixed-CB VA is a GPU/vidmem VA the
+		 * service cannot walk (resolve -2 even through the trusted
+		 * and general fallbacks). Rather than leave the VS reading
+		 * screen data, author a synthetic identity MVP at the
+		 * position block the VS reads (c[0x0][0x80..0xbc], one
+		 * 4x4 row-major matrix). The vertices then pass through in
+		 * model space; the viewport prelude maps NDC to the RT. This
+		 * is a controlled positive control for the raster path: if
+		 * pixels appear, the constant CONTENT is the only remaining
+		 * unknown; if not, the raster/clip path is still wrong.
+		 */
+		memset(buf, 0, sizeof(buf));
+		put_unaligned_le32(0x3f800000u, buf + 0x80);	/* m00 */
+		put_unaligned_le32(0x3f800000u, buf + 0x94);	/* m11 */
+		put_unaligned_le32(0x3f800000u, buf + 0xa8);	/* m22 */
+		put_unaligned_le32(0x3f800000u, buf + 0xbc);	/* m33 */
+		src = "synthetic-id";
+		goto have;
+	}
+	if (svc_read_ap(vgpu, pa, ap, buf, sizeof(buf))) {
+		if (svc_gl_cb_copy_log < 8u) {
+			svc_gl_cb_copy_log++;
+			pr_info("vgsp-sema: sema-gl: cb-copy src=0x%llx dst=0x%llx resolve=0 ap=%u read failed\n",
+				src_va, sema_gl_cb_copy_to, ap);
+		}
+		return;
+	}
+have:
+	hash = svc_gl_cb_hash(buf, sizeof(buf));
+	if (svc_gl_cb_copy_h == h_object && svc_gl_cb_copy_val == val &&
+	    svc_gl_cb_copy_hash == hash)
+		return;
+	rc = svc_grce_inline_write(vgpu, sema_gl_cb_copy_to, buf, sizeof(buf),
+				   h_object);
+	if (svc_gl_cb_copy_log < 8u) {
+		u32 w0, w1;
+
+		memcpy(&w0, buf, 4);
+		memcpy(&w1, buf + 4, 4);
+		svc_gl_cb_copy_log++;
+		pr_info("vgsp-sema: sema-gl: cb-copy src=0x%llx dst=0x%llx resolve=%d ap=%u first=%08x %08x src=%s write=%d (fixed-CB page -> forced seed page)\n",
+			src_va, sema_gl_cb_copy_to, 0, ap, w0, w1, src, rc);
+	}
+	if (!rc) {
+		svc_gl_cb_copy_h = h_object;
+		svc_gl_cb_copy_val = val;
+		svc_gl_cb_copy_hash = hash;
+	}
+}
+
+/*
  * SESSION 153-CONTINUED-94: client init-push capture + local-memory census.
  *
  * CONT-93's "single most concrete next fix" was that the GL driver's
@@ -16481,6 +16716,11 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 	if (ch)
 		svc_scratch18_watch(buf, len, push_va, ch->h_object);
 
+	/* r1475: feed the client's real fixed-CB page into the forced MME
+	 * seed page before this pass's launch (beef GL channels only). */
+	if (ch)
+		svc_gl_cb_copy(vgpu, ch->h_object);
+
 	/* SESSION 153-CONTINUED-94: init-push capture + local-memory census. */
 	if (ch)
 		svc_init_push_scan(buf, len, push_va, ch->h_object);
@@ -16569,6 +16809,17 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 			len >= 4 ? *(const u32 *)(buf + 0) : 0,
 			len >= 8 ? *(const u32 *)(buf + 4) : 0);
 	}
+	/* r1481: cache a raw shader image (SPH magic first word). */
+	if (buf && len >= 128u && len <= sizeof(svc_gl_sph_cache[0])) {
+		u32 w0 = *(const u32 *)buf;
+		int ci = (w0 == 0x020a0481u) ? 1 :
+			 (w0 == 0x000a1482u) ? 0 : -1;
+
+		if (ci >= 0) {
+			memcpy(svc_gl_sph_cache[ci], buf, len);
+			svc_gl_sph_cache_len[ci] = len;
+		}
+	}
 	if (gsp_sema_dbg && svc_dbg_streams < 96) {
 		svc_dbg_streams++;
 		pr_info("sema-dbg: stream h=0x%x eng=%u va=0x%llx len=%u allow=%d w=%08x %08x %08x %08x %08x %08x %08x %08x\n",
@@ -17213,6 +17464,35 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 						svc_gl_spot_dst,
 						buf + off + 8, bytes,
 						ch->h_object);
+					/*
+					 * SESSION 154: the context-init
+					 * upload pair sits OUTSIDE any
+					 * C597 section (start_off is
+					 * later), so it never reaches
+					 * svc_gl.up[]. The program page
+					 * of a later draw still needs
+					 * these bytes: the booter finds
+					 * them through the GRCE
+					 * destination ring and attaches
+					 * them at the bind VA. Record the
+					 * pair there too, or the client's
+					 * real VS image stays invisible
+					 * and every beef window executes
+					 * an empty VTG page.
+					 */
+					{
+						struct svc_grce_rec *rr =
+							&svc_grce_dst_ring
+							[svc_grce_dst_ring_next %
+							 SVC_GRCE_DST_MAX];
+
+						rr->dst_va = svc_gl_spot_dst;
+						rr->len = min_t(u32, bytes,
+							GSP_BRIDGE_INLINE_BYTES);
+						memcpy(rr->data, buf + off + 8,
+						       rr->len);
+						svc_grce_dst_ring_next++;
+					}
 					if (gsp_sema_dbg &&
 					    svc_gl_lid_log < 24) {
 						svc_gl_lid_log++;