vgsp-gl: decode the client fixed-CB upload runs and gate the assembled image

1 files changed, 295 insertions, 0 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index c5a2eda..2f08064 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -15789,6 +15789,272 @@ static void svc_gl_cb_dump_matrix(const u8 *buf, const char *src, u64 pa,
 }
 
 /*
+ * r1515: client fixed-CB upload capture.
+ *
+ * The client's fixed constant buffer is not a page the guest writes with a
+ * plain memory store: the driver publishes its base once per context
+ * (SET_MME_SHADOW_SCRATCH(0x18), method 0x3460, value = VA >> 8) and then
+ * streams the constants through the C597 constant-buffer methods:
+ *   - 0x2380/0x2384/0x2388 SET_CONSTANT_BUFFER_SELECTOR_A/B/C
+ *     (A size, B address[39:32], C address[31:0]);
+ *   - 0x238c LOAD_CONSTANT_BUFFER_OFFSET (byte offset into the store);
+ *   - 0x2390 LOAD_CONSTANT_BUFFER(i) data runs (NON_INC streams the
+ *     constants from the current offset; INC/ONE_INC index them).
+ * r1512/r1514 measured the consequence of not applying them: the page the
+ * VS's c[0x0] bank resolves to holds screen/rotation data, the position
+ * block at 0x80 is not a projection, and 57/64 sampled gear vertices clip.
+ *
+ * This scanner runs on every decoded push of the beef GL channel (the
+ * 60216-byte setup push carries the 0x3460 write at off 0x464c and the
+ * constant stream after it). It maintains a 4 KB image of the first CB
+ * page (the VS reads c[0x0][0x80..0x888], all inside page 0), validates it
+ * with the strict projection gate, and only then marks it authoritative.
+ * svc_gl_cb_copy() prefers this image over any memory-page discovery and
+ * writes it into the forced MME seed page, so the replayed VS reads the
+ * client's real transform. The write is a pure emulated engine store (the
+ * client page is authored through the same inline-write path when it
+ * resolves). No FP in the kernel.
+ */
+static u8 svc_gl_cb_img[SEMA_PAGE_SIZE];
+static bool svc_gl_cb_img_valid;
+static u64 svc_gl_cb_img_base;
+static u64 svc_gl_cb_img_words;
+static u64 svc_gl_cb_img_runs;
+static u32 svc_gl_cb_off;
+static u32 svc_gl_cb_sel_a;
+static u64 svc_gl_cb_sel_bc;
+static bool svc_gl_cb_sel_seen;
+static u32 svc_gl_cb_scan_log;
+static u32 svc_gl_cb_apply_log;
+
+#define SVC_GL_CB_IMG_LIMIT SEMA_PAGE_SIZE
+
+static void svc_gl_cb_img_reset(u64 base)
+{
+	memset(svc_gl_cb_img, 0, sizeof(svc_gl_cb_img));
+	svc_gl_cb_img_valid = false;
+	svc_gl_cb_img_base = base;
+	svc_gl_cb_img_words = 0;
+	svc_gl_cb_img_runs = 0;
+	svc_gl_cb_off = 0;
+}
+
+static void svc_gl_cb_upload_scan(struct nv_vgpu_device *vgpu, u32 h_object,
+				  const u8 *buf, u32 len, u64 push_va)
+{
+	u32 off = 0, runs = 0, words = 0;
+	u64 base = 0;
+	bool dirty = false;
+	u32 i;
+
+	(void)vgpu;
+	if ((h_object & 0xfffff000u) != 0xbeef0000u || !buf || len < 16u)
+		return;
+	/*
+	 * A new SET_MME_SHADOW_SCRATCH(0x18) names a new fixed-CB base:
+	 * discard the previous image so a stale transform can never be
+	 * applied to a new context.
+	 */
+	if (svc_scratch18_seen && READ_ONCE(svc_scratch18_val))
+		base = (u64)READ_ONCE(svc_scratch18_val) << 8;
+	if (base && base != svc_gl_cb_img_base)
+		svc_gl_cb_img_reset(base);
+	while (off + 4u <= len) {
+		u32 hdr, op, cnt, addr, w;
+
+		memcpy(&hdr, buf + off, 4);
+		if (!hdr) {
+			off += 4u;
+			continue;
+		}
+		op = (hdr >> 29) & 7u;
+		cnt = (hdr >> 16) & 0xfffu;
+		addr = (hdr & 0x1fffu) << 2;
+		if (op == 0u || op == 2u || op == 7u) {
+			off += 4u;
+			continue;
+		}
+		if (op == 4u) {
+			/* IMMD: 13-bit payload; offset/selector only. */
+			w = (hdr >> 16) & 0x1fffu;
+			if (addr == 0x3460u) {
+				base = (u64)w << 8;
+				if (base && base != svc_gl_cb_img_base)
+					svc_gl_cb_img_reset(base);
+			} else if (addr == 0x238cu) {
+				svc_gl_cb_off = w;
+				dirty = true;
+			} else if (addr == 0x2384u) {
+				svc_gl_cb_sel_bc = (svc_gl_cb_sel_bc &
+						    ~0xffffffffULL) |
+						   (u64)w;
+				svc_gl_cb_sel_seen = true;
+			} else if (addr == 0x2388u) {
+				svc_gl_cb_sel_bc = (svc_gl_cb_sel_bc &
+						   0xffffffffULL) |
+						   ((u64)w << 32);
+				svc_gl_cb_sel_seen = true;
+			}
+			off += 4u;
+			continue;
+		}
+		if (op != 1u && op != 3u && op != 5u) {
+			off += 4u;
+			continue;
+		}
+		if (!cnt || cnt > 0xfffu ||
+		    off + 4u + cnt * 4u > len) {
+			/*
+			 * A run that overruns this push (or a data word that
+			 * happens to decode as a header) is not the end of the
+			 * stream: the client's setup push interleaves large
+			 * inline payloads with method runs. Resync one word
+			 * later, exactly like svc_scratch18_watch does, so the
+			 * constant stream after the data block is still found.
+			 */
+			off += 4u;
+			continue;
+		}
+		if (addr == 0x238cu) {
+			memcpy(&w, buf + off + 4u, 4);
+			svc_gl_cb_off = w;
+			dirty = true;
+			off += 4u + cnt * 4u;
+			continue;
+		}
+		if (addr >= 0x2380u && addr <= 0x238cu) {
+			u32 j;
+
+			for (j = 0; j < cnt; j++) {
+				u32 m = (op == 3u) ? addr : addr + 4u * j;
+
+				memcpy(&w, buf + off + 4u + 4u * j, 4);
+				switch (m) {
+				case 0x2380u:
+					svc_gl_cb_sel_a = w;
+					break;
+				case 0x2384u:
+					svc_gl_cb_sel_bc =
+						(svc_gl_cb_sel_bc &
+						 ~0xffffffffULL) |
+						(u64)w;
+					svc_gl_cb_sel_seen = true;
+					break;
+				case 0x2388u:
+					svc_gl_cb_sel_bc =
+						(svc_gl_cb_sel_bc &
+						 0xffffffffULL) |
+						((u64)w << 32);
+					svc_gl_cb_sel_seen = true;
+					break;
+				case 0x238cu:
+					svc_gl_cb_off = w;
+					dirty = true;
+					break;
+				default:
+					break;
+				}
+			}
+		} else if (addr == 0x3460u) {
+			memcpy(&w, buf + off + 4u, 4);
+			base = (u64)w << 8;
+			if (base && base != svc_gl_cb_img_base)
+				svc_gl_cb_img_reset(base);
+		} else if (addr >= 0x2390u && addr < 0x2390u + 0x1000u) {
+			u32 idx = (addr - 0x2390u) / 4u;
+
+			for (i = 0; i < cnt; i++) {
+				u64 doff;
+
+				memcpy(&w, buf + off + 4u + 4u * i, 4);
+				/*
+				 * NON_INC streams from the published offset;
+				 * INC/ONE_INC index from the run's method.
+				 */
+				if (op == 3u)
+					doff = (u64)svc_gl_cb_off +
+					       4ULL * (u64)i;
+				else
+					doff = (u64)svc_gl_cb_off +
+					       4ULL * (u64)(idx + i);
+				if (doff + 4ULL > SVC_GL_CB_IMG_LIMIT)
+					continue;
+				memcpy(svc_gl_cb_img + doff, &w, 4);
+				words++;
+				dirty = true;
+			}
+			runs++;
+			if (op == 3u)
+				svc_gl_cb_off += 4u * cnt;
+		}
+		off += 4u + cnt * 4u;
+	}
+	if (!dirty || !words)
+		return;
+	svc_gl_cb_img_words += words;
+	svc_gl_cb_img_runs += runs;
+	/*
+	 * The strict projection gate is the validator: only a page whose
+	 * c[0x0][0x80..0xbc] block is a projection transform becomes the
+	 * authoritative CB source. A partial stream keeps accumulating.
+	 */
+	if (!svc_gl_cb_img_valid && svc_gl_cb_mvp_ok(svc_gl_cb_img)) {
+		svc_gl_cb_img_valid = true;
+		pr_info("vgsp-sema: sema-gl: CB-UPLOAD h=0x%x push=0x%llx base=0x%llx sel=%u a=0x%08x bc=0x%llx off=0x%x runs=%llu words=%llu projection=ok first8=%08x %08x %08x %08x %08x %08x %08x %08x\n",
+			h_object, push_va, svc_gl_cb_img_base,
+			svc_gl_cb_sel_seen ? 1u : 0u, svc_gl_cb_sel_a,
+			svc_gl_cb_sel_bc, svc_gl_cb_off,
+			svc_gl_cb_img_runs, svc_gl_cb_img_words,
+			get_unaligned_le32(svc_gl_cb_img),
+			get_unaligned_le32(svc_gl_cb_img + 4),
+			get_unaligned_le32(svc_gl_cb_img + 8),
+			get_unaligned_le32(svc_gl_cb_img + 12),
+			get_unaligned_le32(svc_gl_cb_img + 16),
+			get_unaligned_le32(svc_gl_cb_img + 20),
+			get_unaligned_le32(svc_gl_cb_img + 24),
+			get_unaligned_le32(svc_gl_cb_img + 28));
+		pr_info("vgsp-sema: sema-gl: CB-UPLOAD m80=%08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n",
+			get_unaligned_le32(svc_gl_cb_img + 0x80),
+			get_unaligned_le32(svc_gl_cb_img + 0x84),
+			get_unaligned_le32(svc_gl_cb_img + 0x88),
+			get_unaligned_le32(svc_gl_cb_img + 0x8c),
+			get_unaligned_le32(svc_gl_cb_img + 0x90),
+			get_unaligned_le32(svc_gl_cb_img + 0x94),
+			get_unaligned_le32(svc_gl_cb_img + 0x98),
+			get_unaligned_le32(svc_gl_cb_img + 0x9c),
+			get_unaligned_le32(svc_gl_cb_img + 0xa0),
+			get_unaligned_le32(svc_gl_cb_img + 0xa4),
+			get_unaligned_le32(svc_gl_cb_img + 0xa8),
+			get_unaligned_le32(svc_gl_cb_img + 0xac),
+			get_unaligned_le32(svc_gl_cb_img + 0xb0),
+			get_unaligned_le32(svc_gl_cb_img + 0xb4),
+			get_unaligned_le32(svc_gl_cb_img + 0xb8),
+			get_unaligned_le32(svc_gl_cb_img + 0xbc));
+	}
+	if (svc_gl_cb_scan_log < 24u) {
+		svc_gl_cb_scan_log++;
+		pr_info("vgsp-sema: sema-gl: CB-UPLOAD-SCAN h=0x%x push=0x%llx base=0x%llx off=0x%x runs=%u words=%u valid=%u total_words=%llu\n",
+			h_object, push_va, svc_gl_cb_img_base, svc_gl_cb_off,
+			runs, words, svc_gl_cb_img_valid ? 1u : 0u,
+			svc_gl_cb_img_words);
+	}
+	if (svc_gl_cb_img_valid && sema_gl_cb_copy_to &&
+	    svc_gl_cb_apply_log < 24u) {
+		svc_gl_cb_apply_log++;
+		pr_info("vgsp-sema: sema-gl: CB-UPLOAD-APPLY h=0x%x base=0x%llx -> seed=0x%llx first8=%08x %08x %08x %08x %08x %08x %08x %08x (client CB image written by cb-copy)\n",
+			h_object, svc_gl_cb_img_base, sema_gl_cb_copy_to,
+			get_unaligned_le32(svc_gl_cb_img),
+			get_unaligned_le32(svc_gl_cb_img + 4),
+			get_unaligned_le32(svc_gl_cb_img + 8),
+			get_unaligned_le32(svc_gl_cb_img + 12),
+			get_unaligned_le32(svc_gl_cb_img + 16),
+			get_unaligned_le32(svc_gl_cb_img + 20),
+			get_unaligned_le32(svc_gl_cb_img + 24),
+			get_unaligned_le32(svc_gl_cb_img + 28));
+	}
+}
+
+/*
  * r1505 (mission task 3): content-based fixed-CB discovery. The client's
  * fixed-CB base VA is boot-dependent and the scratch18/0x0cd70000 chain can
  * miss on a boot (r1504: 96/96 synthetic-id). The CB page is 4 KB whose
@@ -15847,6 +16113,22 @@ static void svc_gl_cb_copy(struct nv_vgpu_device *vgpu, u32 h_object)
 	if ((h_object & 0xfffff000u) != 0xbeef0000u)
 		return;
 	/*
+	 * r1515: the client's own constant-buffer upload is the highest
+	 * authority. Once svc_gl_cb_upload_scan() has assembled the first CB
+	 * page from the setup push's 0x2380/0x238c/0x2390 runs and the
+	 * strict projection gate accepted it, deliver that image directly -
+	 * no memory-page discovery can be more faithful than the bytes the
+	 * client itself streamed into the store.
+	 */
+	if (svc_gl_cb_img_valid) {
+		memcpy(buf, svc_gl_cb_img, sizeof(svc_gl_cb_img));
+		src_va = svc_gl_cb_img_base;
+		val = (u32)(src_va >> 8);
+		src = "const-upload";
+		ssrc = 15;
+		goto have;
+	}
+	/*
 	 * r1513: source selection by CONTENT, in authority order. r1512's
 	 * coverage receipt measured the forced historical page 0x0cd70000
 	 * delivering a rotation block as the "MVP" (clip w ~= 1.1, bottom
@@ -18453,6 +18735,11 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 	if (ch)
 		svc_scratch18_watch(buf, len, push_va, ch->h_object);
 
+	/* r1515: apply the client's C597 constant-buffer upload (0x2390
+	 * runs) into the fixed-CB image before this pass's launch. */
+	if (ch)
+		svc_gl_cb_upload_scan(vgpu, ch->h_object, buf, len, push_va);
+
 	/* r1475: feed the client's real fixed-CB page into the forced MME
 	 * seed page before this pass's launch (beef GL channels only). */
 	if (ch)
@@ -37883,6 +38170,14 @@ static u32 svc_gr_ahead_push(struct nv_vgpu_device *vgpu, struct svc_channel *ch
 	methods = svc_ahead_parse(buf, len, tail, taillen, &used_tail);
 	if (!methods)
 		return 0;
+	/*
+	 * r1515: the init-push read worth catching is exactly the entry the
+	 * main decode may skip as stale. Apply the client's C597 fixed-CB
+	 * upload (0x2390 runs) from this translated push as well, so the
+	 * beef channel's setup entry feeds the CB image even on boots where
+	 * only the util/continuation sweep reads it.
+	 */
+	svc_gl_cb_upload_scan(vgpu, ch->h_object, buf, len, push_va);
 
 	memset(&stq, 0, sizeof(stq));
 	stq.push_va = push_va;