vgsp: synthesized GL launch passes SKEDCHECK - bounded LID blob + QMD scheduling fields (SESSION 149 r889)

- svc_gl_field 0x1b0: skip LAUNCH_DMA cfg word; bound the inline blob by the
  run's own word count (blob 64 -> 12 B, no over-read of following methods).
- svc_gl_flush QMD fields: CTA dims 1/1/1 (w12-14), w18=0x00010023,
  w19=0x00010001, w20=0x00001a00 (REGISTER_COUNT from the stream's own
  REGISTER_COUNT(1) method), barrier bit w29, SASS_VERSION 7.5 (w31), driver
  cache-control block (w5).
- Evidence (r889): SKEDCHECK06/16 zero occurrences; 'GR LAUNCH submitted
  shadow=1 words=239'; SM decodes; FAULT-CAP code_src=win0 ok=1; bounded
  action=proceed + SW release (no stall). Next wall: Illegal Instruction
  Encoding (program content not yet materialized). Gated by sema_gl_xlate.

1 files changed, 134 insertions, 27 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 3cc9936..13d99cc 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -6637,6 +6637,9 @@ static_assert(sizeof(struct svc_gr_page) == 24,
 static u64 svc_push_hash(const u8 *p, u32 len);
 /* SESSION 149 R15: exactly-once guard for the hash-verified replay capture. */
 static bool svc_gr_cand_push_seen(u64 push_va, u64 hash);
+/* SESSION 149 R16 (M3): trusted-walk VA read, defined below the decode. */
+static int svc_read_va_trusted(struct nv_vgpu_device *vgpu, u64 va, void *buf,
+			       size_t len);
 
 /* QMD field decode: the exact shape the host CUDA bridge uses. */
 static void svc_gr_qmd_decode(struct svc_gr_cand_rec *c)
@@ -6826,6 +6829,7 @@ struct svc_gl_sec {
 };
 static struct svc_gl_sec svc_gl;	/* service thread only */
 static u32 svc_gl_rep_log;	/* bounded replay-already-captured witness */
+static u32 svc_gl_prog_log;	/* R16 bounded program-read witness */
 
 static void svc_gl_add_ptr(u64 va)
 {
@@ -6896,14 +6900,26 @@ static void svc_gl_field(u32 addr, u32 count, const u8 *data, u32 dlen)
 			 * data words ARE the observed upload payload. Keep the
 			 * FIRST run's bytes for the M2-S1 inline attachment.
 			 * Observed: a004006c -> [00000041 00000670 00000690
-			 * 000006b0].
+			 * 000006b0] = LAUNCH_DMA cfg + 3 LID words = 12 B.
+			 * SESSION 149 R16 (M3): keep only the LID payload (the
+			 * cfg word is not code) and bound the copy by the run's
+			 * OWN count - dlen reaches the end of the push, so the
+			 * R15 copy over-read the run (blob=64 = 12 real bytes +
+			 * 52 B of the following method words). These bytes are
+			 * only the fallback: svc_gl_flush reads the real
+			 * program bytes from prog_va via the trusted walk.
 			 */
-		if (!svc_gl.blob_len && dlen) {
-			u32 n = dlen < sizeof(svc_gl.blob) ?
-				dlen : (u32)sizeof(svc_gl.blob);
-
-			memcpy(svc_gl.blob, data, n);
-			svc_gl.blob_len = n;
+		if (!svc_gl.blob_len && count > 1 && dlen > 4) {
+			u32 n = (count - 1) * 4;
+
+			if (n > dlen - 4)
+				n = dlen - 4;
+			if (n > sizeof(svc_gl.blob))
+				n = (u32)sizeof(svc_gl.blob);
+			if (n) {
+				memcpy(svc_gl.blob, data + 4, n);
+				svc_gl.blob_len = n;
+			}
 		}
 		return;
 	default:
@@ -6920,7 +6936,8 @@ static void svc_gl_field(u32 addr, u32 count, const u8 *data, u32 dlen)
  * (synthesized - no QMD buffer exists in guest memory) so the backend uses
  * its qmdwin/scratch path.
  */
-static void svc_gl_flush(struct svc_channel *ch,
+static void svc_gl_flush(struct nv_vgpu_device *vgpu,
+			 struct svc_channel *ch,
 			 const struct svc_push_state *st, const u8 *buf,
 			 u32 len, u64 push_va, bool trusted, bool replay,
 			 u32 at_off)
@@ -6928,6 +6945,16 @@ static void svc_gl_flush(struct svc_channel *ch,
 	struct svc_push_state s2;
 	struct gsp_bridge_inline pr;
 	u32 n_recs = 0, k;
+	/*
+	 * SESSION 149 R16 (M3): the inline attachment's source. Priority:
+	 * the REAL program bytes read from prog_va through the trusted walk
+	 * (the GL program page), else the observed LID payload bounded to its
+	 * own run. r888's blob=64 over-read the 12 B LID run.
+	 */
+	u8 blob[SVC_GR_INLINE_BYTES];
+	u32 blob_n = 0;
+	int blob_rc = -1;
+	bool blob_trusted = false;
 
 	if (!svc_gl.seen || !svc_gl.prog_va || !svc_gl.param_va ||
 	    !svc_gl.rel_va || !trusted)
@@ -6965,6 +6992,48 @@ static void svc_gl_flush(struct svc_channel *ch,
 	 * sema service authors the guest word (set it once rel_hpa resolving
 	 * is wanted). qmd[24] b26 (REDUCTION) likewise stays clear.
 	 */
+	/*
+	 * SESSION 149 R16 (M3): the scheduling fields the r888 engine
+	 * refused to schedule without. Word/field offsets from the public
+	 * QMDV02_03 class header (nvidia/inc/class/clc5c0qmd.h; the booter's
+	 * own 48/49 program pair matches its QMDV02_03 offsets, so the
+	 * rest of the layout is the same class):
+	 *   w12  CTA_RASTER_WIDTH  MW(415:384)
+	 *   w13  CTA_RASTER_HEIGHT MW(431:416)
+	 *   w14  CTA_RASTER_DEPTH  MW(463:448)
+	 *   w18  QMD_VERSION MW(579:576), QMD_MAJOR_VERSION MW(583:580),
+	 *        CTA_THREAD_DIMENSION0 MW(607:592) -> w18 hi word
+	 *   w19  CTA_THREAD_DIMENSION1 MW(623:608),
+	 *        CTA_THREAD_DIMENSION2 MW(639:624)
+	 *   w20  REGISTER_COUNT_V MW(656:648)   <- SKEDCHECK06 field
+	 *   w29  BARRIER_COUNT MW(959:955)
+	 *   w31  SASS_VERSION MW(1023:1016)
+	 * SKEDCHECK16 fired on CTA_THREAD_DIMENSION0==0 and SKEDCHECK06 on
+	 * REGISTER_COUNT_V==0. The register count is the stream's own value:
+	 * REGISTER_COUNT(1) = 0x1a = 26 (method 0x200c + 1*0x40 = 0x204c,
+	 * words [0x1a, 0]) - the same number the driver's C5C0 path writes
+	 * ("a NULL QMD will result in SKEDCHECK06_REGISTER_COUNT", cui
+	 * turing_launch.c:781). CTA dims: the 0xC597 stream carries no grid /
+	 * CTA operands we can locate (the 3D-class runs are format/viewport/
+	 * shader config; the C597 SHADER_CONFIG "CTA raster" words are not in
+	 * this push), so a minimal 1x1x1 grid of 1x1x1 CTAs is used and
+	 * documented as the choice: the checks only reject ZERO dims and a
+	 * zero total-thread count, both of which a 1x1x1 launch satisfies.
+	 * SHARED_MEMORY_SIZE (w17) stays 0 = "no shared memory" (valid and
+	 * NOT among the failed checks); the m72 local-memory prelude already
+	 * covers SKEDCHECK05. The driver's cache-control block (w5 top bits)
+	 * is set as every captured guest QMD carries it; the backend turns it
+	 * into its icache-invalidate bit for the shadow line.
+	 */
+	s2.qmd[12] = 1;		/* CTA raster width  (CTAs) */
+	s2.qmd[13] = 1;		/* CTA raster height (CTAs) */
+	s2.qmd[14] = 1;		/* CTA raster depth  (CTAs) */
+	s2.qmd[18] = 0x23u | (1u << 16);	/* QMD 2.3, CTA dim0 = 1 */
+	s2.qmd[19] = 1u | (1u << 16);		/* CTA dim1 = 1, dim2 = 1 */
+	s2.qmd[20] = 0x1au << 8;		/* REGISTER_COUNT_V = 26 */
+	s2.qmd[29] |= 1u << 27;			/* BARRIER_COUNT = 1 */
+	s2.qmd[31] |= 0x75u << 24;		/* SASS_VERSION 7.5 (sm_75) */
+	s2.qmd[5] |= 0x3c000000u;		/* driver cache-control block */
 	for (k = 0; k < svc_gl.n_ptr && k < SVC_GR_POINTER_MAX - 1; k++) {
 		s2.qmd[34 + 2 * k] = (u32)svc_gl.ptr[k];
 		s2.qmd[35 + 2 * k] = ((u32)(svc_gl.ptr[k] >> 32)) & 0x1ffff;
@@ -6974,21 +7043,42 @@ static void svc_gl_flush(struct svc_channel *ch,
 	s2.qmd_end = 0;
 	s2.qmd_have = 0;
 	/*
-	 * SESSION 149 R15 (M2-S1): attach the observed LAUNCH_DMA /
-	 * LOAD_INLINE_DATA payload as an inline record at the synthesized
-	 * program VA. The GL program page (0x121201b00) is NOT among the
-	 * section's own upload destinations, so without this the backend's
-	 * w48 code-source screen sees an empty page, withdraws to the guest
-	 * VA and the launch dies at FAULT_PDE; with it the proven win0
-	 * shadow copy + w48/49 rewrite line runs (the short blob is
-	 * expected to fault in the SM later - that is the next wall, not
-	 * this one).
-	 */
-	if (svc_gl.blob_len) {
+	 * SESSION 149 R15 (M2-S1) + R16 (M3): attach program-page content as
+	 * an inline record at the synthesized program VA. The GL program page
+	 * (0x121201b00) is NOT among the section's own upload destinations,
+	 * so without this the backend's w48 code-source screen sees an empty
+	 * page, withdraws to the guest VA and the launch dies at FAULT_PDE;
+	 * with it the proven win0 shadow copy + w48/49 rewrite line runs.
+	 *
+	 * R16 source order: the REAL program bytes read from prog_va through
+	 * the trusted walk (the read machinery translates GL VAs now; src=4
+	 * answered this channel's ring/pushes in r888), else the observed
+	 * LID payload bounded to its own run (12/24/28 B - "materialize
+	 * exactly that"). A trusted read that lands on an all-zero page is
+	 * treated as "not real" and falls through to the LID bytes.
+	 */
+	memset(blob, 0, sizeof(blob));
+	blob_rc = svc_read_va_trusted(vgpu, svc_gl.prog_va, blob,
+				      sizeof(blob));
+	if (!blob_rc) {
+		u32 z = 0;
+
+		for (k = 0; k < 16; k++)
+			z |= blob[k];
+		if (z)
+			blob_trusted = true;
+	}
+	if (blob_trusted) {
+		blob_n = sizeof(blob);
+	} else if (svc_gl.blob_len) {
+		blob_n = svc_gl.blob_len;
+		memcpy(blob, svc_gl.blob, blob_n);
+	}
+	if (blob_n) {
 		memset(&pr, 0, sizeof(pr));
 		pr.dst_va = svc_gl.prog_va;
-		pr.len = svc_gl.blob_len;
-		memcpy(pr.data, svc_gl.blob, pr.len);
+		pr.len = blob_n;
+		memcpy(pr.data, blob, blob_n);
 		n_recs = 1;
 	}
 	pr_info("sema-gl: section captured h=0x%x push=0x%llx len=%u off=0x%x prog=0x%llx param=0x%llx/0x%x rel=0x%llx/%u ptr=%u blob=%u replay=%u\n",
@@ -6996,6 +7086,22 @@ static void svc_gl_flush(struct svc_channel *ch,
 		svc_gl.param_va, svc_gl.param_size, svc_gl.rel_va,
 		svc_gl.rel_payload, svc_gl.n_ptr, svc_gl.blob_len,
 		replay ? 1u : 0u);
+	/* R16 bounded witness: what the synthesized QMD now schedules and
+	 * where the code bytes came from. */
+	if (gsp_sema_dbg && svc_gl_prog_log < 8) {
+		svc_gl_prog_log++;
+		pr_info("sema-gl: M3 sched w12=%u w13=%u w14=%u w18=0x%08x w19=0x%08x w20=0x%08x w29=0x%08x w31=0x%08x w5=0x%08x\n",
+			s2.qmd[12], s2.qmd[13], s2.qmd[14], s2.qmd[18],
+			s2.qmd[19], s2.qmd[20], s2.qmd[29], s2.qmd[31],
+			s2.qmd[5]);
+		pr_info("sema-gl: M3 code src=%s read_rc=%d len=%u lid=%u first=%08x %08x %08x %08x\n",
+			blob_trusted ? "trusted-prog" : "lid-fallback",
+			blob_rc, blob_n, svc_gl.blob_len,
+			get_unaligned_le32(blob),
+			get_unaligned_le32(blob + 4),
+			get_unaligned_le32(blob + 8),
+			get_unaligned_le32(blob + 12));
+	}
 	svc_gr_cand_capture(ch, &s2, n_recs ? &pr : NULL, n_recs, buf, len,
 			    push_va);
 out:
@@ -11739,17 +11845,17 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 				u32 keep_pay = svc_gl.rel_payload;
 
 				if (svc_gl.seen)
-					svc_gl_flush(ch, &st, buf, len,
-						     push_va, trusted, replay,
-						     off);
+					svc_gl_flush(vgpu, ch, &st, buf,
+						     len, push_va, trusted,
+						     replay, off);
 				memset(&svc_gl, 0, sizeof(svc_gl));
 				svc_gl.seen = true;
 				svc_gl.subch = subch;
 				svc_gl.rel_va = keep_rel;
 				svc_gl.rel_payload = keep_pay;
 			} else if (svc_gl.seen) {
-				svc_gl_flush(ch, &st, buf, len, push_va,
-					     trusted, replay, off);
+				svc_gl_flush(vgpu, ch, &st, buf, len,
+					     push_va, trusted, replay, off);
 			}
 		} else if (gsp_sema_gl_xlate && svc_gl.seen &&
 			   subch == svc_gl.subch) {
@@ -13635,7 +13741,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 	 * this is the normal capture point (one section per large GL push).
 	 */
 	if (gsp_sema_gl_xlate && svc_gl.seen)
-		svc_gl_flush(ch, &st, buf, len, push_va, trusted, replay, off);
+		svc_gl_flush(vgpu, ch, &st, buf, len, push_va, trusted,
+			     replay, off);
 
 	if (gsp_sema_dbg && ch->h_object &&
 	    svc.entry_log < 16384) {