vgsp: staged program delivery (V3 SPH accepted by engine, 0 SPH faults/40 launches), inline bytes 256, per-VA typed slots (SESSION 153 r947-r950)

1 files changed, 197 insertions, 30 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 9d7f8bd..ce008d6 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -6568,7 +6568,7 @@ static struct svc_fwd_cand svc_fwd_cand;
  */
 #define SVC_GR_QMD_WORDS	64
 #define SVC_GR_INLINE_MAX	8
-#define SVC_GR_INLINE_BYTES	64
+#define SVC_GR_INLINE_BYTES	256
 #define SVC_GR_POINTER_MAX	8
 #define SVC_GR_PAGES_MAX	16
 /*
@@ -6821,8 +6821,8 @@ static_assert(SVC_GR_PUSH_WORDS == GSP_PHYS_GR_RAW_WORDS,
 	      "the sema raw push cap must match the backend ABI");
 static_assert(SVC_GR_INLINE_MAX <= GSP_PHYS_GR_MAX_INLINE,
 	      "the sema GR inline capture must fit the backend ABI");
-static_assert(sizeof(struct svc_gr_inline) == 80 &&
-	      offsetof(struct svc_gr_inline, data) == 12,
+static_assert(offsetof(struct svc_gr_inline, data) == 12 &&
+	      sizeof(struct svc_gr_inline) >= 12 + SVC_GR_INLINE_BYTES,
 	      "the sema GR inline record must stay layout-compatible with the backend");
 static_assert(sizeof(struct svc_gr_inline) == sizeof(struct gsp_bridge_inline) &&
 	      offsetof(struct svc_gr_inline, dst_va) ==
@@ -7071,6 +7071,7 @@ struct svc_gl_up {
 	u8 data[SVC_GR_INLINE_BYTES];
 };
 
+#define SVC_GL_PROG_MAX		8
 struct svc_gl_sec {
 	bool seen;
 	u32 subch;
@@ -7080,6 +7081,18 @@ struct svc_gl_sec {
 			 * host-sem/2D ops whose VAs are not mapped in the
 			 * replay channel and FAULT the method walk) */
 	u64 prog_va;	/* SET_PIPELINE_PROGRAM_ADDRESS_A/B(j) upper|lower */
+	/*
+	 * SESSION 153-CONTINUED-11: every SET_PIPELINE_PROGRAM_ADDRESS pair
+	 * the window binds (the engine validates the SPH at EACH bound
+	 * program VA when a draw issues - r944/r946 faults on the draw's
+	 * program pages), with the pipeline slot's shader type collected
+	 * from SET_PIPELINE_SHADER(j) (0x2000+j*0x40, TYPE bits 7:4; 0xff =
+	 * unknown). The per-VA inline records author the program bytes the
+	 * driver's upload never delivered.
+	 */
+	u64 progs[SVC_GL_PROG_MAX];
+	u8 prog_slot[SVC_GL_PROG_MAX];
+	u32 n_progs;
 	u64 param_va;	/* SET_TEX_HEADER_POOL_A/B/C upper|lower */
 	u32 param_size;	/* observed pool value word (0x10 on the r882 stream) */
 	u64 rel_va;	/* last VALID release decoded on the channel */
@@ -7112,6 +7125,17 @@ struct svc_gl_sec {
 };
 static struct svc_gl_sec svc_gl;	/* service thread only */
 /*
+ * SESSION 153-CONTINUED-11: the channel's sticky per-slot shader types
+ * (SET_PIPELINE_SHADER(j) TYPE field). Sticky like the C597 bind: the init
+ * push binds the programs once; later draw pushes reuse the state. 0xff =
+ * never seen. File-static so a section reset cannot wipe it.
+ */
+static u8 svc_gl_shader_type[32] = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
+/*
  * SESSION 153-CONTINUED-10 (task 1): the C597 class ATTACHMENT.
  *
  * A SET_OBJECT bind is sticky on real hardware: the class stays bound to
@@ -7264,39 +7288,74 @@ static u32 svc_gl_up_log;	/* R17 bounded section-upload-apply witness */
 static char *sema_gl_prog_hex;
 module_param_named(sema_gl_prog_hex, sema_gl_prog_hex, charp, 0644);
 MODULE_PARM_DESC(sema_gl_prog_hex,
-	"explicit program bytes (hex) for the synthesized GL launch (<=64 B)");
+	"explicit program bytes (hex) for the synthesized GL launch (<=256 B)");
+/*
+ * SESSION 153-CONTINUED-11: per-stage program images. A Turing graphics
+ * program is [SPH V4 (128 B)][SASS]; the SPH tells the engine the stage
+ * (VTG vs PS) and the I/O mapping. The engine validates the SPH of every
+ * bound pipeline program at draw time, so every program VA the window
+ * binds needs bytes. sema_gl_vs_hex feeds VTG-typed slots, sema_gl_ps_hex
+ * pixel-typed slots, and sema_gl_prog_hex is the generic fallback (also
+ * used for the single-record compute path). All three re-parse whenever
+ * their string changes, so a running module can be iterated against the
+ * "Shader Program Header N Error" fault oracle without a reload.
+ */
+static char *sema_gl_vs_hex;
+module_param_named(sema_gl_vs_hex, sema_gl_vs_hex, charp, 0644);
+MODULE_PARM_DESC(sema_gl_vs_hex,
+	"explicit VTG program image (SPH+SASS hex, <=256 B) for vertex slots");
+static char *sema_gl_ps_hex;
+module_param_named(sema_gl_ps_hex, sema_gl_ps_hex, charp, 0644);
+MODULE_PARM_DESC(sema_gl_ps_hex,
+	"explicit PS program image (SPH+SASS hex, <=256 B) for pixel slots");
 static u8 svc_gl_prog[GSP_BRIDGE_INLINE_BYTES];
 static u32 svc_gl_prog_len;
-static bool svc_gl_prog_parsed;
+static u8 svc_gl_vs[GSP_BRIDGE_INLINE_BYTES];
+static u32 svc_gl_vs_len;
+static u8 svc_gl_ps[GSP_BRIDGE_INLINE_BYTES];
+static u32 svc_gl_ps_len;
+static char svc_gl_prog_seen[GSP_BRIDGE_INLINE_BYTES * 2 + 1];
+static char svc_gl_vs_seen[GSP_BRIDGE_INLINE_BYTES * 2 + 1];
+static char svc_gl_ps_seen[GSP_BRIDGE_INLINE_BYTES * 2 + 1];
 
-static void svc_gl_prog_apply_param(void)
+static void svc_gl_hex_parse(const char *h, const char *tag, u8 *dst,
+			     u32 *dst_len, char *seen)
 {
-	const char *h = sema_gl_prog_hex;
 	u32 n, i;
 
-	if (svc_gl_prog_parsed)
-		return;
-	svc_gl_prog_parsed = true;
+	if (!strcmp(seen, h ? h : ""))
+		return;	/* unchanged since the last parse */
+	strscpy(seen, h ? h : "", GSP_BRIDGE_INLINE_BYTES * 2 + 1);
+	*dst_len = 0;
 	if (!h || !h[0])
 		return;
 	n = (u32)strlen(h);
 	if (n % 2 || n / 2 > GSP_BRIDGE_INLINE_BYTES) {
-		pr_warn("sema-gl: prog-hex param ignored (bad length %u)\n",
-			n);
+		pr_warn("sema-gl: %s param ignored (bad length %u)\n",
+			tag, n);
 		return;
 	}
 	for (i = 0; i < n; i += 2) {
 		unsigned int b;
 
 		if (sscanf(h + i, "%2x", &b) != 1) {
-			pr_warn("sema-gl: prog-hex param bad at %u\n", i);
+			pr_warn("sema-gl: %s param bad at %u\n", tag, i);
 			return;
 		}
-		svc_gl_prog[i / 2] = (u8)b;
+		dst[i / 2] = (u8)b;
 	}
-	svc_gl_prog_len = n / 2;
-	pr_info("sema-gl: prog-hex param: %u bytes stored\n",
-		svc_gl_prog_len);
+	*dst_len = n / 2;
+	pr_info("sema-gl: %s param: %u bytes stored\n", tag, *dst_len);
+}
+
+static void svc_gl_prog_apply_param(void)
+{
+	svc_gl_hex_parse(sema_gl_prog_hex, "prog-hex", svc_gl_prog,
+			 &svc_gl_prog_len, svc_gl_prog_seen);
+	svc_gl_hex_parse(sema_gl_vs_hex, "vs-hex", svc_gl_vs,
+			 &svc_gl_vs_len, svc_gl_vs_seen);
+	svc_gl_hex_parse(sema_gl_ps_hex, "ps-hex", svc_gl_ps,
+			 &svc_gl_ps_len, svc_gl_ps_seen);
 }
 
 /*
@@ -7436,13 +7495,46 @@ static void svc_gl_field(u32 addr, u32 count, const u8 *data, u32 dlen)
 			break;
 		memcpy(&v[k], data + 4 * k, 4);
 	}
+	/*
+	 * SESSION 153-CONTINUED-11: SET_PIPELINE_SHADER(j) = 0x2000+j*0x40,
+	 * [ENABLE(0) | TYPE(7:4)]. Collected per slot so a bound program VA
+	 * can be typed (VTG vs PS) for the program-bytes delivery.
+	 */
+	if (addr >= 0x2000 && addr < 0x2800 && (addr & 0x3f) == 0) {
+		u32 j = (addr - 0x2000) / 0x40;
+
+		if (j < 32 && count >= 1)
+			svc_gl_shader_type[j] = (u8)((v[0] >> 4) & 0xf);
+		return;
+	}
 	/* SET_PIPELINE_PROGRAM_ADDRESS_A/B(j), A=upper(7:0), B=lower(31:0).
 	 * Observed: 0x20020815 (j=1) [00000001 21201b00] -> 0x121201b00.
+	 * SESSION 153-CONTINUED-11: EVERY pair is recorded (the engine
+	 * validates each bound program's SPH at draw time) and the program
+	 * page joins the candidate's page set so the per-VA inline records
+	 * can land there.
 	 */
 	if (addr >= 0x2014 && addr < 0x4000 && (addr & 0x3f) == 0x14 &&
-	    count >= 2 && !svc_gl.prog_va) {
-		svc_gl.prog_va = ((u64)(v[0] & 0xff) << 32) |
-				 (u64)(v[1] & 0xfffffffc);
+	    count >= 2) {
+		u64 pva = ((u64)(v[0] & 0xff) << 32) |
+			  (u64)(v[1] & 0xfffffffc);
+		u32 j = (addr - 0x2014) / 0x40, k;
+
+		if (!svc_gl.prog_va)
+			svc_gl.prog_va = pva;
+		if (pva && j < 32) {
+			bool dup = false;
+
+			for (k = 0; k < svc_gl.n_progs; k++)
+				if (svc_gl.progs[k] == pva)
+					dup = true;
+			if (!dup && svc_gl.n_progs < SVC_GL_PROG_MAX) {
+				svc_gl.progs[svc_gl.n_progs] = pva;
+				svc_gl.prog_slot[svc_gl.n_progs] = (u8)j;
+				svc_gl.n_progs++;
+			}
+			svc_gl_add_ptr(pva);
+		}
 		return;
 	}
 	switch (addr) {
@@ -7612,7 +7704,7 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu,
 			 u32 at_off)
 {
 	struct svc_push_state s2;
-	struct gsp_bridge_inline pr;
+	struct gsp_bridge_inline prs[SVC_GL_PROG_MAX];
 	u32 n_recs = 0, k;
 	/*
 	 * SESSION 149 R16 (M3): the inline attachment's source. Priority:
@@ -7871,12 +7963,69 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu,
 	 * uploads/binds, so the synthesized compute attachments (the inline
 	 * program record, and with it the prog-hex fallback) do not apply.
 	 * Draw-synth/hold captures keep the old compute path unchanged.
-	 */
-	if (blob_n && op_src[0] != 's') {
-		memset(&pr, 0, sizeof(pr));
-		pr.dst_va = svc_gl.prog_va;
-		pr.len = blob_n;
-		memcpy(pr.data, blob, blob_n);
+	 *
+	 * SESSION 153-CONTINUED-11: exception - the GL program-bytes
+	 * delivery. The engine validates the SPH of EVERY pipeline program
+	 * the channel state binds when a draw issues, and the driver's
+	 * upload of those bytes never ran (the program pages read empty).
+	 * For a window that binds program slots (the 0x2014/0x2018 pairs
+	 * collected above) attach one inline record per program VA with the
+	 * stage-typed image (vs-hex for VTG slots, ps-hex for pixel slots,
+	 * prog-hex fallback) - this authors the missing upload. Windows
+	 * without program pairs keep the historical single-record path.
+	 */
+	if (svc_gl.n_progs) {
+		u32 pi;
+
+		for (pi = 0; pi < svc_gl.n_progs && n_recs < GSP_BRIDGE_INLINE_MAX;
+		     pi++) {
+			const u8 *b = NULL;
+			u32 bl = 0;
+			u8 t = svc_gl_shader_type[svc_gl.prog_slot[pi] < 32 ?
+						  svc_gl.prog_slot[pi] : 0];
+
+			if (t == 5 && svc_gl_ps_len) {
+				b = svc_gl_ps;
+				bl = svc_gl_ps_len;
+			} else if (t != 5 && t != 0xff && svc_gl_vs_len) {
+				b = svc_gl_vs;
+				bl = svc_gl_vs_len;
+			} else if (svc_gl_prog_len) {
+				b = svc_gl_prog;
+				bl = svc_gl_prog_len;
+			} else if (blob_n) {
+				b = blob;
+				bl = blob_n;
+			}
+			if (!b || !bl)
+				continue;
+			memset(&prs[n_recs], 0, sizeof(prs[n_recs]));
+			prs[n_recs].dst_va = svc_gl.progs[pi];
+			prs[n_recs].len = bl;
+			memcpy(prs[n_recs].data, b, bl);
+			n_recs++;
+		}
+		if (n_recs && gsp_sema_dbg && svc_gl_prog_log < 64) {
+			u32 pi2;
+
+			svc_gl_prog_log++;
+			pr_info("sema-gl: prog-images h=0x%x push=0x%llx recs=%u n_progs=%u vs=%u ps=%u gen=%u\n",
+				ch->h_object, push_va, n_recs,
+				svc_gl.n_progs,
+				svc_gl_vs_len, svc_gl_ps_len, svc_gl_prog_len);
+			for (pi2 = 0; pi2 < svc_gl.n_progs && pi2 < 8;
+			     pi2++)
+				pr_info("sema-gl: prog-slot[%u] va=0x%llx slot=%u type=%02x\n",
+					pi2, svc_gl.progs[pi2],
+					svc_gl.prog_slot[pi2],
+					svc_gl_shader_type[svc_gl.prog_slot[pi2] < 32 ?
+							  svc_gl.prog_slot[pi2] : 0]);
+		}
+	} else if (blob_n && op_src[0] != 's') {
+		memset(&prs[0], 0, sizeof(prs[0]));
+		prs[0].dst_va = svc_gl.prog_va;
+		prs[0].len = blob_n;
+		memcpy(prs[0].data, blob, blob_n);
 		n_recs = 1;
 	}
 	/*
@@ -7885,11 +8034,11 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu,
 	 * replays of the same (push_va, content hash).
 	 */
 	svc_gl_cap_store(push_va, svc_push_hash(buf, len));
-	pr_info("sema-gl: section captured h=0x%x push=0x%llx len=%u off=0x%x prog=0x%llx param=0x%llx/0x%x rel=0x%llx/%u ptr=%u blob=%u replay=%u src=%s\n",
+	pr_info("sema-gl: section captured h=0x%x push=0x%llx len=%u off=0x%x prog=0x%llx param=0x%llx/0x%x rel=0x%llx/%u ptr=%u blob=%u replay=%u src=%s nprog=%u recs=%u\n",
 		ch->h_object, push_va, len, at_off, svc_gl.prog_va,
 		svc_gl.param_va, svc_gl.param_size, svc_gl.rel_va,
 		svc_gl.rel_payload, svc_gl.n_ptr, svc_gl.blob_len,
-		replay ? 1u : 0u, op_src);
+		replay ? 1u : 0u, op_src, svc_gl.n_progs, n_recs);
 	/*
 	 * SESSION 153: remember real operands for this channel so a later
 	 * draw-only section on the same channel can still capture. A
@@ -7915,7 +8064,7 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu,
 			get_unaligned_le32(blob + 8),
 			get_unaligned_le32(blob + 12));
 	}
-	svc_gr_cand_capture(ch, &s2, n_recs ? &pr : NULL, n_recs, buf, len,
+	svc_gr_cand_capture(ch, &s2, n_recs ? prs : NULL, n_recs, buf, len,
 			    push_va);
 	/*
 	 * r929 (Path A): a stream-derived capture (op_src=stream: the section
@@ -12852,6 +13001,24 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 			}
 		}
 
+		/*
+		 * SESSION 153-CONTINUED-11: SET_PIPELINE_SHADER(j) as an IMMD
+		 * (op 4) method - the GL driver emits it as a 1U method, so
+		 * the TYPE field rides in the header's immediate (bits 27:16,
+		 * value = [ENABLE(0) | TYPE(7:4)]). Sticky per slot, like the
+		 * run-shaped case in svc_gl_field.
+		 */
+		if (gsp_sema_gl_xlate && op == 4 && addr >= 0x2000 &&
+		    addr < 0x2800 && (addr & 0x3f) == 0 &&
+		    ((svc_gl.seen && subch == svc_gl.subch) ||
+		     (svc_gl_att_h == ch->h_object &&
+		      subch == svc_gl_att_subch))) {
+			u32 j = (addr - 0x2000) / 0x40;
+
+			if (j < 32)
+				svc_gl_shader_type[j] =
+					(u8)((((hdr >> 16) & 0xfff) >> 4) & 0xf);
+		}
 		if (op == 4 || op == 7) {
 			/* Header-only methods (immediate data in the header
 			 * for op 4; end-of-push segment for op 7). */