vgsp: sticky C597 attachment + BEGIN recognizer + draw-scan window; draws captured and replayed on 3D lane (engine reaches program header validation); guest-page blob delivery; two mdevs coexist (SESSION 153 r944-r946)

2 files changed, 261 insertions, 23 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index f7b43fb..6228d28 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -19375,6 +19375,29 @@ static u32 gsp_phys_rmcall_gr_3d_patch_window(const u32 *in, u32 n, u32 *out,
 				   addr < 0x4000 && (addr & 0x3fu) == 0x14 &&
 				   cnt >= 2) {
 				pair = true;
+			} else if (op == 1 && (addr == 0x1c00 || addr == 0x1c10) &&
+				   cnt >= 3) {
+				/* SESSION 153-CONTINUED-10: SET_VERTEX_STREAM_
+				 * A_FORMAT(j)+BASE(j) run [fmt, upper, lower]
+				 * (j*0x10 stride) - the vertex data the draw
+				 * reads. Observed [00001000 1 2150b790]. */
+				lo_i = 1;
+				pair = true;
+			} else if (op == 1 && addr == 0x2380 && cnt >= 3) {
+				/* SET_CONSTANT_BUFFER_SELECTOR_A/B:
+				 * [size, upper, lower]; observed
+				 * [00010000 1 2150a500]. */
+				lo_i = 1;
+				pair = true;
+			} else if (op == 1 && addr >= 0x0800 && addr < 0x0c00 &&
+				   (addr & 0x3fu) == 0u && cnt >= 2) {
+				/* SET_COLOR_TARGET_A/B(j) offset pair:
+				 * [upper, lower, w, h, fmt]; the surface the
+				 * draw rasterizes into (must live in guest
+				 * RAM for the present lane to see pixels).
+				 * Observed [1 222fa200 40 4 f7 0]. */
+				lo_i = 0;
+				pair = true;
 			}
 			if (pair && gsp_phys_rmcall_gr_3d_reloc(&d[lo_i],
 								&d[lo_i + 1],
@@ -20177,6 +20200,29 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
 				memcpy(sh_page + (inl->dst_va &
 						  (GSP_PHYS_PAGE_SIZE - 1)),
 				       inl->data, len);
+				/*
+				 * SESSION 153-CONTINUED-10: a stream3d (3D
+				 * method-window) submission maps the engine's
+				 * VAs straight at the GUEST page HPA
+				 * (gsp_phys_rmcall_gr_map_shadow_locked
+				 * installs the guest PTE - there is no
+				 * shadow-chunk copy in that path), so the
+				 * engine's I-fetch reads the guest page, not
+				 * sh_page. The GLX program page is never
+				 * written by the guest itself (the driver's
+				 * uploads never ran), and the r945 evidence is
+				 * exactly that: "Graphics Exception: Shader
+				 * Program Header 11/18 Error" on the draw
+				 * whose inline prog-hex blob had only reached
+				 * this shadow copy. Author the inline bytes
+				 * into the guest page too, emulating the
+				 * upload that never executed.
+				 */
+				if (stream3d && pg->hpa)
+					memcpy((u8 *)__va(pg->hpa) +
+						       (inl->dst_va &
+							(GSP_PHYS_PAGE_SIZE - 1)),
+					       inl->data, len);
 				copied++;
 			}
 			/*
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 2ab1060..9d7f8bd 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -7112,6 +7112,23 @@ struct svc_gl_sec {
 };
 static struct svc_gl_sec svc_gl;	/* service thread only */
 /*
+ * SESSION 153-CONTINUED-10 (task 1): the C597 class ATTACHMENT.
+ *
+ * A SET_OBJECT bind is sticky on real hardware: the class stays bound to
+ * the channel subchannel across pushes until another bind replaces it. The
+ * GLX server's C597 SET_OBJECT arrives once (the driver's init push at
+ * 0x120010004); every later per-frame CONTENT push (588/644/756 B, ~5/s)
+ * carries real C597 draws (NVC597_BEGIN 0x1618 + vertex streams + colour
+ * target + LOAD_CONSTANT_BUFFER payloads) with NO per-push bind, so the
+ * SET_OBJECT-only recognizer below never opens a section for them and the
+ * 3D replay lane never sees a draw. Remember (h_object, subch) when a C597
+ * bind is seen - including catches of the init push - and treat any later
+ * method run on that pair, on engine_type==0, as C597 class space.
+ */
+static u64 svc_gl_att_h;	/* channel whose subch carries the C597 bind */
+static u32 svc_gl_att_subch;
+static u32 svc_gl_draw_log;	/* bounded draw-section-open witness */
+/*
  * SESSION 153 (G-B gate: the client's own C597 draws must open sections).
  * The replay guard at svc_gl_flush consults svc_gr_cand_push_seen(), which
  * returns true when the push VA sits in the GR candidate ring - even when
@@ -7510,12 +7527,76 @@ static void svc_gl_field(u32 addr, u32 count, const u8 *data, u32 dlen)
 			}
 		}
 		return;
+	case 0x1c00:	/* SET_VERTEX_STREAM_A_FORMAT(j) + j*0x10: the run
+			 * carries [fmt, BASE_UPPER, BASE_LOWER] - the vertex
+			 * stream the draw reads positions/colours from.
+			 * Observed 0x1c00 [00001000 00000001 2150b790] and
+			 * 0x1c10 [00001008 00000001 215257c0] (SESSION
+			 * 153-CONTINUED-10 draw-push decode).
+			 */
+	case 0x1c10:
+		if (count >= 3)
+			svc_gl_add_ptr(((u64)(v[1] & 0xff) << 32) |
+				       (u64)(v[2] & 0xfffffffc));
+		return;
+	case 0x2380:	/* SET_CONSTANT_BUFFER_SELECTOR_A/B: [size, upper,
+			 * lower]. Observed [00010000 00000001 2150a500].
+			 */
+		if (count >= 3)
+			svc_gl_add_ptr(((u64)(v[1] & 0xff) << 32) |
+				       (u64)(v[2] & 0xfffffffc));
+		return;
+	case 0x0800:	/* SET_COLOR_TARGET_A/B(j=0): [OFFSET_UPPER,
+			 * OFFSET_LOWER, w, h, fmt, ...] - the surface the
+			 * draw rasterizes into. Observed [00000001 222fa200
+			 * 00000040 00000004 000000f7 00000000].
+			 */
+		if (count >= 2)
+			svc_gl_add_ptr(((u64)(v[0] & 0xff) << 32) |
+				       (u64)(v[1] & 0xfffffffc));
+		return;
 	default:
 		return;
 	}
 }
 
 /*
+ * SESSION 153-CONTINUED-10: re-walk [start, end) of the push and collect
+ * the C597 operand fields of every method run on the section's subchannel.
+ * Used when a draw section opens at its BEGIN marker (0x1618): the state
+ * methods that precede the marker (vertex stream binds, colour target,
+ * constant buffer binds) were decoded before the section existed, so their
+ * VAs must be re-collected or the candidate's page set starves and the
+ * replay's shadow relocation leaves them pointing at unreachable guest
+ * VAs (the r929 v2 FAULT_PDE class). Method shapes mirror the main
+ * decoder: op 1/3/5 carry 1+count dwords, everything else is header-only.
+ */
+static void svc_gl_scan_window(u32 start, u32 end, const u8 *buf, u32 len,
+			       u32 att_subch)
+{
+	u32 o = start;
+
+	while (o + 4 <= end && o + 4 <= len) {
+		u32 hdr, op, count, subch, addr;
+
+		memcpy(&hdr, buf + o, 4);
+		op = (hdr >> 29) & 0x7;
+		count = (hdr >> 16) & 0xfff;
+		subch = (hdr >> 13) & 0x7;
+		addr = (hdr & 0x1fff) << 2;
+		if ((op == 1 || op == 3 || op == 5) && count &&
+		    o + 4 + count * 4 <= len) {
+			if (subch == att_subch)
+				svc_gl_field(addr, count, buf + o + 4,
+					     len - o - 4);
+			o += 4 + count * 4;
+			continue;
+		}
+		o += 4;
+	}
+}
+
+/*
  * Close the open section: synthesize the QMD from the collected fields and
  * capture it as the forwarding candidate. The QMD word layout mirrors
  * svc_gr_qmd_decode() (the CUDA bridge contract): 48/49 program, 32/33
@@ -7748,27 +7829,41 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu,
 		}
 	}
 	memset(blob, 0, sizeof(blob));
-	blob_rc = svc_read_va_trusted(vgpu, svc_gl.prog_va, blob,
-				      sizeof(blob));
-	if (!blob_rc) {
-		u32 z = 0;
-
-		for (k = 0; k < 16; k++)
-			z |= blob[k];
-		if (z)
-			blob_trusted = true;
-	}
-	if (blob_trusted) {
-		blob_n = sizeof(blob);
-		blob_src = "trusted-prog";
-	} else if (svc_gl_prog_len) {
+	/*
+	 * SESSION 153-CONTINUED-10: the explicit operator program override
+	 * (sema_gl_prog_hex) wins over the trusted page read. The GLX
+	 * server's per-frame CONTENT pushes bind their own program slots
+	 * (observed 0x121201f00 / 0x121215280) whose pages hold unrelated
+	 * nonzero bytes at that offset; the R16 trusted-read priority
+	 * accepts any nonzero 16 bytes as "the real program" and attached
+	 * garbage, and the engine rejected the draw with "Graphics
+	 * Exception: Shader Program Header 11/18 Error" (r944 fault
+	 * evidence, push 0x120014578). An unset override keeps the
+	 * historical read priority unchanged.
+	 */
+	if (svc_gl_prog_len) {
 		blob_n = svc_gl_prog_len;
 		memcpy(blob, svc_gl_prog, blob_n);
 		blob_src = "prog-hex";
-	} else if (svc_gl.blob_len) {
-		blob_n = svc_gl.blob_len;
-		memcpy(blob, svc_gl.blob, blob_n);
-		blob_src = "lid-fallback";
+	} else {
+		blob_rc = svc_read_va_trusted(vgpu, svc_gl.prog_va, blob,
+					      sizeof(blob));
+		if (!blob_rc) {
+			u32 z = 0;
+
+			for (k = 0; k < 16; k++)
+				z |= blob[k];
+			if (z)
+				blob_trusted = true;
+		}
+		if (blob_trusted) {
+			blob_n = sizeof(blob);
+			blob_src = "trusted-prog";
+		} else if (svc_gl.blob_len) {
+			blob_n = svc_gl.blob_len;
+			memcpy(blob, svc_gl.blob, blob_n);
+			blob_src = "lid-fallback";
+		}
 	}
 	/*
 	 * r929 (Path A): a stream-derived capture is a 3D method window - the
@@ -7838,8 +7933,18 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu,
 	 * 448-word cap leaves headroom for the backend's 512-word method
 	 * block (2 bind words + the window + the 8-word release tail).
 	 * Draw-synth/hold captures are never marked.
-	 */
-	if (op_src[0] == 's' && svc_gr_cand.valid &&
+	 *
+	 * SESSION 153-CONTINUED-10: op_src=hold is included. The GLX
+	 * server's per-frame CONTENT pushes carry the draw window but no
+	 * program/param operands (the class bind is sticky; no per-push
+	 * SET_OBJECT), so their sections capture with the channel's real
+	 * operands adopted from the hold table (filled by the init
+	 * capture). The window itself is still the REAL push bytes sliced
+	 * from its own first C597 state method to its END - exactly the
+	 * raw 3D method window the r929 replay lane wants; the synthesized
+	 * compute fields are what stream_3d bypasses.
+	 */
+	if ((op_src[0] == 's' || op_src[0] == 'h') && svc_gr_cand.valid &&
 	    len == svc_gr_cand.push_n * 4 &&
 	    svc_gl.start_off + 8u <= at_off &&
 	    (at_off - svc_gl.start_off) / 4u <= 448u) {
@@ -12300,6 +12405,11 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 	u32 ops_runs = 0;
 	u32 ops_seq = 0;	/* ops appended this pass (sequence numbers) */
 	u32 off = 0;
+	/* SESSION 153-CONTINUED-10: draw-window tracker. First method of
+	 * the current C597 run on the attached subchannel after the last
+	 * foreign-subchannel (2D/present) run; see the BEGIN recognizer
+	 * below. Per pass, like the rest of the decode state. */
+	u32 gl_win_start = 0;
 
 	/* m164: fresh CLASS-2 QMD witness for this decode pass. */
 	svc_gr_qmd_seen = 0;
@@ -12441,6 +12551,26 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 		addr = (hdr & 0x1fff) << 2;
 
 		/*
+		 * SESSION 153-CONTINUED-10 (task 1): draw-window tracking on
+		 * the attached C597 channel. The per-frame CONTENT pushes
+		 * interleave the 2D/present ops (subchannel 3) and the 3D
+		 * block (subchannel 0); the replayable 3D window starts at
+		 * the first subchannel-0 method after the last subchannel-3
+		 * run. The SET_REPORT_SEMAPHORE arms (0x1b00..0x1b0c) are
+		 * excluded: they name client semaphore VAs (0x1_2000c000)
+		 * the replay channel cannot translate (the r929 v1 fault
+		 * class), so they must stay outside the sliced window.
+		 */
+		if (gsp_sema_gl_xlate && !ch->engine_type &&
+		    svc_gl_att_h == ch->h_object) {
+			if (subch != svc_gl_att_subch)
+				gl_win_start = 0;
+			else if (!gl_win_start &&
+				 !(op == 1 && addr >= 0x1b00 && addr <= 0x1b0c))
+				gl_win_start = off;
+		}
+
+		/*
 		 * fail4: name every big method-data run and every run whose
 		 * address is the LOAD_INLINE_DATA method (any op/subch - the
 		 * encoding of the 64 KB H2D upload is what we are pinning).
@@ -12676,6 +12806,52 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 				off + 16 <= len ? *(const u32 *)(buf + off + 12) : 0);
 		}
 
+		/*
+		 * SESSION 153-CONTINUED-10 (task 1): the client's per-frame
+		 * CONTENT pushes carry real C597 draws but no per-push
+		 * SET_OBJECT (the class bind is sticky), so the section
+		 * recognizer above never opened for them and the 3D replay
+		 * lane never saw a draw. Recognise the draw marker itself:
+		 * NVC597_BEGIN (0x1618, an IMMD method - its immediate is
+		 * the topology in the count field) on the attached
+		 * subchannel of the attached channel opens the section.
+		 * The window starts at gl_win_start (the first C597 state
+		 * method after the last foreign-subchannel run); the state
+		 * methods before the marker are re-scanned so their operand
+		 * VAs (vertex stream bases 0x1c00/0x1c10, colour target
+		 * 0x0800, constant buffer selectors 0x2380) enter the
+		 * candidate's page set - without them the backend's shadow
+		 * relocation cannot rewrite the window and the engine
+		 * faults on an unreachable guest VA (the r929 v2 class).
+		 * The window then runs from gl_win_start to the pass end
+		 * (the section close at the END method / push tail), which
+		 * svc_gl_flush slices into the verbatim 3D replay.
+		 */
+		if (gsp_sema_gl_xlate && !ch->engine_type && op == 4 &&
+		    addr == 0x1618 && !svc_gl.seen &&
+		    svc_gl_att_h == ch->h_object &&
+		    subch == svc_gl_att_subch) {
+			u64 keep_rel = svc_gl.rel_va;
+			u32 keep_pay = svc_gl.rel_payload;
+			u32 wstart = gl_win_start ? gl_win_start : off;
+
+			memset(&svc_gl, 0, sizeof(svc_gl));
+			svc_gl.seen = true;
+			svc_gl.subch = subch;
+			svc_gl.start_off = wstart;
+			svc_gl.rel_va = keep_rel;
+			svc_gl.rel_payload = keep_pay;
+			if (wstart < off)
+				svc_gl_scan_window(wstart, off, buf, len,
+						   subch);
+			if (gsp_sema_dbg && svc_gl_draw_log < 64) {
+				svc_gl_draw_log++;
+				pr_info("sema-gl: section open (draw) h=0x%x push=0x%llx subch=%u start=0x%x begin=0x%x len=%u trusted=%d\n",
+					ch->h_object, push_va, subch, wstart,
+					off, len, trusted);
+			}
+		}
+
 		if (op == 4 || op == 7) {
 			/* Header-only methods (immediate data in the header
 			 * for op 4; end-of-push segment for op 7). */
@@ -12773,6 +12949,13 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 				svc_gl.start_off = off;
 				svc_gl.rel_va = keep_rel;
 				svc_gl.rel_payload = keep_pay;
+				/* SESSION 153-CONTINUED-10: the bind is sticky -
+				 * remember it so the class-space recognizer
+				 * (BEGIN 0x1618, below) accepts this channel's
+				 * later per-push-CONTENT pushes that carry no
+				 * SET_OBJECT of their own. */
+				svc_gl_att_h = ch->h_object;
+				svc_gl_att_subch = subch;
 				/*
 				 * SESSION 153: witness every C597 section open
 				 * (bounded) - it is the recognizer-side proof
@@ -12786,9 +12969,18 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 						ch->h_object, push_va, subch,
 						trusted);
 				}
-			} else if (svc_gl.seen) {
-				svc_gl_flush(vgpu, ch, &st, buf, len,
-					     push_va, trusted, replay, off);
+			} else {
+				if (svc_gl.seen)
+					svc_gl_flush(vgpu, ch, &st, buf,
+						     len, push_va, trusted,
+						     replay, off);
+				/* A different class bound over the attached
+				 * subchannel replaces the C597 bind: drop the
+				 * attachment so the class-space recognizer
+				 * cannot misread methods of the new class. */
+				if (ch->h_object == svc_gl_att_h &&
+				    subch == svc_gl_att_subch)
+					svc_gl_att_h = 0;
 			}
 		} else if (gsp_sema_gl_xlate && svc_gl.seen &&
 			   subch == svc_gl.subch) {