vgsp: stub IR write real encoding SEC_OP_ONE_INC=5 (op=1 walked 0x114->0x118/0x11c/0x120 clobbering SA[0]=1 = r1131 MISSING_MACRO_DATA), stub PC at IR-end, byte-exact END body, coverage-aware stubbing, ONE_INC ghost CALL fix (18->6 real), MME-WINRUN dump, prelude latch keyed (generation,h_object); mme_ir_end/ir_cov bitmap + USERD adopt latch clear - ATTRIBUTION ANSWERED: non-retire is NONE of the attachments (window-only also ILLEGAL_OPCODE 0/901; all-macro-noop sa_stub_all=1 still ILLEGAL_OPCODE 0/902); r1111 retire was a different frame class (5016B no-CALL) vs draw-class (7148B CALL_MME_MACRO(0)); next: MME reg sample BEFORE WEDGE-RECOVER reset + stub landing verify (stub_pc 0xf0 test) - SESSION 153 r1132-r1141

3 files changed, 296 insertions, 20 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index d099a7e..ed1cb0f 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -1761,6 +1761,19 @@ static inline void gsp_phys_rmcall_wait_quantum(void)
 #define GSP_PHYS_RMCALL_CE_SUBCHANNEL	4u
 
 /*
+ * SESSION 153-CONTINUED-68: SEC_OP_ONE_INC = 5 (class/cl906f.h:181
+ * NV906F_DMA_SEC_OP_ONE_INC, NVC06F_DMA_ONEINCR_* field layout). The
+ * client emits its MME IR uploads (method 0x114) as op=5 runs whose first
+ * data word is the IR instruction address and whose remaining words feed
+ * the IR data port (the r1099/r1116 captures: hdr 0xa0100045 = op 5,
+ * cnt 16, method 0x114, data [pc, code...]). op=1 (INC_METHOD) walks the
+ * METHOD address per data word instead (0x114 -> 0x118 -> 0x11c ->
+ * 0x120), so an IR upload emitted with op=1 writes one stray IR word and
+ * then clobbers the SA pointer/data registers.
+ */
+#define GSP_PHYS_RMCALL_METHOD_ONEINC	5u
+
+/*
  * CE (TURING_DMA_COPY_A 0xc5b5) method offsets and field values, from
  * class/clc5b5.h: SET_OBJECT 0x0, SET_SEMAPHORE_A/B/PAYLOAD 0x240/0x244/0x248,
  * LAUNCH_DMA 0x300, OFFSET_IN_UPPER/LOWER 0x400/0x404, OFFSET_OUT_UPPER/
@@ -17324,12 +17337,38 @@ MODULE_PARM_DESC(sema_gl_mme_prelude,
  * (the r1116 0x238c poison). Default 1; A/B with 0.
  */
 #define GSP_PHYS_RMCALL_GR_MME_STUB_PC	0xC0u
+/*
+ * SESSION 153-CONTINUED-68: the stub body. The r1134/r1136 builds used a
+ * bare [0, 0, 1] (end_next with every other bit zero) and the engine still
+ * raised ILLEGAL_OPCODE with pc_valid=0 - a decode failure at the macro's
+ * FIRST instruction, i.e. the bare END is not an encoding the MME accepts.
+ * This body is a byte-exact copy of a real compiler-emitted instruction
+ * from the client's own captured program (macro(0) instruction 3,
+ * 20208c00 38c40046 038c4301): both ALUs write the discarded ZERO
+ * register (no side effects), no outputs are emitted, end_next=1 - a
+ * known-good macro terminator.
+ */
+static const u32 gsp_phys_gr_mme_stub_code[3] = {
+	0x20208c00u, 0x38c40046u, 0x038c4301u
+};
 static bool gsp_phys_rmcall_gr_sa_stub = true;
 module_param_named(sema_gl_mme_sa_stub, gsp_phys_rmcall_gr_sa_stub, bool,
 		   0644);
 MODULE_PARM_DESC(sema_gl_mme_sa_stub,
 	"Load a harmless end-noop stub into every MME macro slot the window calls but the captured START-ADDRESS RAM table does not cover (default 1; the r1116 0x238c lane poison class)");
 
+/*
+ * SESSION 153-CONTINUED-68: stub EVERY called slot, covered or not
+ * (attribution: with all calls no-op'd the window either still faults
+ * (the fault is in non-call content) or faults differently (the fault is
+ * macro execution). Default 0; A/B knob.
+ */
+static bool gsp_phys_rmcall_gr_sa_stub_all;
+module_param_named(sema_gl_mme_sa_stub_all, gsp_phys_rmcall_gr_sa_stub_all,
+		   bool, 0644);
+MODULE_PARM_DESC(sema_gl_mme_sa_stub_all,
+	"Stub every macro slot the window calls, including slots the captured SA table covers (attribution knob; default 0)");
+
 static inline bool gsp_sema_gr_sa_stub_value(void)
 {
 	return READ_ONCE(gsp_phys_rmcall_gr_sa_stub);
@@ -17345,8 +17384,16 @@ static inline bool gsp_sema_gr_sa_stub_value(void)
  * window replay ran its macros against an unloaded MME (mme_pc=513 class
  * stall, no retire). The recovery clears the latch so the next launch
  * re-preloads the context.
+ *
+ * SESSION 153-CONTINUED-68: the latch is keyed by (generation, channel).
+ * A bare generation latch is SHARED across channels: the X channel's
+ * first post-maturity launch consumed the emission for the whole system
+ * and every beef launch then shipped without the program (r1140: only
+ * h=0x1000013 logged "MME-PRELUDE emitted gen=3", every beef launch
+ * words=400 prelude-less). Re-emitting per channel is idempotent.
  */
 static u32 gsp_phys_gr_mme_prelude_gen;
+static u32 gsp_phys_gr_mme_prelude_h;
 
 /*
  * SESSION 153-CONTINUED-57: attr/pipeline state prelude. The r1091
@@ -17515,14 +17562,17 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb,
 		 */
 		u32 sa_ent[64];
 		u64 sa_mask = 0;
-		u32 sa_stub[24];
-		u32 sa_nstub = 0, sa_block = 0;
+		u32 sa_stub[40];
+		u32 sa_stub_ent[40];
+		u32 sa_cov[40];
+		u32 sa_cov_ent[40];
+		u32 sa_nstub = 0, sa_ncov = 0, sa_block = 0;
 
 		if (gsp_sema_gr_sa_stub_value() && win3d && win3d_n) {
 			u32 so = 0;
 
 			sa_mask = gsp_sema_mme_sa_get(sa_ent, 64);
-			while (so < win3d_n && sa_nstub < 24u) {
+			while (so < win3d_n && (sa_nstub < 40u || sa_ncov < 40u)) {
 				u32 h = win3d[so];
 				u32 sop = (h >> 29) & 7u;
 				u32 scnt = (h >> 16) & 0x1fffu;
@@ -17533,7 +17583,17 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb,
 				    scnt && so + 1u + scnt <= win3d_n) {
 					sspan = 1u + scnt;
 					slo = 0;
-					shi = scnt - 1u;
+					/*
+					 * SESSION 153-CONTINUED-68: ONE_INC
+					 * (op 5) writes only data[0] to the
+					 * header method; the rest go to
+					 * method+4 (the data port, see
+					 * nv3dLoadConstantsHeader). Walking
+					 * mthd+4k for op=5 invented ghost
+					 * CALL_MME_MACRO slots (r1134/1135
+					 * list).
+					 */
+					shi = (sop == 5u) ? 0u : scnt - 1u;
 				} else if (sop == 4u) {
 					slo = 0;
 					shi = 0;
@@ -17541,7 +17601,7 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb,
 					so++;
 					continue;
 				}
-				for (k = slo; k <= shi && sa_nstub < 24u; k++) {
+				for (k = slo; k <= shi; k++) {
 					u32 mthd = ((h & 0xfffu) << 2) + 4u * k;
 					u32 j, t;
 					bool have = false;
@@ -17552,13 +17612,43 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb,
 					j = (mthd - 0x3800u) / 8u;
 					if (j >= 64u)
 						continue;
-					if (sa_mask & (1ull << j))
-						continue;
+					/*
+					 * SESSION 153-CONTINUED-68: a slot is only
+					 * usable when the capture holds its SA entry
+					 * AND the IR instruction that entry names
+					 * actually entered the replay (see
+					 * gsp_sema_mme_ir_covered); a last-wins/
+					 * truncated mismatch makes the engine fetch
+					 * macro data that is not there
+					 * (MISSING_MACRO_DATA). Called-and-covered
+					 * slots are logged too (r1134 ILLEGAL_OPCODE
+					 * attribution: which macros actually run).
+					 */
+					if (sa_mask & (1ull << j)) {
+						if (!READ_ONCE(gsp_phys_rmcall_gr_sa_stub_all) &&
+						    gsp_sema_mme_ir_covered(sa_ent[j])) {
+							for (t = 0; t < sa_ncov; t++)
+								if (sa_cov[t] == j)
+									have = true;
+							if (!have && sa_ncov < 40u) {
+								sa_cov[sa_ncov] = j;
+								sa_cov_ent[sa_ncov] =
+									sa_ent[j];
+								sa_ncov++;
+							}
+							continue;
+						}
+					}
 					for (t = 0; t < sa_nstub; t++)
 						if (sa_stub[t] == j)
 							have = true;
-					if (!have)
-						sa_stub[sa_nstub++] = j;
+					if (!have && sa_nstub < 40u) {
+						sa_stub[sa_nstub] = j;
+						sa_stub_ent[sa_nstub] =
+							(sa_mask & (1ull << j)) ?
+							sa_ent[j] : 0u;
+						sa_nstub++;
+					}
 				}
 				so += sspan;
 			}
@@ -17566,10 +17656,50 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb,
 				sa_block = 5u + 3u * sa_nstub;
 		}
 
+		/*
+		 * SESSION 153-CONTINUED-68: full run-header census of the 3D
+		 * window being replayed (bounded; first 4 windows). The
+		 * r1134 ILLEGAL_OPCODE attribution needs the real call
+		 * structure: which methods the window writes, how many data
+		 * words each run carries, and the CALL_MME_MACRO/DATA
+		 * pattern (op 1 INC vs op 5 ONE_INC) around them.
+		 */
+		{
+			static u32 win_dump_n;
+
+			if (win3d && win3d_n && win_dump_n < 4u) {
+				u32 wo = 0, wr = 0;
+
+				win_dump_n++;
+				pr_info("vgsp-gr: MME-WINRUN dump h=0x%x push=0x%llx window=%u\n",
+					req->h_object, req->push_va, win3d_n);
+				while (wo < win3d_n && wr < 200u) {
+					u32 h = win3d[wo];
+					u32 op = (h >> 29) & 7u;
+					u32 cnt = (h >> 16) & 0x1fffu;
+					u32 addr = (h & 0xfffu) << 2;
+					u32 span = 1u;
+					u32 d0 = (wo + 1u < win3d_n) ?
+						 win3d[wo + 1u] : 0u;
+					u32 d1 = (wo + 2u < win3d_n) ?
+						 win3d[wo + 2u] : 0u;
+
+					if ((op == 1u || op == 3u || op == 5u) &&
+					    cnt && wo + 1u + cnt <= win3d_n)
+						span = 1u + cnt;
+					pr_info("vgsp-gr: MME-WINRUN off=%x op=%u cnt=%u mthd=%x d0=%x d1=%x\n",
+						wo * 4u, op, cnt, addr, d0, d1);
+					wo += span;
+					wr++;
+				}
+			}
+		}
+
 		if (READ_ONCE(gsp_phys_rmcall_gr_mme_prelude)) {
 			pl = gsp_sema_mme_prelude_get(&pl_n, &pl_gen);
 			if (pl && pl_n && pl_gen &&
-			    pl_gen != READ_ONCE(gsp_phys_gr_mme_prelude_gen)) {
+			    (pl_gen != READ_ONCE(gsp_phys_gr_mme_prelude_gen) ||
+			     req->h_object != READ_ONCE(gsp_phys_gr_mme_prelude_h))) {
 				if (win3d_n + 2u + 8u + pl_n + sa_block <=
 				    GSP_PHYS_RMCALL_GR_METHOD_WORDS) {
 					emit_prelude = true;
@@ -17627,6 +17757,8 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb,
 			gsp_sema_mme_prelude_get(NULL, &g1);
 			if (g1 == g0) {
 				WRITE_ONCE(gsp_phys_gr_mme_prelude_gen, g0);
+				WRITE_ONCE(gsp_phys_gr_mme_prelude_h,
+					   req->h_object);
 				pr_info("vgsp-gr: MME-PRELUDE emitted gen=%u words=%u window=%u h=0x%x push=0x%llx sdm_drop=%u\n",
 					g0, pwords, win3d_n, req->h_object,
 					req->push_va, sdrop);
@@ -17645,14 +17777,29 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb,
 		 * and the context keeps it for every later window.
 		 */
 		if (emit_prelude && sa_block) {
-			u32 t;
+			u32 t, stub_pc = gsp_sema_mme_ir_end_get();
 
-			*p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) |
+			/*
+			 * SESSION 153-CONTINUED-68: the stub PC must be past the
+			 * replayed program's last instruction - the window also
+			 * calls COVERED slots, and a fixed PC inside a captured
+			 * macro body would clobber its code. Fall back to 0xC0
+			 * when the capture reports nothing yet.
+			 */
+			if (stub_pc < GSP_PHYS_RMCALL_GR_MME_STUB_PC)
+				stub_pc = GSP_PHYS_RMCALL_GR_MME_STUB_PC;
+			/*
+			 * The IR upload MUST use the client's own SEC_OP (5,
+			 * ONE_INC): with op=1 the method auto-increment walked
+			 * the data words onto 0x11c/0x120 and clobbered SA[0]
+			 * while writing no usable stub code (r1131).
+			 */
+			*p++ = (GSP_PHYS_RMCALL_METHOD_ONEINC << 29) |
 			       (4u << 16) | (0x114u >> 2);
-			*p++ = GSP_PHYS_RMCALL_GR_MME_STUB_PC;
-			*p++ = 0u;
-			*p++ = 0u;
-			*p++ = 1u;	/* end_next, no emits (harmless) */
+			*p++ = stub_pc;
+			*p++ = gsp_phys_gr_mme_stub_code[0];
+			*p++ = gsp_phys_gr_mme_stub_code[1];
+			*p++ = gsp_phys_gr_mme_stub_code[2];
 			/*
 			 * Each [slot, stub] pair is its own INC 0x11c run: the
 			 * pair methods are re-written per slot (a single longer
@@ -17663,11 +17810,36 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb,
 				*p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) |
 				       (2u << 16) | (0x11cu >> 2);
 				*p++ = sa_stub[t];
-				*p++ = GSP_PHYS_RMCALL_GR_MME_STUB_PC;
+				*p++ = stub_pc;
 			}
-			pr_info("vgsp-gr: MME-SA-STUB slots=%u words=%u h=0x%x push=0x%llx\n",
-				sa_nstub, sa_block, req->h_object,
+			pr_info("vgsp-gr: MME-SA-STUB slots=%u words=%u pc=0x%x h=0x%x push=0x%llx\n",
+				sa_nstub, sa_block, stub_pc, req->h_object,
 				req->push_va);
+			{
+				char sb[192];
+				u32 k, o = 0;
+
+				sb[0] = '\0';
+				for (k = 0; k < sa_nstub && o + 8u < sizeof(sb); k++)
+					o += scnprintf(sb + o, sizeof(sb) - o,
+						       "%s%u:%x", k ? "," : "",
+						       sa_stub[k],
+						       sa_stub_ent[k]);
+				pr_info("vgsp-gr: MME-SA-STUB list=[%s]\n", sb);
+			}
+			{
+				char sb[192];
+				u32 k, o = 0;
+
+				sb[0] = '\0';
+				for (k = 0; k < sa_ncov && o + 8u < sizeof(sb); k++)
+					o += scnprintf(sb + o, sizeof(sb) - o,
+						       "%s%u:%x", k ? "," : "",
+						       sa_cov[k],
+						       sa_cov_ent[k]);
+				pr_info("vgsp-gr: MME-SA-COVERED list=[%s]\n",
+					sb);
+			}
 		}
 		/*
 		 * SESSION 153-CONTINUED-57: attr/pipeline state prelude (see the
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 748eae9..2b8b422 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -14212,6 +14212,18 @@ static void svc_mme_census(const u8 *buf, u32 len, u64 push_va, u32 h_object)
  * next window re-preplays. The writer is the single service thread and
  * appends strictly in order; the generation only increments after a
  * capture that appended, giving the booter a seqcheck against torn views.
+ *
+ * SESSION 153-CONTINUED-68: the capture can DROP runs once the buffer is
+ * full ("buffer full=N words=1023" on r1126) - e.g. a second GL program
+ * whose content never entered the replay - while the SA table (recorded
+ * unbounded, last-wins) still points slots at those never-replayed IR
+ * addresses; a call then fetches macro data that is not there (Graphics
+ * Exception: MISSING_MACRO_DATA). The coverage bitmap
+ * (svc_mme_ir_cov, instruction granularity) records exactly which IR
+ * instructions the REPLAY holds so the launch builder can stub every
+ * called slot whose entry is outside that set (see
+ * gsp_sema_mme_ir_covered). The buffer cap itself stays 1024 words: the
+ * launch method block is 2048 words and must also carry the window.
  */
 #define SVC_MME_PRELUDE_WORDS	1024u
 #define SVC_MME_PRELUDE_SIGS	512u
@@ -14253,6 +14265,39 @@ static u64 svc_mme_sa_present;
 static u32 svc_mme_sa_entry[64];
 static u32 svc_mme_sa_log;
 
+/*
+ * SESSION 153-CONTINUED-68: furthest IR instruction any CAPTURED 0x114
+ * upload covers (pc + ceil(code words / 3); the client's upload addresses
+ * are instruction units - slot N's pc equals slot N-1's pc + its code
+ * instructions, verified across the r1099/r1116 captures). The launch
+ * builder's MME stub must land BEYOND the replayed program: a fixed stub
+ * PC inside a captured macro's body would clobber that macro's code (the
+ * window calls covered slots too). Recorded only for runs that entered the
+ * replay buffer, so the value always describes the replayed IR extent.
+ */
+static u32 svc_mme_ir_end;
+
+u32 gsp_sema_mme_ir_end_get(void)
+{
+	return READ_ONCE(svc_mme_ir_end);
+}
+
+/*
+ * SESSION 153-CONTINUED-68: instruction-granular coverage of the IR the
+ * REPLAY buffer actually holds (marked for every 0x114 run that entered
+ * svc_mme_prelude_buf). Values >= SVC_MME_IR_BITS are treated as covered
+ * "elsewhere" so an overflow cannot force every slot to be stubbed.
+ */
+#define SVC_MME_IR_BITS	8192u
+static u8 svc_mme_ir_cov[SVC_MME_IR_BITS / 8u];
+
+bool gsp_sema_mme_ir_covered(u32 pc)
+{
+	if (pc >= SVC_MME_IR_BITS)
+		return true;
+	return !!(svc_mme_ir_cov[pc >> 3] & (1u << (pc & 7u)));
+}
+
 u64 gsp_sema_mme_sa_get(u32 *entries, u32 n)
 {
 	u32 i;
@@ -14378,9 +14423,31 @@ static void svc_mme_prelude_capture(const u8 *buf, u32 len, u64 push_va,
 		 * the 0x114 instruction-RAM uploads are the only runs that
 		 * make the prelude functionally complete; the maturity gate
 		 * keys on them.
+		 *
+		 * SESSION 153-CONTINUED-68: also track the furthest covered
+		 * IR instruction (see svc_mme_ir_end) so the builder's stub
+		 * lands beyond the replayed program.
 		 */
-		if (addr == 0x114u && span > 1u)
+		if (addr == 0x114u && span > 1u) {
+			u32 pc = 0, code_words = span - 2u, ir_end, k;
+
+			if (span >= 2u)
+				memcpy(&pc, buf + off + 4u, 4);
+			ir_end = pc + (code_words + 2u) / 3u;
+			if (ir_end > READ_ONCE(svc_mme_ir_end))
+				WRITE_ONCE(svc_mme_ir_end, ir_end);
+			/*
+			 * SESSION 153-CONTINUED-68: mark the instructions this
+			 * replayed run covers (see svc_mme_ir_cov).
+			 */
+			for (k = 0; k < (code_words + 2u) / 3u; k++) {
+				u32 a = pc + k;
+
+				if (a < SVC_MME_IR_BITS)
+					svc_mme_ir_cov[a >> 3] |= 1u << (a & 7u);
+			}
 			svc_mme_prelude_code_words += span - 1u;
+		}
 		svc_mme_prelude_n += span;
 		svc_mme_prelude_sigs[svc_mme_prelude_sig_n++] = sig;
 		added++;
@@ -21152,6 +21219,24 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 				ch->userd_ok = true;
 				*put = best_put;
 				*get = best_get;
+				/*
+				 * SESSION 153-CONTINUED-68: deliberately do NOT
+				 * seed the speculative latch here. Seeding
+				 * armed the pump's judge on channels that
+				 * previously accepted the VRAM candidate
+				 * quietly; a pool block whose PUT advances in
+				 * jumps (r1133: 219 -> 996) or a stale seed
+				 * from an earlier RAM-scan candidate (r1132:
+				 * latch=2 vs put=6) then rejected on every
+				 * pass - adopt/reject spin at ~10 Hz (4638
+				 * REJECTED lines on r1132, the guest's Xorg
+				 * soft-locked, glxgears starved). The r1129-
+				 * r1131 good boots show the intended shape:
+				 * VRAM adopts logged, no rejects, ring
+				 * consumption via the walk.
+				 */
+				ch->gr_ram_userd_latch_at = 0;
+				ch->gr_ram_userd_confirmed = false;
 				pr_info("sema-chan: GR USERD h=0x%x rank=%u found by VRAM shape scan at fb 0x%llx (put=%u get=%u%s, ring-validated)\n",
 					ch->h_object, ch->gr_index, best,
 					best_put, best_get,
diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h
index 2b99274..d2db39f 100644
--- a/kernel/gsp_sema_service.h
+++ b/kernel/gsp_sema_service.h
@@ -535,6 +535,25 @@ bool gsp_sema_gr_surf_recs(u64 *va, u32 *n);
 u64 gsp_sema_mme_sa_get(u32 *entries, u32 n);
 
 /*
+ * SESSION 153-CONTINUED-68: furthest IR instruction covered by the CAPTURED
+ * 0x114 uploads (pc + ceil(code words / 3)). The GR launch builder places
+ * its end-noop macro stub at this address (aligned past the replayed
+ * program) so the stub cannot clobber a captured macro body; 0 until the
+ * first 0x114 upload enters the prelude buffer.
+ */
+u32 gsp_sema_mme_ir_end_get(void);
+
+/*
+ * SESSION 153-CONTINUED-68: is the IR instruction at address `pc` present
+ * in the replay buffer (i.e. was it uploaded by a captured run that was
+ * NOT dropped as a duplicate/over-cap)? The launch builder stubs every
+ * called macro slot whose SA entry sits outside this set, so a truncated
+ * or generation-mismatched capture becomes a no-op instead of a macro
+ * fetch into never-replayed IR (Graphics Exception: MISSING_MACRO_DATA).
+ */
+bool gsp_sema_mme_ir_covered(u32 pc);
+
+/*
  * r378 write guards. Both re-validate a guest-memory write target that was
  * resolved earlier (ledger/pin/walk/counter registration, or the GR submit's
  * per-page resolution) against the CURRENT resolution before the bytes land,