vgsp: WEDGE-RECOVER race fix (MME-FAST sample returns poison observation; gate = live badf || fast_poison - validated r1151 catches self-cleared poison, recovery #1, retires continue), GR-STATE retire one-shot readback (fed0..3/trap/mthd/mmegen/DS/grfifo, beef <=48), MME-FAST fed= transition lines, force-prelude while keep_mask!=0 - mask train: stores 0 under EVERY mask; kept-macro execution proven (slot-2 macro ran); r1153 cleanest (keep=5, nothing dropped, prelude every launch, fast retires, 0 poison/ILLEGAL, still 0/900) = store wall DOWNSTREAM of macro execution; degrade intrinsic ~66s; next: write witness over relocated operand VAs + MME shadow-scratch (not only 900 RT pages) - SESSION 153 r1151-r1153

1 files changed, 118 insertions, 10 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index 47e5168..640fa55 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -5088,7 +5088,7 @@ module_param_named(sema_mme_fast_ms, gsp_phys_mme_fast_ms, uint, 0644);
 MODULE_PARM_DESC(sema_mme_fast_ms,
 	"Spin-sample the MME fault registers right after each GR launch doorbell for this many ms, logging every transition before the GSP-RM clears them (0 = off; default 400)");
 
-static void gsp_phys_mme_fast_sample(void __iomem *bar0, const char *tag,
+static bool gsp_phys_mme_fast_sample(void __iomem *bar0, const char *tag,
 				     u32 h_object)
 {
 	static u32 fast_boot_n;
@@ -5098,14 +5098,14 @@ static void gsp_phys_mme_fast_sample(void __iomem *bar0, const char *tag,
 	bool poisoned = false;
 
 	if (!bar0 || !ms || fast_boot_n >= 48u)
-		return;
+		return false;
 	/*
 	 * Beef-family channels get the full budget (the draw-class windows
 	 * are the ones with the MME fault); other channels are sampled only
 	 * a few times so a fast X-channel launch train cannot consume it.
 	 */
 	if ((h_object & 0xfffff000u) != 0xbeef0000u && fast_boot_n >= 2u)
-		return;
+		return false;
 	fast_boot_n++;
 	esr_p = ioread32(bar0 + 0x00404490u);
 	ds_p = ioread32(bar0 + 0x00405840u);
@@ -5125,15 +5125,30 @@ static void gsp_phys_mme_fast_sample(void __iomem *bar0, const char *tag,
 				u32 info = ioread32(bar0 + 0x00404494u);
 				u32 info2 = ioread32(bar0 + 0x00404498u);
 				u32 mmeir = ioread32(bar0 + 0x0040a630u);
+				u32 mmeir1 = ioread32(bar0 + 0x0040a634u);
 				u32 mmedata = ioread32(bar0 + 0x00400708u);
 
-				pr_warn("vgsp-gr: MME-FAST %s h=0x%x esr %08x->%08x info=%08x(pc_valid=%u pc=%u mac=0x%x) info2_ir=%08x ds %08x->%08x grfifo %08x->%08x trap=%08x data=%08x fed0=%08x\n",
+				/*
+				 * r1151 (SESSION 153-CONTINUED-72):
+				 * GR-STATE: the FE decode-state EX slot 0
+				 * records the last MME-GENERATED method
+				 * write (bit31 generated, 21:12 macro
+				 * address, 11:0 pc). gen0=1 with the
+				 * kept slot's macro address is the
+				 * "kept macro executes" witness during
+				 * the launch; slot 1 is the second
+				 * lane of the same record set.
+				 */
+				pr_warn("vgsp-gr: MME-FAST %s h=0x%x esr %08x->%08x info=%08x(pc_valid=%u pc=%u mac=0x%x) info2_ir=%08x ds %08x->%08x grfifo %08x->%08x trap=%08x data=%08x fed=%08x,%08x (pc0=%u mac0=0x%x gen0=%u)\n",
 					tag, h_object, esr_p, esr,
 					info, (info >> 28) & 1u,
 					info & 0xfffu,
 					(info >> 12) & 0xffu, info2,
 					ds_p, ds, gf_p, gf, tr,
-					mmedata, mmeir);
+					mmedata, mmeir, mmeir1,
+					mmeir & 0xfffu,
+					(mmeir >> 12) & 0x3ffu,
+					(mmeir >> 31) & 1u);
 			}
 			n++;
 			esr_p = esr;
@@ -5156,6 +5171,15 @@ static void gsp_phys_mme_fast_sample(void __iomem *bar0, const char *tag,
 	}
 	pr_info("vgsp-gr: MME-FAST end %s h=0x%x events=%u%s\n",
 		tag, h_object, n, poisoned ? " (poison)" : "");
+	/*
+	 * SESSION 153-CONTINUED-72: return the poison observation to the
+	 * launch site. The r1149 race: the poison flooded during this spin
+	 * and the FECS cleared it before the poll loop's WEDGE-RECOVER check
+	 * read grfifo, so recovery missed, the prelude latch stayed set and
+	 * the engine wedged for the boot. The caller now latches this flag
+	 * into the recovery gate instead of relying on a live register read.
+	 */
+	return poisoned;
 }
 
 static void gsp_phys_pbdma_fault_dump(u32 chid)
@@ -17961,8 +17985,20 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb,
 
 		if (READ_ONCE(gsp_phys_rmcall_gr_mme_prelude)) {
 			pl = gsp_sema_mme_prelude_get(&pl_n, &pl_gen);
+			/*
+			 * SESSION 153-CONTINUED-72: with keep_mask bits set a
+			 * kept macro can execute, and it needs the MME program
+			 * resident. The r1152 evidence: the prelude was carried
+			 * by one launch of the generation only; when that launch
+			 * did not complete (engine fetch NONE / timeout), the
+			 * later kept-macro launches ran an UNLOADED program and
+			 * priv-errored (ILLEGAL_OPCODE -> poison). Force
+			 * re-emission on every launch while any slot is kept
+			 * (idempotent per the emission contract).
+			 */
 			if (pl && pl_n && pl_gen &&
-			    (pl_gen != READ_ONCE(gsp_phys_gr_mme_prelude_gen) ||
+			    (READ_ONCE(gsp_phys_rmcall_gr_keep_mask) ||
+			     pl_gen != READ_ONCE(gsp_phys_gr_mme_prelude_gen) ||
 			     req->h_object != READ_ONCE(gsp_phys_gr_mme_prelude_h))) {
 				if (win3d_n + 2u + 8u + pl_n + sa_block <=
 				    GSP_PHYS_RMCALL_GR_METHOD_WORDS) {
@@ -20209,6 +20245,51 @@ gsp_phys_gr_poll_break_log(u32 w0, u32 want_qmd, u32 w1, u32 since_us,
 }
 
 /*
+ * SESSION 153-CONTINUED-72: GR-STATE retire readback. The keep-mask A/B
+ * must answer "does the kept macro execute at all" before chasing the
+ * 0/903 stores. The FE decode-state EX records (0x40a630..3c) name the
+ * last MME-GENERATED method write: bit31 GENERATED, 21:12 macro address,
+ * 11:0 pc - gen0=1 with the kept slot's macro address at the retire is
+ * the live witness that the engine fetched and ran the kept macro. The
+ * MME HWW latch (0x404490 + INFO/IR), the trapped method (0x40704 offset
+ * + bit20 mmegen, data 0x40708) and the DS latch catch the write when it
+ * faulted. One line per retire; bounded per boot (beef up to 48, others
+ * 4) so a battery boot cannot flood.
+ */
+static noinline void gsp_phys_gr_state_rb_log(u32 h_object)
+{
+	static u32 gs_beef_n, gs_other_n;
+	void __iomem *bar0 = gsp_phys_gpu_bar0();
+	bool beef = (h_object & 0xfffff000u) == 0xbeef0000u;
+	u32 fed0, fed1, fed2, fed3, esr, info, ir2, trap, data, ds, gf;
+
+	if (!bar0)
+		return;
+	if (beef ? gs_beef_n >= 48u : gs_other_n >= 4u)
+		return;
+	if (beef)
+		gs_beef_n++;
+	else
+		gs_other_n++;
+	fed0 = ioread32(bar0 + 0x0040a630u);
+	fed1 = ioread32(bar0 + 0x0040a634u);
+	fed2 = ioread32(bar0 + 0x0040a638u);
+	fed3 = ioread32(bar0 + 0x0040a63cu);
+	esr = ioread32(bar0 + 0x00404490u);
+	info = ioread32(bar0 + 0x00404494u);
+	ir2 = ioread32(bar0 + 0x00404498u);
+	trap = ioread32(bar0 + 0x00400704u);
+	data = ioread32(bar0 + 0x00400708u);
+	ds = ioread32(bar0 + 0x00405840u);
+	gf = ioread32(bar0 + 0x00400504u);
+	pr_info("vgsp-gr: GR-STATE retire h=0x%x fed=%08x,%08x,%08x,%08x (pc0=%u mac0=0x%x gen0=%u) mme_esr=%08x info=%08x ir=%08x trap=%08x(mthd=0x%x mmegen=%u) data=%08x ds=%08x grfifo=%08x\n",
+		h_object, fed0, fed1, fed2, fed3,
+		fed0 & 0xfffu, (fed0 >> 12) & 0x3ffu, (fed0 >> 31) & 1u,
+		esr, info, ir2, trap, trap & 0x3ffcu, (trap >> 20) & 1u,
+		data, ds, gf);
+}
+
+/*
  * F4-b: one-shot deposit readback at the first timeout (noinline for the same
  * frame reason as above). Reads the qmdwin/shadow destination (or the guest
  * QMD page) with the booter's existing unaligned host readback; w63 is the
@@ -22005,6 +22086,14 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
 	bool wedge_recover_done = false;
 	static u32 wedge_recoveries;
 	/*
+	 * SESSION 153-CONTINUED-72: the MME-FAST pre-reset spin can observe
+	 * the 0xbadf poison and then the FECS clears it before the poll
+	 * loop's first recovery check reads grfifo (the r1149 race: recovery
+	 * missed -> prelude latch stuck -> engine wedged for the boot). The
+	 * observation is latched here and OR-ed into the WEDGE-RECOVER gate.
+	 */
+	bool fast_poison = false;
+	/*
 	 * F2/F4 state: the fast first-timeout re-arm runs once per launch; the
 	 * deposit readback is one-shot per launch (the first timeout is the
 	 * evidence window - the old r198 post-mortem only ran after attempt 3,
@@ -23914,7 +24003,8 @@ gr_page_unmapped:
 	 * caught before the GSP-RM clears them.
 	 */
 	if (stream3d && bar0)
-		gsp_phys_mme_fast_sample(bar0, "launch", req->h_object);
+		fast_poison = gsp_phys_mme_fast_sample(bar0, "launch",
+						       req->h_object);
 
 	poll_start_us = (u32)ktime_to_us(ktime_get());
 	deadline = jiffies + msecs_to_jiffies(poll_ms);
@@ -24068,13 +24158,26 @@ gr_page_unmapped:
 		    !wedge_recover_done) {
 			u32 g0 = ioread32(bar0 + 0x00400504u);
 
-			if ((g0 & 0xffff0000u) == 0xbadf0000u) {
+			/*
+			 * SESSION 153-CONTINUED-72: either the poison is still
+			 * asserted here, or the MME-FAST spin caught it before
+			 * the FECS cleared it (fast_poison latch). Both mean
+			 * the window priv-errored and the engine context is
+			 * gone; the earlier live-register-only check missed
+			 * the self-cleared case (r1149) and wedged the boot.
+			 */
+			if ((g0 & 0xffff0000u) == 0xbadf0000u || fast_poison) {
 				u32 g1;
 
 				wedge_recover_done = true;
 				wedge_recoveries++;
-				pr_warn("vgsp-gr: WEDGE-RECOVER priv-error poison seen (grfifo=0x%08x) h=0x%x push=0x%llx - TSG preempt + resched\n",
-					g0, req->h_object, req->push_va);
+				pr_warn("vgsp-gr: WEDGE-RECOVER priv-error poison seen (grfifo=0x%08x%s) h=0x%x push=0x%llx - TSG preempt + resched\n",
+					g0,
+					fast_poison &&
+					(g0 & 0xffff0000u) != 0xbadf0000u ?
+						" via MME-FAST latch (self-cleared)" :
+						"",
+					req->h_object, req->push_va);
 				gsp_phys_mme_regs_log(bar0, "wedge-poison");
 				mutex_lock(&gsp_phys_rm_lock);
 				gsp_phys_gr_wedge_hard_rearm_locked(bar0);
@@ -24222,6 +24325,11 @@ gr_page_unmapped:
 			/* F4-a: first poll break + USERD witnesses. */
 			gsp_phys_gr_poll_break_log(w0, want_qmd, w1,
 						   poll_start_us, &sub);
+			/*
+			 * SESSION 153-CONTINUED-72: GR-STATE readback at the
+			 * retire (fed MME-generated-write records + latches).
+			 */
+			gsp_phys_gr_state_rb_log(req->h_object);
 			break;
 		}
 		/*