vgsp: CB parser decode fix + pre-replay FE context clear attempt (cwd3 latch survives; zero-store persists)

(a) svc_gl_cb_upload_scan decode now follows the class definitions
(cla16f.h: method 11:0, subchannel 15:13): addr=(hdr&0xfff)<<2, subch=(hdr>>13)&7;
selector B/C swapped (0x2384=B=ADDRESS_UPPER high dword, 0x2388=C=lower,
cla097.h); 0x238c runs apply [offset][constants...] inline data. CB-DIAG
prints subch. Other classes untouched.
(b) Pre-replay CWD poison clear for the stream3d lane: new
gsp_phys_gr_cwd_clear_locked() (checks 0x404490/0x40706c/0x400504 for 0xbadf,
runs the wedge recovery's bounded TSG preempt/resched re-arm + shared latch
wipe) before build_gr_launch; once per launch, stream3d+sema_gr_wedge_recover
gated (CUDA/CE unaffected).

Validation (c250-c254, rich args, ladder armed): the re-arm probes green
('WEDGE-REARM probe rc=0 (engine ALIVE)') but the cwd3 latch does NOT move
('CWD-CLEAR after re-arm ... cwd3=badf5040') and wrote=0 persists (0/1024)
with the 0x1528 trap gone under sema_gl_shader_exc=0 - clause (b) is
falsified as sufficient: the poison is not the TSG/PBDMA context the re-arm
touches. CB-DIAG on these draws shows n3800/n39c8/n2390=0 because bit 12 is
0 on that stream (old/new decodes identical there) - the parser fix stands
on the class definition. X regressed to the DDX DMA wall on these cycles
(DMA-init 0x51 + per-start SIGSEGVs), so clean gl_frames windows were rare.

1 files changed, 97 insertions, 65 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index b548b62..f1cd2cb 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -20713,6 +20713,75 @@ static u32 gsp_phys_gr_fe_status(void)
 }
 
 /*
+ * CONT.474: the MME latch/drain state both context-wiping paths owe after the
+ * r445 hard re-arm (a bWait TSG preempt saves/unloads the engine context, so
+ * the resident MME program is gone). Clearing the emission latch/cursors and
+ * the live/release/pair/window-hold witnesses makes the next launch that can
+ * carry the prelude re-preload the program into the fresh context; without it
+ * every later replay runs its macros against an unloaded MME and stalls
+ * pre-retire (the r1110 mme_pc=513 class; the r1377 split-emitter deadlock and
+ * the r1453 pair/window-hold proofs are the other originals - see the
+ * WEDGE-RECOVER block, whose inline clears this replaces).
+ */
+static void gsp_phys_gr_ctx_wipe_latches(void)
+{
+	WRITE_ONCE(gsp_phys_gr_mme_prelude_gen, 0);
+	WRITE_ONCE(gsp_phys_gr_mme_pre_emit_off, 0);
+	WRITE_ONCE(gsp_phys_gr_mme_pack_off, 0);
+	WRITE_ONCE(gsp_phys_gr_mme_pack_sa, false);
+	WRITE_ONCE(gsp_phys_gr_mme_live_gen, 0);
+	WRITE_ONCE(gsp_phys_gr_mme_release_gen, 0);
+	WRITE_ONCE(gsp_phys_gr_mme_release_h, 0);
+	WRITE_ONCE(gsp_phys_gr_mme_pairs_gen, 0);
+	WRITE_ONCE(gsp_phys_gr_mme_pair_open, false);
+	WRITE_ONCE(gsp_phys_gr_mme_pair_open_split, 0);
+	WRITE_ONCE(gsp_phys_gr_mme_pair_open_entry, false);
+	WRITE_ONCE(gsp_phys_gr_mme_pair_open_entry_split, 0);
+	WRITE_ONCE(gsp_phys_gr_mme_window_hold_gen, 0);
+	WRITE_ONCE(gsp_phys_gr_mme_window_hold_h, 0);
+	WRITE_ONCE(gsp_phys_gr_mme_strict_holds, 0);
+}
+
+/*
+ * CONT.474 CWD-CLEAR: pre-replay priv-error clear for the noQMD stream3d lane.
+ * The c240-c242 receipts show the FE already poisoned when a stream3d replay is
+ * about to be emitted (`MME-FAST begin launch ... esr=0xbadf1201`,
+ * `SKED-WIT ... cwd3=0xbadf5040 act=0`): the 0xbadf latch survives from an
+ * earlier launch's window and this lane carries no context re-init, so the
+ * replay walks a parked CWD and the engine stores nothing
+ * (`NOWRITE-CONFIRMED ... wrote=0`). WEDGE-RECOVER detects the same class but
+ * only AFTER the replay was submitted (so the launch is already lost) and its
+ * re-arm clears grfifo, not the parked CWD. Run the identical bounded re-arm
+ * (bWait TSG preempt -> lane-enable + schedules -> warm-sem probe) BEFORE the
+ * replay is built, then wipe the MME latches so the build below re-preloads
+ * the program into the fresh context. One attempt per launch (caller's flag);
+ * stream3d lane only, so CUDA/QMD and CE submissions are untouched.
+ */
+static u32 gsp_phys_gr_cwd_clears;
+static void gsp_phys_gr_cwd_clear_locked(void __iomem *bar0, u32 h_object,
+					 u64 push_va)
+{
+	u32 esr = ioread32(bar0 + 0x00404490u);
+	u32 cwd3 = ioread32(bar0 + 0x0040706cu);
+	u32 gf = ioread32(bar0 + 0x00400504u);
+
+	if ((esr & 0xffff0000u) != 0xbadf0000u &&
+	    (cwd3 & 0xffff0000u) != 0xbadf0000u &&
+	    (gf & 0xffff0000u) != 0xbadf0000u)
+		return;
+	gsp_phys_gr_cwd_clears++;
+	pr_warn("vgsp-gr: CWD-CLEAR pre-replay poison (esr=%08x cwd3=%08x grfifo=%08x) h=0x%x push=0x%llx - TSG preempt + resched before the replay; clear #%u this boot\n",
+		esr, cwd3, gf, h_object, push_va, gsp_phys_gr_cwd_clears);
+	gsp_phys_mme_regs_log(bar0, "cwd-clear");
+	gsp_phys_gr_wedge_hard_rearm_locked(bar0);
+	gsp_phys_gr_ctx_wipe_latches();
+	pr_warn("vgsp-gr: CWD-CLEAR after re-arm esr=%08x cwd3=%08x grfifo=%08x (MME latches cleared, re-preload owed)\n",
+		ioread32(bar0 + 0x00404490u),
+		ioread32(bar0 + 0x0040706cu),
+		ioread32(bar0 + 0x00400504u));
+}
+
+/*
  * r1445 5c: submit-side FE-busy deferral of the load-only carrier. The
  * r1444 first-poison class submitted its first load-only carrier while the
  * front-end was mid-fetch and the packed-program fetch poisoned the lane.
@@ -28362,6 +28431,11 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
 	 * ordered work item is the only caller.
 	 */
 	bool wedge_recover_done = false;
+	/*
+	 * CONT.474 CWD-CLEAR: one pre-replay priv-error clear attempt per
+	 * launch (gsp_phys_gr_cwd_clear_locked; stream3d lane only).
+	 */
+	bool cwd_clear_done = false;
 	static u32 wedge_recoveries;
 	/*
 	 * SESSION 153-CONTINUED-72: the MME-FAST pre-reset spin can observe
@@ -30943,6 +31017,19 @@ gr_page_unmapped:
 					GSP_PHYS_RMCALL_GR_METHOD_WORDS);
 		win3d_n = pn;
 	}
+	/*
+	 * CONT.474 CWD-CLEAR: clear a pre-existing 0xbadf FE priv-error latch
+	 * BEFORE this replay is emitted (see gsp_phys_gr_cwd_clear_locked).
+	 * Bounded to one attempt per launch; stream3d lane only, so the
+	 * CUDA/QMD and CE paths never reach this site. Gated by the same
+	 * sema_gr_wedge_recover knob as the mid-launch recovery, which stays
+	 * available for a poison the replay itself causes.
+	 */
+	if (stream3d && bar0 && !cwd_clear_done &&
+	    READ_ONCE(gsp_phys_gr_wedge_recover)) {
+		cwd_clear_done = true;
+		gsp_phys_gr_cwd_clear_locked(bar0, req->h_object, req->push_va);
+	}
 	words = gsp_phys_rmcall_build_gr_launch(methods, req, qmd,
 						qmd_dst,
 						ch_sem_va, want_ch,
@@ -31569,72 +31656,17 @@ gr_page_unmapped:
 				mutex_unlock(&gsp_phys_rm_lock);
 				g1 = ioread32(bar0 + 0x00400504u);
 				/*
-				 * SESSION 153-CONTINUED-61: the TSG preempt +
-				 * resched above resets the engine context, so
-				 * the MME program the emitted prelude loaded is
-				 * gone. Clear the emission latch: the next
-				 * launch that can carry the prelude re-preloads
-				 * it (without this every later replay ran its
-				 * macros against an unloaded MME and stalled
-				 * pre-retire - the r1110 mme_pc=513 class).
-				 */
-				WRITE_ONCE(gsp_phys_gr_mme_prelude_gen, 0);
-				/*
-				 * SESSION 153-CONTINUED-79: also rewind the
-				 * split-emission drain cursor, else a cleared
-				 * latch with an exhausted cursor would skip
-				 * the owed re-preload.
-				 */
-				WRITE_ONCE(gsp_phys_gr_mme_pre_emit_off, 0);
-				/*
-				 * SESSION 153-CONTINUED-109: rewind the PACK
-				 * cursor too (the r1377 deadlock: recovery reset
-				 * pre_emit_off but left pack_off at its end, so
-				 * the split emitter hit `poff >= maxa` -> can=0
-				 * -> packed=1 with zero words on every launch:
-				 * `MME-PRELUDE part gen=3 off=0 words=0/3083`
-				 * forever, latch never re-set, gates never
-				 * release, MME never starts). The gate-fail
-				 * path (gsp_phys_gr_mme_iram_gate_fail) already
-				 * rewinds both cursor pairs.
-				 */
-				WRITE_ONCE(gsp_phys_gr_mme_pack_off, 0);
-				WRITE_ONCE(gsp_phys_gr_mme_pack_sa, false);
-				/*
-				 * SESSION 153-CONTINUED-109: the hard re-arm
-				 * wiped the engine context, so the generation
-				 * whose carrier executed is no longer resident.
-				 * Invalidate it; the re-preload's own GP_GET
-				 * advance (readback-free witness) re-promotes
-				 * (fetch_miss/poll-break paths). Without this
-				 * the stale live_gen would mark a wiped context
-				 * live and the next window+calls launch would
-				 * run against an unloaded MME.
-				 */
-				WRITE_ONCE(gsp_phys_gr_mme_live_gen, 0);
-				WRITE_ONCE(gsp_phys_gr_mme_release_gen, 0);
-				WRITE_ONCE(gsp_phys_gr_mme_release_h, 0);
-				/* r1453 PAIR-CHAIN: context wipe voids the
-				 * pair-install proof; re-earn it. */
-				WRITE_ONCE(gsp_phys_gr_mme_pairs_gen, 0);
-				/* r1454 PAIR-CHAIN-ENTRY-ELIDE: the wiped
-				 * context has no open FE pair; clear the
-				 * cross-launch tracker too. */
-				WRITE_ONCE(gsp_phys_gr_mme_pair_open, false);
-				WRITE_ONCE(gsp_phys_gr_mme_pair_open_split,
-					   0);
-				WRITE_ONCE(gsp_phys_gr_mme_pair_open_entry,
-					   false);
-				WRITE_ONCE(
-					 gsp_phys_gr_mme_pair_open_entry_split,
-					 0);
-				/*
-				 * r1447: the wiped context voids the arming
-				 * window-hold and the strict grace budget too.
+				 * SESSION 153-CONTINUED-61/79/109 + r1447/r1453/
+				 * r1454: the TSG preempt + resched above resets
+				 * the engine context, so the MME program the
+				 * emitted prelude loaded is gone; the shared
+				 * wipe clears the emission latch/cursors and
+				 * the live/release/pair/window-hold witnesses
+				 * so the next launch re-preloads it (the r1110
+				 * mme_pc=513, r1377 deadlock and r1453/r1454
+				 * proof-void evidence live at the helper).
 				 */
-				WRITE_ONCE(gsp_phys_gr_mme_window_hold_gen, 0);
-				WRITE_ONCE(gsp_phys_gr_mme_window_hold_h, 0);
-				WRITE_ONCE(gsp_phys_gr_mme_strict_holds, 0);
+				gsp_phys_gr_ctx_wipe_latches();
 				pr_warn("vgsp-gr: WEDGE-RECOVER after re-arm grfifo=0x%08x (%s); recovery #%u this boot; MME prelude latch cleared (context re-preload owed)\n",
 					g1,
 					(g1 & 0xffff0000u) == 0xbadf0000u ?