vgsp: CB parser decode fix + pre-replay FE context clear attempt (cwd3 latch survives; zero-store persists)

(a) svc_gl_cb_upload_scan decode now follows the class definitions
(cla16f.h: method 11:0, subchannel 15:13): addr=(hdr&0xfff)<<2, subch=(hdr>>13)&7;
selector B/C swapped (0x2384=B=ADDRESS_UPPER high dword, 0x2388=C=lower,
cla097.h); 0x238c runs apply [offset][constants...] inline data. CB-DIAG
prints subch. Other classes untouched.
(b) Pre-replay CWD poison clear for the stream3d lane: new
gsp_phys_gr_cwd_clear_locked() (checks 0x404490/0x40706c/0x400504 for 0xbadf,
runs the wedge recovery's bounded TSG preempt/resched re-arm + shared latch
wipe) before build_gr_launch; once per launch, stream3d+sema_gr_wedge_recover
gated (CUDA/CE unaffected).

Validation (c250-c254, rich args, ladder armed): the re-arm probes green
('WEDGE-REARM probe rc=0 (engine ALIVE)') but the cwd3 latch does NOT move
('CWD-CLEAR after re-arm ... cwd3=badf5040') and wrote=0 persists (0/1024)
with the 0x1528 trap gone under sema_gl_shader_exc=0 - clause (b) is
falsified as sufficient: the poison is not the TSG/PBDMA context the re-arm
touches. CB-DIAG on these draws shows n3800/n39c8/n2390=0 because bit 12 is
0 on that stream (old/new decodes identical there) - the parser fix stands
on the class definition. X regressed to the DDX DMA wall on these cycles
(DMA-init 0x51 + per-start SIGSEGVs), so clean gl_frames windows were rare.

2 files changed, 161 insertions, 76 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index b548b62..f1cd2cb 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -20713,6 +20713,75 @@ static u32 gsp_phys_gr_fe_status(void)
 }
 
 /*
+ * CONT.474: the MME latch/drain state both context-wiping paths owe after the
+ * r445 hard re-arm (a bWait TSG preempt saves/unloads the engine context, so
+ * the resident MME program is gone). Clearing the emission latch/cursors and
+ * the live/release/pair/window-hold witnesses makes the next launch that can
+ * carry the prelude re-preload the program into the fresh context; without it
+ * every later replay runs its macros against an unloaded MME and stalls
+ * pre-retire (the r1110 mme_pc=513 class; the r1377 split-emitter deadlock and
+ * the r1453 pair/window-hold proofs are the other originals - see the
+ * WEDGE-RECOVER block, whose inline clears this replaces).
+ */
+static void gsp_phys_gr_ctx_wipe_latches(void)
+{
+	WRITE_ONCE(gsp_phys_gr_mme_prelude_gen, 0);
+	WRITE_ONCE(gsp_phys_gr_mme_pre_emit_off, 0);
+	WRITE_ONCE(gsp_phys_gr_mme_pack_off, 0);
+	WRITE_ONCE(gsp_phys_gr_mme_pack_sa, false);
+	WRITE_ONCE(gsp_phys_gr_mme_live_gen, 0);
+	WRITE_ONCE(gsp_phys_gr_mme_release_gen, 0);
+	WRITE_ONCE(gsp_phys_gr_mme_release_h, 0);
+	WRITE_ONCE(gsp_phys_gr_mme_pairs_gen, 0);
+	WRITE_ONCE(gsp_phys_gr_mme_pair_open, false);
+	WRITE_ONCE(gsp_phys_gr_mme_pair_open_split, 0);
+	WRITE_ONCE(gsp_phys_gr_mme_pair_open_entry, false);
+	WRITE_ONCE(gsp_phys_gr_mme_pair_open_entry_split, 0);
+	WRITE_ONCE(gsp_phys_gr_mme_window_hold_gen, 0);
+	WRITE_ONCE(gsp_phys_gr_mme_window_hold_h, 0);
+	WRITE_ONCE(gsp_phys_gr_mme_strict_holds, 0);
+}
+
+/*
+ * CONT.474 CWD-CLEAR: pre-replay priv-error clear for the noQMD stream3d lane.
+ * The c240-c242 receipts show the FE already poisoned when a stream3d replay is
+ * about to be emitted (`MME-FAST begin launch ... esr=0xbadf1201`,
+ * `SKED-WIT ... cwd3=0xbadf5040 act=0`): the 0xbadf latch survives from an
+ * earlier launch's window and this lane carries no context re-init, so the
+ * replay walks a parked CWD and the engine stores nothing
+ * (`NOWRITE-CONFIRMED ... wrote=0`). WEDGE-RECOVER detects the same class but
+ * only AFTER the replay was submitted (so the launch is already lost) and its
+ * re-arm clears grfifo, not the parked CWD. Run the identical bounded re-arm
+ * (bWait TSG preempt -> lane-enable + schedules -> warm-sem probe) BEFORE the
+ * replay is built, then wipe the MME latches so the build below re-preloads
+ * the program into the fresh context. One attempt per launch (caller's flag);
+ * stream3d lane only, so CUDA/QMD and CE submissions are untouched.
+ */
+static u32 gsp_phys_gr_cwd_clears;
+static void gsp_phys_gr_cwd_clear_locked(void __iomem *bar0, u32 h_object,
+					 u64 push_va)
+{
+	u32 esr = ioread32(bar0 + 0x00404490u);
+	u32 cwd3 = ioread32(bar0 + 0x0040706cu);
+	u32 gf = ioread32(bar0 + 0x00400504u);
+
+	if ((esr & 0xffff0000u) != 0xbadf0000u &&
+	    (cwd3 & 0xffff0000u) != 0xbadf0000u &&
+	    (gf & 0xffff0000u) != 0xbadf0000u)
+		return;
+	gsp_phys_gr_cwd_clears++;
+	pr_warn("vgsp-gr: CWD-CLEAR pre-replay poison (esr=%08x cwd3=%08x grfifo=%08x) h=0x%x push=0x%llx - TSG preempt + resched before the replay; clear #%u this boot\n",
+		esr, cwd3, gf, h_object, push_va, gsp_phys_gr_cwd_clears);
+	gsp_phys_mme_regs_log(bar0, "cwd-clear");
+	gsp_phys_gr_wedge_hard_rearm_locked(bar0);
+	gsp_phys_gr_ctx_wipe_latches();
+	pr_warn("vgsp-gr: CWD-CLEAR after re-arm esr=%08x cwd3=%08x grfifo=%08x (MME latches cleared, re-preload owed)\n",
+		ioread32(bar0 + 0x00404490u),
+		ioread32(bar0 + 0x0040706cu),
+		ioread32(bar0 + 0x00400504u));
+}
+
+/*
  * r1445 5c: submit-side FE-busy deferral of the load-only carrier. The
  * r1444 first-poison class submitted its first load-only carrier while the
  * front-end was mid-fetch and the packed-program fetch poisoned the lane.
@@ -28362,6 +28431,11 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
 	 * ordered work item is the only caller.
 	 */
 	bool wedge_recover_done = false;
+	/*
+	 * CONT.474 CWD-CLEAR: one pre-replay priv-error clear attempt per
+	 * launch (gsp_phys_gr_cwd_clear_locked; stream3d lane only).
+	 */
+	bool cwd_clear_done = false;
 	static u32 wedge_recoveries;
 	/*
 	 * SESSION 153-CONTINUED-72: the MME-FAST pre-reset spin can observe
@@ -30943,6 +31017,19 @@ gr_page_unmapped:
 					GSP_PHYS_RMCALL_GR_METHOD_WORDS);
 		win3d_n = pn;
 	}
+	/*
+	 * CONT.474 CWD-CLEAR: clear a pre-existing 0xbadf FE priv-error latch
+	 * BEFORE this replay is emitted (see gsp_phys_gr_cwd_clear_locked).
+	 * Bounded to one attempt per launch; stream3d lane only, so the
+	 * CUDA/QMD and CE paths never reach this site. Gated by the same
+	 * sema_gr_wedge_recover knob as the mid-launch recovery, which stays
+	 * available for a poison the replay itself causes.
+	 */
+	if (stream3d && bar0 && !cwd_clear_done &&
+	    READ_ONCE(gsp_phys_gr_wedge_recover)) {
+		cwd_clear_done = true;
+		gsp_phys_gr_cwd_clear_locked(bar0, req->h_object, req->push_va);
+	}
 	words = gsp_phys_rmcall_build_gr_launch(methods, req, qmd,
 						qmd_dst,
 						ch_sem_va, want_ch,
@@ -31569,72 +31656,17 @@ gr_page_unmapped:
 				mutex_unlock(&gsp_phys_rm_lock);
 				g1 = ioread32(bar0 + 0x00400504u);
 				/*
-				 * SESSION 153-CONTINUED-61: the TSG preempt +
-				 * resched above resets the engine context, so
-				 * the MME program the emitted prelude loaded is
-				 * gone. Clear the emission latch: the next
-				 * launch that can carry the prelude re-preloads
-				 * it (without this every later replay ran its
-				 * macros against an unloaded MME and stalled
-				 * pre-retire - the r1110 mme_pc=513 class).
-				 */
-				WRITE_ONCE(gsp_phys_gr_mme_prelude_gen, 0);
-				/*
-				 * SESSION 153-CONTINUED-79: also rewind the
-				 * split-emission drain cursor, else a cleared
-				 * latch with an exhausted cursor would skip
-				 * the owed re-preload.
-				 */
-				WRITE_ONCE(gsp_phys_gr_mme_pre_emit_off, 0);
-				/*
-				 * SESSION 153-CONTINUED-109: rewind the PACK
-				 * cursor too (the r1377 deadlock: recovery reset
-				 * pre_emit_off but left pack_off at its end, so
-				 * the split emitter hit `poff >= maxa` -> can=0
-				 * -> packed=1 with zero words on every launch:
-				 * `MME-PRELUDE part gen=3 off=0 words=0/3083`
-				 * forever, latch never re-set, gates never
-				 * release, MME never starts). The gate-fail
-				 * path (gsp_phys_gr_mme_iram_gate_fail) already
-				 * rewinds both cursor pairs.
-				 */
-				WRITE_ONCE(gsp_phys_gr_mme_pack_off, 0);
-				WRITE_ONCE(gsp_phys_gr_mme_pack_sa, false);
-				/*
-				 * SESSION 153-CONTINUED-109: the hard re-arm
-				 * wiped the engine context, so the generation
-				 * whose carrier executed is no longer resident.
-				 * Invalidate it; the re-preload's own GP_GET
-				 * advance (readback-free witness) re-promotes
-				 * (fetch_miss/poll-break paths). Without this
-				 * the stale live_gen would mark a wiped context
-				 * live and the next window+calls launch would
-				 * run against an unloaded MME.
-				 */
-				WRITE_ONCE(gsp_phys_gr_mme_live_gen, 0);
-				WRITE_ONCE(gsp_phys_gr_mme_release_gen, 0);
-				WRITE_ONCE(gsp_phys_gr_mme_release_h, 0);
-				/* r1453 PAIR-CHAIN: context wipe voids the
-				 * pair-install proof; re-earn it. */
-				WRITE_ONCE(gsp_phys_gr_mme_pairs_gen, 0);
-				/* r1454 PAIR-CHAIN-ENTRY-ELIDE: the wiped
-				 * context has no open FE pair; clear the
-				 * cross-launch tracker too. */
-				WRITE_ONCE(gsp_phys_gr_mme_pair_open, false);
-				WRITE_ONCE(gsp_phys_gr_mme_pair_open_split,
-					   0);
-				WRITE_ONCE(gsp_phys_gr_mme_pair_open_entry,
-					   false);
-				WRITE_ONCE(
-					 gsp_phys_gr_mme_pair_open_entry_split,
-					 0);
-				/*
-				 * r1447: the wiped context voids the arming
-				 * window-hold and the strict grace budget too.
+				 * SESSION 153-CONTINUED-61/79/109 + r1447/r1453/
+				 * r1454: the TSG preempt + resched above resets
+				 * the engine context, so the MME program the
+				 * emitted prelude loaded is gone; the shared
+				 * wipe clears the emission latch/cursors and
+				 * the live/release/pair/window-hold witnesses
+				 * so the next launch re-preloads it (the r1110
+				 * mme_pc=513, r1377 deadlock and r1453/r1454
+				 * proof-void evidence live at the helper).
 				 */
-				WRITE_ONCE(gsp_phys_gr_mme_window_hold_gen, 0);
-				WRITE_ONCE(gsp_phys_gr_mme_window_hold_h, 0);
-				WRITE_ONCE(gsp_phys_gr_mme_strict_holds, 0);
+				gsp_phys_gr_ctx_wipe_latches();
 				pr_warn("vgsp-gr: WEDGE-RECOVER after re-arm grfifo=0x%08x (%s); recovery #%u this boot; MME prelude latch cleared (context re-preload owed)\n",
 					g1,
 					(g1 & 0xffff0000u) == 0xbadf0000u ?
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 6ce84f1..ebbe3bd 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -18098,6 +18098,7 @@ static void svc_gl_cb_upload_scan(struct nv_vgpu_device *vgpu, u32 h_object,
 	u32 n3800 = 0, n39c8 = 0, n39d0 = 0, n2380 = 0;
 	u32 n238c = 0, n2390 = 0, n3460 = 0;
 	u32 b3800_first = 0xffffffffu;
+	u32 subch_seen = 0xffffffffu;
 
 	(void)vgpu;
 	/*
@@ -18119,7 +18120,7 @@ static void svc_gl_cb_upload_scan(struct nv_vgpu_device *vgpu, u32 h_object,
 	if (base && base != svc_gl_cb_img_base)
 		svc_gl_cb_img_reset(base);
 	while (off + 4u <= len) {
-		u32 hdr, op, cnt, addr, w;
+		u32 hdr, op, cnt, addr, subch, w;
 
 		memcpy(&hdr, buf + off, 4);
 		nhdr++;
@@ -18129,7 +18130,18 @@ static void svc_gl_cb_upload_scan(struct nv_vgpu_device *vgpu, u32 h_object,
 		}
 		op = (hdr >> 29) & 7u;
 		cnt = (hdr >> 16) & 0xfffu;
-		addr = (hdr & 0x1fffu) << 2;
+		/*
+		 * The method header is the A16F DMA encoding: method address
+		 * 11:0 (dwords), subchannel 15:13, count 28:16. The old
+		 * 0x1fff mask folded the reserved bit 12 into the address, so
+		 * a header whose bit 12 is set decoded 0x4000 high and the
+		 * 0x3800/0x39c8/0x2390 targets were never matched (CB-DIAG
+		 * n3800/n39c8/n2390 = 0 on every push).
+		 */
+		addr = (hdr & 0xfffu) << 2;
+		subch = (hdr >> 13) & 0x7u;
+		if (subch_seen == 0xffffffffu)
+			subch_seen = subch;
 		if (op == 0u || op == 2u || op == 7u) {
 			off += 4u;
 			continue;
@@ -18156,14 +18168,22 @@ static void svc_gl_cb_upload_scan(struct nv_vgpu_device *vgpu, u32 h_object,
 				svc_gl_cb_off = w;
 				dirty = true;
 			} else if (addr == 0x2384u) {
+				/*
+				 * SELECTOR_B = ADDRESS_UPPER (bits 7:0 of the
+				 * word are cb addr[39:32]); it is the HIGH half
+				 * of the 64-bit address, not the low one. The
+				 * historical packing put B in the low dword and
+				 * C in the high dword (swapped).
+				 */
 				svc_gl_cb_sel_bc = (svc_gl_cb_sel_bc &
-						    ~0xffffffffULL) |
-						   (u64)w;
+						   0xffffffffULL) |
+						   ((u64)w << 32);
 				svc_gl_cb_sel_seen = true;
 			} else if (addr == 0x2388u) {
+				/* SELECTOR_C = ADDRESS_LOWER (addr[31:0]). */
 				svc_gl_cb_sel_bc = (svc_gl_cb_sel_bc &
-						   0xffffffffULL) |
-						   ((u64)w << 32);
+						    ~0xffffffffULL) |
+						   (u64)w;
 				svc_gl_cb_sel_seen = true;
 			} else if (addr == 0x3800u) {
 				/*
@@ -18212,9 +18232,37 @@ static void svc_gl_cb_upload_scan(struct nv_vgpu_device *vgpu, u32 h_object,
 		else if (addr >= 0x2390u && addr < 0x2390u + 0x1000u)
 			n2390++;
 		if (addr == 0x238cu) {
+			/*
+			 * LOAD_CONSTANT_BUFFER_OFFSET with inline data (the
+			 * driver's own emission, nv3dLoadConstantsHeader /
+			 * nv3dLoadSingleConstant): the run payload is
+			 * [offset][constant dwords...]. Word 0 sets the CB
+			 * load offset; the rest stream through the
+			 * LOAD_CONSTANT_BUFFER data port at successive
+			 * offsets. Only word 0 used to be applied, so an
+			 * upload carried this way landed nothing.
+			 */
 			memcpy(&w, buf + off + 4u, 4);
 			svc_gl_cb_off = w;
 			dirty = true;
+			if (op != 3u && cnt > 1u) {
+				u32 j;
+
+				for (j = 1; j < cnt; j++) {
+					u64 doff = (u64)svc_gl_cb_off +
+						   4ULL * (u64)(j - 1u);
+
+					if (doff + 4ULL > SVC_GL_CB_IMG_LIMIT)
+						continue;
+					memcpy(&w, buf + off + 4u +
+						   4u * j, 4);
+					memcpy(svc_gl_cb_img + doff, &w, 4);
+					words++;
+					dirty = true;
+				}
+				svc_gl_cb_off += 4u * (cnt - 1u);
+				runs++;
+			}
 			off += 4u + cnt * 4u;
 			continue;
 		}
@@ -18230,17 +18278,21 @@ static void svc_gl_cb_upload_scan(struct nv_vgpu_device *vgpu, u32 h_object,
 					svc_gl_cb_sel_a = w;
 					break;
 				case 0x2384u:
+					/* SELECTOR_B = ADDRESS_UPPER: high
+					 * dword of the cb address (see the
+					 * IMMD branch above). */
 					svc_gl_cb_sel_bc =
 						(svc_gl_cb_sel_bc &
-						 ~0xffffffffULL) |
-						(u64)w;
+						 0xffffffffULL) |
+						((u64)w << 32);
 					svc_gl_cb_sel_seen = true;
 					break;
 				case 0x2388u:
+					/* SELECTOR_C = ADDRESS_LOWER. */
 					svc_gl_cb_sel_bc =
 						(svc_gl_cb_sel_bc &
-						 0xffffffffULL) |
-						((u64)w << 32);
+						 ~0xffffffffULL) |
+						(u64)w;
 					svc_gl_cb_sel_seen = true;
 					break;
 				case 0x238cu:
@@ -18333,9 +18385,10 @@ static void svc_gl_cb_upload_scan(struct nv_vgpu_device *vgpu, u32 h_object,
 	 */
 	if (svc_gl_cb_diag_log < SVC_GL_CB_DIAG_MAX) {
 		svc_gl_cb_diag_log++;
-		pr_info("vgsp-sema: sema-gl: CB-DIAG h=0x%x push=0x%llx len=%u hdr=%u run=%u over=%u match=%u[3800=%u 39c8=%u 39d0=%u 2380=%u 238c=%u 2390=%u 3460=%u] b3800first=0x%x words=%u off=0x%x base=0x%llx valid=%u mvp=%u m80=%08x %08x %08x %08x row3=%08x %08x %08x %08x\n",
+		pr_info("vgsp-sema: sema-gl: CB-DIAG h=0x%x push=0x%llx len=%u hdr=%u run=%u over=%u match=%u[3800=%u 39c8=%u 39d0=%u 2380=%u 238c=%u 2390=%u 3460=%u] subch=%u b3800first=0x%x words=%u off=0x%x base=0x%llx valid=%u mvp=%u m80=%08x %08x %08x %08x row3=%08x %08x %08x %08x\n",
 			h_object, push_va, len, nhdr, runs, nover, nmatch,
 			n3800, n39c8, n39d0, n2380, n238c, n2390, n3460,
+			subch_seen == 0xffffffffu ? 0u : subch_seen,
 			b3800_first, words, svc_gl_cb_off, svc_gl_cb_img_base,
 			svc_gl_cb_img_valid ? 1u : 0u,
 			svc_gl_cb_mvp_ok(svc_gl_cb_img) ? 1u : 0u,