From ac741b2a993d4ff8f4b67e385df00d1a58fe3a4f Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Fri, 13 Nov 2026 05:49:55 -0500 Subject: vgsp: CB parser decode fix + pre-replay FE context clear attempt (cwd3 latch survives; zero-store persists) (a) svc_gl_cb_upload_scan decode now follows the class definitions (cla16f.h: method 11:0, subchannel 15:13): addr=(hdr&0xfff)<<2, subch=(hdr>>13)&7; selector B/C swapped (0x2384=B=ADDRESS_UPPER high dword, 0x2388=C=lower, cla097.h); 0x238c runs apply [offset][constants...] inline data. CB-DIAG prints subch. Other classes untouched. (b) Pre-replay CWD poison clear for the stream3d lane: new gsp_phys_gr_cwd_clear_locked() (checks 0x404490/0x40706c/0x400504 for 0xbadf, runs the wedge recovery's bounded TSG preempt/resched re-arm + shared latch wipe) before build_gr_launch; once per launch, stream3d+sema_gr_wedge_recover gated (CUDA/CE unaffected). Validation (c250-c254, rich args, ladder armed): the re-arm probes green ('WEDGE-REARM probe rc=0 (engine ALIVE)') but the cwd3 latch does NOT move ('CWD-CLEAR after re-arm ... cwd3=badf5040') and wrote=0 persists (0/1024) with the 0x1528 trap gone under sema_gl_shader_exc=0 - clause (b) is falsified as sufficient: the poison is not the TSG/PBDMA context the re-arm touches. CB-DIAG on these draws shows n3800/n39c8/n2390=0 because bit 12 is 0 on that stream (old/new decodes identical there) - the parser fix stands on the class definition. X regressed to the DDX DMA wall on these cycles (DMA-init 0x51 + per-start SIGSEGVs), so clean gl_frames windows were rare. --- kernel/gsp_phys_booter.c | 162 +++++++++++++++++++++++++++------------------- kernel/gsp_sema_service.c | 75 +++++++++++++++++---- 2 files changed, 161 insertions(+), 76 deletions(-) diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index b548b62..f1cd2cb 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -20712,6 +20712,75 @@ static u32 gsp_phys_gr_fe_status(void) return bar0 ? ioread32(bar0 + 0x00400504u) : 0u; } +/* + * CONT.474: the MME latch/drain state both context-wiping paths owe after the + * r445 hard re-arm (a bWait TSG preempt saves/unloads the engine context, so + * the resident MME program is gone). Clearing the emission latch/cursors and + * the live/release/pair/window-hold witnesses makes the next launch that can + * carry the prelude re-preload the program into the fresh context; without it + * every later replay runs its macros against an unloaded MME and stalls + * pre-retire (the r1110 mme_pc=513 class; the r1377 split-emitter deadlock and + * the r1453 pair/window-hold proofs are the other originals - see the + * WEDGE-RECOVER block, whose inline clears this replaces). + */ +static void gsp_phys_gr_ctx_wipe_latches(void) +{ + WRITE_ONCE(gsp_phys_gr_mme_prelude_gen, 0); + WRITE_ONCE(gsp_phys_gr_mme_pre_emit_off, 0); + WRITE_ONCE(gsp_phys_gr_mme_pack_off, 0); + WRITE_ONCE(gsp_phys_gr_mme_pack_sa, false); + WRITE_ONCE(gsp_phys_gr_mme_live_gen, 0); + WRITE_ONCE(gsp_phys_gr_mme_release_gen, 0); + WRITE_ONCE(gsp_phys_gr_mme_release_h, 0); + WRITE_ONCE(gsp_phys_gr_mme_pairs_gen, 0); + WRITE_ONCE(gsp_phys_gr_mme_pair_open, false); + WRITE_ONCE(gsp_phys_gr_mme_pair_open_split, 0); + WRITE_ONCE(gsp_phys_gr_mme_pair_open_entry, false); + WRITE_ONCE(gsp_phys_gr_mme_pair_open_entry_split, 0); + WRITE_ONCE(gsp_phys_gr_mme_window_hold_gen, 0); + WRITE_ONCE(gsp_phys_gr_mme_window_hold_h, 0); + WRITE_ONCE(gsp_phys_gr_mme_strict_holds, 0); +} + +/* + * CONT.474 CWD-CLEAR: pre-replay priv-error clear for the noQMD stream3d lane. + * The c240-c242 receipts show the FE already poisoned when a stream3d replay is + * about to be emitted (`MME-FAST begin launch ... esr=0xbadf1201`, + * `SKED-WIT ... cwd3=0xbadf5040 act=0`): the 0xbadf latch survives from an + * earlier launch's window and this lane carries no context re-init, so the + * replay walks a parked CWD and the engine stores nothing + * (`NOWRITE-CONFIRMED ... wrote=0`). WEDGE-RECOVER detects the same class but + * only AFTER the replay was submitted (so the launch is already lost) and its + * re-arm clears grfifo, not the parked CWD. Run the identical bounded re-arm + * (bWait TSG preempt -> lane-enable + schedules -> warm-sem probe) BEFORE the + * replay is built, then wipe the MME latches so the build below re-preloads + * the program into the fresh context. One attempt per launch (caller's flag); + * stream3d lane only, so CUDA/QMD and CE submissions are untouched. + */ +static u32 gsp_phys_gr_cwd_clears; +static void gsp_phys_gr_cwd_clear_locked(void __iomem *bar0, u32 h_object, + u64 push_va) +{ + u32 esr = ioread32(bar0 + 0x00404490u); + u32 cwd3 = ioread32(bar0 + 0x0040706cu); + u32 gf = ioread32(bar0 + 0x00400504u); + + if ((esr & 0xffff0000u) != 0xbadf0000u && + (cwd3 & 0xffff0000u) != 0xbadf0000u && + (gf & 0xffff0000u) != 0xbadf0000u) + return; + gsp_phys_gr_cwd_clears++; + pr_warn("vgsp-gr: CWD-CLEAR pre-replay poison (esr=%08x cwd3=%08x grfifo=%08x) h=0x%x push=0x%llx - TSG preempt + resched before the replay; clear #%u this boot\n", + esr, cwd3, gf, h_object, push_va, gsp_phys_gr_cwd_clears); + gsp_phys_mme_regs_log(bar0, "cwd-clear"); + gsp_phys_gr_wedge_hard_rearm_locked(bar0); + gsp_phys_gr_ctx_wipe_latches(); + pr_warn("vgsp-gr: CWD-CLEAR after re-arm esr=%08x cwd3=%08x grfifo=%08x (MME latches cleared, re-preload owed)\n", + ioread32(bar0 + 0x00404490u), + ioread32(bar0 + 0x0040706cu), + ioread32(bar0 + 0x00400504u)); +} + /* * r1445 5c: submit-side FE-busy deferral of the load-only carrier. The * r1444 first-poison class submitted its first load-only carrier while the @@ -28362,6 +28431,11 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, * ordered work item is the only caller. */ bool wedge_recover_done = false; + /* + * CONT.474 CWD-CLEAR: one pre-replay priv-error clear attempt per + * launch (gsp_phys_gr_cwd_clear_locked; stream3d lane only). + */ + bool cwd_clear_done = false; static u32 wedge_recoveries; /* * SESSION 153-CONTINUED-72: the MME-FAST pre-reset spin can observe @@ -30943,6 +31017,19 @@ gr_page_unmapped: GSP_PHYS_RMCALL_GR_METHOD_WORDS); win3d_n = pn; } + /* + * CONT.474 CWD-CLEAR: clear a pre-existing 0xbadf FE priv-error latch + * BEFORE this replay is emitted (see gsp_phys_gr_cwd_clear_locked). + * Bounded to one attempt per launch; stream3d lane only, so the + * CUDA/QMD and CE paths never reach this site. Gated by the same + * sema_gr_wedge_recover knob as the mid-launch recovery, which stays + * available for a poison the replay itself causes. + */ + if (stream3d && bar0 && !cwd_clear_done && + READ_ONCE(gsp_phys_gr_wedge_recover)) { + cwd_clear_done = true; + gsp_phys_gr_cwd_clear_locked(bar0, req->h_object, req->push_va); + } words = gsp_phys_rmcall_build_gr_launch(methods, req, qmd, qmd_dst, ch_sem_va, want_ch, @@ -31569,72 +31656,17 @@ gr_page_unmapped: mutex_unlock(&gsp_phys_rm_lock); g1 = ioread32(bar0 + 0x00400504u); /* - * SESSION 153-CONTINUED-61: the TSG preempt + - * resched above resets the engine context, so - * the MME program the emitted prelude loaded is - * gone. Clear the emission latch: the next - * launch that can carry the prelude re-preloads - * it (without this every later replay ran its - * macros against an unloaded MME and stalled - * pre-retire - the r1110 mme_pc=513 class). - */ - WRITE_ONCE(gsp_phys_gr_mme_prelude_gen, 0); - /* - * SESSION 153-CONTINUED-79: also rewind the - * split-emission drain cursor, else a cleared - * latch with an exhausted cursor would skip - * the owed re-preload. - */ - WRITE_ONCE(gsp_phys_gr_mme_pre_emit_off, 0); - /* - * SESSION 153-CONTINUED-109: rewind the PACK - * cursor too (the r1377 deadlock: recovery reset - * pre_emit_off but left pack_off at its end, so - * the split emitter hit `poff >= maxa` -> can=0 - * -> packed=1 with zero words on every launch: - * `MME-PRELUDE part gen=3 off=0 words=0/3083` - * forever, latch never re-set, gates never - * release, MME never starts). The gate-fail - * path (gsp_phys_gr_mme_iram_gate_fail) already - * rewinds both cursor pairs. - */ - WRITE_ONCE(gsp_phys_gr_mme_pack_off, 0); - WRITE_ONCE(gsp_phys_gr_mme_pack_sa, false); - /* - * SESSION 153-CONTINUED-109: the hard re-arm - * wiped the engine context, so the generation - * whose carrier executed is no longer resident. - * Invalidate it; the re-preload's own GP_GET - * advance (readback-free witness) re-promotes - * (fetch_miss/poll-break paths). Without this - * the stale live_gen would mark a wiped context - * live and the next window+calls launch would - * run against an unloaded MME. - */ - WRITE_ONCE(gsp_phys_gr_mme_live_gen, 0); - WRITE_ONCE(gsp_phys_gr_mme_release_gen, 0); - WRITE_ONCE(gsp_phys_gr_mme_release_h, 0); - /* r1453 PAIR-CHAIN: context wipe voids the - * pair-install proof; re-earn it. */ - WRITE_ONCE(gsp_phys_gr_mme_pairs_gen, 0); - /* r1454 PAIR-CHAIN-ENTRY-ELIDE: the wiped - * context has no open FE pair; clear the - * cross-launch tracker too. */ - WRITE_ONCE(gsp_phys_gr_mme_pair_open, false); - WRITE_ONCE(gsp_phys_gr_mme_pair_open_split, - 0); - WRITE_ONCE(gsp_phys_gr_mme_pair_open_entry, - false); - WRITE_ONCE( - gsp_phys_gr_mme_pair_open_entry_split, - 0); - /* - * r1447: the wiped context voids the arming - * window-hold and the strict grace budget too. + * SESSION 153-CONTINUED-61/79/109 + r1447/r1453/ + * r1454: the TSG preempt + resched above resets + * the engine context, so the MME program the + * emitted prelude loaded is gone; the shared + * wipe clears the emission latch/cursors and + * the live/release/pair/window-hold witnesses + * so the next launch re-preloads it (the r1110 + * mme_pc=513, r1377 deadlock and r1453/r1454 + * proof-void evidence live at the helper). */ - WRITE_ONCE(gsp_phys_gr_mme_window_hold_gen, 0); - WRITE_ONCE(gsp_phys_gr_mme_window_hold_h, 0); - WRITE_ONCE(gsp_phys_gr_mme_strict_holds, 0); + gsp_phys_gr_ctx_wipe_latches(); pr_warn("vgsp-gr: WEDGE-RECOVER after re-arm grfifo=0x%08x (%s); recovery #%u this boot; MME prelude latch cleared (context re-preload owed)\n", g1, (g1 & 0xffff0000u) == 0xbadf0000u ? diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 6ce84f1..ebbe3bd 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -18098,6 +18098,7 @@ static void svc_gl_cb_upload_scan(struct nv_vgpu_device *vgpu, u32 h_object, u32 n3800 = 0, n39c8 = 0, n39d0 = 0, n2380 = 0; u32 n238c = 0, n2390 = 0, n3460 = 0; u32 b3800_first = 0xffffffffu; + u32 subch_seen = 0xffffffffu; (void)vgpu; /* @@ -18119,7 +18120,7 @@ static void svc_gl_cb_upload_scan(struct nv_vgpu_device *vgpu, u32 h_object, if (base && base != svc_gl_cb_img_base) svc_gl_cb_img_reset(base); while (off + 4u <= len) { - u32 hdr, op, cnt, addr, w; + u32 hdr, op, cnt, addr, subch, w; memcpy(&hdr, buf + off, 4); nhdr++; @@ -18129,7 +18130,18 @@ static void svc_gl_cb_upload_scan(struct nv_vgpu_device *vgpu, u32 h_object, } op = (hdr >> 29) & 7u; cnt = (hdr >> 16) & 0xfffu; - addr = (hdr & 0x1fffu) << 2; + /* + * The method header is the A16F DMA encoding: method address + * 11:0 (dwords), subchannel 15:13, count 28:16. The old + * 0x1fff mask folded the reserved bit 12 into the address, so + * a header whose bit 12 is set decoded 0x4000 high and the + * 0x3800/0x39c8/0x2390 targets were never matched (CB-DIAG + * n3800/n39c8/n2390 = 0 on every push). + */ + addr = (hdr & 0xfffu) << 2; + subch = (hdr >> 13) & 0x7u; + if (subch_seen == 0xffffffffu) + subch_seen = subch; if (op == 0u || op == 2u || op == 7u) { off += 4u; continue; @@ -18156,14 +18168,22 @@ static void svc_gl_cb_upload_scan(struct nv_vgpu_device *vgpu, u32 h_object, svc_gl_cb_off = w; dirty = true; } else if (addr == 0x2384u) { + /* + * SELECTOR_B = ADDRESS_UPPER (bits 7:0 of the + * word are cb addr[39:32]); it is the HIGH half + * of the 64-bit address, not the low one. The + * historical packing put B in the low dword and + * C in the high dword (swapped). + */ svc_gl_cb_sel_bc = (svc_gl_cb_sel_bc & - ~0xffffffffULL) | - (u64)w; + 0xffffffffULL) | + ((u64)w << 32); svc_gl_cb_sel_seen = true; } else if (addr == 0x2388u) { + /* SELECTOR_C = ADDRESS_LOWER (addr[31:0]). */ svc_gl_cb_sel_bc = (svc_gl_cb_sel_bc & - 0xffffffffULL) | - ((u64)w << 32); + ~0xffffffffULL) | + (u64)w; svc_gl_cb_sel_seen = true; } else if (addr == 0x3800u) { /* @@ -18212,9 +18232,37 @@ static void svc_gl_cb_upload_scan(struct nv_vgpu_device *vgpu, u32 h_object, else if (addr >= 0x2390u && addr < 0x2390u + 0x1000u) n2390++; if (addr == 0x238cu) { + /* + * LOAD_CONSTANT_BUFFER_OFFSET with inline data (the + * driver's own emission, nv3dLoadConstantsHeader / + * nv3dLoadSingleConstant): the run payload is + * [offset][constant dwords...]. Word 0 sets the CB + * load offset; the rest stream through the + * LOAD_CONSTANT_BUFFER data port at successive + * offsets. Only word 0 used to be applied, so an + * upload carried this way landed nothing. + */ memcpy(&w, buf + off + 4u, 4); svc_gl_cb_off = w; dirty = true; + if (op != 3u && cnt > 1u) { + u32 j; + + for (j = 1; j < cnt; j++) { + u64 doff = (u64)svc_gl_cb_off + + 4ULL * (u64)(j - 1u); + + if (doff + 4ULL > SVC_GL_CB_IMG_LIMIT) + continue; + memcpy(&w, buf + off + 4u + + 4u * j, 4); + memcpy(svc_gl_cb_img + doff, &w, 4); + words++; + dirty = true; + } + svc_gl_cb_off += 4u * (cnt - 1u); + runs++; + } off += 4u + cnt * 4u; continue; } @@ -18230,17 +18278,21 @@ static void svc_gl_cb_upload_scan(struct nv_vgpu_device *vgpu, u32 h_object, svc_gl_cb_sel_a = w; break; case 0x2384u: + /* SELECTOR_B = ADDRESS_UPPER: high + * dword of the cb address (see the + * IMMD branch above). */ svc_gl_cb_sel_bc = (svc_gl_cb_sel_bc & - ~0xffffffffULL) | - (u64)w; + 0xffffffffULL) | + ((u64)w << 32); svc_gl_cb_sel_seen = true; break; case 0x2388u: + /* SELECTOR_C = ADDRESS_LOWER. */ svc_gl_cb_sel_bc = (svc_gl_cb_sel_bc & - 0xffffffffULL) | - ((u64)w << 32); + ~0xffffffffULL) | + (u64)w; svc_gl_cb_sel_seen = true; break; case 0x238cu: @@ -18333,9 +18385,10 @@ static void svc_gl_cb_upload_scan(struct nv_vgpu_device *vgpu, u32 h_object, */ if (svc_gl_cb_diag_log < SVC_GL_CB_DIAG_MAX) { svc_gl_cb_diag_log++; - pr_info("vgsp-sema: sema-gl: CB-DIAG h=0x%x push=0x%llx len=%u hdr=%u run=%u over=%u match=%u[3800=%u 39c8=%u 39d0=%u 2380=%u 238c=%u 2390=%u 3460=%u] b3800first=0x%x words=%u off=0x%x base=0x%llx valid=%u mvp=%u m80=%08x %08x %08x %08x row3=%08x %08x %08x %08x\n", + pr_info("vgsp-sema: sema-gl: CB-DIAG h=0x%x push=0x%llx len=%u hdr=%u run=%u over=%u match=%u[3800=%u 39c8=%u 39d0=%u 2380=%u 238c=%u 2390=%u 3460=%u] subch=%u b3800first=0x%x words=%u off=0x%x base=0x%llx valid=%u mvp=%u m80=%08x %08x %08x %08x row3=%08x %08x %08x %08x\n", h_object, push_va, len, nhdr, runs, nover, nmatch, n3800, n39c8, n39d0, n2380, n238c, n2390, n3460, + subch_seen == 0xffffffffu ? 0u : subch_seen, b3800_first, words, svc_gl_cb_off, svc_gl_cb_img_base, svc_gl_cb_img_valid ? 1u : 0u, svc_gl_cb_mvp_ok(svc_gl_cb_img) ? 1u : 0u, -- cgit v1.2.3