diff options
Diffstat (limited to 'kernel/gsp_phys_booter.c')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 162 |
1 files changed, 97 insertions, 65 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index b548b62..f1cd2cb 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -20713,6 +20713,75 @@ static u32 gsp_phys_gr_fe_status(void) } /* + * CONT.474: the MME latch/drain state both context-wiping paths owe after the + * r445 hard re-arm (a bWait TSG preempt saves/unloads the engine context, so + * the resident MME program is gone). Clearing the emission latch/cursors and + * the live/release/pair/window-hold witnesses makes the next launch that can + * carry the prelude re-preload the program into the fresh context; without it + * every later replay runs its macros against an unloaded MME and stalls + * pre-retire (the r1110 mme_pc=513 class; the r1377 split-emitter deadlock and + * the r1453 pair/window-hold proofs are the other originals - see the + * WEDGE-RECOVER block, whose inline clears this replaces). + */ +static void gsp_phys_gr_ctx_wipe_latches(void) +{ + WRITE_ONCE(gsp_phys_gr_mme_prelude_gen, 0); + WRITE_ONCE(gsp_phys_gr_mme_pre_emit_off, 0); + WRITE_ONCE(gsp_phys_gr_mme_pack_off, 0); + WRITE_ONCE(gsp_phys_gr_mme_pack_sa, false); + WRITE_ONCE(gsp_phys_gr_mme_live_gen, 0); + WRITE_ONCE(gsp_phys_gr_mme_release_gen, 0); + WRITE_ONCE(gsp_phys_gr_mme_release_h, 0); + WRITE_ONCE(gsp_phys_gr_mme_pairs_gen, 0); + WRITE_ONCE(gsp_phys_gr_mme_pair_open, false); + WRITE_ONCE(gsp_phys_gr_mme_pair_open_split, 0); + WRITE_ONCE(gsp_phys_gr_mme_pair_open_entry, false); + WRITE_ONCE(gsp_phys_gr_mme_pair_open_entry_split, 0); + WRITE_ONCE(gsp_phys_gr_mme_window_hold_gen, 0); + WRITE_ONCE(gsp_phys_gr_mme_window_hold_h, 0); + WRITE_ONCE(gsp_phys_gr_mme_strict_holds, 0); +} + +/* + * CONT.474 CWD-CLEAR: pre-replay priv-error clear for the noQMD stream3d lane. + * The c240-c242 receipts show the FE already poisoned when a stream3d replay is + * about to be emitted (`MME-FAST begin launch ... esr=0xbadf1201`, + * `SKED-WIT ... cwd3=0xbadf5040 act=0`): the 0xbadf latch survives from an + * earlier launch's window and this lane carries no context re-init, so the + * replay walks a parked CWD and the engine stores nothing + * (`NOWRITE-CONFIRMED ... wrote=0`). WEDGE-RECOVER detects the same class but + * only AFTER the replay was submitted (so the launch is already lost) and its + * re-arm clears grfifo, not the parked CWD. Run the identical bounded re-arm + * (bWait TSG preempt -> lane-enable + schedules -> warm-sem probe) BEFORE the + * replay is built, then wipe the MME latches so the build below re-preloads + * the program into the fresh context. One attempt per launch (caller's flag); + * stream3d lane only, so CUDA/QMD and CE submissions are untouched. + */ +static u32 gsp_phys_gr_cwd_clears; +static void gsp_phys_gr_cwd_clear_locked(void __iomem *bar0, u32 h_object, + u64 push_va) +{ + u32 esr = ioread32(bar0 + 0x00404490u); + u32 cwd3 = ioread32(bar0 + 0x0040706cu); + u32 gf = ioread32(bar0 + 0x00400504u); + + if ((esr & 0xffff0000u) != 0xbadf0000u && + (cwd3 & 0xffff0000u) != 0xbadf0000u && + (gf & 0xffff0000u) != 0xbadf0000u) + return; + gsp_phys_gr_cwd_clears++; + pr_warn("vgsp-gr: CWD-CLEAR pre-replay poison (esr=%08x cwd3=%08x grfifo=%08x) h=0x%x push=0x%llx - TSG preempt + resched before the replay; clear #%u this boot\n", + esr, cwd3, gf, h_object, push_va, gsp_phys_gr_cwd_clears); + gsp_phys_mme_regs_log(bar0, "cwd-clear"); + gsp_phys_gr_wedge_hard_rearm_locked(bar0); + gsp_phys_gr_ctx_wipe_latches(); + pr_warn("vgsp-gr: CWD-CLEAR after re-arm esr=%08x cwd3=%08x grfifo=%08x (MME latches cleared, re-preload owed)\n", + ioread32(bar0 + 0x00404490u), + ioread32(bar0 + 0x0040706cu), + ioread32(bar0 + 0x00400504u)); +} + +/* * r1445 5c: submit-side FE-busy deferral of the load-only carrier. The * r1444 first-poison class submitted its first load-only carrier while the * front-end was mid-fetch and the packed-program fetch poisoned the lane. @@ -28362,6 +28431,11 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, * ordered work item is the only caller. */ bool wedge_recover_done = false; + /* + * CONT.474 CWD-CLEAR: one pre-replay priv-error clear attempt per + * launch (gsp_phys_gr_cwd_clear_locked; stream3d lane only). + */ + bool cwd_clear_done = false; static u32 wedge_recoveries; /* * SESSION 153-CONTINUED-72: the MME-FAST pre-reset spin can observe @@ -30943,6 +31017,19 @@ gr_page_unmapped: GSP_PHYS_RMCALL_GR_METHOD_WORDS); win3d_n = pn; } + /* + * CONT.474 CWD-CLEAR: clear a pre-existing 0xbadf FE priv-error latch + * BEFORE this replay is emitted (see gsp_phys_gr_cwd_clear_locked). + * Bounded to one attempt per launch; stream3d lane only, so the + * CUDA/QMD and CE paths never reach this site. Gated by the same + * sema_gr_wedge_recover knob as the mid-launch recovery, which stays + * available for a poison the replay itself causes. + */ + if (stream3d && bar0 && !cwd_clear_done && + READ_ONCE(gsp_phys_gr_wedge_recover)) { + cwd_clear_done = true; + gsp_phys_gr_cwd_clear_locked(bar0, req->h_object, req->push_va); + } words = gsp_phys_rmcall_build_gr_launch(methods, req, qmd, qmd_dst, ch_sem_va, want_ch, @@ -31569,72 +31656,17 @@ gr_page_unmapped: mutex_unlock(&gsp_phys_rm_lock); g1 = ioread32(bar0 + 0x00400504u); /* - * SESSION 153-CONTINUED-61: the TSG preempt + - * resched above resets the engine context, so - * the MME program the emitted prelude loaded is - * gone. Clear the emission latch: the next - * launch that can carry the prelude re-preloads - * it (without this every later replay ran its - * macros against an unloaded MME and stalled - * pre-retire - the r1110 mme_pc=513 class). - */ - WRITE_ONCE(gsp_phys_gr_mme_prelude_gen, 0); - /* - * SESSION 153-CONTINUED-79: also rewind the - * split-emission drain cursor, else a cleared - * latch with an exhausted cursor would skip - * the owed re-preload. - */ - WRITE_ONCE(gsp_phys_gr_mme_pre_emit_off, 0); - /* - * SESSION 153-CONTINUED-109: rewind the PACK - * cursor too (the r1377 deadlock: recovery reset - * pre_emit_off but left pack_off at its end, so - * the split emitter hit `poff >= maxa` -> can=0 - * -> packed=1 with zero words on every launch: - * `MME-PRELUDE part gen=3 off=0 words=0/3083` - * forever, latch never re-set, gates never - * release, MME never starts). The gate-fail - * path (gsp_phys_gr_mme_iram_gate_fail) already - * rewinds both cursor pairs. - */ - WRITE_ONCE(gsp_phys_gr_mme_pack_off, 0); - WRITE_ONCE(gsp_phys_gr_mme_pack_sa, false); - /* - * SESSION 153-CONTINUED-109: the hard re-arm - * wiped the engine context, so the generation - * whose carrier executed is no longer resident. - * Invalidate it; the re-preload's own GP_GET - * advance (readback-free witness) re-promotes - * (fetch_miss/poll-break paths). Without this - * the stale live_gen would mark a wiped context - * live and the next window+calls launch would - * run against an unloaded MME. - */ - WRITE_ONCE(gsp_phys_gr_mme_live_gen, 0); - WRITE_ONCE(gsp_phys_gr_mme_release_gen, 0); - WRITE_ONCE(gsp_phys_gr_mme_release_h, 0); - /* r1453 PAIR-CHAIN: context wipe voids the - * pair-install proof; re-earn it. */ - WRITE_ONCE(gsp_phys_gr_mme_pairs_gen, 0); - /* r1454 PAIR-CHAIN-ENTRY-ELIDE: the wiped - * context has no open FE pair; clear the - * cross-launch tracker too. */ - WRITE_ONCE(gsp_phys_gr_mme_pair_open, false); - WRITE_ONCE(gsp_phys_gr_mme_pair_open_split, - 0); - WRITE_ONCE(gsp_phys_gr_mme_pair_open_entry, - false); - WRITE_ONCE( - gsp_phys_gr_mme_pair_open_entry_split, - 0); - /* - * r1447: the wiped context voids the arming - * window-hold and the strict grace budget too. + * SESSION 153-CONTINUED-61/79/109 + r1447/r1453/ + * r1454: the TSG preempt + resched above resets + * the engine context, so the MME program the + * emitted prelude loaded is gone; the shared + * wipe clears the emission latch/cursors and + * the live/release/pair/window-hold witnesses + * so the next launch re-preloads it (the r1110 + * mme_pc=513, r1377 deadlock and r1453/r1454 + * proof-void evidence live at the helper). */ - WRITE_ONCE(gsp_phys_gr_mme_window_hold_gen, 0); - WRITE_ONCE(gsp_phys_gr_mme_window_hold_h, 0); - WRITE_ONCE(gsp_phys_gr_mme_strict_holds, 0); + gsp_phys_gr_ctx_wipe_latches(); pr_warn("vgsp-gr: WEDGE-RECOVER after re-arm grfifo=0x%08x (%s); recovery #%u this boot; MME prelude latch cleared (context re-preload owed)\n", g1, (g1 & 0xffff0000u) == 0xbadf0000u ? |