diff options
| -rw-r--r-- | kernel/gsp_phys_booter.c | 139 | ||||
| -rw-r--r-- | kernel/gsp_phys_booter.h | 5 | ||||
| -rw-r--r-- | kernel/gsp_rm_emulation.c | 8 |
3 files changed, 152 insertions, 0 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index bfc9149..35cfc79 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -18505,6 +18505,23 @@ static u32 gsp_phys_gr_mme_pack_off; static bool gsp_phys_gr_mme_pack_sa; /* + * eng40: per-channel pack-load accounting. The pack cursor above is global + * (the packed image is byte-identical across channels and generations under + * sema_gl_mme_golden_force), but the 0x114/0x11c IRAM writes land in EACH + * window's own channel context. A channel that emitted part 1 and then saw + * the completion emitted elsewhere holds only a PREFIX of the program; only + * a channel whose emissions were contiguous from word 0 to maxa holds the + * whole program (c489: beef 1692 + 1107 = 2799; c487: X-lane completion + * after a beef part 1 -> the X context held only the last chunk). pack_done_h + * names the channel whose context holds the complete program; the post- + * completion call release keys on it, and every context wipe / channel + * re-register invalidates it (see gsp_phys_gr_mme_chan_ctx_reset). + */ +static u32 gsp_phys_gr_mme_pack_load_h; +static u32 gsp_phys_gr_mme_pack_load_words; +static u32 gsp_phys_gr_mme_pack_done_h; + +/* * CONT-103: prelude residency witness + calls-kept block shrink. * The CONT-79 split-emission re-carries the whole prelude prefix on EVERY * launch while keep_mask is set (idempotent re-emission, CONT-72), so the @@ -20588,6 +20605,9 @@ static void gsp_phys_gr_mme_iram_gate_fail(void) WRITE_ONCE(gsp_phys_gr_mme_pre_emit_off, 0); WRITE_ONCE(gsp_phys_gr_mme_pack_off, 0); WRITE_ONCE(gsp_phys_gr_mme_pack_sa, false); + WRITE_ONCE(gsp_phys_gr_mme_pack_load_h, 0); + WRITE_ONCE(gsp_phys_gr_mme_pack_load_words, 0); + WRITE_ONCE(gsp_phys_gr_mme_pack_done_h, 0); /* * CONT-118: residency is unproven again, so a settled release armed * for this generation is void; the re-emission re-earns it. @@ -20805,6 +20825,9 @@ static void gsp_phys_gr_ctx_wipe_latches(void) WRITE_ONCE(gsp_phys_gr_mme_pre_emit_off, 0); WRITE_ONCE(gsp_phys_gr_mme_pack_off, 0); WRITE_ONCE(gsp_phys_gr_mme_pack_sa, false); + WRITE_ONCE(gsp_phys_gr_mme_pack_load_h, 0); + WRITE_ONCE(gsp_phys_gr_mme_pack_load_words, 0); + WRITE_ONCE(gsp_phys_gr_mme_pack_done_h, 0); WRITE_ONCE(gsp_phys_gr_mme_live_gen, 0); WRITE_ONCE(gsp_phys_gr_mme_release_gen, 0); WRITE_ONCE(gsp_phys_gr_mme_release_h, 0); @@ -20819,6 +20842,45 @@ static void gsp_phys_gr_ctx_wipe_latches(void) } /* + * eng40: a channel re-registration gives that channel a FRESH engine + * context - the MME program loaded earlier is gone. Invalidate the + * per-channel completion state along with the pack cursors and the + * latch/live/release witnesses, so the next window on the new context + * re-emits the packed program from word 0 (the pre-eng40 behaviour) and + * the post-completion call release cannot fire against a non-resident + * program. Gated by the same A/B knobs as the release itself; with the + * knobs off this is a no-op (legacy). + */ +void gsp_phys_gr_mme_chan_ctx_reset(u32 h) +{ + if (!(READ_ONCE(gsp_phys_rmcall_gr_mme_pack_keep_gen) && + READ_ONCE(gsp_phys_rmcall_gr_mme_golden_force))) + return; + /* + * Only the channel that owns the in-flight or completed packed load + * loses its context on (re-)registration; an unrelated channel's + * recycle (the client re-registers its whole family per fire) must + * not rewind a pack another channel holds. + */ + if (h != READ_ONCE(gsp_phys_gr_mme_pack_load_h) && + h != READ_ONCE(gsp_phys_gr_mme_pack_done_h)) + return; + WRITE_ONCE(gsp_phys_gr_mme_pack_off, 0); + WRITE_ONCE(gsp_phys_gr_mme_pack_sa, false); + WRITE_ONCE(gsp_phys_gr_mme_pack_load_h, 0); + WRITE_ONCE(gsp_phys_gr_mme_pack_load_words, 0); + WRITE_ONCE(gsp_phys_gr_mme_pack_done_h, 0); + WRITE_ONCE(gsp_phys_gr_mme_prelude_gen, 0); + WRITE_ONCE(gsp_phys_gr_mme_pre_emit_off, 0); + WRITE_ONCE(gsp_phys_gr_mme_pending_gen, 0); + WRITE_ONCE(gsp_phys_gr_mme_live_gen, 0); + WRITE_ONCE(gsp_phys_gr_mme_release_gen, 0); + WRITE_ONCE(gsp_phys_gr_mme_release_h, 0); + WRITE_ONCE(gsp_phys_gr_mme_window_hold_gen, 0); + WRITE_ONCE(gsp_phys_gr_mme_window_hold_h, 0); +} + +/* * CONT.474 CWD-CLEAR: pre-replay priv-error clear for the noQMD stream3d lane. * The c240-c242 receipts show the FE already poisoned when a stream3d replay is * about to be emitted (`MME-FAST begin launch ... esr=0xbadf1201`, @@ -21521,6 +21583,12 @@ if (gsp_phys_gr_slot_kept(j, sa_mask, WRITE_ONCE(gsp_phys_gr_mme_pack_off, 0); WRITE_ONCE(gsp_phys_gr_mme_pack_sa, false); + WRITE_ONCE(gsp_phys_gr_mme_pack_load_h, + 0); + WRITE_ONCE(gsp_phys_gr_mme_pack_load_words, + 0); + WRITE_ONCE(gsp_phys_gr_mme_pack_done_h, + 0); } } /* @@ -22476,6 +22544,46 @@ if (gsp_phys_gr_slot_kept(j, sa_mask, WRITE_ONCE( gsp_phys_gr_mme_pack_off, poff); + /* + * eng40: contiguous-load + * accounting (see + * gsp_phys_gr_mme_pack_done_h). + */ + if (poff - chunk == 0u) { + WRITE_ONCE( + gsp_phys_gr_mme_pack_load_h, + req->h_object); + WRITE_ONCE( + gsp_phys_gr_mme_pack_load_words, + chunk); + } else if ( + READ_ONCE( + gsp_phys_gr_mme_pack_load_h) == + req->h_object && + poff - chunk == + READ_ONCE( + gsp_phys_gr_mme_pack_load_words)) { + WRITE_ONCE( + gsp_phys_gr_mme_pack_load_words, + poff); + } else { + WRITE_ONCE( + gsp_phys_gr_mme_pack_load_h, + 0); + WRITE_ONCE( + gsp_phys_gr_mme_pack_load_words, + 0); + } + if (poff >= maxa) + WRITE_ONCE( + gsp_phys_gr_mme_pack_done_h, + (READ_ONCE( + gsp_phys_gr_mme_pack_load_h) == + req->h_object && + READ_ONCE( + gsp_phys_gr_mme_pack_load_words) >= + maxa) ? + req->h_object : 0); } pwords = (u32)(pp - p); p = pp; @@ -22836,6 +22944,34 @@ if (gsp_phys_gr_slot_kept(j, sa_mask, } } /* + * eng40: post-completion per-channel re-arm. Once the packed + * program has completed ON THIS CHANNEL - contiguous load, + * deterministic golden image - the channel context keeps it + * until a wipe or re-register. Every later frame bumps the + * capture generation, the pack branch cannot re-emit (poff == + * maxa), and the completion carrier dropped its own window, so + * without this release the real calls stay held forever (c489: + * pcv=0 on every post-completion draw). Only non-emitting + * launches release (pre_words_out == 0 - the r1222 same-launch + * clause stays intact), and the per-channel flag is invalidated + * by any context wipe or channel registration, so a fresh context + * can never run calls against a non-resident program. + */ + if (mme_hold_calls && pre_words_out == 0 && + READ_ONCE(gsp_phys_rmcall_gr_mme_pack_keep_gen) && + READ_ONCE(gsp_phys_rmcall_gr_mme_golden_force) && + READ_ONCE(gsp_phys_gr_mme_pack_done_h) == req->h_object) { + static u32 pack_rearm_log; + + if (pack_rearm_log < 8u) { + pack_rearm_log++; + pr_info("vgsp-gr: MME-PACK-REARM h=0x%x gen=%u (pack complete + contiguous load; calls released) push=0x%llx\n", + req->h_object, pl_gen, + (unsigned long long)req->push_va); + } + mme_hold_calls = false; + } + /* * r1487 STREAM-DAMAGE: an earlier chunk of this same push was * dropped (LOAD-ONLY carrier or the arming window hold), so * the channel no longer holds the state that chunk carried. @@ -32345,6 +32481,9 @@ gr_page_unmapped: WRITE_ONCE(gsp_phys_gr_mme_pre_emit_off, 0); WRITE_ONCE(gsp_phys_gr_mme_pack_off, 0); WRITE_ONCE(gsp_phys_gr_mme_pack_sa, false); + WRITE_ONCE(gsp_phys_gr_mme_pack_load_h, 0); + WRITE_ONCE(gsp_phys_gr_mme_pack_load_words, 0); + WRITE_ONCE(gsp_phys_gr_mme_pack_done_h, 0); WRITE_ONCE(gsp_phys_gr_mme_live_gen, 0); WRITE_ONCE( gsp_phys_gr_mme_release_gen, diff --git a/kernel/gsp_phys_booter.h b/kernel/gsp_phys_booter.h index f821a85..92c8e69 100644 --- a/kernel/gsp_phys_booter.h +++ b/kernel/gsp_phys_booter.h @@ -714,4 +714,9 @@ int gsp_phys_boot_rm_unload(void __iomem *bar0); int gsp_phys_boot_debugfs_init(void); void gsp_phys_boot_debugfs_fini(void); +/* eng40: invalidate the per-channel MME pack-completion state on a channel + * registration (fresh engine context); no-op unless the pack_keep_gen + + * golden_force A/B knobs are on. */ +void gsp_phys_gr_mme_chan_ctx_reset(u32 h); + #endif /* GSP_PHYS_BOOTER_H */ diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c index b928faf..d4390b9 100644 --- a/kernel/gsp_rm_emulation.c +++ b/kernel/gsp_rm_emulation.c @@ -3675,6 +3675,14 @@ static void gsp_rm_register_channel(struct gsp_emu_context *ctx, u32 h_client, u64 userd_wire_base = 0; /* + * eng40: a (re-)registration gives this channel a fresh engine + * context; the MME program it may have held is gone, so invalidate + * the per-channel pack-completion state (no-op unless the + * pack_keep_gen + golden_force knobs are on). + */ + gsp_phys_gr_mme_chan_ctx_reset(h_object); + + /* * CONT.405: name the registration attempt for the direct-GL client's * channel families. A "beef"/"bfef" handle that appears more than * once is the client's alloc -> setup -> free -> retry loop; the |