diff options
Diffstat (limited to 'kernel/gsp_phys_booter.c')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 111 |
1 files changed, 99 insertions, 12 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 35cfc79..9f1efdd 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -18520,6 +18520,26 @@ static bool gsp_phys_gr_mme_pack_sa; static u32 gsp_phys_gr_mme_pack_load_h; static u32 gsp_phys_gr_mme_pack_load_words; static u32 gsp_phys_gr_mme_pack_done_h; +/* + * eng42: the packed program's residency is an ENGINE property, not a + * channel property. The 0x114/0x11c IRAM writes load the engine context + * that every window's calls execute against - a completion on ANY lane + * means the (deterministic golden) image is resident. c552 proved the + * per-channel proof above can never fire for a cross-lane completion: + * part 1 on h=0x1000013, final part on h=0xbeef0100 -> the contiguous + * check zeroed load_h -> pack_done_h=0 -> the eng40 release + * (done_h == req->h_object) was unreachable although the engine held the + * whole program, and the cursor wedged at maxa for the rest of the boot + * (no gen rewind under keep_gen=1; chan_ctx_reset no-ops without an + * owner). pack_done_gen/ch record the completion GLOBALLY (generation + * and completing channel, forensics only); every genuine engine-context + * wipe clears them beside pack_done_h, and a re-register of a channel + * that owns the load invalidates them via chan_ctx_reset. The post- + * completion call release then fires for a draw on ANY channel whose + * window will execute against the resident engine program. + */ +static u32 gsp_phys_gr_mme_pack_done_gen; +static u32 gsp_phys_gr_mme_pack_done_ch; /* * CONT-103: prelude residency witness + calls-kept block shrink. @@ -20608,6 +20628,8 @@ static void gsp_phys_gr_mme_iram_gate_fail(void) WRITE_ONCE(gsp_phys_gr_mme_pack_load_h, 0); WRITE_ONCE(gsp_phys_gr_mme_pack_load_words, 0); WRITE_ONCE(gsp_phys_gr_mme_pack_done_h, 0); + WRITE_ONCE(gsp_phys_gr_mme_pack_done_gen, 0); + WRITE_ONCE(gsp_phys_gr_mme_pack_done_ch, 0); /* * CONT-118: residency is unproven again, so a settled release armed * for this generation is void; the re-emission re-earns it. @@ -20828,6 +20850,8 @@ static void gsp_phys_gr_ctx_wipe_latches(void) WRITE_ONCE(gsp_phys_gr_mme_pack_load_h, 0); WRITE_ONCE(gsp_phys_gr_mme_pack_load_words, 0); WRITE_ONCE(gsp_phys_gr_mme_pack_done_h, 0); + WRITE_ONCE(gsp_phys_gr_mme_pack_done_gen, 0); + WRITE_ONCE(gsp_phys_gr_mme_pack_done_ch, 0); WRITE_ONCE(gsp_phys_gr_mme_live_gen, 0); WRITE_ONCE(gsp_phys_gr_mme_release_gen, 0); WRITE_ONCE(gsp_phys_gr_mme_release_h, 0); @@ -20870,6 +20894,8 @@ void gsp_phys_gr_mme_chan_ctx_reset(u32 h) WRITE_ONCE(gsp_phys_gr_mme_pack_load_h, 0); WRITE_ONCE(gsp_phys_gr_mme_pack_load_words, 0); WRITE_ONCE(gsp_phys_gr_mme_pack_done_h, 0); + WRITE_ONCE(gsp_phys_gr_mme_pack_done_gen, 0); + WRITE_ONCE(gsp_phys_gr_mme_pack_done_ch, 0); WRITE_ONCE(gsp_phys_gr_mme_prelude_gen, 0); WRITE_ONCE(gsp_phys_gr_mme_pre_emit_off, 0); WRITE_ONCE(gsp_phys_gr_mme_pending_gen, 0); @@ -21589,6 +21615,10 @@ if (gsp_phys_gr_slot_kept(j, sa_mask, 0); WRITE_ONCE(gsp_phys_gr_mme_pack_done_h, 0); + WRITE_ONCE(gsp_phys_gr_mme_pack_done_gen, + 0); + WRITE_ONCE(gsp_phys_gr_mme_pack_done_ch, + 0); } } /* @@ -22501,6 +22531,37 @@ if (gsp_phys_gr_slot_kept(j, sa_mask, if (can && split && !sa_done && 3u * npair > left) can = false; + /* + * eng42 (b): an owner-less completion + * (c552 cross-lane: done_h==0) wedges + * the cursor at maxa for the rest of + * the boot - the generation edge does + * not rewind it (keep_gen=1) and + * chan_ctx_reset no-ops without an + * owner, so no later window could + * re-emit the deterministic golden + * image. Restart the drain here (this + * launch stays non-emitting; the REARM + * clause below still releases its held + * calls against the global completion + * record above). + */ + if (!can && poff >= maxa && maxa >= 3u && + READ_ONCE(gsp_phys_gr_mme_pack_done_h) == 0u && + READ_ONCE(gsp_phys_rmcall_gr_mme_pack_keep_gen) && + READ_ONCE(gsp_phys_rmcall_gr_mme_golden_force)) { + WRITE_ONCE(gsp_phys_gr_mme_pack_off, + 0); + WRITE_ONCE(gsp_phys_gr_mme_pack_sa, + false); + WRITE_ONCE(gsp_phys_gr_mme_pack_load_h, + 0); + WRITE_ONCE(gsp_phys_gr_mme_pack_load_words, + 0); + pr_info("vgsp-gr: MME-PACK-WEDGE-RESET h=0x%x maxa=%u (owner-less completion; cursor restarted, calls released below if held) push=0x%llx\n", + req->h_object, maxa, + req->push_va); + } if (can) { pp = p; if (!sa_done) { @@ -22574,7 +22635,7 @@ if (gsp_phys_gr_slot_kept(j, sa_mask, gsp_phys_gr_mme_pack_load_words, 0); } - if (poff >= maxa) + if (poff >= maxa) { WRITE_ONCE( gsp_phys_gr_mme_pack_done_h, (READ_ONCE( @@ -22584,6 +22645,20 @@ if (gsp_phys_gr_slot_kept(j, sa_mask, gsp_phys_gr_mme_pack_load_words) >= maxa) ? req->h_object : 0); + /* + * eng42: completion is + * engine-global - record it + * whatever lane carried the + * final part (see the + * pack_done_gen comment). + */ + WRITE_ONCE( + gsp_phys_gr_mme_pack_done_gen, + pl_gen); + WRITE_ONCE( + gsp_phys_gr_mme_pack_done_ch, + req->h_object); + } } pwords = (u32)(pp - p); p = pp; @@ -22944,29 +23019,39 @@ if (gsp_phys_gr_slot_kept(j, sa_mask, } } /* - * eng40: post-completion per-channel re-arm. Once the packed - * program has completed ON THIS CHANNEL - contiguous load, - * deterministic golden image - the channel context keeps it - * until a wipe or re-register. Every later frame bumps the + * eng40/eng42: post-completion re-arm. Once the packed + * program completed (deterministic golden image), the engine + * context keeps it until a wipe. Every later frame bumps the * capture generation, the pack branch cannot re-emit (poff == * maxa), and the completion carrier dropped its own window, so * without this release the real calls stay held forever (c489: - * pcv=0 on every post-completion draw). Only non-emitting - * launches release (pre_words_out == 0 - the r1222 same-launch - * clause stays intact), and the per-channel flag is invalidated - * by any context wipe or channel registration, so a fresh context - * can never run calls against a non-resident program. + * pcv=0 on every post-completion draw). eng42: the completion + * is engine-global - the release also fires when the + * contiguous per-channel proof is absent but the global + * record stands (c552: part 1 on X, completion on beef -> + * pack_done_h withheld, engine still holds the image; the + * completing beeves had no draw while the X lane kept + * drawing). Only non-emitting launches release (pre_words_out + * == 0 - the r1222 same-launch clause stays intact), and the + * records are invalidated by every context wipe / owning- + * channel re-register, so a fresh context can never run calls + * against a non-resident program. */ if (mme_hold_calls && pre_words_out == 0 && READ_ONCE(gsp_phys_rmcall_gr_mme_pack_keep_gen) && READ_ONCE(gsp_phys_rmcall_gr_mme_golden_force) && - READ_ONCE(gsp_phys_gr_mme_pack_done_h) == req->h_object) { + (READ_ONCE(gsp_phys_gr_mme_pack_done_h) == req->h_object || + READ_ONCE(gsp_phys_gr_mme_pack_done_gen) != 0u)) { static u32 pack_rearm_log; if (pack_rearm_log < 8u) { pack_rearm_log++; - pr_info("vgsp-gr: MME-PACK-REARM h=0x%x gen=%u (pack complete + contiguous load; calls released) push=0x%llx\n", + pr_info("vgsp-gr: MME-PACK-REARM h=0x%x gen=%u (pack complete%s; calls released) push=0x%llx\n", req->h_object, pl_gen, + READ_ONCE(gsp_phys_gr_mme_pack_done_h) == + req->h_object ? + " + contiguous load" : + " (engine-resident, cross-lane completion)", (unsigned long long)req->push_va); } mme_hold_calls = false; @@ -32484,6 +32569,8 @@ gr_page_unmapped: WRITE_ONCE(gsp_phys_gr_mme_pack_load_h, 0); WRITE_ONCE(gsp_phys_gr_mme_pack_load_words, 0); WRITE_ONCE(gsp_phys_gr_mme_pack_done_h, 0); + WRITE_ONCE(gsp_phys_gr_mme_pack_done_gen, 0); + WRITE_ONCE(gsp_phys_gr_mme_pack_done_ch, 0); WRITE_ONCE(gsp_phys_gr_mme_live_gen, 0); WRITE_ONCE( gsp_phys_gr_mme_release_gen, |