diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 82 |
1 files changed, 53 insertions, 29 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index af62e5c..0180da1 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -19025,6 +19025,19 @@ static void gsp_phys_gr_mme_release_arm(u32 h, u64 push, u32 gen) pdg = READ_ONCE(gsp_phys_gr_mme_pack_done_gen); pack_done = (READ_ONCE(gsp_phys_gr_mme_pack_done_h) == h_object || pdg != 0u); + if (!pack_done) { + /* + * The held-call record is saved but the pack is still partial + * (or the completion has not landed yet): defer, never + * consume. The pump calls this every pass, so the record is + * re-armed the moment pack_done_gen turns non-zero. + */ + spin_unlock_irqrestore(&gsp_phys_gr_mme_release_lock, flags); + if (s->valid && held && win_n) + pr_info_ratelimited("vgsp-gr: MME-CARRY-HOLD reason=pack-not-resident mode=release h=0x%x held=%u win=%u\n", + h_object, held, win_n); + return; + } golden = gsp_phys_gr_golden_ready(); /* * One fire per pack completion: a completion generation already @@ -19061,9 +19074,14 @@ void gsp_phys_gr_mme_release_rearm(void) u32 arm_h = READ_ONCE(s->arm_h); u64 arm_push = READ_ONCE(s->arm_push); - if (!READ_ONCE(gsp_phys_rmcall_gr_mme_pack_release) || - !READ_ONCE(gsp_phys_gr_mme_pack_done_gen)) + if (!READ_ONCE(gsp_phys_rmcall_gr_mme_pack_release)) return; + /* + * Unconditional on a valid, non-consumed record: release_arm() is the + * gate (it no-ops until pack_done_gen != 0 and the record is still + * owed). This keeps the saved record alive across the 9 s gap between + * the held-call save and the pack completion. + */ gsp_phys_gr_mme_release_arm(arm_h ? arm_h : READ_ONCE(s->h_object), arm_push ? arm_push : READ_ONCE(s->push_va), @@ -19084,20 +19102,6 @@ bool gsp_phys_gr_mme_release_is_arming(u32 h, u64 push) return ret; } -/* True when the pending release belongs to channel h (re-register cancel). */ -static bool gsp_phys_gr_mme_release_on_channel(u32 h) -{ - struct gsp_phys_gr_mme_release_state *s = &gsp_phys_gr_mme_release; - unsigned long flags; - bool ret; - - spin_lock_irqsave(&gsp_phys_gr_mme_release_lock, flags); - ret = (s->armed || s->fired) && - (s->arm_h == h || s->h_object == h); - spin_unlock_irqrestore(&gsp_phys_gr_mme_release_lock, flags); - return ret; -} - bool gsp_phys_gr_mme_release_peek(u32 *h_object) { struct gsp_phys_gr_mme_release_state *s = &gsp_phys_gr_mme_release; @@ -19121,15 +19125,28 @@ bool gsp_phys_gr_mme_release_take(u32 h_object, u32 *held, u32 *win) struct gsp_phys_gr_mme_release_state *s = &gsp_phys_gr_mme_release; const char *reason = NULL; unsigned long flags; - bool pack_done; + bool pack_done, retryable = false; u32 sh, sheld, swin, pdg; spin_lock_irqsave(&gsp_phys_gr_mme_release_lock, flags); - if (!s->armed || s->consumed || - !time_after_eq(jiffies, s->due_j)) { + if (s->consumed) { spin_unlock_irqrestore(&gsp_phys_gr_mme_release_lock, flags); return false; } + if (!s->armed || !time_after_eq(jiffies, s->due_j)) { + const char *defer = s->armed ? "grace" : "not-armed"; + + sh = s->h_object; + spin_unlock_irqrestore(&gsp_phys_gr_mme_release_lock, flags); + /* + * Deferral, not a verdict: the record stays valid and armed + * (or is re-armed by the pump) until the grace/arm condition + * is met. + */ + pr_info_ratelimited("vgsp-gr: MME-CARRY-HOLD reason=%s mode=release h=0x%x\n", + defer, sh); + return false; + } sh = s->h_object; sheld = s->held; swin = s->win_n; @@ -19144,11 +19161,25 @@ bool gsp_phys_gr_mme_release_take(u32 h_object, u32 *held, u32 *win) reason = "not-beef"; else if (!gsp_phys_gr_golden_ready()) reason = "golden-lost"; - else if (!pack_done) - reason = "pack-lost"; else if (pdg != 0u && pdg == s->fired_gen) reason = "budget"; + else if (!pack_done) { + /* + * pack-not-resident is retryable: the pack completion (which + * may still be in flight) will arm/fire later. Never consume + * the saved record on it. + */ + reason = "pack-lost"; + retryable = true; + } if (reason) { + if (retryable) { + spin_unlock_irqrestore(&gsp_phys_gr_mme_release_lock, + flags); + pr_info_ratelimited("vgsp-gr: MME-CARRY-HOLD reason=pack-not-resident mode=release h=0x%x\n", + sh); + return false; + } s->armed = false; s->fired = false; s->consumed = true; @@ -21757,13 +21788,6 @@ void gsp_phys_gr_mme_chan_ctx_reset(u32 h) READ_ONCE(gsp_phys_rmcall_gr_mme_golden_force))) return; /* - * A re-register of the saved-window release's own channel is a fresh - * context for the window it would replay: void the pending release - * even when another channel owns the packed load. - */ - if (gsp_phys_gr_mme_release_on_channel(h)) - gsp_phys_gr_mme_release_cancel("chan-ctx-reset"); - /* * Only the channel that owns the in-flight or completed packed load * loses its context on (re-)registration; an unrelated channel's * recycle (the client re-registers its whole family per fire) must @@ -21773,7 +21797,6 @@ void gsp_phys_gr_mme_chan_ctx_reset(u32 h) h != READ_ONCE(gsp_phys_gr_mme_pack_done_h)) return; gsp_phys_gr_mme_carry_cancel("chan-ctx-reset"); - gsp_phys_gr_mme_release_cancel("chan-ctx-reset"); /* * eng43: sema_gl_mme_keep_done_across_recycle=1 - the re-register * recycles the client handle but leaves the GR engine context (and @@ -21788,6 +21811,7 @@ void gsp_phys_gr_mme_chan_ctx_reset(u32 h) * Default 0 = legacy (identical to the pre-knob behaviour). */ if (!READ_ONCE(gsp_phys_rmcall_gr_mme_keep_done_across_recycle)) { + gsp_phys_gr_mme_release_cancel("chan-ctx-reset"); WRITE_ONCE(gsp_phys_gr_mme_pack_off, 0); WRITE_ONCE(gsp_phys_gr_mme_pack_sa, false); WRITE_ONCE(gsp_phys_gr_mme_pack_load_h, 0); |