diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 123 | ||||
| -rw-r--r-- | kernel/gsp_phys_booter.h | 2 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 29 |
3 files changed, 125 insertions, 29 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index cc33e31..af62e5c 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -18856,6 +18856,7 @@ struct gsp_phys_gr_mme_release_state { bool valid; bool armed; bool fired; + bool consumed; /* release no longer owed (released/ctx/err) */ u32 h_object; u32 gen; u32 held; @@ -18864,6 +18865,11 @@ struct gsp_phys_gr_mme_release_state { unsigned long due_j; u32 boot_n; u32 fired_gen; /* pack_done_gen already released (no retries) */ + /* The launch that armed this record (h + push); a later launch of the + * same identity is "the arming launch", never a client that consumed + * the release. */ + u32 arm_h; + u64 arm_push; /* capture snapshot (see struct gsp_phys_gr_submit) */ u32 qmd[GSP_PHYS_GR_QMD_WORDS]; u32 n_pages; @@ -18945,6 +18951,9 @@ static void gsp_phys_gr_mme_release_save(const struct gsp_phys_gr_submit *req, s->valid = false; s->armed = false; s->fired = false; + s->consumed = false; + s->arm_h = 0; + s->arm_push = 0; s->h_object = req->h_object; s->gen = gen; s->held = held; @@ -18989,15 +18998,17 @@ static void gsp_phys_gr_mme_release_save(const struct gsp_phys_gr_submit *req, /* * Arm once the packed program is complete/resident. The record survives * partial emissions, so the completing launch (which may carry no held CALL - * runs of its own) arms the last held-call window. A previously armed release - * is superseded by a newer record; a fired one is never re-armed (one fire - * per boot, no retries). + * runs of its own) arms the last held-call window. Idempotent: an already + * armed release keeps its deadline (the pump calls this every pass as a + * re-arm safety net). `h`/`push` identify the arming launch, so a later + * launch of that same identity is never mistaken for a client that consumed + * the release. A fired or consumed release is never re-armed (no retries). */ -static void gsp_phys_gr_mme_release_arm(u32 gen) +static void gsp_phys_gr_mme_release_arm(u32 h, u64 push, u32 gen) { struct gsp_phys_gr_mme_release_state *s = &gsp_phys_gr_mme_release; unsigned long flags; - bool pack_done, golden, superseded = false, ok = false, fresh; + bool pack_done, golden, ok = false, fresh; u32 ms, h_object, held, win_n, pdg; if (!READ_ONCE(gsp_phys_rmcall_gr_mme_pack_release)) @@ -19007,14 +19018,10 @@ static void gsp_phys_gr_mme_release_arm(u32 gen) h_object = s->h_object; held = s->held; win_n = s->win_n; - if (s->fired) { + if (s->fired || s->consumed || s->armed) { spin_unlock_irqrestore(&gsp_phys_gr_mme_release_lock, flags); return; } - if (s->armed) { - s->armed = false; - superseded = true; - } pdg = READ_ONCE(gsp_phys_gr_mme_pack_done_gen); pack_done = (READ_ONCE(gsp_phys_gr_mme_pack_done_h) == h_object || pdg != 0u); @@ -19030,18 +19037,67 @@ static void gsp_phys_gr_mme_release_arm(u32 gen) fresh && golden) { s->armed = true; s->gen = gen; + s->arm_h = h; + s->arm_push = push; s->due_j = jiffies + msecs_to_jiffies(ms); ok = true; } spin_unlock_irqrestore(&gsp_phys_gr_mme_release_lock, flags); - if (superseded) - pr_info("vgsp-gr: MME-CARRY-CANCEL reason=superseded mode=release h=0x%x held=%u win=%u\n", - h_object, held, win_n); if (ok) pr_info("vgsp-gr: MME-CARRY-ARM mode=release h=0x%x held=%u win=%u due_ms=%u\n", h_object, held, win_n, ms); } +/* + * Pump-side re-arm safety net: the builder arms at the pack completion and at + * the held-call snapshot, but a launch that races the grace can leave the + * record valid and disarmed. Re-arm it here (idempotent) so the release still + * fires at the deadline. Terminal cancels set consumed, so a released or + * context-voided record is never revived. + */ +void gsp_phys_gr_mme_release_rearm(void) +{ + struct gsp_phys_gr_mme_release_state *s = &gsp_phys_gr_mme_release; + u32 arm_h = READ_ONCE(s->arm_h); + u64 arm_push = READ_ONCE(s->arm_push); + + if (!READ_ONCE(gsp_phys_rmcall_gr_mme_pack_release) || + !READ_ONCE(gsp_phys_gr_mme_pack_done_gen)) + return; + gsp_phys_gr_mme_release_arm(arm_h ? arm_h : READ_ONCE(s->h_object), + arm_push ? arm_push : + READ_ONCE(s->push_va), + READ_ONCE(s->gen)); +} + +/* True when (h, push) is the launch that armed the pending release. */ +bool gsp_phys_gr_mme_release_is_arming(u32 h, u64 push) +{ + struct gsp_phys_gr_mme_release_state *s = &gsp_phys_gr_mme_release; + unsigned long flags; + bool ret; + + spin_lock_irqsave(&gsp_phys_gr_mme_release_lock, flags); + ret = (s->armed || s->fired) && s->arm_h == h && + s->arm_push == push; + spin_unlock_irqrestore(&gsp_phys_gr_mme_release_lock, flags); + return ret; +} + +/* True when the pending release belongs to channel h (re-register cancel). */ +static bool gsp_phys_gr_mme_release_on_channel(u32 h) +{ + struct gsp_phys_gr_mme_release_state *s = &gsp_phys_gr_mme_release; + unsigned long flags; + bool ret; + + spin_lock_irqsave(&gsp_phys_gr_mme_release_lock, flags); + ret = (s->armed || s->fired) && + (s->arm_h == h || s->h_object == h); + spin_unlock_irqrestore(&gsp_phys_gr_mme_release_lock, flags); + return ret; +} + bool gsp_phys_gr_mme_release_peek(u32 *h_object) { struct gsp_phys_gr_mme_release_state *s = &gsp_phys_gr_mme_release; @@ -19069,7 +19125,8 @@ bool gsp_phys_gr_mme_release_take(u32 h_object, u32 *held, u32 *win) u32 sh, sheld, swin, pdg; spin_lock_irqsave(&gsp_phys_gr_mme_release_lock, flags); - if (!s->armed || !time_after_eq(jiffies, s->due_j)) { + if (!s->armed || s->consumed || + !time_after_eq(jiffies, s->due_j)) { spin_unlock_irqrestore(&gsp_phys_gr_mme_release_lock, flags); return false; } @@ -19094,6 +19151,8 @@ bool gsp_phys_gr_mme_release_take(u32 h_object, u32 *held, u32 *win) if (reason) { s->armed = false; s->fired = false; + s->consumed = true; + s->valid = false; spin_unlock_irqrestore(&gsp_phys_gr_mme_release_lock, flags); pr_info("vgsp-gr: MME-CARRY-CANCEL reason=%s mode=release h=0x%x held=%u win=%u\n", reason, sh, sheld, swin); @@ -19135,6 +19194,7 @@ void gsp_phys_gr_mme_release_complete(void) s->armed = false; s->fired = false; s->valid = false; + s->consumed = true; spin_unlock_irqrestore(&gsp_phys_gr_mme_release_lock, flags); if (fired) pr_info("vgsp-gr: MME-CARRY-COMPLETE mode=release\n"); @@ -19154,6 +19214,8 @@ void gsp_phys_gr_mme_release_cancel(const char *reason) swin = s->win_n; s->armed = false; s->fired = false; + s->consumed = true; + s->valid = false; spin_unlock_irqrestore(&gsp_phys_gr_mme_release_lock, flags); if (had) pr_info("vgsp-gr: MME-CARRY-CANCEL reason=%s mode=release h=0x%x held=%u win=%u\n", @@ -19174,6 +19236,8 @@ void gsp_phys_gr_mme_release_failed(int rc) swin = s->win_n; s->armed = false; s->fired = false; + s->consumed = true; + s->valid = false; spin_unlock_irqrestore(&gsp_phys_gr_mme_release_lock, flags); if (!had) return; @@ -21693,6 +21757,13 @@ void gsp_phys_gr_mme_chan_ctx_reset(u32 h) READ_ONCE(gsp_phys_rmcall_gr_mme_golden_force))) return; /* + * A re-register of the saved-window release's own channel is a fresh + * context for the window it would replay: void the pending release + * even when another channel owns the packed load. + */ + if (gsp_phys_gr_mme_release_on_channel(h)) + gsp_phys_gr_mme_release_cancel("chan-ctx-reset"); + /* * Only the channel that owns the in-flight or completed packed load * loses its context on (re-)registration; an unrelated channel's * recycle (the client re-registers its whole family per fire) must @@ -22103,14 +22174,18 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, stream_damaged = gsp_phys_gr_stream_damaged(req->push_va, req->stream_rel); /* - * A real client 3D window is the "next launch" the saved-window - * release carrier was waiting for: cancel a pending release so - * the client's own window (which may run or hold the calls) - * wins. The hold point below can re-arm from this launch's own - * snapshot when the pack is complete. + * The launch that armed the saved-window release is not a + * "next client launch" that consumed it: it is the same build + * (and a duplicate/retry of it) that left the held CALL runs + * unrun. Never cancel on it - just name the hold. Other client + * windows also leave the release armed; the only "calls + * actually released" cancel is the MME-PACK-REARM edge below. */ - if (!carrier && !release && req->stream_n) - gsp_phys_gr_mme_release_cancel("client-first"); + if (!carrier && !release && req->stream_n && + gsp_phys_gr_mme_release_is_arming(req->h_object, + req->push_va)) + pr_info_ratelimited("vgsp-gr: MME-CARRY-HOLD reason=arming-launch h=0x%x push=0x%llx\n", + req->h_object, req->push_va); /* * SESSION 153-CONTINUED-63: MME START-ADDRESS RAM completion. * The prelude capture is capped at 512 words, so the client's @@ -23557,6 +23632,8 @@ if (gsp_phys_gr_slot_kept(j, sa_mask, * launch may carry none). */ gsp_phys_gr_mme_release_arm( + req->h_object, + req->push_va, pl_gen); pr_info("vgsp-gr: MME-PRELUDE pack ir_words=%u runs=%u sa=%u h=0x%x push=0x%llx\n", maxa, nruns, @@ -24476,7 +24553,9 @@ if (gsp_phys_gr_slot_kept(j, sa_mask, if (held) { gsp_phys_gr_mme_release_save(req, held, win3d_n, pl_gen); - gsp_phys_gr_mme_release_arm(pl_gen); + gsp_phys_gr_mme_release_arm(req->h_object, + req->push_va, + pl_gen); } if (pelided) pr_info_ratelimited("vgsp-gr: PAIR-CHAIN held-elide %u word(s) of program-pair runs dropped (pairs installed gen=%u h=0x%x; 0x2058 other-writer) push=0x%llx\n", diff --git a/kernel/gsp_phys_booter.h b/kernel/gsp_phys_booter.h index a27225b..21eb1fc 100644 --- a/kernel/gsp_phys_booter.h +++ b/kernel/gsp_phys_booter.h @@ -597,6 +597,8 @@ void gsp_phys_gr_mme_carry_failed(int rc); */ bool gsp_phys_gr_mme_release_peek(u32 *h_object); bool gsp_phys_gr_mme_release_take(u32 h_object, u32 *held, u32 *win); +void gsp_phys_gr_mme_release_rearm(void); +bool gsp_phys_gr_mme_release_is_arming(u32 h, u64 push); void gsp_phys_gr_mme_release_cancel(const char *reason); void gsp_phys_gr_mme_release_failed(int rc); void gsp_phys_gr_mme_release_complete(void); diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 6d72a62..26d3b1a 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -50031,15 +50031,19 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) /* * Saved-window release carrier (sema_gl_mme_pack_release). After the - * channel loop (a client window decoded in this pass wins and cancels - * the pending release), fire the saved held-call window once the pack - * completed and the settle grace expired. Same gates as the - * continuation carrier, except the FE-busy defer (this launch carries - * a real window, not a LOAD-ONLY carrier). + * channel loop (so a client window decoded in this pass is offered + * first), re-arm the saved held-call window if a race left the record + * valid but disarmed, then fire it once the pack completed and the + * settle grace expired. Same gates as the continuation carrier, except + * the FE-busy defer (this launch carries a real window, not a + * LOAD-ONLY carrier). A real client window never cancels the release: + * only the MME-PACK-REARM edge (calls actually released) does. */ { u32 rel_h = 0; + bool found = false; + gsp_phys_gr_mme_release_rearm(); if (gsp_phys_gr_mme_release_peek(&rel_h)) { int ci; @@ -50048,14 +50052,25 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (c->h_object != rel_h || c->engine_type) continue; - if (svc_gr_cand_unoffered_for(rel_h)) + found = true; + if (svc_gr_cand_unoffered_for(rel_h)) { + pr_info_ratelimited("vgsp-gr: MME-CARRY-HOLD reason=client-candidate h=0x%x\n", + rel_h); break; + } if (svc_fwd_hold.active || - test_bit(SVC_FWD_INFLIGHT, &svc_fwd_state)) + test_bit(SVC_FWD_INFLIGHT, + &svc_fwd_state)) { + pr_info_ratelimited("vgsp-gr: MME-CARRY-HOLD reason=request-inflight h=0x%x\n", + rel_h); break; + } (void)svc_gr_carrier_request(vgpu, c, true); break; } + if (!found) + pr_info_ratelimited("vgsp-gr: MME-CARRY-HOLD reason=no-channel h=0x%x\n", + rel_h); } } |