diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/gsp_sema_service.c | 97 |
1 files changed, 58 insertions, 39 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 55482cd..0520c4f 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -417,7 +417,6 @@ struct svc_channel { u64 wd_put; /* put at the window start */ u64 wd_sub; /* submissions at the window start */ unsigned long wd_since; /* window start (0 = no window) */ - unsigned long wd_rewind_off_until; /* pin-batch rewind cooldown */ bool wd_epoch_done; /* the stale-epoch reset ran for this put */ u32 wd_nocap_put; /* force-replay (no capture) for slots below * this while the stale range is re-walked; 0 = @@ -426,6 +425,9 @@ struct svc_channel { * must reconcile the binding before the walk * (drop dead opdef slots whose since predates * the binding; log the transition) */ + unsigned long wd_newbind_hold_until; /* short hold window inside + * wd_newbind_until (residual-only, + * put==get; never delays fresh work) */ unsigned long wd_newbind_until; /* STALLFREE: freshness window of a * new binding; inside it an * unmapped/undecoded GR entry is held @@ -16572,12 +16574,11 @@ static void svc_userd_sweep(struct nv_vgpu_device *vgpu, struct svc_channel *ch, * forward-only) and never re-capture/re-forward a dead launch. * 2. stale replay: last_get pinned behind put while put == get (the * producer has nothing pending) AND the loop consumed entries during - * the window (submissions moved) - i.e. the rewind is re-opening an - * already-consumed window faster than the loop closes it. The window - * is dropped (last_get = put) and the pin-batch rewind is put on a - * cooldown (wd_rewind_off_until) so the futile retry cannot pin the - * pump; the rewind budget returns later and a translation that lands - * can still retry (bounded duty cycle). + * the window (submissions moved). The cursor is clamped to put once + * per put epoch (bookkeeping only). The pin-batch rewind is NEVER + * touched: r235 proved disabling it kills the next client's + * context-init (its pins land after the first decode; the rewind is + * the retry path). A clamp that a rewind undoes is harmless. * * Nothing here can fire on a green channel: a green channel either moves * put or consumes its first entries within milliseconds (submissions > 0), @@ -16587,6 +16588,15 @@ static void svc_userd_sweep(struct nv_vgpu_device *vgpu, struct svc_channel *ch, */ #define SVC_GR_STALE_WEDGE_MS 3000u /* + * STALLFREE r235: the replay/clamp leg needs a MUCH longer stalled window + * than the epoch legs. A fresh client's context-init legitimately shows a + * 3 s position oscillation (rewind replays a just-consumed entry while its + * pins land); treating that as a livelock killed the second client (r235: + * 8 channels classified at waited_ms=3491). Only a position stalled for + * ~10 s with consumption happening is a livelock worth clamping. + */ +#define SVC_GR_STALE_REPLAY_MS 10000u +/* * STALLFREE r229/r230: freshness window after a genuinely new channel * binding. Within it, a GR entry whose page is unmapped or whose decode * yields nothing is NOT silently consumed: the pump requests an immediate @@ -16597,6 +16607,15 @@ static void svc_userd_sweep(struct nv_vgpu_device *vgpu, struct svc_channel *ch, * window of extra latency on first contact. */ #define SVC_GR_NEWBIND_MS 3000u +/* + * STALLFREE r237: the HOLD part of the window must be shorter than the + * window itself. r237-2: the 3 s hold on T2's residual ring delayed its + * GP_GET past the guest's ~3 s ctx-init patience (T2 unwound with a + * zeroed PROMOTE_CTX at binding+3.46 s, exactly when the host finally + * consumed the residual). The guest needs GP_GET to advance promptly; + * the rebuild lands in ~0.8 s, so 1.2 s is enough for one attempt. + */ +#define SVC_GR_NEWBIND_HOLD_MS 1200u static void svc_gr_stale_wedge_check(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u32 put, u32 get) @@ -16613,11 +16632,8 @@ static void svc_gr_stale_wedge_check(struct nv_vgpu_device *vgpu, * and the replay cooldown: the next stale region gets its own * recovery. */ - if (ch->wd_put != put) { - if (ch->wd_rewind_off_until) - ch->wd_rewind_off_until = 0; + if (ch->wd_put != put) ch->wd_epoch_done = false; - } /* The forced-replay re-walk is done once the walk catches up. */ if (ch->wd_nocap_put && ch->last_get == put) ch->wd_nocap_put = 0; @@ -16627,7 +16643,7 @@ static void svc_gr_stale_wedge_check(struct nv_vgpu_device *vgpu, ch->wd_since = 0; /* progress (or a new position): re-arm */ catchup = (ch->last_get == put); - replay_cand = (!catchup && put == get); + replay_cand = (!catchup && put == get && !ch->wd_epoch_done); /* * Stale epoch: the host caught up (last_get == put) but the ring * still carries a session's residue: put > 1 and the epoch reset @@ -16693,7 +16709,8 @@ static void svc_gr_stale_wedge_check(struct nv_vgpu_device *vgpu, * proof this is a producer-pending channel on an unreadable entry, * which the existing grace paths own. */ - fire_replay = replay_cand && ch->submissions != ch->wd_sub; + fire_replay = replay_cand && ch->submissions != ch->wd_sub && + waited >= SVC_GR_STALE_REPLAY_MS; /* * The stalled-entry leg fires only with proof of no progress at all: * nothing was consumed during the window (submissions static), so @@ -16701,7 +16718,12 @@ static void svc_gr_stale_wedge_check(struct nv_vgpu_device *vgpu, */ fire_stall = stalled_cand && ch->submissions == ch->wd_sub; if (!residual && !fire_reset && !fire_replay && !fire_stall) { - ch->wd_since = 0; + /* + * Keep an accumulating replay window alive (its evidence needs + * SVC_GR_STALE_REPLAY_MS); anything else restarts. + */ + if (!replay_cand) + ch->wd_since = 0; return; } if (residual || fire_reset) { @@ -16739,12 +16761,19 @@ static void svc_gr_stale_wedge_check(struct nv_vgpu_device *vgpu, ch->wd_nocap_put = put; } if (fire_replay) { - /* Drop the rewind-reopened consumed window; cool the rewind. */ + /* + * STALLFREE r235: clamp the stale software cursor to the + * producer position ONCE per put epoch. The pin-batch rewind + * is deliberately NOT touched: disabling it killed every + * second client (r235-2: 8 channels got a 24 s rewind + * cooldown at ctx-init while their pins were landing, so + * their init pushes were never retried). The clamp is + * bookkeeping only and rewind retries stay available. + */ ch->last_get = put; - ch->wd_rewind_off_until = jiffies + - msecs_to_jiffies(8u * SVC_GR_STALE_WEDGE_MS); + ch->wd_epoch_done = true; } - if (residual || fire_reset) + if (residual || fire_reset || fire_replay) ch->wd_epoch_done = true; ch->wd_since = 0; ch->wd_put = put; @@ -16758,7 +16787,7 @@ static void svc_gr_stale_wedge_check(struct nv_vgpu_device *vgpu, reason = "stale-entry"; else reason = "stale-epoch"; - pr_warn_ratelimited("vgsp-gr: GR-STALL-stale-wedge h=0x%x va=0x%llx last=%u put=%u get=%u sub=%llu waited_ms=%u action=recover reason=%s reset=%d rewind=%d entry=%d cpa=0x%llx rpa=0x%llx\n", + pr_warn_ratelimited("vgsp-gr: GR-STALL-stale-wedge h=0x%x va=0x%llx last=%u put=%u get=%u sub=%llu waited_ms=%u action=recover reason=%s reset=%d clamp=%d entry=%d cpa=0x%llx rpa=0x%llx\n", ch->h_object, ch->gpfifo_va, (u32)ch->last_get, put, get, ch->submissions, waited, reason, (fire_reset || fire_stall) ? 1 : 0, @@ -17178,6 +17207,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc_opdef_drop_channel(vgpu, ch->h_object, jiffies); ch->wd_newbind_until = jiffies + msecs_to_jiffies(SVC_GR_NEWBIND_MS); + ch->wd_newbind_hold_until = jiffies + + msecs_to_jiffies(SVC_GR_NEWBIND_HOLD_MS); /* * Bounded proof (r234 asks): the window was armed for * THIS binding; if a later stall shows no such line, @@ -17702,9 +17733,10 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) !entry_replay && svc_rel_count == rel_n_iter && svc_gr_qmd_seen == 0 && - ch->wd_newbind_until && + put == get && + ch->wd_newbind_hold_until && time_before(jiffies, - ch->wd_newbind_until)) { + ch->wd_newbind_hold_until)) { if (!ch->defer_since) ch->defer_since = jiffies; @@ -18788,9 +18820,10 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * the rebuild caught up). */ if (!ch->engine_type && - ch->wd_newbind_until && + put == get && + ch->wd_newbind_hold_until && time_before(jiffies, - ch->wd_newbind_until)) { + ch->wd_newbind_hold_until)) { svc.rebuild_now = true; if (svc.gr_pending_map < 16) { svc.gr_pending_map++; @@ -19024,12 +19057,12 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc.chan[k].wd_put = ch->wd_put; svc.chan[k].wd_sub = ch->wd_sub; svc.chan[k].wd_since = ch->wd_since; - svc.chan[k].wd_rewind_off_until = - ch->wd_rewind_off_until; svc.chan[k].wd_epoch_done = ch->wd_epoch_done; svc.chan[k].wd_nocap_put = ch->wd_nocap_put; svc.chan[k].wd_chanswitch = ch->wd_chanswitch; svc.chan[k].wd_newbind_until = ch->wd_newbind_until; + svc.chan[k].wd_newbind_hold_until = + ch->wd_newbind_hold_until; svc.chan[k].pushpg_va = ch->pushpg_va; svc.chan[k].pushpg_n = ch->pushpg_n; svc.chan[k].content_max = ch->content_max; @@ -19499,20 +19532,6 @@ static void svc_handle_map_add(void) if (!ch->valid || !ch->gpfifo_entries) continue; /* - * STALLFREE r225: a channel under the stale-replay - * watchdog's rewind cooldown is NOT rewound. Without - * this the rewind re-opened the same consumed window - * every batch while the producer had nothing pending, - * pinning last_get and re-walking stale entries - * forever (r225: slots 93..100 of 0x5c000059 for - * 42+ s). The cooldown expires on its own (bounded - * duty cycle) and is cleared by new work, so a - * translation landing later can still retry. - */ - if (ch->wd_rewind_off_until && - time_before(jiffies, ch->wd_rewind_off_until)) - continue; - /* * Clamp the rewind at 0: wrapping below zero (get=0 -> * entries - N) makes the pump read ring slots past the * producer that hold stale entries, so replay only |