From 478ac03d52a7ed7328f8eb832d22778da4ecf52f Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Sun, 18 Oct 2026 14:51:00 -0400 Subject: vgsp: stale-wedge recovery round 2 - dead-range reset, no residual publish r227 field result: the watchdog fired 10x (reason=stale-epoch, arm+publish) but the wedge survived, and victory_mail after the stalled rerun also hung. Root causes fixed here: 1. Do NOT arm/publish on a residual epoch. Arming let the next pass's svc_counter_publish write the residual put (35/101) into the fresh client's completion word - the r207/mm2 far-jump wedge (UVM_GPU_SEMAPHORE_MAX_JUMP), the exact poison gr_pub_armed exists to prevent. The epoch reset now only zeroes a word the HOST published this session (counter_last != 0, m181 semantics); the arm stays at m181's put-drop and the pristine put <= 1 discovery. 2. The stale-replay leg could never fire: its predicate required wd_since (set only when a candidate armed the window) - a circular dependency. The window now arms on either candidate and the action is decided at fire time with proof of the livelock (submissions moved during the stalled window). 3. Dead-range reset (reason=stale-dead-range): when the host caught up but the whole consumed range delivered nothing (counter_last == 0, submissions >= 8) and the epoch was never armed, rewind last_get to slot 0 so the fresh session's own entries (which start at slot 0) can be walked, and force replay for those dead slots (wd_nocap_put) so the re-walk decodes releases only (idempotent, forward-only) and never re-captures/re-forwards a dead launch. Once per put epoch (wd_epoch_done), cleared when new producer work arrives. 4. Log extended: reason=stale-epoch|stale-dead-range|stale-replay plus reset=/rewind=/cpa=/rpa= so the next run shows exactly which recovery ran and whether a publish target even exists. Green safety: both epoch legs require !gr_pub_armed (a channel that never proved a fresh epoch); a green mid-stream-discovered unarmed channel can at worst pay one forced-replay re-read of its consumed range per put epoch (no capture, no word writes). The r220 hold-storm guard and the rewind cooldown are untouched. make kernel green. (cherry picked from commit d2ba3cadecf1ca0e72358024f80dbe7a8c468842) --- kernel/gsp_sema_service.c | 161 +++++++++++++++++++++++++++++++++------------- 1 file changed, 118 insertions(+), 43 deletions(-) (limited to 'kernel') diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 0e66d73..57a6ba5 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -418,6 +418,10 @@ struct svc_channel { u64 wd_sub; /* submissions at the window start */ unsigned long wd_since; /* window start (0 = no window) */ unsigned long wd_rewind_off_until; /* pin-batch rewind cooldown */ + bool wd_epoch_done; /* the stale-epoch reset ran for this put */ + u32 wd_nocap_put; /* force-replay (no capture) for slots below + * this while the stale range is re-walked; 0 = + * none */ }; /* @@ -16426,17 +16430,18 @@ static void svc_userd_sweep(struct nv_vgpu_device *vgpu, struct svc_channel *ch, * SVC_GR_STALE_WEDGE_MS (the (last_get, put) pair unchanged across the * window): * - * 1. stale-epoch: a GR channel that still carries a residual put - * (put > 1 at this pass, publish withheld) and whose ring the host - * believes is fully consumed (last_get == put) - the fresh session - * never proved its switch with a put drop. The host applies the - * m181 recovery on a TIMEOUT instead: arm the counter publish and - * void the withheld publish state (no last_get rewind - a timeout is - * not proof the ring is the dead session's; the proven m181 rewind - * stays reserved for its put-drop signature). This is the recovery - * the killed-client rerun needs. - * - * 2. stale-replay: last_get pinned behind put while put == get (the + * 1. stale epoch (host caught up, last_get == put): the ring still + * carries a session's residue (put > 1). The host resets the stale + * publish state - zero the host-owned completion word and + * counter_last (m181's resync; the residual put is NEVER published, + * see the r207 far-jump note in svc_counter_publish) - and, when the + * whole consumed range delivered nothing (counter_last == 0 and + * submissions >= 8), additionally rewinds last_get to slot 0 + * (reason=stale-dead-range) so the fresh session's own entries, which + * start at slot 0, are walked. The forced-replay marker + * (wd_nocap_put) makes that re-walk decode releases only (idempotent, + * forward-only) and never re-capture/re-forward a dead launch. + * 2. stale replay: last_get pinned behind put while put == get (the * producer has nothing pending) AND the loop consumed entries during * the window (submissions moved) - i.e. the rewind is re-opening an * already-consumed window faster than the loop closes it. The window @@ -16445,33 +16450,64 @@ static void svc_userd_sweep(struct nv_vgpu_device *vgpu, struct svc_channel *ch, * pump; the rewind budget returns later and a translation that lands * can still retry (bounded duty cycle). * - * Nothing here can fire on a green channel: a fresh channel has put <= 1 - * (signature 1) or consumes its first entries within milliseconds - * (submissions > 0); signature 2 additionally requires BOTH a stalled - * (last_get, put) pair for 3 s with no pending producer work AND consumed - * entries during that window, which only the rewind re-opening of a - * consumed window produces. + * Nothing here can fire on a green channel: a green channel either moves + * put or consumes its first entries within milliseconds (submissions > 0), + * so the no-progress window never forms; the stale-replay leg + * additionally requires entries consumed during a stalled window, which + * only the rewind re-opening of a consumed window produces. */ #define SVC_GR_STALE_WEDGE_MS 3000u static void svc_gr_stale_wedge_check(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u32 put, u32 get) { - bool residual, replay; + bool catchup, residual, dead_reset, replay_cand; + bool fire_replay, fire_reset; const char *reason = ""; unsigned long waited; if (ch->engine_type || !ch->gpfifo_entries) return; - /* New work (put advanced) clears the replay cooldown immediately. */ - if (ch->wd_rewind_off_until && ch->wd_put != put) - ch->wd_rewind_off_until = 0; + /* + * New producer work (put advanced) clears the per-put-epoch latches + * and the replay cooldown: the next stale region gets its own + * recovery. + */ + if (ch->wd_put != put) { + if (ch->wd_rewind_off_until) + ch->wd_rewind_off_until = 0; + ch->wd_epoch_done = false; + } + /* The forced-replay re-walk is done once the walk catches up. */ + if (ch->wd_nocap_put && ch->last_get == put) + ch->wd_nocap_put = 0; + if (ch->wd_since && (ch->wd_put != put || ch->wd_last_get != ch->last_get)) ch->wd_since = 0; /* progress (or a new position): re-arm */ - residual = (!ch->gr_pub_armed && put > 1 && ch->last_get == put); - replay = (ch->last_get != put && put == get && - ch->wd_since && ch->submissions != ch->wd_sub); - if (!residual && !replay) { + + catchup = (ch->last_get == put); + replay_cand = (!catchup && put == get); + /* + * Stale epoch: the host caught up (last_get == put) but the ring + * still carries a session's residue: put > 1 and the epoch reset + * latch is unset. Two additive recoveries: + * - residual: the host published its own value for this channel at + * some point (counter_last != 0), so reset the stale publish + * state (zero the word, m181 semantics). + * - dead-reset: nothing was ever published for this channel + * (counter_last == 0) and the consumed range delivered nothing + * while being non-trivial (submissions >= 8): the whole walked + * range was the dead session's ring. Rewind last_get to 0 so the + * fresh session's own entries (which start at slot 0) get walked; + * the re-walk forces replay so a dead launch is never re-captured + * or re-forwarded. + */ + residual = (catchup && put > 1 && !ch->wd_epoch_done && + !ch->gr_pub_armed && ch->counter_last != 0); + dead_reset = (catchup && put > 1 && !ch->wd_epoch_done && + !ch->gr_pub_armed && ch->counter_last == 0 && + ch->submissions >= 8); + if (!residual && !dead_reset && !replay_cand) { ch->wd_since = 0; ch->wd_put = put; ch->wd_last_get = ch->last_get; @@ -16488,21 +16524,31 @@ static void svc_gr_stale_wedge_check(struct nv_vgpu_device *vgpu, msecs_to_jiffies(SVC_GR_STALE_WEDGE_MS))) return; waited = jiffies_to_msecs(jiffies - ch->wd_since); - if (residual) { + fire_reset = dead_reset; + /* + * The replay leg only fires with proof of a rewind livelock: entries + * were consumed during the window while the position did not move + * (the rewind re-opened an already-consumed window). Without that + * proof this is a producer-pending channel on an unreadable entry, + * which the existing grace paths own. + */ + fire_replay = replay_cand && ch->submissions != ch->wd_sub; + if (!residual && !fire_reset && !fire_replay) { + ch->wd_since = 0; + return; + } + if (residual || fire_reset) { /* - * m181 recovery on timeout: the fresh session never dropped - * the residual put, so prove the epoch switch here. Arm the - * publish (the withheld completion counter was the plausible - * ctx-init wait - r225's client polled service-interrupts 1/s - * with no host edge) and void any publish state the stale - * pass advanced, exactly like m181; the guest word itself is - * only zeroed when the host authored it. No last_get rewind: - * a timeout is not proof that the ring is the dead session's - * (re-walking it could re-capture a stale launch), so the - * proven m181 rewind stays reserved for its put-drop - * signature; the client's own first put write will trigger it. + * Reset the stale publish state. The residual put is NEVER + * published: writing a far-forward value into the fresh + * client's completion word is the r207/mM2 far-jump wedge + * (UVM_GPU_SEMAPHORE_MAX_JUMP), which is exactly what + * gr_pub_armed exists to prevent. Only a word the host itself + * published this session is zeroed (m181 semantics) so a + * channel the host never wrote (counter_last == 0) is left + * untouched; the arm stays reserved for the proven put-drop + * (m181) and the pristine put <= 1 discovery. */ - ch->gr_pub_armed = true; if (ch->counter_last) { u64 upa = ch->release_pa ? ch->release_pa : ch->counter_pa; @@ -16513,21 +16559,35 @@ static void svc_gr_stale_wedge_check(struct nv_vgpu_device *vgpu, &zero, sizeof(zero)); ch->counter_last = 0; } - reason = "stale-epoch"; - } else { + } + if (fire_reset) { + ch->wd_epoch_done = true; + ch->last_get = 0; + ch->wd_nocap_put = put; + } + if (fire_replay) { /* Drop the rewind-reopened consumed window; cool the rewind. */ ch->last_get = put; ch->wd_rewind_off_until = jiffies + msecs_to_jiffies(8u * SVC_GR_STALE_WEDGE_MS); - reason = "stale-replay"; } + if (residual || fire_reset) + ch->wd_epoch_done = true; ch->wd_since = 0; ch->wd_put = put; ch->wd_last_get = ch->last_get; svc.gr_stale_wedge++; - pr_warn_ratelimited("vgsp-gr: GR-STALL-stale-wedge h=0x%x va=0x%llx last=%u put=%u get=%u sub=%llu waited_ms=%u action=recover reason=%s\n", + if (fire_replay) + reason = "stale-replay"; + else if (fire_reset) + reason = "stale-dead-range"; + else + reason = "stale-epoch"; + pr_warn_ratelimited("vgsp-gr: GR-STALL-stale-wedge h=0x%x va=0x%llx last=%u put=%u get=%u sub=%llu waited_ms=%u action=recover reason=%s reset=%d rewind=%d cpa=0x%llx rpa=0x%llx\n", ch->h_object, ch->gpfifo_va, (u32)ch->last_get, - put, get, ch->submissions, waited, reason); + put, get, ch->submissions, waited, reason, + fire_reset ? 1 : 0, fire_replay ? 1 : 0, + ch->counter_pa, ch->release_pa); } static void svc_pump_channels(struct nv_vgpu_device *vgpu) @@ -17088,6 +17148,19 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) */ entry_replay = same && ch->slot_ops_complete[idx]; + /* + * STALLFREE r227: while a stale-epoch recovery + * re-walks a dead session's consumed range + * (wd_nocap_put set by the watchdog), force + * replay for the dead slots: their releases + * are re-decoded/applied forward-only + * (idempotent) but a dead launch is never + * re-captured or re-forwarded. Cleared by the + * watchdog when the walk catches up. + */ + if (!entry_replay && ch->wd_nocap_put && + (u32)idx < ch->wd_nocap_put) + entry_replay = true; if (same) entry_done = ch->slot_ops_done[idx]; } @@ -18677,6 +18750,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc.chan[k].wd_since = ch->wd_since; svc.chan[k].wd_rewind_off_until = ch->wd_rewind_off_until; + svc.chan[k].wd_epoch_done = ch->wd_epoch_done; + svc.chan[k].wd_nocap_put = ch->wd_nocap_put; svc.chan[k].pushpg_va = ch->pushpg_va; svc.chan[k].pushpg_n = ch->pushpg_n; svc.chan[k].content_max = ch->content_max; -- cgit v1.2.3