diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 181 |
1 files changed, 179 insertions, 2 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index b354437..0e66d73 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -406,6 +406,18 @@ struct svc_channel { u32 gr_nocap_replay;/* 1 = window opened on a replay decode */ u64 gr_nocap_va; /* push VA the window was opened for */ unsigned long gr_nocap_since; /* absolute first-seen stamp */ + /* + * STALLFREE r225: stale-epoch / stale-replay wedge watchdog. A + * channel that cannot make forward progress (last_get pinned behind + * put, or a residual session put that never drops) is detected here + * and recovered instead of spinning. All fields MUST be in the pump + * snapshot write-back list (GRFIX NOTE). + */ + u64 wd_last_get; /* last_get at the window start */ + u64 wd_put; /* put at the window start */ + u64 wd_sub; /* submissions at the window start */ + unsigned long wd_since; /* window start (0 = no window) */ + unsigned long wd_rewind_off_until; /* pin-batch rewind cooldown */ }; /* @@ -990,6 +1002,11 @@ struct sema_svc { * bounded-completion guarantee; zero on a green run). */ u64 fwd_gr_stall_swrel; + /* + * STALLFREE r225: GR channels recovered by the stale-epoch / + * stale-replay wedge watchdog (zero on a green run). + */ + u64 gr_stale_wedge; u64 fwd_gr_hold_exp; /* GRFIX FIX1: data-stalled holds * that reached the hard bound and * were consumed with the software @@ -4011,14 +4028,14 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) svc.false_trans, svc.gpfifo_skipped, svc.gpfifo_rescued, svc.releases_faked, svc.releases_recovered, svc.counter_rel_applied); - pr_info("sema-svc: drain held=%llu starved=%llu forwarded=%llu faked=%llu rec=%llu ops_once=%llu reident=%llu opdef=%llu/%llu/%llu gr_nocap=%llu/%llu gr_hold_exp=%llu gr_swrel=%llu pend_ovf=%llu\n", + pr_info("sema-svc: drain held=%llu starved=%llu forwarded=%llu faked=%llu rec=%llu ops_once=%llu reident=%llu opdef=%llu/%llu/%llu gr_nocap=%llu/%llu gr_hold_exp=%llu gr_swrel=%llu gr_wedge=%llu pend_ovf=%llu\n", svc.gpfifo_held, svc.gpfifo_starved, svc.releases_forwarded, svc.releases_faked, svc.releases_recovered, svc.ops_once_skipped, svc.ops_reident, svc.opdef_held, svc.opdef_applied, svc.opdef_expired, svc.gr_nocap_held, svc.gr_nocap_consumed, svc.fwd_gr_hold_exp, svc.fwd_gr_stall_swrel, - svc_pending_ovf); + svc.gr_stale_wedge, svc_pending_ovf); pr_info("sema-svc: counter pub_calls=%llu gr_rel=%llu reresolve=%llu pushpg_fb=%llu altw=%llu residue=%llu\n", svc.counter_pub_calls, svc.gr_rel_applied, svc.sem_write_reresolve, svc.pushpg_fallback, @@ -16387,6 +16404,132 @@ static void svc_userd_sweep(struct nv_vgpu_device *vgpu, struct svc_channel *ch, ch->h_object, hi, lo); } +/* + * STALLFREE r225: stale-epoch / stale-replay wedge watchdog. + * + * r225 evidence (witness-r225b.log): after a client was killed mid-run, a + * fresh client's GR channel (0x5c000059) was registered with a RESIDUAL + * USERD put=101 left by the dead session. Its fresh session wrote no put + * (it was stuck at ctx-init, polling MC_SERVICE_INTERRUPTS once a second), + * so the m181 fresh-put-drop recovery - which requires put < put_prev - + * never fired, and the host-side completion-counter publish stayed + * withheld (gr_pub_armed false). Meanwhile the pin-batch rewind + * (svc_handle_map_add, SVC_PIN_REWIND=8) kept yanking last_get from 101 + * back to 93 on every injection batch, so the entry loop re-walked slots + * 93..100 forever: 8 stale entries per batch, all decoding to nothing + * (rel=0, no capture) and consuming with zero delivery. The whole stall + * window shows zero captures, zero GR requests and therefore zero + * GR-STALL lines - the launch never reached the bounded submit path + * because the session epoch itself was wedged. + * + * Two signatures are detected, both keyed to NO FORWARD PROGRESS for + * SVC_GR_STALE_WEDGE_MS (the (last_get, put) pair unchanged across the + * window): + * + * 1. stale-epoch: a GR channel that still carries a residual put + * (put > 1 at this pass, publish withheld) and whose ring the host + * believes is fully consumed (last_get == put) - the fresh session + * never proved its switch with a put drop. The host applies the + * m181 recovery on a TIMEOUT instead: arm the counter publish and + * void the withheld publish state (no last_get rewind - a timeout is + * not proof the ring is the dead session's; the proven m181 rewind + * stays reserved for its put-drop signature). This is the recovery + * the killed-client rerun needs. + * + * 2. stale-replay: last_get pinned behind put while put == get (the + * producer has nothing pending) AND the loop consumed entries during + * the window (submissions moved) - i.e. the rewind is re-opening an + * already-consumed window faster than the loop closes it. The window + * is dropped (last_get = put) and the pin-batch rewind is put on a + * cooldown (wd_rewind_off_until) so the futile retry cannot pin the + * pump; the rewind budget returns later and a translation that lands + * can still retry (bounded duty cycle). + * + * Nothing here can fire on a green channel: a fresh channel has put <= 1 + * (signature 1) or consumes its first entries within milliseconds + * (submissions > 0); signature 2 additionally requires BOTH a stalled + * (last_get, put) pair for 3 s with no pending producer work AND consumed + * entries during that window, which only the rewind re-opening of a + * consumed window produces. + */ +#define SVC_GR_STALE_WEDGE_MS 3000u +static void svc_gr_stale_wedge_check(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u32 put, u32 get) +{ + bool residual, replay; + const char *reason = ""; + unsigned long waited; + + if (ch->engine_type || !ch->gpfifo_entries) + return; + /* New work (put advanced) clears the replay cooldown immediately. */ + if (ch->wd_rewind_off_until && ch->wd_put != put) + ch->wd_rewind_off_until = 0; + if (ch->wd_since && + (ch->wd_put != put || ch->wd_last_get != ch->last_get)) + ch->wd_since = 0; /* progress (or a new position): re-arm */ + residual = (!ch->gr_pub_armed && put > 1 && ch->last_get == put); + replay = (ch->last_get != put && put == get && + ch->wd_since && ch->submissions != ch->wd_sub); + if (!residual && !replay) { + ch->wd_since = 0; + ch->wd_put = put; + ch->wd_last_get = ch->last_get; + return; + } + if (!ch->wd_since) { + ch->wd_since = jiffies; + ch->wd_put = put; + ch->wd_last_get = ch->last_get; + ch->wd_sub = ch->submissions; + return; + } + if (time_before(jiffies, ch->wd_since + + msecs_to_jiffies(SVC_GR_STALE_WEDGE_MS))) + return; + waited = jiffies_to_msecs(jiffies - ch->wd_since); + if (residual) { + /* + * m181 recovery on timeout: the fresh session never dropped + * the residual put, so prove the epoch switch here. Arm the + * publish (the withheld completion counter was the plausible + * ctx-init wait - r225's client polled service-interrupts 1/s + * with no host edge) and void any publish state the stale + * pass advanced, exactly like m181; the guest word itself is + * only zeroed when the host authored it. No last_get rewind: + * a timeout is not proof that the ring is the dead session's + * (re-walking it could re-capture a stale launch), so the + * proven m181 rewind stays reserved for its put-drop + * signature; the client's own first put write will trigger it. + */ + ch->gr_pub_armed = true; + if (ch->counter_last) { + u64 upa = ch->release_pa ? ch->release_pa : + ch->counter_pa; + u32 zero = 0; + + if (upa) + (void)svc_write_ap(vgpu, upa, SVC_APERTURE_SYS, + &zero, sizeof(zero)); + ch->counter_last = 0; + } + reason = "stale-epoch"; + } else { + /* Drop the rewind-reopened consumed window; cool the rewind. */ + ch->last_get = put; + ch->wd_rewind_off_until = jiffies + + msecs_to_jiffies(8u * SVC_GR_STALE_WEDGE_MS); + reason = "stale-replay"; + } + ch->wd_since = 0; + ch->wd_put = put; + ch->wd_last_get = ch->last_get; + svc.gr_stale_wedge++; + pr_warn_ratelimited("vgsp-gr: GR-STALL-stale-wedge h=0x%x va=0x%llx last=%u put=%u get=%u sub=%llu waited_ms=%u action=recover reason=%s\n", + ch->h_object, ch->gpfifo_va, (u32)ch->last_get, + put, get, ch->submissions, waited, reason); +} + static void svc_pump_channels(struct nv_vgpu_device *vgpu) { /* Only ever called from the service thread. */ @@ -16773,6 +16916,14 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->put_prev = put; /* + * STALLFREE r225: stale-epoch / stale-replay wedge watchdog. + * Runs before the entry loop so a wedged session epoch is + * recovered (and logged) instead of spinning: see + * svc_gr_stale_wedge_check. + */ + svc_gr_stale_wedge_check(vgpu, ch, put, get); + + /* * m136 (SA-STALL M2): liveness watchdog for a held CE entry. * When this channel's entry loop is not re-entered (early * continue paths above, or no fresh pushes), a deferred entry @@ -18514,6 +18665,18 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->gr_nocap_replay; svc.chan[k].gr_nocap_va = ch->gr_nocap_va; svc.chan[k].gr_nocap_since = ch->gr_nocap_since; + /* + * STALLFREE r225: the stale-wedge watchdog + * state is snapshot state too - without the + * write-back the window would restart every + * pass and never expire. + */ + svc.chan[k].wd_last_get = ch->wd_last_get; + svc.chan[k].wd_put = ch->wd_put; + svc.chan[k].wd_sub = ch->wd_sub; + svc.chan[k].wd_since = ch->wd_since; + svc.chan[k].wd_rewind_off_until = + ch->wd_rewind_off_until; svc.chan[k].pushpg_va = ch->pushpg_va; svc.chan[k].pushpg_n = ch->pushpg_n; svc.chan[k].content_max = ch->content_max; @@ -18983,6 +19146,20 @@ static void svc_handle_map_add(void) if (!ch->valid || !ch->gpfifo_entries) continue; /* + * STALLFREE r225: a channel under the stale-replay + * watchdog's rewind cooldown is NOT rewound. Without + * this the rewind re-opened the same consumed window + * every batch while the producer had nothing pending, + * pinning last_get and re-walking stale entries + * forever (r225: slots 93..100 of 0x5c000059 for + * 42+ s). The cooldown expires on its own (bounded + * duty cycle) and is cleared by new work, so a + * translation landing later can still retry. + */ + if (ch->wd_rewind_off_until && + time_before(jiffies, ch->wd_rewind_off_until)) + continue; + /* * Clamp the rewind at 0: wrapping below zero (get=0 -> * entries - N) makes the pump read ring slots past the * producer that hold stale entries, so replay only |