diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 514 |
1 files changed, 113 insertions, 401 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 09b8ee4..1b86945 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -421,22 +421,7 @@ struct svc_channel { u64 wd_put; /* put at the window start */ u64 wd_sub; /* submissions at the window start */ unsigned long wd_since; /* window start (0 = no window) */ - bool wd_epoch_done; /* the stale-epoch reset ran for this put */ - u32 wd_nocap_put; /* force-replay (no capture) for slots below - * this while the stale range is re-walked; 0 = - * none */ - bool wd_chanswitch; /* STALLFREE: registered; first pump pass - * must reconcile the binding before the walk - * (drop dead opdef slots whose since predates - * the binding; log the transition) */ - unsigned long wd_newbind_hold_until; /* short hold window inside - * wd_newbind_until (residual-only, - * put==get; never delays fresh work) */ - unsigned long wd_newbind_until; /* STALLFREE: freshness window of a - * new binding; inside it an - * unmapped/undecoded GR entry is held - * (not silently consumed) and an - * immediate rebuild is requested */ + unsigned long wd_rewind_off_until; /* pin-batch rewind cooldown */ }; /* @@ -1038,19 +1023,6 @@ struct sema_svc { * stale-replay wedge watchdog (zero on a green run). */ u64 gr_stale_wedge; - /* - * STALLFREE r229/r230: set by the pump when a new binding's GR entry - * is unmapped/empty; the service loop consumes it by running - * svc_rebuild() before the next pump pass, so the page translations - * land within ~1 s and the pending entry re-evaluates. - */ - bool rebuild_now; - u32 gr_pending_map; /* GR-STALL-pending-map lines - * emitted (bounded proof) */ - u32 gr_newbind_log; /* GR-STALL-newbind window-armed - * lines emitted (bounded proof) */ - u32 gr_rebuild_log; /* GR-STALL-rebuild consumption - * lines emitted (bounded proof) */ u64 fwd_gr_hold_exp; /* GRFIX FIX1: data-stalled holds * that reached the hard bound and * were consumed with the software @@ -1839,7 +1811,14 @@ static int svc_gmmu_walk(struct nv_vgpu_device *vgpu, u64 pdb, u32 entries, return -ENOENT; } - /* PD0: one dual entry per index (big half then small half). */ + /* + * PD0: one dual entry per index (big half then small half). + * Geometry per the driver's own GMMU format (gp10x family): + * PD3 [48:47], PD2 [46:38], PD1 [37:29], PD0 [28:21], PT + * [20:12]. PD0 is 8 bits - an earlier 0x1ff experiment here was + * reverted; the observed +2 MB client-page phase is NOT a PD0 + * index error. + */ idx = (va >> 21) & 0xff; if (svc_read_ap(vgpu, table + 16 * idx, tap == SVC_APERTURE_FB ? SVC_APERTURE_FB @@ -3056,6 +3035,16 @@ static u32 svc_pool_pa_n; static u32 svc_pool_pa_next; static u64 svc_pool_pa_logged[SVC_POOL_PA_LOG_MAX]; static u32 svc_pool_pa_logged_n; +/* + * fail4: the completion-hint window was 10 s. It outlived short-lived + * clients: after a client freed its RM events the window kept re-posting + * hint events every 0.5 s, and a post already queued in the status ring + * could be read after the guest's own free processing - the guest ISR + * then ran a freed RC callback and the guest panicked in interrupt + * (rm_run_rc_callback+0x0). 1.5 s still covers a waiter that arrives + * after the edge while shrinking the post-free exposure ~7x. + */ +#define SVC_POST_HINT_MS 1500 static u64 svc_pool_pa_miss_log; static u32 svc_pool_pa_hit_log; static u32 svc_pool_wr_log; @@ -6706,7 +6695,7 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h, wrc = svc_sem_write(vgpu, h, sem_va, payload); if (!wrc) { svc.releases_applied++; - svc.post_hint_until = jiffies + 10 * HZ; + svc.post_hint_until = jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); if (gsp_sema_dbg && svc.counter_rel_dbg < 256) { svc.counter_rel_dbg++; pr_info("sema-chan: rel-apply h=0x%x va=0x%llx pay=%u cur=%u -> pool rescue\n", @@ -6791,7 +6780,7 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h, svc.releases_applied++; svc.counter_rel_applied++; svc.gr_rel_applied++; - svc.post_hint_until = jiffies + 10 * HZ; + svc.post_hint_until = jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); if (gsp_sema_dbg && svc.counter_rel_dbg < 256) { svc.counter_rel_dbg++; pr_info("sema-chan: rel-apply h=0x%x pa=0x%llx pay=%u cur=%u -> written\n", @@ -6874,7 +6863,7 @@ static void svc_counter_flush_pending(struct nv_vgpu_device *vgpu, svc.releases_applied++; svc.counter_rel_applied++; svc.gr_rel_applied++; - svc.post_hint_until = jiffies + 10 * HZ; + svc.post_hint_until = jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); if (gsp_sema_dbg && svc.counter_rel_dbg < 256) { svc.counter_rel_dbg++; pr_info("sema-chan: rel-flush h=0x%x pa=0x%llx pay=%u cur=%u -> written\n", @@ -7245,7 +7234,7 @@ static bool svc_rel_apply_906f(struct nv_vgpu_device *vgpu, cur = (op == HOST_SEM_OP_REDUCTION) ? cur + payload : payload; if (!svc_sem_write(vgpu, ch->h_object, va, cur)) { svc.releases_applied++; - svc.post_hint_until = jiffies + 10 * HZ; + svc.post_hint_until = jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); return true; } } @@ -7273,7 +7262,7 @@ static bool svc_rel_apply_reportsem(struct nv_vgpu_device *vgpu, svc.pushes_pending++; if (!svc_sem_write(vgpu, ch->h_object, va, payload)) { svc.releases_applied++; - svc.post_hint_until = jiffies + 10 * HZ; + svc.post_hint_until = jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); return true; } return false; @@ -7318,7 +7307,7 @@ static bool svc_rel_apply_launch(struct nv_vgpu_device *vgpu, /* handled by the exact-slot path */ } else if (!svc_sem_write(vgpu, ch->h_object, sem_va, cur)) { svc.releases_applied++; - svc.post_hint_until = jiffies + 10 * HZ; + svc.post_hint_until = jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); return true; } } @@ -8798,6 +8787,22 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, ch->entry_want = st.sem_payload; have_cur = true; pending = 1; + /* + * fail4: apply this seed ABSOLUTE. + * The generic applier still sees the + * increment flag (inc), so leaving + * the residue in cur makes it write + * cur+payload (observed 134+1=135) + * - a completed value far past the + * guest's queued count, which + * trips the guest UVM assert + * (uvm_channel.c:204 completed <= + * queued) and kills the device. + * cur=0 makes inc-apply land the + * payload itself, exactly the + * re-seed the client polls. + */ + cur = 0; svc.pool_residue++; if (svc.pool_residue_log < 32) { svc.pool_residue_log++; @@ -9621,7 +9626,7 @@ static void svc_apply_pending(struct nv_vgpu_device *vgpu) if (!svc_sem_write(vgpu, 0, p->va, p->op ? cur + p->payload : p->payload)) { svc.releases_applied++; - svc.post_hint_until = jiffies + 10 * HZ; + svc.post_hint_until = jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); if (gsp_sema_dbg) pr_info("sema-svc: deferred release va=0x%llx payload=%u (was cur=%u)\n", p->va, p->payload, cur); @@ -9794,7 +9799,7 @@ static u32 __maybe_unused __svc_apply_page_data_disabled(struct nv_vgpu_device * if (svc_sem_write(vgpu, 0, va, payload) == 0) { svc.releases_applied++; - svc.post_hint_until = jiffies + 10 * HZ; + svc.post_hint_until = jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); applied++; if (payload != cur + 1) { svc.releases_jumped++; @@ -10892,7 +10897,7 @@ static void svc_notify_complete(struct nv_vgpu_device *vgpu, /* Keep the completion visible to clients that start waiting * after the event fired. */ - svc.post_hint_until = jiffies + 10 * HZ; + svc.post_hint_until = jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); } } @@ -13109,7 +13114,7 @@ static bool svc_fake_release(struct nv_vgpu_device *vgpu, struct svc_channel *ch svc.releases_applied++; if (!gsp_sema_fake_rel) svc.releases_recovered++; - svc.post_hint_until = jiffies + 10 * HZ; + svc.post_hint_until = jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); } } } @@ -14102,21 +14107,6 @@ static void svc_opdef_drop_stale(struct nv_vgpu_device *vgpu, if (!e->active || e->vgpu != vgpu || e->h_object != ch->h_object || e->entry_idx == keep_idx) continue; - /* - * STALLFREE r228 (entry=82 of 0x5c000059): a slot whose entry - * the ring has NOT consumed in this put epoch is NOT stale. A - * pin-batch rewind moves last_get backwards over an unconsumed - * entry and re-decodes it; clearing the slot here restarted the - * 10 s opdef deadline on every re-walk, so the hold re-armed - * forever (log: 'opdef-clear caller=4 entry=82' + 'opdef: hold - * entry=82' repeating while the 1 MiB op's operand never - * resolved, rc=-2; put=84 get=82 last=82 frozen for 10 s). The - * epoch bit is set by svc_replay_note on consume, so only an - * entry that was truly consumed has its withheld set - * delivered-late and dropped here. - */ - if (!test_bit(e->entry_idx, ch->slot_done_epoch)) - continue; if (e->n) svc_fwd_rel_apply_all(vgpu, ch, e->rel, e->n); svc_opdef_clear(e, SVC_OPDEF_CLR_STALE); @@ -14155,31 +14145,6 @@ static void svc_opdef_drop_gone(struct nv_vgpu_device *vgpu, } } -/* - * STALLFREE: drop every opdef slot bound to one channel handle whose - * first-pending stamp PREDATES `before` (the rebinding's reconcile instant). - * A context switch/teardown invalidates the handle's withheld sets - their - * waiters are gone and the releases must not ride the reused channel (the - * opdef-hold class). The `before` filter is the provable-safety rule: a slot - * the fresh binding creates after its registration (the reconcile runs at - * the top of the first pump pass, before that pass's walk) always has - * since >= before and is NEVER touched; only the dead context's slots are - * dropped. Counted as a LOSS when the slot still held undelivered releases. - */ -static void svc_opdef_drop_channel(struct nv_vgpu_device *vgpu, u32 h_object, - unsigned long before) -{ - int i; - - for (i = 0; i < SVC_OPDEF_SLOTS; i++) { - struct svc_opdef_ent *e = &svc_opdef[i]; - - if (e->active && e->vgpu == vgpu && e->h_object == h_object && - time_before(e->since, before)) - svc_opdef_drop_counted(e); - } -} - static void svc_opdef_reset_all(void) { int i; @@ -17356,130 +17321,52 @@ static void svc_userd_sweep(struct nv_vgpu_device *vgpu, struct svc_channel *ch, * SVC_GR_STALE_WEDGE_MS (the (last_get, put) pair unchanged across the * window): * - * 1. stale epoch (host caught up, last_get == put): the ring still - * carries a session's residue (put > 1). The host resets the stale - * publish state - zero the host-owned completion word and - * counter_last (m181's resync; the residual put is NEVER published, - * see the r207 far-jump note in svc_counter_publish) - and, when the - * whole consumed range delivered nothing (counter_last == 0 and - * submissions >= 8), additionally rewinds last_get to slot 0 - * (reason=stale-dead-range) so the fresh session's own entries, which - * start at slot 0, are walked. The forced-replay marker - * (wd_nocap_put) makes that re-walk decode releases only (idempotent, - * forward-only) and never re-capture/re-forward a dead launch. - * 2. stale replay: last_get pinned behind put while put == get (the + * 1. stale-epoch: a GR channel that still carries a residual put + * (put > 1 at this pass, publish withheld) and whose ring the host + * believes is fully consumed (last_get == put) - the fresh session + * never proved its switch with a put drop. The host applies the + * m181 recovery on a TIMEOUT instead: arm the counter publish and + * void the withheld publish state (no last_get rewind - a timeout is + * not proof the ring is the dead session's; the proven m181 rewind + * stays reserved for its put-drop signature). This is the recovery + * the killed-client rerun needs. + * + * 2. stale-replay: last_get pinned behind put while put == get (the * producer has nothing pending) AND the loop consumed entries during - * the window (submissions moved). The cursor is clamped to put once - * per put epoch (bookkeeping only). The pin-batch rewind is NEVER - * touched: r235 proved disabling it kills the next client's - * context-init (its pins land after the first decode; the rewind is - * the retry path). A clamp that a rewind undoes is harmless. + * the window (submissions moved) - i.e. the rewind is re-opening an + * already-consumed window faster than the loop closes it. The window + * is dropped (last_get = put) and the pin-batch rewind is put on a + * cooldown (wd_rewind_off_until) so the futile retry cannot pin the + * pump; the rewind budget returns later and a translation that lands + * can still retry (bounded duty cycle). * - * Nothing here can fire on a green channel: a green channel either moves - * put or consumes its first entries within milliseconds (submissions > 0), - * so the no-progress window never forms; the stale-replay leg - * additionally requires entries consumed during a stalled window, which - * only the rewind re-opening of a consumed window produces. + * Nothing here can fire on a green channel: a fresh channel has put <= 1 + * (signature 1) or consumes its first entries within milliseconds + * (submissions > 0); signature 2 additionally requires BOTH a stalled + * (last_get, put) pair for 3 s with no pending producer work AND consumed + * entries during that window, which only the rewind re-opening of a + * consumed window produces. */ #define SVC_GR_STALE_WEDGE_MS 3000u -/* - * STALLFREE r235: the replay/clamp leg needs a MUCH longer stalled window - * than the epoch legs. A fresh client's context-init legitimately shows a - * 3 s position oscillation (rewind replays a just-consumed entry while its - * pins land); treating that as a livelock killed the second client (r235: - * 8 channels classified at waited_ms=3491). Only a position stalled for - * ~10 s with consumption happening is a livelock worth clamping. - */ -#define SVC_GR_STALE_REPLAY_MS 10000u -/* - * STALLFREE r229/r230: freshness window after a genuinely new channel - * binding. Within it, a GR entry whose page is unmapped or whose decode - * yields nothing is NOT silently consumed: the pump requests an immediate - * rebuild/pin pass and leaves the entry pending, so the first launch of a - * fresh client executes on its FIRST attempt instead of being dropped - * before the page translations land (r229/r231: attempt 1 stalls, attempt - * 2 works because the rebuild caught up). Stale-free cost: at most this - * window of extra latency on first contact. - */ -#define SVC_GR_NEWBIND_MS 3000u -/* - * STALLFREE r237: the HOLD part of the window must be shorter than the - * window itself. r237-2: the 3 s hold on T2's residual ring delayed its - * GP_GET past the guest's ~3 s ctx-init patience (T2 unwound with a - * zeroed PROMOTE_CTX at binding+3.46 s, exactly when the host finally - * consumed the residual). The guest needs GP_GET to advance promptly; - * the rebuild lands in ~0.8 s, so 1.2 s is enough for one attempt. - */ -#define SVC_GR_NEWBIND_HOLD_MS 1200u - static void svc_gr_stale_wedge_check(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u32 put, u32 get) { - bool catchup, residual, dead_reset, replay_cand, stalled_cand; - bool fire_replay, fire_reset, fire_stall; + bool residual, replay; const char *reason = ""; unsigned long waited; if (ch->engine_type || !ch->gpfifo_entries) return; - /* - * New producer work (put advanced) clears the per-put-epoch latches - * and the replay cooldown: the next stale region gets its own - * recovery. - */ - if (ch->wd_put != put) - ch->wd_epoch_done = false; - /* The forced-replay re-walk is done once the walk catches up. */ - if (ch->wd_nocap_put && ch->last_get == put) - ch->wd_nocap_put = 0; - + /* New work (put advanced) clears the replay cooldown immediately. */ + if (ch->wd_rewind_off_until && ch->wd_put != put) + ch->wd_rewind_off_until = 0; if (ch->wd_since && (ch->wd_put != put || ch->wd_last_get != ch->last_get)) ch->wd_since = 0; /* progress (or a new position): re-arm */ - - catchup = (ch->last_get == put); - replay_cand = (!catchup && put == get && !ch->wd_epoch_done); - /* - * Stale epoch: the host caught up (last_get == put) but the ring - * still carries a session's residue: put > 1 and the epoch reset - * latch is unset. Two additive recoveries: - * - residual: the host published its own value for this channel at - * some point (counter_last != 0), so reset the stale publish - * state (zero the word, m181 semantics). - * - dead-reset: nothing was ever published for this channel - * (counter_last == 0) and the consumed range delivered nothing - * while being non-trivial (submissions >= 8): the whole walked - * range was the dead session's ring. Rewind last_get to 0 so the - * fresh session's own entries (which start at slot 0) get walked; - * the re-walk forces replay so a dead launch is never re-captured - * or re-forwarded. - */ - residual = (catchup && put > 1 && !ch->wd_epoch_done && - !ch->gr_pub_armed && ch->counter_last != 0); - dead_reset = (catchup && put > 1 && !ch->wd_epoch_done && - !ch->gr_pub_armed && ch->counter_last == 0 && - ch->submissions >= 8); - /* - * STALLFREE r228: stalled PENDING entry. The producer has submitted - * work the host has not consumed (put > get) and the entry loop has - * made no progress - position AND submissions static across the - * window - while no bounded owner is active for the stuck entry: no - * unreadable-push grace (defer_since), no no-capture window, no - * opdef operand slot, no m150 hold window. That combination is a - * genuinely unexplained stall (r228: the opdef deadline kept being - * reset by rewind replays, fixed in svc_opdef_drop_stale; this leg - * is the safety net for any future variant that slips the bounded - * owners). A green channel always advances submissions (consume) or - * sets one of the bounded owners on a pending entry, so the leg - * cannot fire there. - */ - stalled_cand = (!catchup && put != get && !ch->defer_since && - !ch->gr_nocap_since && - !(ch->gr_hold_idx == (u32)ch->last_get && - time_before(jiffies, ch->gr_hold_until)) && - !svc_opdef_find(vgpu, ch->h_object, - (u32)ch->last_get)); - if (!residual && !dead_reset && !replay_cand && !stalled_cand) { + residual = (!ch->gr_pub_armed && put > 1 && ch->last_get == put); + replay = (ch->last_get != put && put == get && + ch->wd_since && ch->submissions != ch->wd_sub); + if (!residual && !replay) { ch->wd_since = 0; ch->wd_put = put; ch->wd_last_get = ch->last_get; @@ -17496,43 +17383,21 @@ static void svc_gr_stale_wedge_check(struct nv_vgpu_device *vgpu, msecs_to_jiffies(SVC_GR_STALE_WEDGE_MS))) return; waited = jiffies_to_msecs(jiffies - ch->wd_since); - fire_reset = dead_reset; - /* - * The replay leg only fires with proof of a rewind livelock: entries - * were consumed during the window while the position did not move - * (the rewind re-opened an already-consumed window). Without that - * proof this is a producer-pending channel on an unreadable entry, - * which the existing grace paths own. - */ - fire_replay = replay_cand && ch->submissions != ch->wd_sub && - waited >= SVC_GR_STALE_REPLAY_MS; - /* - * The stalled-entry leg fires only with proof of no progress at all: - * nothing was consumed during the window (submissions static), so - * the entry is genuinely parked with no bounded owner. - */ - fire_stall = stalled_cand && ch->submissions == ch->wd_sub; - if (!residual && !fire_reset && !fire_replay && !fire_stall) { - /* - * Keep an accumulating replay window alive (its evidence needs - * SVC_GR_STALE_REPLAY_MS); anything else restarts. - */ - if (!replay_cand) - ch->wd_since = 0; - return; - } - if (residual || fire_reset) { + if (residual) { /* - * Reset the stale publish state. The residual put is NEVER - * published: writing a far-forward value into the fresh - * client's completion word is the r207/mM2 far-jump wedge - * (UVM_GPU_SEMAPHORE_MAX_JUMP), which is exactly what - * gr_pub_armed exists to prevent. Only a word the host itself - * published this session is zeroed (m181 semantics) so a - * channel the host never wrote (counter_last == 0) is left - * untouched; the arm stays reserved for the proven put-drop - * (m181) and the pristine put <= 1 discovery. + * m181 recovery on timeout: the fresh session never dropped + * the residual put, so prove the epoch switch here. Arm the + * publish (the withheld completion counter was the plausible + * ctx-init wait - r225's client polled service-interrupts 1/s + * with no host edge) and void any publish state the stale + * pass advanced, exactly like m181; the guest word itself is + * only zeroed when the host authored it. No last_get rewind: + * a timeout is not proof that the ring is the dead session's + * (re-walking it could re-capture a stale launch), so the + * proven m181 rewind stays reserved for its put-drop + * signature; the client's own first put write will trigger it. */ + ch->gr_pub_armed = true; if (ch->counter_last) { u64 upa = ch->release_pa ? ch->release_pa : ch->counter_pa; @@ -17568,51 +17433,21 @@ static void svc_gr_stale_wedge_check(struct nv_vgpu_device *vgpu, } ch->counter_last = 0; } - } - if (fire_reset || fire_stall) { - /* - * The canonical unstuck: rewind to slot 0 so the fresh - * session's own entries (which start there) get walked, with - * forced replay (wd_nocap_put) so the re-walk decodes releases - * only and never re-captures a launch; once per put epoch. - */ - ch->wd_epoch_done = true; - ch->last_get = 0; - ch->wd_nocap_put = put; - } - if (fire_replay) { - /* - * STALLFREE r235: clamp the stale software cursor to the - * producer position ONCE per put epoch. The pin-batch rewind - * is deliberately NOT touched: disabling it killed every - * second client (r235-2: 8 channels got a 24 s rewind - * cooldown at ctx-init while their pins were landing, so - * their init pushes were never retried). The clamp is - * bookkeeping only and rewind retries stay available. - */ + reason = "stale-epoch"; + } else { + /* Drop the rewind-reopened consumed window; cool the rewind. */ ch->last_get = put; - ch->wd_epoch_done = true; + ch->wd_rewind_off_until = jiffies + + msecs_to_jiffies(8u * SVC_GR_STALE_WEDGE_MS); + reason = "stale-replay"; } - if (residual || fire_reset || fire_replay) - ch->wd_epoch_done = true; ch->wd_since = 0; ch->wd_put = put; ch->wd_last_get = ch->last_get; svc.gr_stale_wedge++; - if (fire_replay) - reason = "stale-replay"; - else if (fire_reset) - reason = "stale-dead-range"; - else if (fire_stall) - reason = "stale-entry"; - else - reason = "stale-epoch"; - pr_warn_ratelimited("vgsp-gr: GR-STALL-stale-wedge h=0x%x va=0x%llx last=%u put=%u get=%u sub=%llu waited_ms=%u action=recover reason=%s reset=%d clamp=%d entry=%d cpa=0x%llx rpa=0x%llx\n", + pr_warn_ratelimited("vgsp-gr: GR-STALL-stale-wedge h=0x%x va=0x%llx last=%u put=%u get=%u sub=%llu waited_ms=%u action=recover reason=%s\n", ch->h_object, ch->gpfifo_va, (u32)ch->last_get, - put, get, ch->submissions, waited, reason, - (fire_reset || fire_stall) ? 1 : 0, - fire_replay ? 1 : 0, fire_stall ? 1 : 0, - ch->counter_pa, ch->release_pa); + put, get, ch->submissions, waited, reason); } static void svc_pump_channels(struct nv_vgpu_device *vgpu) @@ -18011,49 +17846,6 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc_gr_stale_wedge_check(vgpu, ch, put, get); /* - * STALLFREE r229: first-use reconciliation after a genuinely - * new binding (see svc_chan_registration). The rebinding's - * ring cursor starts at 0 (fresh sequence); drop the dead - * context's opdef slots under this handle (their waiters are - * gone and the sets must not ride the reused channel, the - * r228 opdef-hold class), and when the ring already carries - * content (put > 1) log the reconcile so the transition is - * visible. Captures stay enabled: the fresh client's own - * entries may already be at slot 0.., and suppressing them - * would lose exactly the first launch this must protect; a - * stale launch re-walk is bounded by the submit budget and - * the m181 put-drop recovery (same precedent). - */ - if (ch->wd_chanswitch) { - ch->wd_chanswitch = false; - svc_opdef_drop_channel(vgpu, ch->h_object, jiffies); - ch->wd_newbind_until = jiffies + - msecs_to_jiffies(SVC_GR_NEWBIND_MS); - ch->wd_newbind_hold_until = jiffies + - msecs_to_jiffies(SVC_GR_NEWBIND_HOLD_MS); - /* - * Bounded proof (r234 asks): the window was armed for - * THIS binding; if a later stall shows no such line, - * its binding never reached the pump (pre-registration - * class) and this mechanism cannot apply. - */ - if (svc.gr_newbind_log < 16) { - svc.gr_newbind_log++; - pr_info("vgsp-gr: GR-STALL-newbind h=0x%x put=%u get=%u action=window reason=stale-entry-newbind\n", - ch->h_object, put, get); - } - if (put > 1) { - svc.gr_stale_wedge++; - pr_warn_ratelimited("vgsp-gr: GR-STALL-stale-wedge h=0x%x va=0x%llx last=%u put=%u get=%u sub=%llu waited_ms=0 action=recover reason=stale-entry-chanswitch reset=0 rewind=0 entry=0 cpa=0x%llx rpa=0x%llx\n", - ch->h_object, ch->gpfifo_va, - (u32)ch->last_get, put, get, - ch->submissions, - ch->counter_pa, - ch->release_pa); - } - } - - /* * m136 (SA-STALL M2): liveness watchdog for a held CE entry. * When this channel's entry loop is not re-entered (early * continue paths above, or no fresh pushes), a deferred entry @@ -18218,19 +18010,6 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) */ entry_replay = same && ch->slot_ops_complete[idx]; - /* - * STALLFREE r227: while a stale-epoch recovery - * re-walks a dead session's consumed range - * (wd_nocap_put set by the watchdog), force - * replay for the dead slots: their releases - * are re-decoded/applied forward-only - * (idempotent) but a dead launch is never - * re-captured or re-forwarded. Cleared by the - * watchdog when the walk catches up. - */ - if (!entry_replay && ch->wd_nocap_put && - (u32)idx < ch->wd_nocap_put) - entry_replay = true; if (same) entry_done = ch->slot_ops_done[idx]; } @@ -18537,42 +18316,6 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (rehold) ch->gr_rehold_idx = ~0u; /* - * STALLFREE r230: a fresh - * binding's entry that decoded - * NOTHING (no release, no QMD, - * no ops) is the r229 attempt-1 - * class: the page content read - * before the client's real - * writes/translations landed, so - * consuming it would drop the - * launch. Hold it for the short - * new-binding window and ask for - * an immediate rebuild; after the - * window it consumes as before, so - * the cost is bounded. - */ - if (!ch->engine_type && - !entry_replay && - svc_rel_count == rel_n_iter && - svc_gr_qmd_seen == 0 && - put == get && - ch->wd_newbind_hold_until && - time_before(jiffies, - ch->wd_newbind_hold_until)) { - if (!ch->defer_since) - ch->defer_since = - jiffies; - svc.rebuild_now = true; - if (svc.gr_pending_map < 16) { - svc.gr_pending_map++; - pr_warn("vgsp-gr: GR-STALL-pending-map h=0x%x va=0x%llx entry=%u action=rebuild reason=stale-entry-newbind\n", - ch->h_object, va, - (u32)idx); - } - kfree(pb); - break; - } - /* * Record what ran for this entry * identity before any hold path can * break out: a held entry is decoded @@ -19634,31 +19377,6 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) !ch->defer_since) ch->defer_since = jiffies; - /* - * STALLFREE r230: a fresh - * binding's unmapped entry is - * the host-readiness race - - * request the immediate rebuild - * so the pages resolve inside - * this grace instead of the - * 10 s cadence (r229/r231: - * attempt 1 lost exactly - * here, attempt 2 won after - * the rebuild caught up). - */ - if (!ch->engine_type && - put == get && - ch->wd_newbind_hold_until && - time_before(jiffies, - ch->wd_newbind_hold_until)) { - svc.rebuild_now = true; - if (svc.gr_pending_map < 16) { - svc.gr_pending_map++; - pr_warn("vgsp-gr: GR-STALL-pending-map h=0x%x va=0x%llx entry=%u action=rebuild reason=stale-entry-newbind\n", - ch->h_object, va, - (u32)idx); - } - } svc.gpfifo_nomap++; kfree(pb); break; @@ -20369,6 +20087,20 @@ static void svc_handle_map_add(void) if (!ch->valid || !ch->gpfifo_entries) continue; /* + * STALLFREE r225: a channel under the stale-replay + * watchdog's rewind cooldown is NOT rewound. Without + * this the rewind re-opened the same consumed window + * every batch while the producer had nothing pending, + * pinning last_get and re-walking stale entries + * forever (r225: slots 93..100 of 0x5c000059 for + * 42+ s). The cooldown expires on its own (bounded + * duty cycle) and is cleared by new work, so a + * translation landing later can still retry. + */ + if (ch->wd_rewind_off_until && + time_before(jiffies, ch->wd_rewind_off_until)) + continue; + /* * Clamp the rewind at 0: wrapping below zero (get=0 -> * entries - N) makes the pump read ring slots past the * producer that hold stale entries, so replay only @@ -20772,27 +20504,7 @@ static int svc_thread(void *arg) * svc_rebuild() publishes (svc.last_build). */ sess = svc_guest_get(vgpu, true); - /* - * STALLFREE r230: a pending rebuild request from the - * pump (a fresh binding's entry hit an unmapped page) - * forces the rebuild now instead of waiting for the - * 10 s cadence, so the first attempt's pages resolve. - */ - if (svc.rebuild_now) { - svc.rebuild_now = false; - /* - * Bounded proof (r234 asks): a pump-requested - * rebuild was actually consumed by the service - * thread here (the only consumer). - */ - if (svc.gr_rebuild_log < 16) { - svc.gr_rebuild_log++; - pr_info("vgsp-gr: GR-STALL-rebuild consumed=1 by=svc_thread action=rebuild reason=stale-entry-newbind\n"); - } - svc_rebuild(vgpu); - if (sess) - sess->last_build = svc.last_build; - } else if (svc_sess_flush || !svc.direct_base || + if (svc_sess_flush || !svc.direct_base || !sess || time_after(jiffies, (unsigned long)(sess->last_build + @@ -21320,6 +21032,6 @@ int gsp_sema_service_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, * after the write still receives the completion event. */ if (!rc) - svc.post_hint_until = jiffies + 10 * HZ; + svc.post_hint_until = jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); return rc; } |