diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 397 |
1 files changed, 369 insertions, 28 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 8ae11ec..fb00685 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -984,6 +984,12 @@ struct sema_svc { * abort, or a release0 page the backend could not observe). */ u64 fwd_gr_sw_rel; + /* + * STALLFREE: failed forwarded launches the pass-top outcome check + * completed in software because no hold owned their release (the + * bounded-completion guarantee; zero on a green run). + */ + u64 fwd_gr_stall_swrel; u64 fwd_gr_hold_exp; /* GRFIX FIX1: data-stalled holds * that reached the hard bound and * were consumed with the software @@ -3875,13 +3881,33 @@ static void svc_session_flush(void); /* defined next to the pending queue */ #define SVC_LEDGER_MAX (1u << 14) #define SVC_LEDGER_MASK (SVC_LEDGER_MAX - 1u) #define SVC_LEDGER_PROBE 16u +/* + * STALLFREE: eviction burst threshold for the cap-and-degrade reset. A + * cluster keeps 16 same-home VA pages; EXIT_RESET evictions inside one session + * means the table is thrashing (pin churn / leftover crashed clients), and + * the half-evicted table silently strands the evicted pages' landing + * authority. At the threshold the cache is cleared and rebuilt from live + * writes (uniform resolver fallback instead of silent partial authority). + */ +#define SVC_LEDGER_EVICT_RESET 256u static u32 svc_ledger_n; static u64 svc_ledger_diverges; static u64 svc_ledger_stores; static u64 svc_ledger_evictions; +static u64 svc_ledger_evict_since_reset; +static u64 svc_ledger_evict_resets; static u64 svc_ledger_rewrites; static u64 svc_ledger_reloc_diverges; static u32 svc_ledger_store_log; +/* + * STALLFREE: pending-ring overflow telemetry (the negative queue silently + * overwrote its oldest entry when full). Hoisted here so svc_rebuild's diag + * can print it; the ring itself is defined with the pending queue below. + */ +static u64 svc_pending_ovf; +static u32 svc_pending_ovf_log; +/* STALLFREE: rescue scans that hit their wall-clock cap (bounded service). */ +static u64 svc_rescue_scan_capped; static void svc_rebuild(struct nv_vgpu_device *vgpu) { @@ -3985,20 +4011,21 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) svc.false_trans, svc.gpfifo_skipped, svc.gpfifo_rescued, svc.releases_faked, svc.releases_recovered, svc.counter_rel_applied); - pr_info("sema-svc: drain held=%llu starved=%llu forwarded=%llu faked=%llu rec=%llu ops_once=%llu reident=%llu opdef=%llu/%llu/%llu gr_nocap=%llu/%llu gr_hold_exp=%llu\n", + pr_info("sema-svc: drain held=%llu starved=%llu forwarded=%llu faked=%llu rec=%llu ops_once=%llu reident=%llu opdef=%llu/%llu/%llu gr_nocap=%llu/%llu gr_hold_exp=%llu gr_swrel=%llu pend_ovf=%llu\n", svc.gpfifo_held, svc.gpfifo_starved, svc.releases_forwarded, svc.releases_faked, svc.releases_recovered, svc.ops_once_skipped, svc.ops_reident, svc.opdef_held, svc.opdef_applied, svc.opdef_expired, svc.gr_nocap_held, svc.gr_nocap_consumed, - svc.fwd_gr_hold_exp); + svc.fwd_gr_hold_exp, svc.fwd_gr_stall_swrel, + svc_pending_ovf); pr_info("sema-svc: counter pub_calls=%llu gr_rel=%llu reresolve=%llu pushpg_fb=%llu altw=%llu residue=%llu\n", svc.counter_pub_calls, svc.gr_rel_applied, svc.sem_write_reresolve, svc.pushpg_fallback, svc.pool_alt_applied, svc.pool_residue); - if (svc.resc_run) - pr_info("sema-svc: rescue_run=%llu (multi-page content pins)\n", - svc.resc_run); + if (svc.resc_run || svc_rescue_scan_capped) + pr_info("sema-svc: rescue_run=%llu scan_capped=%llu (multi-page content pins)\n", + svc.resc_run, svc_rescue_scan_capped); pr_info("sema-svc: gpfifo done=%llu nomap=%llu vidmem_writes=%llu ring_auto=%u pb=%u/%u pins=%u/%llu ipins=%u/%llu\n", svc.gpfifo_entries_done, svc.gpfifo_nomap, svc.vidmem_writes, svc_ring_n, svc_pb_n, svc_pb_passes, @@ -4013,9 +4040,10 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) * content" window. n approaching SVC_LEDGER_MAX or evict > 0 in a * client's window is the tell. */ - pr_info("sema-svc: ledger n=%u/%u stores=%llu evict=%llu diverges=%llu reloc_div=%llu rewrites=%llu\n", + pr_info("sema-svc: ledger n=%u/%u stores=%llu evict=%llu resets=%llu diverges=%llu reloc_div=%llu rewrites=%llu\n", svc_ledger_n, SVC_LEDGER_MAX, svc_ledger_stores, - svc_ledger_evictions, svc_ledger_diverges, + svc_ledger_evictions, svc_ledger_evict_resets, + svc_ledger_diverges, svc_ledger_reloc_diverges, svc_ledger_rewrites); /* * P6b wave 1: guest-forwarding summary. Printed whenever any counter @@ -4374,6 +4402,7 @@ static void svc_fill_audit(struct nv_vgpu_device *vgpu, u64 va0, u64 addr, */ static void svc_ledger_store(u64 va, u64 pa, u32 ap); /* R2 ledger, defined below */ static int svc_ledger_lookup(u64 va, u64 *pa, u32 *ap); /* R2 ledger, defined below */ +static void svc_ledger_clear(void); /* STALLFREE: cap-and-degrade reset */ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, u32 count, u32 phys, u32 aperture, u64 *done_out) @@ -4614,6 +4643,15 @@ struct svc_gr_data_win { struct svc_gr_cand { bool valid; bool offered; /* consumed by a request or a fallback */ + /* + * STALLFREE: exactly-once latch for the candidate's QMD completion + * release when the launch is consumed without ever being forwarded + * (a data-stall/busy/nocap hold fell through). Zeroed with the rest + * of the candidate at capture. + */ + bool rel_done; + unsigned long captured_jiffies; /* STALLFREE: capture stamp for the + * GR-STALL waited_ms accounting */ u32 h_object; u64 push_va; u32 push_len; @@ -4726,6 +4764,7 @@ static void svc_gr_cand_capture(struct svc_channel *ch, } memset(&svc_gr_cand, 0, sizeof(svc_gr_cand)); svc_gr_cand.valid = true; + svc_gr_cand.captured_jiffies = jiffies; svc_gr_cand.h_object = ch->h_object; svc_gr_cand.push_va = push_va; svc_gr_cand.push_len = push_len; @@ -4979,8 +5018,27 @@ static void svc_ledger_store(u64 va, u64 pa, u32 ap) * VA's landing authority (its later readers fall back to the * resolver, possibly stale), so the first occurrences are logged and * counted into the diag line. + * + * STALLFREE cap-and-degrade: a sustained eviction burst means the + * table is being thrashed (a pin storm / leftover crashed client + * churning VA pages faster than the working set), and a half-evicted + * table silently strands authority for exactly the evicted pages. + * Past SVC_LEDGER_EVICT_RESET evictions in one session the cache is + * cleared wholesale and rebuilt from live writes: every reader + * uniformly falls back to the resolver until its page is rewritten, + * which removes the stranded-authority class at a known, logged + * cost. The clear is a cache operation only (no control flow, no + * wait); the current store then lands in the empty table. */ svc_ledger_evictions++; + svc_ledger_evict_since_reset++; + if (svc_ledger_evict_since_reset >= SVC_LEDGER_EVICT_RESET) { + svc_ledger_evict_since_reset = 0; + svc_ledger_evict_resets++; + pr_warn("sema-ledger: LEDGER-RESET evict=%llu resets=%llu (cluster thrash under churn; cache cleared, readers fall back to the resolver until rewritten)\n", + svc_ledger_evictions, svc_ledger_evict_resets); + svc_ledger_clear(); + } if (svc_ledger_evictions <= 32) pr_warn("sema-ledger: EVICT home=%u old_va=0x%llx new_va=0x%llx (cluster full; old landing authority lost until rewritten)\n", home, (u64)svc_ledger[home].va_page, page); @@ -5017,6 +5075,7 @@ static void svc_ledger_clear(void) svc_ledger_n = 0; svc_ledger_rewrites = 0; svc_ledger_reloc_diverges = 0; + svc_ledger_evict_since_reset = 0; } /* @@ -8363,6 +8422,13 @@ static u32 svc_pending_next; static u64 svc_pending_rescued; /* stall-class: entries the old >= dedup would have dropped */ static u32 svc_pending_rescued_log; +/* + * STALLFREE: the pending ring silently overwrote its oldest entry when full + * (an evicted release is a client wait that never clears). The counters are + * hoisted next to the ledger counters so the periodic diag (above) can print + * them; the ring is a bounded queue, so under churn this is the back-pressure + * signal, not an error path. + */ static void svc_pending_add(u64 va, u32 payload, u32 op) { @@ -8401,6 +8467,14 @@ static void svc_pending_add(u64 va, u32 payload, u32 op) i = svc_pending_n++; } else { i = svc_pending_next++ % SVC_PENDING_MAX; + svc_pending_ovf++; + if (svc_pending_ovf_log < 8) { + svc_pending_ovf_log++; + pr_warn("sema-svc: PENDING-OVF ring full (%u); overwriting oldest entry va=0x%llx pay=%u op=%u (n=%llu)\n", + SVC_PENDING_MAX, svc_pending[i].va, + svc_pending[i].payload, svc_pending[i].op, + svc_pending_ovf); + } } svc_pending[i].va = va; svc_pending[i].payload = payload; @@ -10704,10 +10778,11 @@ unsigned gsp_phys_gr_hold_max_ms = 3000; * pages - the drops are logged (GRFILL-DROP) and the launch takes the * historical forward path immediately (exactly the pre-8b058884 behaviour); * the GRFILL-HOLD/GRFILL-EXPIRED machinery and the per-launch absolute - * bound are skipped entirely. When set to 1 the page-count hold + absolute - * bound apply (r213-r215 showed the bound never fired: the per-channel - * record fields are not in the svc_pump_channels snapshot write-back list, - * so they were zeroed every pass - see the GRFIX NOTE at that list). + * bound are skipped entirely (the pump's own m150/m156 hold and its + * absolute bound are NOT - they run regardless of this gate; the record + * fields are now in the snapshot write-back list, see the GRFIX NOTE). + * When set to 1 the page-count hold + absolute bound apply on the fill + * path too. */ unsigned gsp_phys_gr_fill_hold; /* default 0: log-only, never hold */ @@ -11627,6 +11702,19 @@ static int svc_rescue_run_try(struct nv_vgpu_device *vgpu, } /* + * STALLFREE: wall-clock cap for one content-rescue scan range. The scan is + * bounded by the guest-RAM range, not by time: the full fallback walks up to + * 6 GB in 4 KB vfio reads, and under a pin storm / leftover crashed client's + * churn that walk can run for tens of seconds on the service thread, + * delaying every channel (GR launches included). Green scans answer from the + * hot set or the near window in milliseconds, so a 3 s cap never fires on a + * healthy boot; when it does fire the entry falls through its existing grace + * path (hold, then consume with the software release) instead of pinning the + * whole pump. + */ +#define SVC_RESCUE_SCAN_MAX_MS 3000u + +/* * Scan a guest-RAM range descending for a page that validates as this entry's * push (method stream + expected release payload), pinning it and returning 0 * on the first hit. `hi` and `lo` are inclusive chunk start addresses. @@ -11639,10 +11727,20 @@ static int svc_rescue_scan_range(struct nv_vgpu_device *vgpu, { u64 gpa = hi; unsigned long t0 = jiffies; + unsigned long deadline = + t0 + msecs_to_jiffies(SVC_RESCUE_SCAN_MAX_MS); u32 total = 0; u64 bytes = 0; for (;;) { + if (time_after_eq(jiffies, deadline)) { + svc_rescue_scan_capped++; + if (svc_rescue_scan_capped <= 8) + pr_warn("sema-svc: RESCUE-SCAN-CAP h=0x%x va=0x%llx [0x%llx..0x%llx] %llu MB in %u ms (capped; entry keeps its grace path)\n", + ch->h_object, entry_va, hi, lo, + bytes >> 20, SVC_RESCUE_SCAN_MAX_MS); + break; + } if (!svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK)) { u32 ncand, n; @@ -12379,6 +12477,25 @@ struct svc_fwd_req { bool gr; bool gr_bridge_done; bool gr_page_overflow; + /* + * STALLFREE: bounded-completion state for the forwarded launch. The + * request's backend outcome is the only host edge that can complete a + * launch whose release records engage no hold (svc_fwd_rel_n == 0); + * gr_swrel_done is the exactly-once latch for the software authorship + * of the QMD completion word, and gr_fill_jiffies stamps the request + * so the GR-STALL line reports the real waited time. + */ + bool gr_swrel_done; + /* + * STALLFREE: set by the work item when release0 could not target the + * guest page (unresolved release PA / HPA), so a successful launch's + * engine completion lands only on the module semaphore and the guest + * QMD word stays behind. The pump's bounded outcome check then + * authors it in software (the same write the hold's success edge + * performs), so the client always completes. + */ + bool gr_rel_unobserved; + unsigned long gr_fill_jiffies; u32 gr_h_object; u32 gr_dropped; /* GRFIX: pages the fill's last pass could not * resolve (0 = launch fully resolved); read @@ -13320,6 +13437,73 @@ static void svc_fwd_hold_gr_release(struct nv_vgpu_device *vgpu, svc_fwd_req.gr_rel_payload); } +/* + * STALLFREE: bounded completion for a failed forwarded GR launch that no hold + * completes. A GR push whose decode recorded no push-level release records + * (svc_fwd_rel_n == 0) engages no release hold, so when the backend fails + * (ladder timeout, fault abort, refusal) nothing authors the QMD completion + * word the guest client polls after the kick: the client's launch wait then + * has no host edge and lasts until the guest's own timeout (the r219/r222 + * stall class, qmd_rel frozen at the previous launch's value). + * + * This runs on the service thread at the top of every pump pass, before any + * request fill of that pass (svc_fwd_request requires SVC_FWD_INFLIGHT free, + * which the work item only releases after publishing done, and the fill's + * first write is done=false), so a pass that sees done also sees this + * request's fields, and every finished request is observed before its slot + * can be reused. Authors the release once (forward-only, full resolver - the + * same write the hold's failure edge performs), latches gr_swrel_done, and + * logs the distinguishable GR-STALL line. + * + * While a hold is active the hold owns the completion (its failure edge + * applies the recorded releases AND the QMD word, bounded by the existing + * hold timeout), so this check stands down for it. + */ +static void svc_fwd_outcome_check(struct nv_vgpu_device *vgpu) +{ + bool done = smp_load_acquire(&svc_fwd_req.done); + unsigned long waited; + + if (!svc_fwd_req.gr || svc_fwd_req.vgpu != vgpu || + svc_fwd_req.gr_swrel_done || svc_fwd_hold.active || !done) + return; + /* + * Complete when the backend failed (the never-completing r219/r222 + * class) OR when a successful launch's release0 could not target the + * guest page, so the engine's completion landed only on the module + * semaphore and the client's QMD word is still behind (the same edge + * the hold's success path covers for held requests). + */ + if (svc_fwd_req.hw_rc == 0 && !svc_fwd_req.gr_rel_unobserved) + return; + /* + * The outcome is final; latch it first so no later pass, hold edge or + * consume tail accounts it again (the release write itself is + * forward-only, so even a duplicate could not corrupt the word). + */ + svc_fwd_req.gr_swrel_done = true; + waited = jiffies_to_msecs(jiffies - svc_fwd_req.gr_fill_jiffies); + if (!svc_fwd_req.gr_rel_va || !svc_fwd_req.gr_rel_payload) { + pr_warn("vgsp-gr: GR-STALL-backend va=0x%llx waited_ms=%lu action=proceed (launch carries no release record; no word to author)\n", + svc_fwd_req.gr_push_va, waited); + return; + } + svc_gr_sw_release(vgpu, svc_fwd_req.gr_h_object, + svc_fwd_req.gr_rel_va, + svc_fwd_req.gr_rel_payload); + svc.fwd_gr_stall_swrel++; + if (svc_fwd_req.hw_rc != 0) + pr_warn("vgsp-gr: GR-STALL-backend va=0x%llx waited_ms=%lu action=proceed (failed forwarded launch, rc=%d; QMD release authored in software)\n", + svc_fwd_req.gr_push_va, waited, svc_fwd_req.hw_rc); + else + pr_warn("vgsp-gr: GR-STALL-backend va=0x%llx waited_ms=%lu action=proceed (successful launch whose release0 could not target the guest page; QMD release authored in software)\n", + svc_fwd_req.gr_push_va, waited); + pr_info("vgsp-gr: GR-COMPLETE-SW h=0x%x push=0x%llx rel=0x%llx pay=0x%x waited_ms=%lu n=%llu (backend outcome without a hold)\n", + svc_fwd_req.gr_h_object, svc_fwd_req.gr_push_va, + svc_fwd_req.gr_rel_va, svc_fwd_req.gr_rel_payload, waited, + svc.fwd_gr_stall_swrel); +} + static bool svc_fwd_hold_expired(void) { if (svc_fwd_req.gr && !smp_load_acquire(&svc_fwd_req.done)) @@ -13537,6 +13721,15 @@ static void svc_fwd_hold_expire_check(struct nv_vgpu_device *vgpu, svc_fwd_hold.relay_armed) svc_fwd_fault_deliver(hv, ch); /* + * STALLFREE: the unpumpable-channel expiry edge (the entry loop never + * ran for this hold). The releases and the QMD completion word are + * software-applied below; exactly one line per hold. + */ + if (svc_fwd_req.gr) + pr_warn("vgsp-gr: GR-STALL-fwd-hold va=0x%llx waited_ms=%u action=proceed (hold expired on an unpumpable channel; releases + QMD word software-applied)\n", + svc_fwd_req.gr_push_va, + jiffies_to_msecs(jiffies - svc_fwd_hold.since)); + /* * P6b wave 2: a skipped candidate whose hardware copy did not complete * successfully is authored by one software copy here, BEFORE the * recorded releases below. A successful hardware copy needs none. @@ -13666,6 +13859,17 @@ static bool svc_fwd_hold_poll(struct nv_vgpu_device *vgpu, if (done && svc_fwd_req.hw_rc == -ETIMEDOUT && svc_fwd_hold.relay_armed) svc_fwd_fault_deliver(hv, ch); + /* + * STALLFREE: the bounded failure edge of a held forwarded GR + * launch - the releases AND the QMD completion word are + * software-applied below, so the client always completes. + * Exactly one line per hold (the hold is cleared at the end). + */ + if (svc_fwd_req.gr) + pr_warn("vgsp-gr: GR-STALL-fwd-fail va=0x%llx waited_ms=%u action=proceed (hold resolved on backend failure/expiry; releases + QMD word software-applied)\n", + svc_fwd_req.gr_push_va, + jiffies_to_msecs(jiffies - + svc_fwd_hold.since)); svc_fwd_hold_soft_copy(hv); if (svc_fwd_hold.relay_armed) svc.fwd_relay_fallback++; @@ -13924,6 +14128,7 @@ static void svc_fwd_gr_work(void) (SEMA_PAGE_SIZE - 1)); u8 rel_kind = (svc_fwd_req.gr_rel_ap == SVC_APERTURE_FB) ? SVC_APERTURE_FB : SVC_APERTURE_SYS; + bool rel_unobserved = false; if (svc_fwd_req.gr_rel_pa && (svc_fwd_operand_hpa(svc_fwd_req.vgpu, rel_kind, @@ -13936,11 +14141,24 @@ static void svc_fwd_gr_work(void) * means the engine would author nothing this launch; point * release0 at the module semaphore instead so the host still * observes completion. + * + * STALLFREE: when the guest page could not be targeted at all + * (unresolved release PA or HPA), the engine's completion will + * land only on the module semaphore and the guest QMD word + * stays behind - a launch whose client would otherwise wait + * forever even on a SUCCESSFUL submit. Record it so the pump's + * bounded outcome check authors the word in software (the + * same write the hold's success edge performs). A resolved + * page whose word is already ahead needs nothing. */ - if (rel_hpa && - READ_ONCE(*(u32 *)((u8 *)__va(rel_hpa) + roff)) >= + if (rel_hpa) { + if (READ_ONCE(*(u32 *)((u8 *)__va(rel_hpa) + roff)) >= svc_fwd_req.gr_rel_payload) - rel_hpa = 0; + rel_hpa = 0; + } else { + rel_unobserved = true; + } + svc_fwd_req.gr_rel_unobserved = rel_unobserved; } gs.qmd = svc_fwd_req.gr_qmd; @@ -14745,6 +14963,23 @@ static void svc_gr_scan_inline(u64 *wins, u32 *n_cand, u64 page_va) } /* + * STALLFREE: identity of the launch currently held/retried on this channel - + * the capture's push VA when it belongs to this channel (the busy-refusal + * path never reaches the fill, so the request slot can still hold a previous + * launch's VA), else the request slot's when it names this channel. The + * absolute hold bound is keyed to this identity so a pin-batch rewind that + * replays the entry at a different slot index can never re-arm it. + */ +static u64 svc_gr_launch_va(const struct svc_channel *ch) +{ + if (svc_gr_cand.valid && svc_gr_cand.h_object == ch->h_object) + return svc_gr_cand.push_va; + if (svc_fwd_req.gr && svc_fwd_req.gr_h_object == ch->h_object) + return svc_fwd_req.gr_push_va; + return 0; +} + +/* * GRFIX FIX1b (r213): absolute per-launch hold bound. The m150/m156 data * hold had no working bound: the 8b058884 deadline was keyed by the GPFIFO * slot index, and a pin-batch rewind/replay re-decodes the same launch push @@ -14760,6 +14995,24 @@ static bool svc_gr_hold_bound_expired(struct svc_channel *ch, u64 launch_va) { u32 cap_ms = gsp_phys_gr_hold_max_ms; + /* + * STALLFREE (r213-r215 follow-up): the absolute per-launch bound is + * now effective on the default path. Before the write-back fix the + * bound's record fields were excluded from the pump snapshot and + * zeroed every pass, so this returned false forever and the legacy + * m150/m156 hold window (gr_hold_until) was the only expiry - and + * that window is keyed to the GPFIFO slot index, which a pin-batch + * rewind changes on every replay (the r213 identity bug). Under pin + * churn the same launch was therefore re-held indefinitely: r220 + * held one launch from 155221 to 155417 (196 s, 1631 re-fills of + * entry 106) until the guest's 240 s kill. The bound is keyed to the + * LAUNCH PUSH VA and nothing re-arms it, so a replayed slot no + * longer extends the hold; at the cap the pump falls through to the + * historical consume, where STALLFREE authors the QMD completion + * release and logs GR-STALL-hold action=proceed. Default cap 3000 ms + * (gsp_phys_gr_hold_max_ms) sits above the observed 0.5-2 s pin + * latency, so the legitimate pin-late retry still wins. + */ if (!launch_va) launch_va = 1; /* anonymous launch: still absolutely bound */ if (!cap_ms) @@ -15388,6 +15641,11 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, svc_fwd_req.guest = false; svc_fwd_req.gr = true; svc_fwd_req.gr_bridge_done = false; + /* STALLFREE: fresh request - re-arm the software-completion latch and + * stamp the fill time for the GR-STALL waited_ms accounting. */ + svc_fwd_req.gr_swrel_done = false; + svc_fwd_req.gr_rel_unobserved = false; + svc_fwd_req.gr_fill_jiffies = jiffies; svc_fwd_req.defer = true; svc_fwd_req.relay = false; svc_fwd_req.relay_index = 0; @@ -16141,6 +16399,14 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) n_gr = SVC_GR_COUNTER_TOTAL; /* + * STALLFREE: complete a failed forwarded GR launch no hold owns (see + * svc_fwd_outcome_check). Runs before this pass's channel loop, so a + * finished request is always observed before any fill can reuse its + * slot. + */ + svc_fwd_outcome_check(vgpu); + + /* * H1 last resort: one self-gated content-discovery pass for the GR * ctx-init rings before this pass's channel loop. The pins it * installs (ring VA + push VA pages, both validated from content and @@ -17135,9 +17401,11 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch); if (fwd_guest_queued) { svc.fwd_gr_defer++; - } else if (ch->gr_hold_idx != (u32)idx || - time_before(jiffies, - ch->gr_hold_until)) { + } else if (!svc_gr_hold_bound_expired(ch, + svc_gr_launch_va(ch)) && + (ch->gr_hold_idx != (u32)idx || + time_before(jiffies, + ch->gr_hold_until))) { u32 hold_ms = gsp_sema_gr_grace_ms ? gsp_sema_gr_grace_ms : 8000; @@ -17213,7 +17481,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) !ch->engine_type && svc_gr_data_stalled && !svc_gr_hold_bound_expired(ch, - svc_fwd_req.gr_push_va) && + svc_gr_launch_va(ch)) && (ch->gr_hold_idx != (u32)idx || time_before(jiffies, ch->gr_hold_until))) { @@ -17252,6 +17520,31 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) break; } /* + * STALLFREE: the absolute per-launch + * hold bound just fired - the entry + * falls through to the historical + * consume, where the tail authors the + * QMD completion release and logs + * GR-STALL-hold. This line pins the + * bound expiry itself (the r220 hold + * storm's fix: one line per launch, + * nothing re-arms it across rewind + * replays). + */ + if (!fwd_guest_queued && + !ch->engine_type && + svc_gr_data_stalled && + svc_gr_hold_bound_expired(ch, + svc_gr_launch_va(ch)) && + !ch->gr_hold_exp_logged) { + ch->gr_hold_exp_logged = true; + svc.fwd_gr_hold_exp++; + pr_warn("vgsp-gr: GR-STALL-hold-bound va=0x%llx waited_ms=%u action=proceed (data-stall hold bound expired; entry consumed, QMD release authored)\n", + svc_gr_launch_va(ch), + jiffies_to_msecs(jiffies - + ch->gr_hold_dead_since)); + } + /* * P6b wave 1.5: when * this pass recorded * releases and the @@ -17327,11 +17620,16 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * bridge job owns the release. */ if (!ch->engine_type && - !gsp_cuda_bridge_enabled()) + !gsp_cuda_bridge_enabled()) { svc_gr_sw_release(vgpu, ch->h_object, svc_gr_cand.rel_va, svc_gr_cand.rel_payload); + /* STALLFREE: latch so the + * consume tail cannot author + * the same release twice. */ + svc_gr_cand.rel_done = true; + } } } /* @@ -18007,6 +18305,36 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) */ if (svc_rel_count != rel_n_iter || !ch->engine_type) svc_replay_note(ch, idx, put); + /* + * STALLFREE: bounded completion for a GR capture that was + * consumed without ever being forwarded - a data-stall, + * busy-defer or no-capture hold window expired and the + * entry fell through to the historical consume. No + * hardware copy is coming and no release hold owns the + * launch, so without this the QMD completion word the + * client polls is never authored by the host and the + * launch wait runs to the guest's own timeout. Author + * it now, forward-only, latched once per capture. + * svc_gr_cand.push_va is the entry's push VA, so a + * candidate captured for a later entry of the same + * channel can never be completed by this entry's + * consume. + */ + if (!ch->engine_type && svc_gr_cand.valid && + svc_gr_cand.h_object == ch->h_object && + !svc_gr_cand.offered && !svc_gr_cand.rel_done && + svc_gr_cand.rel_va && + svc_gr_cand.push_va == va) { + svc_gr_cand.rel_done = true; + svc_gr_sw_release(vgpu, ch->h_object, + svc_gr_cand.rel_va, + svc_gr_cand.rel_payload); + svc.fwd_gr_stall_swrel++; + pr_warn("vgsp-gr: GR-STALL-hold va=0x%llx waited_ms=%u action=proceed (unforwarded GR capture consumed; QMD release authored in software)\n", + va, + jiffies_to_msecs(jiffies - + svc_gr_cand.captured_jiffies)); + } ch->last_get = (idx + 1) % ch->gpfifo_entries; ch->rescue_fail = 0; ch->defer_since = 0; @@ -18143,15 +18471,28 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * record field MUST be added to this snapshot * write-back list or it is zeroed on every pass * (the pump works on the local[] snapshot copied - * at the top of svc_pump_channels). The GRFIX - * absolute-bound fields (gr_hold_dead_va / - * gr_hold_dead_since / gr_hold_deadline / - * gr_hold_exp_logged) are deliberately NOT - * written back here - the proven record - * destroyer - which is why - * gsp_phys_gr_fill_hold defaults to 0. If the - * hold is ever re-enabled, add those four fields - * here first. */ + * at the top of svc_pump_channels). + * + * STALLFREE (r213-r215 follow-up): the GRFIX + * absolute-bound fields were the proven record + * destroyer - they were intentionally left out, + * which is why gsp_phys_gr_fill_hold defaulted + * to 0 (the bound could never fire). They are now + * written back, so the per-launch bound survives + * the snapshot and is effective on the default + * path: a data-stalled launch is held for at most + * gsp_phys_gr_hold_max_ms (default 3000) and then + * is consumed with a software completion, instead + * of being re-held forever by a rewind replay at + * a different slot index (the r220 196 s hold). + */ + svc.chan[k].gr_hold_dead_va = ch->gr_hold_dead_va; + svc.chan[k].gr_hold_dead_since = + ch->gr_hold_dead_since; + svc.chan[k].gr_hold_deadline = + ch->gr_hold_deadline; + svc.chan[k].gr_hold_exp_logged = + ch->gr_hold_exp_logged; /* m164: the CLASS-2 no-capture window is snapshot * state too - without the write-back every pass * would re-arm it and rehold the entry forever |