diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 82 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.c | 35 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.h | 6 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 397 |
4 files changed, 490 insertions, 30 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 5e16630..6a66a16 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -16202,6 +16202,16 @@ static u64 gsp_phys_gr_pool_exhausted; /* m156 (E5): zero-write retries where the warm sem push proved the engine * wedged (the persistent-lottery class); further retries are skipped. */ static u64 gsp_phys_gr_sem_wedged; +/* + * STALLFREE: bounded-completion telemetry. stall_wait_max counts launches + * that crossed gsp_phys_gr_wait_max_ms and failed out of the submit ladder + * (the sema service then authors the QMD release in software); stall_retry + * counts launches whose retirement retry loop stopped early on the same + * budget (the launch proceeds with its current outcome). Both are zero on a + * green run. + */ +static u64 gsp_phys_gr_stall_wait_max; +static u64 gsp_phys_gr_stall_retry; /* m86: program-page override blob (see gsp-phys-gr-override). */ static u8 gsp_phys_gr_override[GSP_PHYS_PAGE_SIZE]; static u32 gsp_phys_gr_override_len; @@ -18249,6 +18259,37 @@ gsp_phys_gr_code_src(const struct gsp_phys_gr_submit *req, return mode; } +/* + * STALLFREE: the absolute per-launch wait budget. Every ladder phase is + * individually bounded, but a launch can chain them (fast re-arm + fresh + * entries + retirement retries), so the sum - and with it the time the + * single-inflight work item holds the client's completion - is not. This + * predicate bounds the whole submit: once it is true, no further re-arm, + * re-publish or re-execution runs. Green launches complete in ~0.2-11 s and + * never approach the default 60 s. + */ +static bool gsp_phys_gr_wait_budget_over(unsigned long start) +{ + unsigned int budget = gsp_phys_gr_wait_max_ms_value(); + + return budget && time_after_eq(jiffies, + start + msecs_to_jiffies(budget)); +} + +/* + * STALLFREE: one distinguishable bounded-completion line per stall episode. + * `action=fail` means the submit gives up (the sema service's failure edge + * authors the QMD completion release in software, so the guest proceeds); + * `action=proceed` means the phase gave up but the launch continues with its + * current outcome. + */ +static void gsp_phys_gr_stall_log(const char *reason, u64 push_va, + unsigned long start, const char *action) +{ + pr_warn("vgsp-gr: GR-STALL-%s va=0x%llx waited_ms=%u action=%s\n", + reason, push_va, jiffies_to_msecs(jiffies - start), action); +} + int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, struct gsp_phys_backend_copy_result *out) { @@ -19554,6 +19595,24 @@ gr_page_unmapped: poll_start_us, &sub); break; } + /* + * STALLFREE: the whole-launch budget. Checked after the release + * acks (a completed launch always wins) and before any further + * ladder work; on expiry the submit fails out now and the sema + * service's failure edge authors the QMD completion release in + * software, so the guest client is never left waiting on a + * launch this host will not finish. + */ + if (gsp_phys_gr_wait_budget_over(start)) { + gsp_phys_gr_stall_wait_max++; + gsp_phys_gr_stall_log("wait-max", req->push_va, start, + "fail"); + pr_err("vgsp-gr: GR LAUNCH FAILED: wait budget crossed (qmd_rel=0x%08x want=0x%08x ch_rel=0x%08x want=0x%08x) h=0x%x push=0x%llx; ladder stopped\n", + w0, want_qmd, w1, want_ch, req->h_object, + req->push_va); + ret = -ETIMEDOUT; + goto out; + } if ((++grpoll_n % 100) == 0) pr_info("vgsp-gr: poll +%ums qmd_rel=0x%08x ch_rel=0x%08x\n", grpoll_n * GSP_PHYS_RMCALL_POLL_MS, w0, w1); @@ -20066,6 +20125,22 @@ gr_page_unmapped: (u32)gsp_phys_gr_nowrite_retries_value(); while (wrote == 0 && nw_attempts < nw_retries) { + /* + * STALLFREE: the same whole-launch budget caps + * the retirement retry chain. When it is crossed + * the loop stops and the launch PROCEEDS with + * its current outcome (the zero-write accept + * below restores the release words and logs + * loudly), so a chain of slow dud retries can + * never hold the client past the budget. + */ + if (gsp_phys_gr_wait_budget_over(start)) { + gsp_phys_gr_stall_retry++; + gsp_phys_gr_stall_log("retry", + req->push_va, start, + "proceed"); + break; + } nw_attempts++; pr_info("vgsp-gr: outcome retry %u/%u (wrote=%u tracked=%u): refill + re-publish\n", nw_attempts, nw_retries, @@ -21756,12 +21831,15 @@ static int gsp_phys_state_show(struct seq_file *m, void *v) gsp_phys_backend_gr_counters(&b_sub, &b_ok, &b_refused, &b_timeout, &b_pool); seq_printf(m, - "work_gr: req=%llu fallback=%llu ok=%llu refused=%llu timeout=%llu fault=%llu submitted=%llu backend_ok=%llu backend_refused=%llu backend_timeout=%llu pool_exhausted=%llu forward_param=%u exec_ok=%u sem_wedged=%llu\n", + "work_gr: req=%llu fallback=%llu ok=%llu refused=%llu timeout=%llu fault=%llu submitted=%llu backend_ok=%llu backend_refused=%llu backend_timeout=%llu pool_exhausted=%llu forward_param=%u exec_ok=%u sem_wedged=%llu stall_wait_max=%llu stall_retry=%llu wait_max_ms=%u\n", gr_req, gr_fallback, gr_ok, gr_refused, gr_timeout, gr_fault, b_sub, b_ok, b_refused, b_timeout, b_pool, gsp_phys_gr_forward_enabled(), boot_stage.rmcall_gr_exec_ok, - READ_ONCE(gsp_phys_gr_sem_wedged)); + READ_ONCE(gsp_phys_gr_sem_wedged), + READ_ONCE(gsp_phys_gr_stall_wait_max), + READ_ONCE(gsp_phys_gr_stall_retry), + gsp_phys_gr_wait_max_ms_value()); } { /* diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c index 9568d22..e845493 100644 --- a/kernel/gsp_phys_gpu.c +++ b/kernel/gsp_phys_gpu.c @@ -1069,6 +1069,41 @@ module_param_named(gsp_phys_gr_nowrite_retries, MODULE_PARM_DESC(gsp_phys_gr_nowrite_retries, "Dedicated re-execution budget for a data-bearing GR launch that wrote nothing tracked, clamped [0,4] (default 3)"); +/* + * STALLFREE: absolute per-launch wall-clock budget (gsp_phys_gr_wait_max_ms; + * default 60000, clamped [10000, 600000]; 0 disables the extra cap and the + * built-in ladder bounds stand alone). The GR submit ladder and its retirement + * (no-write) retries are individually bounded, but their SUM grows with every + * re-arm/re-publish: a launch whose engine ack never arrives could hold the + * single-inflight work item for tens of seconds while the guest client waits + * on its QMD completion word. When this budget is crossed the submit stops its + * ladder immediately (no further re-arm/publish/retry), logs + * 'vgsp-gr: GR-STALL-wait-max ... action=fail' and returns -ETIMEDOUT; the + * sema service's failure edge then authors the QMD completion release in + * software, so the client always completes. A green launch (~0.2-11 s end to + * end, including the fast-re-arm recovery) never approaches the default, so + * the default does not change green behavior. + */ +static unsigned int gsp_phys_gr_wait_max_ms_param = 60000u; +module_param_named(gsp_phys_gr_wait_max_ms, + gsp_phys_gr_wait_max_ms_param, uint, 0644); +MODULE_PARM_DESC(gsp_phys_gr_wait_max_ms, + "Absolute per-launch GR submit budget in ms, clamped [10000,600000]; 0 = only the built-in per-ladder bounds (default 60000)"); + +unsigned int gsp_phys_gr_wait_max_ms_value(void) +{ + unsigned int v = gsp_phys_gr_wait_max_ms_param; + + if (!v) + return 0; + if (v < 10000u) + v = 10000u; + if (v > 600000u) + v = 600000u; + return v; +} +EXPORT_SYMBOL_GPL(gsp_phys_gr_wait_max_ms_value); + int gsp_phys_gr_nowrite_retries_value(void) { int v = gsp_phys_gr_nowrite_retries_param; diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h index 5243eba..c664e12 100644 --- a/kernel/gsp_phys_gpu.h +++ b/kernel/gsp_phys_gpu.h @@ -393,6 +393,12 @@ int gsp_phys_gr_icache_inv_value(void); * attempt counter (default 3, clamped [0,4]). */ int gsp_phys_gr_nowrite_retries_value(void); +/* STALLFREE: absolute per-launch GR submit wall-clock budget in ms, clamped + * [10000,600000]; 0 = only the built-in per-ladder bounds (default 60000). + * On expiry the submit fails out immediately with a GR-STALL-wait-max line + * and the sema service authors the QMD completion release in software. */ +unsigned int gsp_phys_gr_wait_max_ms_value(void); + /* m73: use host-visible sysmem chunks for V7 instance/RAMFC (m62 shape). */ int gsp_phys_rmcall_gr_inst_sys_value(void); diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 8ae11ec..fb00685 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -984,6 +984,12 @@ struct sema_svc { * abort, or a release0 page the backend could not observe). */ u64 fwd_gr_sw_rel; + /* + * STALLFREE: failed forwarded launches the pass-top outcome check + * completed in software because no hold owned their release (the + * bounded-completion guarantee; zero on a green run). + */ + u64 fwd_gr_stall_swrel; u64 fwd_gr_hold_exp; /* GRFIX FIX1: data-stalled holds * that reached the hard bound and * were consumed with the software @@ -3875,13 +3881,33 @@ static void svc_session_flush(void); /* defined next to the pending queue */ #define SVC_LEDGER_MAX (1u << 14) #define SVC_LEDGER_MASK (SVC_LEDGER_MAX - 1u) #define SVC_LEDGER_PROBE 16u +/* + * STALLFREE: eviction burst threshold for the cap-and-degrade reset. A + * cluster keeps 16 same-home VA pages; EXIT_RESET evictions inside one session + * means the table is thrashing (pin churn / leftover crashed clients), and + * the half-evicted table silently strands the evicted pages' landing + * authority. At the threshold the cache is cleared and rebuilt from live + * writes (uniform resolver fallback instead of silent partial authority). + */ +#define SVC_LEDGER_EVICT_RESET 256u static u32 svc_ledger_n; static u64 svc_ledger_diverges; static u64 svc_ledger_stores; static u64 svc_ledger_evictions; +static u64 svc_ledger_evict_since_reset; +static u64 svc_ledger_evict_resets; static u64 svc_ledger_rewrites; static u64 svc_ledger_reloc_diverges; static u32 svc_ledger_store_log; +/* + * STALLFREE: pending-ring overflow telemetry (the negative queue silently + * overwrote its oldest entry when full). Hoisted here so svc_rebuild's diag + * can print it; the ring itself is defined with the pending queue below. + */ +static u64 svc_pending_ovf; +static u32 svc_pending_ovf_log; +/* STALLFREE: rescue scans that hit their wall-clock cap (bounded service). */ +static u64 svc_rescue_scan_capped; static void svc_rebuild(struct nv_vgpu_device *vgpu) { @@ -3985,20 +4011,21 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) svc.false_trans, svc.gpfifo_skipped, svc.gpfifo_rescued, svc.releases_faked, svc.releases_recovered, svc.counter_rel_applied); - pr_info("sema-svc: drain held=%llu starved=%llu forwarded=%llu faked=%llu rec=%llu ops_once=%llu reident=%llu opdef=%llu/%llu/%llu gr_nocap=%llu/%llu gr_hold_exp=%llu\n", + pr_info("sema-svc: drain held=%llu starved=%llu forwarded=%llu faked=%llu rec=%llu ops_once=%llu reident=%llu opdef=%llu/%llu/%llu gr_nocap=%llu/%llu gr_hold_exp=%llu gr_swrel=%llu pend_ovf=%llu\n", svc.gpfifo_held, svc.gpfifo_starved, svc.releases_forwarded, svc.releases_faked, svc.releases_recovered, svc.ops_once_skipped, svc.ops_reident, svc.opdef_held, svc.opdef_applied, svc.opdef_expired, svc.gr_nocap_held, svc.gr_nocap_consumed, - svc.fwd_gr_hold_exp); + svc.fwd_gr_hold_exp, svc.fwd_gr_stall_swrel, + svc_pending_ovf); pr_info("sema-svc: counter pub_calls=%llu gr_rel=%llu reresolve=%llu pushpg_fb=%llu altw=%llu residue=%llu\n", svc.counter_pub_calls, svc.gr_rel_applied, svc.sem_write_reresolve, svc.pushpg_fallback, svc.pool_alt_applied, svc.pool_residue); - if (svc.resc_run) - pr_info("sema-svc: rescue_run=%llu (multi-page content pins)\n", - svc.resc_run); + if (svc.resc_run || svc_rescue_scan_capped) + pr_info("sema-svc: rescue_run=%llu scan_capped=%llu (multi-page content pins)\n", + svc.resc_run, svc_rescue_scan_capped); pr_info("sema-svc: gpfifo done=%llu nomap=%llu vidmem_writes=%llu ring_auto=%u pb=%u/%u pins=%u/%llu ipins=%u/%llu\n", svc.gpfifo_entries_done, svc.gpfifo_nomap, svc.vidmem_writes, svc_ring_n, svc_pb_n, svc_pb_passes, @@ -4013,9 +4040,10 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) * content" window. n approaching SVC_LEDGER_MAX or evict > 0 in a * client's window is the tell. */ - pr_info("sema-svc: ledger n=%u/%u stores=%llu evict=%llu diverges=%llu reloc_div=%llu rewrites=%llu\n", + pr_info("sema-svc: ledger n=%u/%u stores=%llu evict=%llu resets=%llu diverges=%llu reloc_div=%llu rewrites=%llu\n", svc_ledger_n, SVC_LEDGER_MAX, svc_ledger_stores, - svc_ledger_evictions, svc_ledger_diverges, + svc_ledger_evictions, svc_ledger_evict_resets, + svc_ledger_diverges, svc_ledger_reloc_diverges, svc_ledger_rewrites); /* * P6b wave 1: guest-forwarding summary. Printed whenever any counter @@ -4374,6 +4402,7 @@ static void svc_fill_audit(struct nv_vgpu_device *vgpu, u64 va0, u64 addr, */ static void svc_ledger_store(u64 va, u64 pa, u32 ap); /* R2 ledger, defined below */ static int svc_ledger_lookup(u64 va, u64 *pa, u32 *ap); /* R2 ledger, defined below */ +static void svc_ledger_clear(void); /* STALLFREE: cap-and-degrade reset */ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, u32 count, u32 phys, u32 aperture, u64 *done_out) @@ -4614,6 +4643,15 @@ struct svc_gr_data_win { struct svc_gr_cand { bool valid; bool offered; /* consumed by a request or a fallback */ + /* + * STALLFREE: exactly-once latch for the candidate's QMD completion + * release when the launch is consumed without ever being forwarded + * (a data-stall/busy/nocap hold fell through). Zeroed with the rest + * of the candidate at capture. + */ + bool rel_done; + unsigned long captured_jiffies; /* STALLFREE: capture stamp for the + * GR-STALL waited_ms accounting */ u32 h_object; u64 push_va; u32 push_len; @@ -4726,6 +4764,7 @@ static void svc_gr_cand_capture(struct svc_channel *ch, } memset(&svc_gr_cand, 0, sizeof(svc_gr_cand)); svc_gr_cand.valid = true; + svc_gr_cand.captured_jiffies = jiffies; svc_gr_cand.h_object = ch->h_object; svc_gr_cand.push_va = push_va; svc_gr_cand.push_len = push_len; @@ -4979,8 +5018,27 @@ static void svc_ledger_store(u64 va, u64 pa, u32 ap) * VA's landing authority (its later readers fall back to the * resolver, possibly stale), so the first occurrences are logged and * counted into the diag line. + * + * STALLFREE cap-and-degrade: a sustained eviction burst means the + * table is being thrashed (a pin storm / leftover crashed client + * churning VA pages faster than the working set), and a half-evicted + * table silently strands authority for exactly the evicted pages. + * Past SVC_LEDGER_EVICT_RESET evictions in one session the cache is + * cleared wholesale and rebuilt from live writes: every reader + * uniformly falls back to the resolver until its page is rewritten, + * which removes the stranded-authority class at a known, logged + * cost. The clear is a cache operation only (no control flow, no + * wait); the current store then lands in the empty table. */ svc_ledger_evictions++; + svc_ledger_evict_since_reset++; + if (svc_ledger_evict_since_reset >= SVC_LEDGER_EVICT_RESET) { + svc_ledger_evict_since_reset = 0; + svc_ledger_evict_resets++; + pr_warn("sema-ledger: LEDGER-RESET evict=%llu resets=%llu (cluster thrash under churn; cache cleared, readers fall back to the resolver until rewritten)\n", + svc_ledger_evictions, svc_ledger_evict_resets); + svc_ledger_clear(); + } if (svc_ledger_evictions <= 32) pr_warn("sema-ledger: EVICT home=%u old_va=0x%llx new_va=0x%llx (cluster full; old landing authority lost until rewritten)\n", home, (u64)svc_ledger[home].va_page, page); @@ -5017,6 +5075,7 @@ static void svc_ledger_clear(void) svc_ledger_n = 0; svc_ledger_rewrites = 0; svc_ledger_reloc_diverges = 0; + svc_ledger_evict_since_reset = 0; } /* @@ -8363,6 +8422,13 @@ static u32 svc_pending_next; static u64 svc_pending_rescued; /* stall-class: entries the old >= dedup would have dropped */ static u32 svc_pending_rescued_log; +/* + * STALLFREE: the pending ring silently overwrote its oldest entry when full + * (an evicted release is a client wait that never clears). The counters are + * hoisted next to the ledger counters so the periodic diag (above) can print + * them; the ring is a bounded queue, so under churn this is the back-pressure + * signal, not an error path. + */ static void svc_pending_add(u64 va, u32 payload, u32 op) { @@ -8401,6 +8467,14 @@ static void svc_pending_add(u64 va, u32 payload, u32 op) i = svc_pending_n++; } else { i = svc_pending_next++ % SVC_PENDING_MAX; + svc_pending_ovf++; + if (svc_pending_ovf_log < 8) { + svc_pending_ovf_log++; + pr_warn("sema-svc: PENDING-OVF ring full (%u); overwriting oldest entry va=0x%llx pay=%u op=%u (n=%llu)\n", + SVC_PENDING_MAX, svc_pending[i].va, + svc_pending[i].payload, svc_pending[i].op, + svc_pending_ovf); + } } svc_pending[i].va = va; svc_pending[i].payload = payload; @@ -10704,10 +10778,11 @@ unsigned gsp_phys_gr_hold_max_ms = 3000; * pages - the drops are logged (GRFILL-DROP) and the launch takes the * historical forward path immediately (exactly the pre-8b058884 behaviour); * the GRFILL-HOLD/GRFILL-EXPIRED machinery and the per-launch absolute - * bound are skipped entirely. When set to 1 the page-count hold + absolute - * bound apply (r213-r215 showed the bound never fired: the per-channel - * record fields are not in the svc_pump_channels snapshot write-back list, - * so they were zeroed every pass - see the GRFIX NOTE at that list). + * bound are skipped entirely (the pump's own m150/m156 hold and its + * absolute bound are NOT - they run regardless of this gate; the record + * fields are now in the snapshot write-back list, see the GRFIX NOTE). + * When set to 1 the page-count hold + absolute bound apply on the fill + * path too. */ unsigned gsp_phys_gr_fill_hold; /* default 0: log-only, never hold */ @@ -11627,6 +11702,19 @@ static int svc_rescue_run_try(struct nv_vgpu_device *vgpu, } /* + * STALLFREE: wall-clock cap for one content-rescue scan range. The scan is + * bounded by the guest-RAM range, not by time: the full fallback walks up to + * 6 GB in 4 KB vfio reads, and under a pin storm / leftover crashed client's + * churn that walk can run for tens of seconds on the service thread, + * delaying every channel (GR launches included). Green scans answer from the + * hot set or the near window in milliseconds, so a 3 s cap never fires on a + * healthy boot; when it does fire the entry falls through its existing grace + * path (hold, then consume with the software release) instead of pinning the + * whole pump. + */ +#define SVC_RESCUE_SCAN_MAX_MS 3000u + +/* * Scan a guest-RAM range descending for a page that validates as this entry's * push (method stream + expected release payload), pinning it and returning 0 * on the first hit. `hi` and `lo` are inclusive chunk start addresses. @@ -11639,10 +11727,20 @@ static int svc_rescue_scan_range(struct nv_vgpu_device *vgpu, { u64 gpa = hi; unsigned long t0 = jiffies; + unsigned long deadline = + t0 + msecs_to_jiffies(SVC_RESCUE_SCAN_MAX_MS); u32 total = 0; u64 bytes = 0; for (;;) { + if (time_after_eq(jiffies, deadline)) { + svc_rescue_scan_capped++; + if (svc_rescue_scan_capped <= 8) + pr_warn("sema-svc: RESCUE-SCAN-CAP h=0x%x va=0x%llx [0x%llx..0x%llx] %llu MB in %u ms (capped; entry keeps its grace path)\n", + ch->h_object, entry_va, hi, lo, + bytes >> 20, SVC_RESCUE_SCAN_MAX_MS); + break; + } if (!svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK)) { u32 ncand, n; @@ -12379,6 +12477,25 @@ struct svc_fwd_req { bool gr; bool gr_bridge_done; bool gr_page_overflow; + /* + * STALLFREE: bounded-completion state for the forwarded launch. The + * request's backend outcome is the only host edge that can complete a + * launch whose release records engage no hold (svc_fwd_rel_n == 0); + * gr_swrel_done is the exactly-once latch for the software authorship + * of the QMD completion word, and gr_fill_jiffies stamps the request + * so the GR-STALL line reports the real waited time. + */ + bool gr_swrel_done; + /* + * STALLFREE: set by the work item when release0 could not target the + * guest page (unresolved release PA / HPA), so a successful launch's + * engine completion lands only on the module semaphore and the guest + * QMD word stays behind. The pump's bounded outcome check then + * authors it in software (the same write the hold's success edge + * performs), so the client always completes. + */ + bool gr_rel_unobserved; + unsigned long gr_fill_jiffies; u32 gr_h_object; u32 gr_dropped; /* GRFIX: pages the fill's last pass could not * resolve (0 = launch fully resolved); read @@ -13320,6 +13437,73 @@ static void svc_fwd_hold_gr_release(struct nv_vgpu_device *vgpu, svc_fwd_req.gr_rel_payload); } +/* + * STALLFREE: bounded completion for a failed forwarded GR launch that no hold + * completes. A GR push whose decode recorded no push-level release records + * (svc_fwd_rel_n == 0) engages no release hold, so when the backend fails + * (ladder timeout, fault abort, refusal) nothing authors the QMD completion + * word the guest client polls after the kick: the client's launch wait then + * has no host edge and lasts until the guest's own timeout (the r219/r222 + * stall class, qmd_rel frozen at the previous launch's value). + * + * This runs on the service thread at the top of every pump pass, before any + * request fill of that pass (svc_fwd_request requires SVC_FWD_INFLIGHT free, + * which the work item only releases after publishing done, and the fill's + * first write is done=false), so a pass that sees done also sees this + * request's fields, and every finished request is observed before its slot + * can be reused. Authors the release once (forward-only, full resolver - the + * same write the hold's failure edge performs), latches gr_swrel_done, and + * logs the distinguishable GR-STALL line. + * + * While a hold is active the hold owns the completion (its failure edge + * applies the recorded releases AND the QMD word, bounded by the existing + * hold timeout), so this check stands down for it. + */ +static void svc_fwd_outcome_check(struct nv_vgpu_device *vgpu) +{ + bool done = smp_load_acquire(&svc_fwd_req.done); + unsigned long waited; + + if (!svc_fwd_req.gr || svc_fwd_req.vgpu != vgpu || + svc_fwd_req.gr_swrel_done || svc_fwd_hold.active || !done) + return; + /* + * Complete when the backend failed (the never-completing r219/r222 + * class) OR when a successful launch's release0 could not target the + * guest page, so the engine's completion landed only on the module + * semaphore and the client's QMD word is still behind (the same edge + * the hold's success path covers for held requests). + */ + if (svc_fwd_req.hw_rc == 0 && !svc_fwd_req.gr_rel_unobserved) + return; + /* + * The outcome is final; latch it first so no later pass, hold edge or + * consume tail accounts it again (the release write itself is + * forward-only, so even a duplicate could not corrupt the word). + */ + svc_fwd_req.gr_swrel_done = true; + waited = jiffies_to_msecs(jiffies - svc_fwd_req.gr_fill_jiffies); + if (!svc_fwd_req.gr_rel_va || !svc_fwd_req.gr_rel_payload) { + pr_warn("vgsp-gr: GR-STALL-backend va=0x%llx waited_ms=%lu action=proceed (launch carries no release record; no word to author)\n", + svc_fwd_req.gr_push_va, waited); + return; + } + svc_gr_sw_release(vgpu, svc_fwd_req.gr_h_object, + svc_fwd_req.gr_rel_va, + svc_fwd_req.gr_rel_payload); + svc.fwd_gr_stall_swrel++; + if (svc_fwd_req.hw_rc != 0) + pr_warn("vgsp-gr: GR-STALL-backend va=0x%llx waited_ms=%lu action=proceed (failed forwarded launch, rc=%d; QMD release authored in software)\n", + svc_fwd_req.gr_push_va, waited, svc_fwd_req.hw_rc); + else + pr_warn("vgsp-gr: GR-STALL-backend va=0x%llx waited_ms=%lu action=proceed (successful launch whose release0 could not target the guest page; QMD release authored in software)\n", + svc_fwd_req.gr_push_va, waited); + pr_info("vgsp-gr: GR-COMPLETE-SW h=0x%x push=0x%llx rel=0x%llx pay=0x%x waited_ms=%lu n=%llu (backend outcome without a hold)\n", + svc_fwd_req.gr_h_object, svc_fwd_req.gr_push_va, + svc_fwd_req.gr_rel_va, svc_fwd_req.gr_rel_payload, waited, + svc.fwd_gr_stall_swrel); +} + static bool svc_fwd_hold_expired(void) { if (svc_fwd_req.gr && !smp_load_acquire(&svc_fwd_req.done)) @@ -13537,6 +13721,15 @@ static void svc_fwd_hold_expire_check(struct nv_vgpu_device *vgpu, svc_fwd_hold.relay_armed) svc_fwd_fault_deliver(hv, ch); /* + * STALLFREE: the unpumpable-channel expiry edge (the entry loop never + * ran for this hold). The releases and the QMD completion word are + * software-applied below; exactly one line per hold. + */ + if (svc_fwd_req.gr) + pr_warn("vgsp-gr: GR-STALL-fwd-hold va=0x%llx waited_ms=%u action=proceed (hold expired on an unpumpable channel; releases + QMD word software-applied)\n", + svc_fwd_req.gr_push_va, + jiffies_to_msecs(jiffies - svc_fwd_hold.since)); + /* * P6b wave 2: a skipped candidate whose hardware copy did not complete * successfully is authored by one software copy here, BEFORE the * recorded releases below. A successful hardware copy needs none. @@ -13666,6 +13859,17 @@ static bool svc_fwd_hold_poll(struct nv_vgpu_device *vgpu, if (done && svc_fwd_req.hw_rc == -ETIMEDOUT && svc_fwd_hold.relay_armed) svc_fwd_fault_deliver(hv, ch); + /* + * STALLFREE: the bounded failure edge of a held forwarded GR + * launch - the releases AND the QMD completion word are + * software-applied below, so the client always completes. + * Exactly one line per hold (the hold is cleared at the end). + */ + if (svc_fwd_req.gr) + pr_warn("vgsp-gr: GR-STALL-fwd-fail va=0x%llx waited_ms=%u action=proceed (hold resolved on backend failure/expiry; releases + QMD word software-applied)\n", + svc_fwd_req.gr_push_va, + jiffies_to_msecs(jiffies - + svc_fwd_hold.since)); svc_fwd_hold_soft_copy(hv); if (svc_fwd_hold.relay_armed) svc.fwd_relay_fallback++; @@ -13924,6 +14128,7 @@ static void svc_fwd_gr_work(void) (SEMA_PAGE_SIZE - 1)); u8 rel_kind = (svc_fwd_req.gr_rel_ap == SVC_APERTURE_FB) ? SVC_APERTURE_FB : SVC_APERTURE_SYS; + bool rel_unobserved = false; if (svc_fwd_req.gr_rel_pa && (svc_fwd_operand_hpa(svc_fwd_req.vgpu, rel_kind, @@ -13936,11 +14141,24 @@ static void svc_fwd_gr_work(void) * means the engine would author nothing this launch; point * release0 at the module semaphore instead so the host still * observes completion. + * + * STALLFREE: when the guest page could not be targeted at all + * (unresolved release PA or HPA), the engine's completion will + * land only on the module semaphore and the guest QMD word + * stays behind - a launch whose client would otherwise wait + * forever even on a SUCCESSFUL submit. Record it so the pump's + * bounded outcome check authors the word in software (the + * same write the hold's success edge performs). A resolved + * page whose word is already ahead needs nothing. */ - if (rel_hpa && - READ_ONCE(*(u32 *)((u8 *)__va(rel_hpa) + roff)) >= + if (rel_hpa) { + if (READ_ONCE(*(u32 *)((u8 *)__va(rel_hpa) + roff)) >= svc_fwd_req.gr_rel_payload) - rel_hpa = 0; + rel_hpa = 0; + } else { + rel_unobserved = true; + } + svc_fwd_req.gr_rel_unobserved = rel_unobserved; } gs.qmd = svc_fwd_req.gr_qmd; @@ -14745,6 +14963,23 @@ static void svc_gr_scan_inline(u64 *wins, u32 *n_cand, u64 page_va) } /* + * STALLFREE: identity of the launch currently held/retried on this channel - + * the capture's push VA when it belongs to this channel (the busy-refusal + * path never reaches the fill, so the request slot can still hold a previous + * launch's VA), else the request slot's when it names this channel. The + * absolute hold bound is keyed to this identity so a pin-batch rewind that + * replays the entry at a different slot index can never re-arm it. + */ +static u64 svc_gr_launch_va(const struct svc_channel *ch) +{ + if (svc_gr_cand.valid && svc_gr_cand.h_object == ch->h_object) + return svc_gr_cand.push_va; + if (svc_fwd_req.gr && svc_fwd_req.gr_h_object == ch->h_object) + return svc_fwd_req.gr_push_va; + return 0; +} + +/* * GRFIX FIX1b (r213): absolute per-launch hold bound. The m150/m156 data * hold had no working bound: the 8b058884 deadline was keyed by the GPFIFO * slot index, and a pin-batch rewind/replay re-decodes the same launch push @@ -14760,6 +14995,24 @@ static bool svc_gr_hold_bound_expired(struct svc_channel *ch, u64 launch_va) { u32 cap_ms = gsp_phys_gr_hold_max_ms; + /* + * STALLFREE (r213-r215 follow-up): the absolute per-launch bound is + * now effective on the default path. Before the write-back fix the + * bound's record fields were excluded from the pump snapshot and + * zeroed every pass, so this returned false forever and the legacy + * m150/m156 hold window (gr_hold_until) was the only expiry - and + * that window is keyed to the GPFIFO slot index, which a pin-batch + * rewind changes on every replay (the r213 identity bug). Under pin + * churn the same launch was therefore re-held indefinitely: r220 + * held one launch from 155221 to 155417 (196 s, 1631 re-fills of + * entry 106) until the guest's 240 s kill. The bound is keyed to the + * LAUNCH PUSH VA and nothing re-arms it, so a replayed slot no + * longer extends the hold; at the cap the pump falls through to the + * historical consume, where STALLFREE authors the QMD completion + * release and logs GR-STALL-hold action=proceed. Default cap 3000 ms + * (gsp_phys_gr_hold_max_ms) sits above the observed 0.5-2 s pin + * latency, so the legitimate pin-late retry still wins. + */ if (!launch_va) launch_va = 1; /* anonymous launch: still absolutely bound */ if (!cap_ms) @@ -15388,6 +15641,11 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, svc_fwd_req.guest = false; svc_fwd_req.gr = true; svc_fwd_req.gr_bridge_done = false; + /* STALLFREE: fresh request - re-arm the software-completion latch and + * stamp the fill time for the GR-STALL waited_ms accounting. */ + svc_fwd_req.gr_swrel_done = false; + svc_fwd_req.gr_rel_unobserved = false; + svc_fwd_req.gr_fill_jiffies = jiffies; svc_fwd_req.defer = true; svc_fwd_req.relay = false; svc_fwd_req.relay_index = 0; @@ -16141,6 +16399,14 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) n_gr = SVC_GR_COUNTER_TOTAL; /* + * STALLFREE: complete a failed forwarded GR launch no hold owns (see + * svc_fwd_outcome_check). Runs before this pass's channel loop, so a + * finished request is always observed before any fill can reuse its + * slot. + */ + svc_fwd_outcome_check(vgpu); + + /* * H1 last resort: one self-gated content-discovery pass for the GR * ctx-init rings before this pass's channel loop. The pins it * installs (ring VA + push VA pages, both validated from content and @@ -17135,9 +17401,11 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch); if (fwd_guest_queued) { svc.fwd_gr_defer++; - } else if (ch->gr_hold_idx != (u32)idx || - time_before(jiffies, - ch->gr_hold_until)) { + } else if (!svc_gr_hold_bound_expired(ch, + svc_gr_launch_va(ch)) && + (ch->gr_hold_idx != (u32)idx || + time_before(jiffies, + ch->gr_hold_until))) { u32 hold_ms = gsp_sema_gr_grace_ms ? gsp_sema_gr_grace_ms : 8000; @@ -17213,7 +17481,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) !ch->engine_type && svc_gr_data_stalled && !svc_gr_hold_bound_expired(ch, - svc_fwd_req.gr_push_va) && + svc_gr_launch_va(ch)) && (ch->gr_hold_idx != (u32)idx || time_before(jiffies, ch->gr_hold_until))) { @@ -17252,6 +17520,31 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) break; } /* + * STALLFREE: the absolute per-launch + * hold bound just fired - the entry + * falls through to the historical + * consume, where the tail authors the + * QMD completion release and logs + * GR-STALL-hold. This line pins the + * bound expiry itself (the r220 hold + * storm's fix: one line per launch, + * nothing re-arms it across rewind + * replays). + */ + if (!fwd_guest_queued && + !ch->engine_type && + svc_gr_data_stalled && + svc_gr_hold_bound_expired(ch, + svc_gr_launch_va(ch)) && + !ch->gr_hold_exp_logged) { + ch->gr_hold_exp_logged = true; + svc.fwd_gr_hold_exp++; + pr_warn("vgsp-gr: GR-STALL-hold-bound va=0x%llx waited_ms=%u action=proceed (data-stall hold bound expired; entry consumed, QMD release authored)\n", + svc_gr_launch_va(ch), + jiffies_to_msecs(jiffies - + ch->gr_hold_dead_since)); + } + /* * P6b wave 1.5: when * this pass recorded * releases and the @@ -17327,11 +17620,16 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * bridge job owns the release. */ if (!ch->engine_type && - !gsp_cuda_bridge_enabled()) + !gsp_cuda_bridge_enabled()) { svc_gr_sw_release(vgpu, ch->h_object, svc_gr_cand.rel_va, svc_gr_cand.rel_payload); + /* STALLFREE: latch so the + * consume tail cannot author + * the same release twice. */ + svc_gr_cand.rel_done = true; + } } } /* @@ -18007,6 +18305,36 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) */ if (svc_rel_count != rel_n_iter || !ch->engine_type) svc_replay_note(ch, idx, put); + /* + * STALLFREE: bounded completion for a GR capture that was + * consumed without ever being forwarded - a data-stall, + * busy-defer or no-capture hold window expired and the + * entry fell through to the historical consume. No + * hardware copy is coming and no release hold owns the + * launch, so without this the QMD completion word the + * client polls is never authored by the host and the + * launch wait runs to the guest's own timeout. Author + * it now, forward-only, latched once per capture. + * svc_gr_cand.push_va is the entry's push VA, so a + * candidate captured for a later entry of the same + * channel can never be completed by this entry's + * consume. + */ + if (!ch->engine_type && svc_gr_cand.valid && + svc_gr_cand.h_object == ch->h_object && + !svc_gr_cand.offered && !svc_gr_cand.rel_done && + svc_gr_cand.rel_va && + svc_gr_cand.push_va == va) { + svc_gr_cand.rel_done = true; + svc_gr_sw_release(vgpu, ch->h_object, + svc_gr_cand.rel_va, + svc_gr_cand.rel_payload); + svc.fwd_gr_stall_swrel++; + pr_warn("vgsp-gr: GR-STALL-hold va=0x%llx waited_ms=%u action=proceed (unforwarded GR capture consumed; QMD release authored in software)\n", + va, + jiffies_to_msecs(jiffies - + svc_gr_cand.captured_jiffies)); + } ch->last_get = (idx + 1) % ch->gpfifo_entries; ch->rescue_fail = 0; ch->defer_since = 0; @@ -18143,15 +18471,28 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * record field MUST be added to this snapshot * write-back list or it is zeroed on every pass * (the pump works on the local[] snapshot copied - * at the top of svc_pump_channels). The GRFIX - * absolute-bound fields (gr_hold_dead_va / - * gr_hold_dead_since / gr_hold_deadline / - * gr_hold_exp_logged) are deliberately NOT - * written back here - the proven record - * destroyer - which is why - * gsp_phys_gr_fill_hold defaults to 0. If the - * hold is ever re-enabled, add those four fields - * here first. */ + * at the top of svc_pump_channels). + * + * STALLFREE (r213-r215 follow-up): the GRFIX + * absolute-bound fields were the proven record + * destroyer - they were intentionally left out, + * which is why gsp_phys_gr_fill_hold defaulted + * to 0 (the bound could never fire). They are now + * written back, so the per-launch bound survives + * the snapshot and is effective on the default + * path: a data-stalled launch is held for at most + * gsp_phys_gr_hold_max_ms (default 3000) and then + * is consumed with a software completion, instead + * of being re-held forever by a rewind replay at + * a different slot index (the r220 196 s hold). + */ + svc.chan[k].gr_hold_dead_va = ch->gr_hold_dead_va; + svc.chan[k].gr_hold_dead_since = + ch->gr_hold_dead_since; + svc.chan[k].gr_hold_deadline = + ch->gr_hold_deadline; + svc.chan[k].gr_hold_exp_logged = + ch->gr_hold_exp_logged; /* m164: the CLASS-2 no-capture window is snapshot * state too - without the write-back every pass * would re-arm it and rehold the entry forever |