From e99040b9ca2acc10a9d4b57fe45626331e203a7c Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Sun, 18 Oct 2026 06:17:50 -0400 Subject: vgsp: GR fill hold - absolute deadline + expiry-proceed + dropped-page detail --- kernel/gsp_sema_service.c | 312 +++++++++++++++++++++++++++++++--------------- 1 file changed, 211 insertions(+), 101 deletions(-) (limited to 'kernel/gsp_sema_service.c') diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 51310a2..6286d38 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -377,18 +377,23 @@ struct svc_channel { * even if still unresolved - bounded zeros, never * a hang); m156: re-held while the window is * live so a busy forward gets its real retry */ - unsigned long gr_hold_deadline; /* GRFIX FIX1: hard expiry for the data- - * stalled hold of gr_hold_dead_idx. The - * m156 re-arm restarts gr_hold_until on - * every pass, so a page that never - * resolves would otherwise keep the - * entry held forever (the r176 stall - * class). Fixed once on the first arm; - * re-arms never move it; at the deadline - * the entry is consumed with the - * historical software release (bounded, - * never a hang). */ - u32 gr_hold_dead_idx; /* entry gr_hold_deadline was armed for */ + unsigned long gr_hold_deadline; /* GRFIX FIX1b: hard expiry of the data- + * stalled hold for gr_hold_dead_va. + * Fixed once on the first hold of that + * launch push VA and NEVER moved while + * the launch stays held; when it passes + * the launch proceeds (pre-8b058884 + * behaviour), so the guest can never + * wait unboundedly. Keyed by the launch + * VA, not the GPFIFO slot: pin-batch + * rewinds re-decode one launch at + * different slot indices (r213: + * entry 107 -> 107 -> 99), which + * re-armed a slot-keyed deadline on + * every pass. */ + u64 gr_hold_dead_va; /* launch push VA the bound belongs to */ + unsigned long gr_hold_dead_since; /* first-hold stamp (waited_ms) */ + bool gr_hold_exp_logged; /* one GRFILL-EXPIRED line per launch */ u32 gr_rehold_idx; /* m156: slot whose next decode must force a * fresh launch capture (a held entry is * re-decoded as a replay, which suppresses @@ -1271,7 +1276,8 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) memset(ch, 0, sizeof(*ch)); ch->gr_hold_idx = ~0u; /* m156: no hold armed yet */ - ch->gr_hold_dead_idx = ~0u; /* GRFIX: no hard bound armed */ + ch->gr_hold_dead_va = 0; /* GRFIX: no launch bound armed */ + ch->gr_hold_exp_logged = false; ch->gr_rehold_idx = ~0u; ch->gr_nocap_idx = ~0u; /* m164: no no-capture window armed */ ch->session_gen = svc_sess_gen; /* m176 */ @@ -10423,6 +10429,19 @@ bool gsp_sema_counter_derive = true; */ unsigned gsp_sema_gr_grace_ms = 8000; +/* + * GRFIX FIX1b (r213): absolute per-LAUNCH bound (ms) for the data-stalled + * hold of svc_gr_data_fill. The identity is the launch push VA, not the + * GPFIFO slot: pin-batch rewinds/replays re-decode the same launch at + * different slot indices (r213: entry 107 -> 107 -> 99 -> ... on one + * channel), so an idx-keyed deadline re-armed on every pass and never + * fired - 410+ holds over 216 s, gr_hold_exp=0, client killed at its 240 s + * timeout. The deadline is fixed at the first hold of a launch; when it + * passes, svc_gr_request proceeds with the launch exactly as the + * pre-8b058884 path did (forward best-effort - never block the guest). + */ +unsigned gsp_phys_gr_hold_max_ms = 3000; + /* * F1-b: GR release-word authorship gate (default 1). While a forwarded GR * launch is in flight, the RAM-content sweep must not author the very QMD @@ -12092,6 +12111,9 @@ struct svc_fwd_req { bool gr_bridge_done; bool gr_page_overflow; u32 gr_h_object; + u32 gr_dropped; /* GRFIX: pages the fill's last pass could not + * resolve (0 = launch fully resolved); read + * by the absolute-bound expiry log */ u64 gr_push_va; u32 gr_push_len; u32 gr_push_n; @@ -13400,6 +13422,10 @@ static bool svc_fwd_hold_poll(struct nv_vgpu_device *vgpu, ch->last_get = (idx + 1) % ch->gpfifo_entries; ch->rescue_fail = 0; ch->defer_since = 0; + /* GRFIX FIX1b: this entry is consumed; its per-launch absolute-bound + * record resets so a later launch (slot/VA reuse) arms a fresh one. */ + ch->gr_hold_dead_va = 0; + ch->gr_hold_exp_logged = false; ch->submissions++; svc.gpfifo_entries_done++; if (ch->engine_type && ch->expected_sem_payload) @@ -14192,12 +14218,17 @@ static bool svc_gr_data_stalled; /* m150: high data args unresolved */ */ static bool svc_gr_busy_defer; static u64 svc_gr_hold_log; /* m150: bounded GR-hold proof lines */ +static u64 svc_gr_drop_log_va; /* GRFIX: launch whose dropped-page detail + * (GRFILL-DROP) was already logged - the + * detail prints once per launch episode, + * not once per ~100 ms re-decode */ /* m150: re-arm both fill diagnostics for each client session. */ static void svc_gr_diag_reset(void) { svc_gr_data_log = 0; svc_gr_src_log = 0; + svc_gr_drop_log_va = 0; } /* @@ -14312,6 +14343,35 @@ static void svc_gr_scan_inline(u64 *wins, u32 *n_cand, u64 page_va) } } +/* + * GRFIX FIX1b (r213): absolute per-launch hold bound. The m150/m156 data + * hold had no working bound: the 8b058884 deadline was keyed by the GPFIFO + * slot index, and a pin-batch rewind/replay re-decodes the same launch push + * at different slots (r213: entry 107 -> 107 -> 99 on one channel, other + * passes at yet other indices), so "first arm" re-armed on every pass; the + * launch held for 216+ s (410+ GRFILL-HOLD lines) until the client's 240 s + * timeout killed it (gr_hold_exp=0). The identity is now the launch push VA: + * the first hold of a VA arms an ABSOLUTE deadline that nothing moves while + * the launch is held. Returns true when the bound has passed (caller must + * let the launch proceed); false leaves the record armed. + */ +static bool svc_gr_hold_bound_expired(struct svc_channel *ch, u64 launch_va) +{ + u32 cap_ms = gsp_phys_gr_hold_max_ms; + + if (!launch_va) + launch_va = 1; /* anonymous launch: still absolutely bound */ + if (!cap_ms) + cap_ms = 3000; + if (ch->gr_hold_dead_va != launch_va) { + ch->gr_hold_dead_va = launch_va; + ch->gr_hold_dead_since = jiffies; + ch->gr_hold_deadline = jiffies + msecs_to_jiffies(cap_ms); + ch->gr_hold_exp_logged = false; + } + return time_after_eq(jiffies, ch->gr_hold_deadline); +} + static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) { u64 wins[16]; @@ -14348,10 +14408,25 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) * first-page guard below, so a QMD is never forwarded with a data * pointer left as an unresolvable guest VA (the zero-store class) * and the bounce window is never served stale for this launch. + * FIX1b: the hold carries an absolute per-launch bound + * (gsp_phys_gr_hold_max_ms) - when it passes the launch proceeds. */ u32 dropped_pages = 0; + /* + * GRFIX FIX1c (r213): detail for the first (up to 4) dropped pages, + * so the cycle records WHICH VAs and which candidate class dropped. + * There is no read/write direction anywhere in the candidate or + * data[] structures (both are va/pa/reloc_va only), so kind is the + * truthful candidate origin: 'a' = argument page (c[0x0] param / + * inline), 'b' = const-bank page; idx is the candidate index (the + * same index the per-candidate probe lines use). + */ + u64 drop_va[4]; + char drop_kind[4]; + u32 drop_idx[4]; svc_fwd_req.gr_n_data = 0; + svc_fwd_req.gr_dropped = 0; svc_fwd_req.gr_n_data_wins = 0; svc_gr_data_stalled = false; memset(is_arg, 0, sizeof(is_arg)); @@ -14586,10 +14661,20 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) * span) for the launch-level * hold; the span still stops * here exactly as before. + * FIX1c: capture the first 4 + * (VA + candidate origin + + * candidate index) for the + * GRFILL-DROP detail. */ if (p == 0 && svc_fwd_req.gr_n_data == before) first_unresolved = true; + if (dropped_pages < 4) { + drop_va[dropped_pages] = va; + drop_kind[dropped_pages] = + is_arg[w] ? 'a' : 'b'; + drop_idx[dropped_pages] = w; + } dropped_pages++; break; } @@ -14740,10 +14825,28 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) */ svc_gr_data_stalled = (dropped_pages > 0) || (n_arg_hi > 0 && n_arg_hi_res == 0); - if (dropped_pages > 0) - pr_info_ratelimited("vgsp-sema: vgsp-gr: GRFILL-HOLD launch=0x%llx dropped=%u kept=%u\n", + svc_fwd_req.gr_dropped = dropped_pages; + if (dropped_pages > 0) { + pr_info_ratelimited("vgsp-gr: GRFILL-HOLD launch=0x%llx dropped=%u kept=%u\n", svc_fwd_req.gr_push_va, dropped_pages, svc_fwd_req.gr_n_data); + /* + * GRFIX FIX1c: name the dropped pages once per launch + * episode (the fill re-runs every pass for a held launch; + * the marker makes the detail a first-pass-only record). + * kind 'a' = argument-page candidate, 'b' = bank candidate; + * idx = candidate index (same indexing as the probe lines). + */ + if (svc_gr_drop_log_va != svc_fwd_req.gr_push_va) { + u32 dl; + + svc_gr_drop_log_va = svc_fwd_req.gr_push_va; + for (dl = 0; dl < 4 && dl < dropped_pages; dl++) + pr_info("vgsp-gr: GRFILL-DROP va=0x%llx kind=%c idx=%u\n", + drop_va[dl], drop_kind[dl], + drop_idx[dl]); + } + } if (svc_gr_data_stalled) pr_info("vgsp-sema: svc-gr data fill: %u candidate ptrs, %u pages resolved; %u high argument ptr(s), %u resolved; %u dropped page(s) -> launch held for a bounded retry\n", n_cand, svc_fwd_req.gr_n_data, n_arg_hi, n_arg_hi_res, @@ -14996,6 +15099,40 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, * the channel's gr_hold window at the call site. Once the window * expires the entry is consumed exactly as before. */ + if (svc_gr_data_stalled) { + u64 lva = svc_fwd_req.gr_push_va; + /* + * The channel is the pump's mutable live bookkeeping object + * (the const view exists for the read-only gates); these + * record fields are the same ones the pump's hold writes, + * on this same service thread. + */ + struct svc_channel *mch = (struct svc_channel *)ch; + + /* + * GRFIX FIX1b (r213): the ABSOLUTE per-launch bound. The + * first hold of this push VA arms a fixed deadline + * (svc_gr_hold_bound_expired) that nothing re-arms; once it + * passes, the launch PROCEEDS exactly as the pre-8b058884 + * path - the candidate is consumed below and forwarded + * best-effort - so the guest is never blocked by a page that + * never resolves. r213: a slot-keyed deadline was re-armed + * on every re-decode (the same launch is replayed at + * different GPFIFO slots after rewinds) and held one launch + * for 216+ s until the client's 240 s timeout killed it. + */ + if (svc_gr_hold_bound_expired(mch, lva)) { + svc.fwd_gr_hold_exp++; + if (!mch->gr_hold_exp_logged) { + mch->gr_hold_exp_logged = true; + pr_info("vgsp-gr: GRFILL-EXPIRED launch=0x%llx dropped=%u waited_ms=%u\n", + lva, svc_fwd_req.gr_dropped, + jiffies_to_msecs(jiffies - + mch->gr_hold_dead_since)); + } + svc_gr_data_stalled = false; /* proceed: forward */ + } + } if (svc_gr_data_stalled) { clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); mutex_unlock(&svc_fwd_lock); @@ -15816,11 +15953,12 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) } } bitmap_zero(ch->slot_done_epoch, SVC_GPFIFO_MAX_BITS); - /* GRFIX FIX1: a recycled slot in the next put epoch - * gets a fresh hard-bound episode for its hold; - * without this a wrapped slot would inherit a stale - * expired deadline and never retry. */ - ch->gr_hold_dead_idx = ~0u; + /* GRFIX FIX1b: a recycled slot in the next put epoch + * gets a fresh absolute-bound record; without this a + * wrapped slot would inherit a stale expired launch + * record. */ + ch->gr_hold_dead_va = 0; + ch->gr_hold_exp_logged = false; } put_prev_wit = ch->put_prev; ch->put_prev = put; @@ -16404,10 +16542,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) fwd_guest_queued = svc_fwd_request(vgpu, ch); if (fwd_guest_queued && - !ch->engine_type) { + !ch->engine_type) ch->gr_hold_idx = ~0u; - ch->gr_hold_dead_idx = ~0u; - } /* * m159: a busy refusal (forward * in flight / release hold @@ -16519,23 +16655,34 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) } /* * m150: the GR fill held this - * launch because its high - * argument pointers do not - * translate yet. Leave the - * entry unconsumed (do not - * write GP_GET) so the pump - * re-captures and retries it; - * one bounded window per entry, - * then fall through and consume - * exactly as before. The slot's - * identity makes the hold - * one-shot per entry: after - * expiry the same entry is - * never re-held. + * launch because it carries + * data pages that do not + * resolve yet. Leave the entry + * unconsumed (do not write + * GP_GET) so the pump + * re-captures and retries it. + * GRFIX FIX1b (r213): the hold + * is bounded ABSOLUTELY per + * launch push VA + * (svc_gr_hold_bound_expired + * fixes the deadline on the + * first hold and nothing + * re-arms it; svc_gr_request + * enforces it and forwards the + * launch once it passes). The + * extra check here is the + * defence for a stale stalled + * verdict when the request + * path did not run this pass - + * past the bound the entry + * falls through to the + * historical consume. */ if (!fwd_guest_queued && !ch->engine_type && svc_gr_data_stalled && + !svc_gr_hold_bound_expired(ch, + svc_fwd_req.gr_push_va) && (ch->gr_hold_idx != (u32)idx || time_before(jiffies, ch->gr_hold_until))) { @@ -16543,76 +16690,35 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) gsp_sema_gr_grace_ms : 8000; /* - * GRFIX FIX1 hard bound: the - * m156 re-arm below restarts - * gr_hold_until on every - * pass, so a page that never - * resolves would keep this - * entry held forever (the - * r176 stall class). Fix the - * deadline on the first arm; - * re-arms never move it. + * m156: arm the forced + * re-capture so the next + * decode of this entry + * rebuilds the GR candidate + * (a replay decode would + * suppress it) and the + * deferred launch is + * forwarded for real once + * the in-flight work has + * completed. */ - if (ch->gr_hold_dead_idx != - (u32)idx) { - ch->gr_hold_dead_idx = - (u32)idx; - ch->gr_hold_deadline = - jiffies + - msecs_to_jiffies( - hold_ms); - } - if (time_after_eq(jiffies, - ch->gr_hold_deadline)) { - /* - * Bound reached: fall - * through to the - * historical consume - * (the software release - * is applied below) - the - * guest never stalls. - */ - svc.fwd_gr_hold_exp++; - pr_info_ratelimited("sema-svc: GR data-hold hard bound h=0x%x entry=%u (%u ms); consuming with software release\n", + ch->gr_rehold_idx = + (u32)idx; + ch->gr_hold_idx = (u32)idx; + ch->gr_hold_until = jiffies + + msecs_to_jiffies(hold_ms); + svc.fwd_gr_defer++; + if (svc_gr_hold_log < 16) { + svc_gr_hold_log++; + pr_info("sema-svc: GR hold h=0x%x entry=%u (%s; retry window %u ms)\n", ch->h_object, (u32)idx, + svc_gr_data_stalled ? + "data args unresolved" : + "forward busy (m156)", hold_ms); - } else { - /* - * m156: arm the - * forced re-capture so - * the next decode of - * this entry rebuilds - * the GR candidate (a - * replay decode would - * suppress it) and the - * deferred launch is - * forwarded for real - * once the in-flight - * work has completed. - */ - ch->gr_rehold_idx = - (u32)idx; - ch->gr_hold_idx = - (u32)idx; - ch->gr_hold_until = - jiffies + - msecs_to_jiffies( - hold_ms); - svc.fwd_gr_defer++; - if (svc_gr_hold_log < 16) { - svc_gr_hold_log++; - pr_info("sema-svc: GR hold h=0x%x entry=%u (%s; retry window %u ms)\n", - ch->h_object, - (u32)idx, - svc_gr_data_stalled ? - "data args unresolved" : - "forward busy (m156)", - hold_ms); - } - kfree(pb); - break; } + kfree(pb); + break; } /* * P6b wave 1.5: when @@ -17381,6 +17487,10 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->gr_nocap_va = 0; ch->unread_va = 0; ch->unread_until = 0; + /* GRFIX FIX1b: the entry is consumed; the per-launch + * absolute-bound record resets with it. */ + ch->gr_hold_dead_va = 0; + ch->gr_hold_exp_logged = false; /* Publish progress the way the GPU would. */ g = ch->last_get; -- cgit v1.2.3