diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 174 |
1 files changed, 138 insertions, 36 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 143ec54..51310a2 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -377,6 +377,18 @@ struct svc_channel { * even if still unresolved - bounded zeros, never * a hang); m156: re-held while the window is * live so a busy forward gets its real retry */ + unsigned long gr_hold_deadline; /* GRFIX FIX1: hard expiry for the data- + * stalled hold of gr_hold_dead_idx. The + * m156 re-arm restarts gr_hold_until on + * every pass, so a page that never + * resolves would otherwise keep the + * entry held forever (the r176 stall + * class). Fixed once on the first arm; + * re-arms never move it; at the deadline + * the entry is consumed with the + * historical software release (bounded, + * never a hang). */ + u32 gr_hold_dead_idx; /* entry gr_hold_deadline was armed for */ u32 gr_rehold_idx; /* m156: slot whose next decode must force a * fresh launch capture (a held entry is * re-decoded as a replay, which suppresses @@ -967,6 +979,11 @@ struct sema_svc { * abort, or a release0 page the backend could not observe). */ u64 fwd_gr_sw_rel; + u64 fwd_gr_hold_exp; /* GRFIX FIX1: data-stalled holds + * that reached the hard bound and + * were consumed with the software + * release (proof the bound fires + * instead of a stall) */ /* Distinct stalled VA tracking for rate-limited logging. */ #define SVC_STALLED_MAX 256 @@ -1254,6 +1271,7 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) memset(ch, 0, sizeof(*ch)); ch->gr_hold_idx = ~0u; /* m156: no hold armed yet */ + ch->gr_hold_dead_idx = ~0u; /* GRFIX: no hard bound armed */ ch->gr_rehold_idx = ~0u; ch->gr_nocap_idx = ~0u; /* m164: no no-capture window armed */ ch->session_gen = svc_sess_gen; /* m176 */ @@ -3944,12 +3962,13 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) svc.false_trans, svc.gpfifo_skipped, svc.gpfifo_rescued, svc.releases_faked, svc.releases_recovered, svc.counter_rel_applied); - pr_info("sema-svc: drain held=%llu starved=%llu forwarded=%llu faked=%llu rec=%llu ops_once=%llu reident=%llu opdef=%llu/%llu/%llu gr_nocap=%llu/%llu\n", + pr_info("sema-svc: drain held=%llu starved=%llu forwarded=%llu faked=%llu rec=%llu ops_once=%llu reident=%llu opdef=%llu/%llu/%llu gr_nocap=%llu/%llu gr_hold_exp=%llu\n", svc.gpfifo_held, svc.gpfifo_starved, svc.releases_forwarded, svc.releases_faked, svc.releases_recovered, svc.ops_once_skipped, svc.ops_reident, svc.opdef_held, svc.opdef_applied, svc.opdef_expired, - svc.gr_nocap_held, svc.gr_nocap_consumed); + svc.gr_nocap_held, svc.gr_nocap_consumed, + svc.fwd_gr_hold_exp); pr_info("sema-svc: counter pub_calls=%llu gr_rel=%llu reresolve=%llu pushpg_fb=%llu altw=%llu residue=%llu\n", svc.counter_pub_calls, svc.gr_rel_applied, svc.sem_write_reresolve, svc.pushpg_fallback, @@ -14321,6 +14340,16 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) * retry instead of forwarded (svc_gr_data_stalled). */ u32 n_arg_hi = 0, n_arg_hi_res = 0; + /* + * GRFIX FIX1 (full-page fail-closed): every page of every tracked + * span whose resolution chain (ipin > resolve > ledger) misses is + * counted here, first page or mid-span alike. Any drop holds the + * launch through the same bounded m150/m156 retry path as the + * first-page guard below, so a QMD is never forwarded with a data + * pointer left as an unresolvable guest VA (the zero-store class) + * and the bounce window is never served stale for this launch. + */ + u32 dropped_pages = 0; svc_fwd_req.gr_n_data = 0; svc_fwd_req.gr_n_data_wins = 0; @@ -14551,10 +14580,17 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) * it from the benign "0 new * pages" of a span whose * pages were already mapped. + * + * GRFIX FIX1: every failed + * page counts (first or mid- + * span) for the launch-level + * hold; the span still stops + * here exactly as before. */ if (p == 0 && svc_fwd_req.gr_n_data == before) first_unresolved = true; + dropped_pages++; break; } } @@ -14683,17 +14719,35 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) /* * m170 revert: the per-argument fail-closed signal is kept as * DIAGNOSTIC only (the held fail-closed line above identifies the - * exact unresolved operand), but the launch-level stall keeps the - * historical aggregate semantics: a fresh client's launch stream - * carries BENIGN 0-page candidates (junk pointer words that never - * translate), and persisting the per-arg flag held mail's first - * launch forever (r176). Only a launch with NO resolvable high - * argument holds for the bounded m150 retry. + * exact unresolved operand); the historical aggregate rule held only + * a launch with NO resolvable high argument, because a fresh client's + * launch stream carries BENIGN 0-page candidates (junk pointer words + * that never translate) and persisting the per-arg flag held mail's + * first launch forever (r176). + * + * GRFIX FIX1: the stall is now page-count based - ANY page in a + * tracked span that failed resolution (or any tracked data VA whose + * resolution returned nothing) holds the launch, so a QMD can never + * be forwarded with a data pointer left as an unresolvable guest VA + * and no bounce page is ever served stale for this launch. The + * response is the unchanged bounded m150/m156 hold (leave the GPFIFO + * entry unconsumed, retry the fill while the pins land); the pump + * side now carries a hard wall-clock bound (gr_hold_deadline) so a + * word that never resolves can no longer re-arm the window forever: + * at the bound the entry is consumed with the historical software + * release - bounded, never a hang, never a silent stale-operand + * submission. */ - svc_gr_data_stalled = (n_arg_hi > 0 && n_arg_hi_res == 0); + svc_gr_data_stalled = (dropped_pages > 0) || + (n_arg_hi > 0 && n_arg_hi_res == 0); + if (dropped_pages > 0) + pr_info_ratelimited("vgsp-sema: vgsp-gr: GRFILL-HOLD launch=0x%llx dropped=%u kept=%u\n", + svc_fwd_req.gr_push_va, dropped_pages, + svc_fwd_req.gr_n_data); if (svc_gr_data_stalled) - pr_info("vgsp-sema: svc-gr data fill: %u candidate ptrs, %u pages resolved; %u high argument ptr(s), %u resolved -> launch held for a bounded retry\n", - n_cand, svc_fwd_req.gr_n_data, n_arg_hi, n_arg_hi_res); + pr_info("vgsp-sema: svc-gr data fill: %u candidate ptrs, %u pages resolved; %u high argument ptr(s), %u resolved; %u dropped page(s) -> launch held for a bounded retry\n", + n_cand, svc_fwd_req.gr_n_data, n_arg_hi, n_arg_hi_res, + dropped_pages); else pr_info("vgsp-sema: svc-gr data fill: %u candidate ptrs, %u pages resolved (reloc 0x%llx+)\n", n_cand, svc_fwd_req.gr_n_data, reloc_base); @@ -15762,6 +15816,11 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) } } bitmap_zero(ch->slot_done_epoch, SVC_GPFIFO_MAX_BITS); + /* GRFIX FIX1: a recycled slot in the next put epoch + * gets a fresh hard-bound episode for its hold; + * without this a wrapped slot would inherit a stale + * expired deadline and never retry. */ + ch->gr_hold_dead_idx = ~0u; } put_prev_wit = ch->put_prev; ch->put_prev = put; @@ -16345,8 +16404,10 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) fwd_guest_queued = svc_fwd_request(vgpu, ch); if (fwd_guest_queued && - !ch->engine_type) + !ch->engine_type) { ch->gr_hold_idx = ~0u; + ch->gr_hold_dead_idx = ~0u; + } /* * m159: a busy refusal (forward * in flight / release hold @@ -16482,35 +16543,76 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) gsp_sema_gr_grace_ms : 8000; /* - * m156: arm the forced - * re-capture so the next - * decode of this entry - * rebuilds the GR candidate - * (a replay decode would - * suppress it) and the - * deferred launch is - * forwarded for real once - * the in-flight work has - * completed. + * GRFIX FIX1 hard bound: the + * m156 re-arm below restarts + * gr_hold_until on every + * pass, so a page that never + * resolves would keep this + * entry held forever (the + * r176 stall class). Fix the + * deadline on the first arm; + * re-arms never move it. */ - ch->gr_rehold_idx = - (u32)idx; - ch->gr_hold_idx = (u32)idx; - ch->gr_hold_until = jiffies + - msecs_to_jiffies(hold_ms); - svc.fwd_gr_defer++; - if (svc_gr_hold_log < 16) { - svc_gr_hold_log++; - pr_info("sema-svc: GR hold h=0x%x entry=%u (%s; retry window %u ms)\n", + if (ch->gr_hold_dead_idx != + (u32)idx) { + ch->gr_hold_dead_idx = + (u32)idx; + ch->gr_hold_deadline = + jiffies + + msecs_to_jiffies( + hold_ms); + } + if (time_after_eq(jiffies, + ch->gr_hold_deadline)) { + /* + * Bound reached: fall + * through to the + * historical consume + * (the software release + * is applied below) - the + * guest never stalls. + */ + svc.fwd_gr_hold_exp++; + pr_info_ratelimited("sema-svc: GR data-hold hard bound h=0x%x entry=%u (%u ms); consuming with software release\n", ch->h_object, (u32)idx, - svc_gr_data_stalled ? - "data args unresolved" : - "forward busy (m156)", hold_ms); + } else { + /* + * m156: arm the + * forced re-capture so + * the next decode of + * this entry rebuilds + * the GR candidate (a + * replay decode would + * suppress it) and the + * deferred launch is + * forwarded for real + * once the in-flight + * work has completed. + */ + ch->gr_rehold_idx = + (u32)idx; + ch->gr_hold_idx = + (u32)idx; + ch->gr_hold_until = + jiffies + + msecs_to_jiffies( + hold_ms); + svc.fwd_gr_defer++; + if (svc_gr_hold_log < 16) { + svc_gr_hold_log++; + pr_info("sema-svc: GR hold h=0x%x entry=%u (%s; retry window %u ms)\n", + ch->h_object, + (u32)idx, + svc_gr_data_stalled ? + "data args unresolved" : + "forward busy (m156)", + hold_ms); + } + kfree(pb); + break; } - kfree(pb); - break; } /* * P6b wave 1.5: when |