diff options
| -rw-r--r-- | kernel/gsp_phys_booter.c | 158 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 174 |
2 files changed, 282 insertions, 50 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index c5870ce..b3e095a 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -74,6 +74,25 @@ module_param_named(gsp_phys_nocat, gsp_phys_nocat_param, bool, 0644); MODULE_PARM_DESC(gsp_phys_nocat, "Log every GSP-RM NOCAT journal record (status-queue function 0x1020) individually during the stage-3e RM-init poll (default 0; the unique-record summary is always accumulated and dumped when the poll ends)"); +/* + * GRFIX FIX3 (relocation coverage, default 0): the QMD/bank rewrite scan + * counts 8-byte words that pass the rewrite's own device-VA shape test + * (>= 0x10000000, < 2^47, 256-byte aligned) but match NO tracked data page + * (req->data[x].va). Such a word is left as an unrelocated guest VA for the + * engine to walk, which is the multi-pointer QMD failure signature. With the + * parameter clear the scan only logs the per-page count (RELOC-GAP, + * rate-limited); with it set a launch carrying any such word is held + * fail-closed before publish (same backend-refusal path as the copy-in hold; + * the sema hold applies the software releases - bounded, never a hang). + * The hold is opt-in because the shape heuristic can false-positive on + * legitimate non-pointer data words; the RELOC-GAP count measures that rate. + */ +static bool gsp_phys_gr_strict_reloc_param; +module_param_named(gsp_phys_gr_strict_reloc, gsp_phys_gr_strict_reloc_param, + bool, 0644); +MODULE_PARM_DESC(gsp_phys_gr_strict_reloc, + "Hold a GR launch fail-closed when the QMD rewrite finds device-VA-shaped words that match no tracked data page (default 0: RELOC-GAP log only; 1: hold the launch)"); + bool gsp_phys_boot_enabled(void) { return gsp_phys_boot_param; @@ -16751,14 +16770,14 @@ static void gsp_phys_rmcall_gr_gpc_mmu_init(void __iomem *bar0) * the affine-relocated server-managed VA (the engine cannot walk the * client-managed guest VAs: UPDATE_PDE_2 refuses them with 0x1f). */ -static void gsp_phys_gr_rewrite_data_vars(u32 *w, u32 n_words, - const struct gsp_phys_gr_submit *req, - u32 bank, u32 *mirror) +static u32 gsp_phys_gr_rewrite_data_vars(u32 *w, u32 n_words, + const struct gsp_phys_gr_submit *req, + u32 bank, u32 *mirror, u64 *gap_first) { - u32 rew = 0, skipped = 0, k; + u32 rew = 0, skipped = 0, gap = 0, k; if (!req->n_data) - return; + return 0; /* * m117: 8-byte stride with a 256-byte alignment guard. The m103 * 4-byte stride mis-matched arbitrary adjacent 4-byte words (the @@ -16839,10 +16858,24 @@ static void gsp_phys_gr_rewrite_data_vars(u32 *w, u32 n_words, rew++; break; } + /* + * GRFIX FIX3: the inner scan ran out (x reached n_data) + * without matching req->data[x].va - the word passes the + * exact shape heuristic the match test above uses, so it + * will be left as an unrelocated guest VA in this page. + * Count it; the caller decides (RELOC-GAP log with the + * parameter off, fail-closed hold with it on). + */ + if (x == req->n_data) { + gap++; + if (gap_first && !*gap_first) + *gap_first = v; + } } if (rew) pr_info("vgsp-gr: bank %u: %u data pointers relocated\n", bank, rew); + return gap; } /* @@ -17420,10 +17453,19 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, static bool deposit_rb_done; static u16 poll_ms; static u32 poll_start_us; + /* + * GRFIX FIX3: per-launch relocation coverage. Static like the other + * submit scratch: the single-inflight ordered work item is the only + * caller, and statics keep the frame at its warning-bound size. + */ + static u32 reloc_gap; + static u64 reloc_gap_first; int ret = 0; fast_rearmed = false; deposit_rb_done = false; + reloc_gap = 0; + reloc_gap_first = 0; /* * The fault record is static scratch (frame bound): reset it per * submission so a previous launch's fault can never leak into this @@ -17748,10 +17790,11 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, } if (pg->hpa && pg->qmd_word >= 32 && pg->qmd_word <= 47 && !(pg->qmd_word & 1)) - gsp_phys_gr_rewrite_data_vars((u32 *)sh_page, + (void)gsp_phys_gr_rewrite_data_vars( + (u32 *)sh_page, GSP_PHYS_PAGE_SIZE / 4, req, (pg->qmd_word - 32) / 2, - (u32 *)__va(pg->hpa)); + (u32 *)__va(pg->hpa), NULL); if (pg->hpa && pg->qmd_word == 40u && gsp_phys_rmcall_gr_mail_patch_value()) { u32 *w = (u32 *)sh_page; @@ -17862,11 +17905,31 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, * relocated values no longer match a guest data VA. */ if (pg->hpa && pg->qmd_word >= 32 && pg->qmd_word <= 47 && - !(pg->qmd_word & 1)) - gsp_phys_gr_rewrite_data_vars((u32 *)sh_page, + !(pg->qmd_word & 1)) { + u64 gf = 0; + u32 gg; + + /* + * GRFIX FIX3: this is the final authoring + * of the engine-visible page (after the + * inline replay), so its gap count is the + * launch's relocation-coverage measure. + * Log it (rate-limited) and accumulate it + * for the strict hold below. + */ + gg = gsp_phys_gr_rewrite_data_vars( + (u32 *)sh_page, GSP_PHYS_PAGE_SIZE / 4, req, (pg->qmd_word - 32) / 2, - (u32 *)__va(pg->hpa)); + (u32 *)__va(pg->hpa), &gf); + if (gg) { + reloc_gap += gg; + if (!reloc_gap_first) + reloc_gap_first = gf; + pr_info_ratelimited("vgsp-gr: RELOC-GAP words=%u first=0x%llx\n", + gg, gf); + } + } if (copied) { u32 sh_line = boot_stage.rmcall_guest_pt_next_line; @@ -18050,6 +18113,27 @@ gr_page_unmapped: } /* + * GRFIX FIX3 strict hold: with gsp_phys_gr_strict_reloc=1, a launch + * whose rewritten pages carry device-VA-shaped words that match no + * tracked data page is refused before publish - the word stays what + * it was (an unrelocated guest VA), but the engine never sees the + * QMD, so it can never walk an untranslatable pointer. The refusal + * rides the existing backend-failure path (caller bridge fallback + + * the sema service's recorded-release hold; bounded, never a hang). + * The hold is opt-in because the shape heuristic can false-positive + * on non-pointer data words; the RELOC-GAP line (emitted with the + * parameter off as well) is the false-positive evidence. + */ + if (reloc_gap && gsp_phys_gr_strict_reloc_param) { + mutex_unlock(&gsp_phys_rm_lock); + gsp_phys_gr_refused++; + pr_err("vgsp-gr: RELOC-GAP-STRICT words=%u first=0x%llx; launch held fail-closed (unrelocated device-VA words never reach the engine)\n", + reloc_gap, reloc_gap_first); + ret = -EAGAIN; + goto out; + } + + /* * Rank-2 qmdwin (gsp_phys_rmcall_gr_qmdwin=1): relocate the inline-QMD * destination to the dedicated module page planted in the ladder's * window-0 small-PT line (VA 0x107000). The PTE was written by @@ -18245,16 +18329,36 @@ gr_page_unmapped: struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); u32 dcopied = 0, dfailed = 0, di; - gsp_phys_gr_fill_bounce(req, bounce_csum_in); + /* + * GRFIX FIX2 (copy-in fail-closed): classify every data page + * BEFORE the bounce copy. An entry with no resolved HPA, no + * bounce index or no backing bounce page cannot be freshly + * filled for this launch; publishing it would hand the engine + * a QMD data pointer into a bounce page still holding a + * previous launch's bytes (the multi-page stale-output + * class). Hold on the existing backend-refusal path: the + * caller's bridge fallback and the sema service's recorded- + * release hold apply the software releases - bounded, never + * a hang - and the engine never sees the QMD. + */ for (di = 0; di < req->n_data; di++) { u32 bi = gsp_phys_gr_data_bi(req->data[di].reloc_va); if (bi == GSP_PHYS_GR_DATA_BI_NONE || - !req->data[di].hpa) + !req->data[di].hpa || + !gsp_phys_chunk_page_va(dchunk, bi)) dfailed++; - else if (gsp_phys_chunk_page_va(dchunk, bi)) - dcopied++; } + if (dfailed) { + mutex_unlock(&gsp_phys_rm_lock); + gsp_phys_gr_refused++; + pr_err("vgsp-gr: copy-in hold: %u/%u data pages unresolved at copy-in (no fresh bounce content); launch held fail-closed, no publish\n", + dfailed, req->n_data); + ret = -EAGAIN; + goto out; + } + gsp_phys_gr_fill_bounce(req, bounce_csum_in); + dcopied = req->n_data; pr_info("vgsp-gr: bounce copy-in: %u pages, %u skipped (pre-publish)\n", dcopied, dfailed); for (di = 0; di < req->n_data && di < 3; di++) @@ -19303,6 +19407,32 @@ gr_page_unmapped: */ gsp_phys_gr_deposit_rb_log(req, qmd_dst_mem); /* + * GRFIX FIX4 (agent-A decisive experiment): bounded + * real-GR register dump at the zero-write accept, one + * line per classified launch, using the same readl + * accessor and the same offsets as the read-only + * debugfs probe (gsp_phys_reg_probe_show): FE_HWW_ESR + * 0x404000, SKED_HWW_ESR 0x407020, PGRAPH_TRAPPED_ADDR + * 0x400704 + TRAPPED_DATA_LO/HI 0x400708/70c, + * FECS_IRQSTAT 0x409008, FECS_CUR/NEW_CTX 0x409b00/04. + * No new locks or sleeps: the submit context already + * holds gsp_phys_rm_lock exactly as the surrounding + * post-mortem reads do. Zero is a legitimate value for + * several of these (no context resident), so the line + * is classified with the sentinel/faultbuf evidence, + * not on its own. + */ + if (bar0) + pr_warn("vgsp-gr: NOWRITE-REGS fe_esr=0x%08x sked_esr=0x%08x trap=0x%08x/%08x/%08x fecs_irq=0x%08x ctx=0x%08x->0x%08x\n", + readl(bar0 + 0x00404000u), + readl(bar0 + 0x00407020u), + readl(bar0 + 0x00400704u), + readl(bar0 + 0x00400708u), + readl(bar0 + 0x0040070cu), + readl(bar0 + 0x00409008u), + readl(bar0 + 0x00409b00u), + readl(bar0 + 0x00409b04u)); + /* * (c) Late-store check: a store ramp that retires after * the host's window is indistinguishable from a lost * dispatch at the accept moment. Re-scan once at +1 s diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 143ec54..51310a2 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -377,6 +377,18 @@ struct svc_channel { * even if still unresolved - bounded zeros, never * a hang); m156: re-held while the window is * live so a busy forward gets its real retry */ + unsigned long gr_hold_deadline; /* GRFIX FIX1: hard expiry for the data- + * stalled hold of gr_hold_dead_idx. The + * m156 re-arm restarts gr_hold_until on + * every pass, so a page that never + * resolves would otherwise keep the + * entry held forever (the r176 stall + * class). Fixed once on the first arm; + * re-arms never move it; at the deadline + * the entry is consumed with the + * historical software release (bounded, + * never a hang). */ + u32 gr_hold_dead_idx; /* entry gr_hold_deadline was armed for */ u32 gr_rehold_idx; /* m156: slot whose next decode must force a * fresh launch capture (a held entry is * re-decoded as a replay, which suppresses @@ -967,6 +979,11 @@ struct sema_svc { * abort, or a release0 page the backend could not observe). */ u64 fwd_gr_sw_rel; + u64 fwd_gr_hold_exp; /* GRFIX FIX1: data-stalled holds + * that reached the hard bound and + * were consumed with the software + * release (proof the bound fires + * instead of a stall) */ /* Distinct stalled VA tracking for rate-limited logging. */ #define SVC_STALLED_MAX 256 @@ -1254,6 +1271,7 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) memset(ch, 0, sizeof(*ch)); ch->gr_hold_idx = ~0u; /* m156: no hold armed yet */ + ch->gr_hold_dead_idx = ~0u; /* GRFIX: no hard bound armed */ ch->gr_rehold_idx = ~0u; ch->gr_nocap_idx = ~0u; /* m164: no no-capture window armed */ ch->session_gen = svc_sess_gen; /* m176 */ @@ -3944,12 +3962,13 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) svc.false_trans, svc.gpfifo_skipped, svc.gpfifo_rescued, svc.releases_faked, svc.releases_recovered, svc.counter_rel_applied); - pr_info("sema-svc: drain held=%llu starved=%llu forwarded=%llu faked=%llu rec=%llu ops_once=%llu reident=%llu opdef=%llu/%llu/%llu gr_nocap=%llu/%llu\n", + pr_info("sema-svc: drain held=%llu starved=%llu forwarded=%llu faked=%llu rec=%llu ops_once=%llu reident=%llu opdef=%llu/%llu/%llu gr_nocap=%llu/%llu gr_hold_exp=%llu\n", svc.gpfifo_held, svc.gpfifo_starved, svc.releases_forwarded, svc.releases_faked, svc.releases_recovered, svc.ops_once_skipped, svc.ops_reident, svc.opdef_held, svc.opdef_applied, svc.opdef_expired, - svc.gr_nocap_held, svc.gr_nocap_consumed); + svc.gr_nocap_held, svc.gr_nocap_consumed, + svc.fwd_gr_hold_exp); pr_info("sema-svc: counter pub_calls=%llu gr_rel=%llu reresolve=%llu pushpg_fb=%llu altw=%llu residue=%llu\n", svc.counter_pub_calls, svc.gr_rel_applied, svc.sem_write_reresolve, svc.pushpg_fallback, @@ -14321,6 +14340,16 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) * retry instead of forwarded (svc_gr_data_stalled). */ u32 n_arg_hi = 0, n_arg_hi_res = 0; + /* + * GRFIX FIX1 (full-page fail-closed): every page of every tracked + * span whose resolution chain (ipin > resolve > ledger) misses is + * counted here, first page or mid-span alike. Any drop holds the + * launch through the same bounded m150/m156 retry path as the + * first-page guard below, so a QMD is never forwarded with a data + * pointer left as an unresolvable guest VA (the zero-store class) + * and the bounce window is never served stale for this launch. + */ + u32 dropped_pages = 0; svc_fwd_req.gr_n_data = 0; svc_fwd_req.gr_n_data_wins = 0; @@ -14551,10 +14580,17 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) * it from the benign "0 new * pages" of a span whose * pages were already mapped. + * + * GRFIX FIX1: every failed + * page counts (first or mid- + * span) for the launch-level + * hold; the span still stops + * here exactly as before. */ if (p == 0 && svc_fwd_req.gr_n_data == before) first_unresolved = true; + dropped_pages++; break; } } @@ -14683,17 +14719,35 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) /* * m170 revert: the per-argument fail-closed signal is kept as * DIAGNOSTIC only (the held fail-closed line above identifies the - * exact unresolved operand), but the launch-level stall keeps the - * historical aggregate semantics: a fresh client's launch stream - * carries BENIGN 0-page candidates (junk pointer words that never - * translate), and persisting the per-arg flag held mail's first - * launch forever (r176). Only a launch with NO resolvable high - * argument holds for the bounded m150 retry. + * exact unresolved operand); the historical aggregate rule held only + * a launch with NO resolvable high argument, because a fresh client's + * launch stream carries BENIGN 0-page candidates (junk pointer words + * that never translate) and persisting the per-arg flag held mail's + * first launch forever (r176). + * + * GRFIX FIX1: the stall is now page-count based - ANY page in a + * tracked span that failed resolution (or any tracked data VA whose + * resolution returned nothing) holds the launch, so a QMD can never + * be forwarded with a data pointer left as an unresolvable guest VA + * and no bounce page is ever served stale for this launch. The + * response is the unchanged bounded m150/m156 hold (leave the GPFIFO + * entry unconsumed, retry the fill while the pins land); the pump + * side now carries a hard wall-clock bound (gr_hold_deadline) so a + * word that never resolves can no longer re-arm the window forever: + * at the bound the entry is consumed with the historical software + * release - bounded, never a hang, never a silent stale-operand + * submission. */ - svc_gr_data_stalled = (n_arg_hi > 0 && n_arg_hi_res == 0); + svc_gr_data_stalled = (dropped_pages > 0) || + (n_arg_hi > 0 && n_arg_hi_res == 0); + if (dropped_pages > 0) + pr_info_ratelimited("vgsp-sema: vgsp-gr: GRFILL-HOLD launch=0x%llx dropped=%u kept=%u\n", + svc_fwd_req.gr_push_va, dropped_pages, + svc_fwd_req.gr_n_data); if (svc_gr_data_stalled) - pr_info("vgsp-sema: svc-gr data fill: %u candidate ptrs, %u pages resolved; %u high argument ptr(s), %u resolved -> launch held for a bounded retry\n", - n_cand, svc_fwd_req.gr_n_data, n_arg_hi, n_arg_hi_res); + pr_info("vgsp-sema: svc-gr data fill: %u candidate ptrs, %u pages resolved; %u high argument ptr(s), %u resolved; %u dropped page(s) -> launch held for a bounded retry\n", + n_cand, svc_fwd_req.gr_n_data, n_arg_hi, n_arg_hi_res, + dropped_pages); else pr_info("vgsp-sema: svc-gr data fill: %u candidate ptrs, %u pages resolved (reloc 0x%llx+)\n", n_cand, svc_fwd_req.gr_n_data, reloc_base); @@ -15762,6 +15816,11 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) } } bitmap_zero(ch->slot_done_epoch, SVC_GPFIFO_MAX_BITS); + /* GRFIX FIX1: a recycled slot in the next put epoch + * gets a fresh hard-bound episode for its hold; + * without this a wrapped slot would inherit a stale + * expired deadline and never retry. */ + ch->gr_hold_dead_idx = ~0u; } put_prev_wit = ch->put_prev; ch->put_prev = put; @@ -16345,8 +16404,10 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) fwd_guest_queued = svc_fwd_request(vgpu, ch); if (fwd_guest_queued && - !ch->engine_type) + !ch->engine_type) { ch->gr_hold_idx = ~0u; + ch->gr_hold_dead_idx = ~0u; + } /* * m159: a busy refusal (forward * in flight / release hold @@ -16482,35 +16543,76 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) gsp_sema_gr_grace_ms : 8000; /* - * m156: arm the forced - * re-capture so the next - * decode of this entry - * rebuilds the GR candidate - * (a replay decode would - * suppress it) and the - * deferred launch is - * forwarded for real once - * the in-flight work has - * completed. + * GRFIX FIX1 hard bound: the + * m156 re-arm below restarts + * gr_hold_until on every + * pass, so a page that never + * resolves would keep this + * entry held forever (the + * r176 stall class). Fix the + * deadline on the first arm; + * re-arms never move it. */ - ch->gr_rehold_idx = - (u32)idx; - ch->gr_hold_idx = (u32)idx; - ch->gr_hold_until = jiffies + - msecs_to_jiffies(hold_ms); - svc.fwd_gr_defer++; - if (svc_gr_hold_log < 16) { - svc_gr_hold_log++; - pr_info("sema-svc: GR hold h=0x%x entry=%u (%s; retry window %u ms)\n", + if (ch->gr_hold_dead_idx != + (u32)idx) { + ch->gr_hold_dead_idx = + (u32)idx; + ch->gr_hold_deadline = + jiffies + + msecs_to_jiffies( + hold_ms); + } + if (time_after_eq(jiffies, + ch->gr_hold_deadline)) { + /* + * Bound reached: fall + * through to the + * historical consume + * (the software release + * is applied below) - the + * guest never stalls. + */ + svc.fwd_gr_hold_exp++; + pr_info_ratelimited("sema-svc: GR data-hold hard bound h=0x%x entry=%u (%u ms); consuming with software release\n", ch->h_object, (u32)idx, - svc_gr_data_stalled ? - "data args unresolved" : - "forward busy (m156)", hold_ms); + } else { + /* + * m156: arm the + * forced re-capture so + * the next decode of + * this entry rebuilds + * the GR candidate (a + * replay decode would + * suppress it) and the + * deferred launch is + * forwarded for real + * once the in-flight + * work has completed. + */ + ch->gr_rehold_idx = + (u32)idx; + ch->gr_hold_idx = + (u32)idx; + ch->gr_hold_until = + jiffies + + msecs_to_jiffies( + hold_ms); + svc.fwd_gr_defer++; + if (svc_gr_hold_log < 16) { + svc_gr_hold_log++; + pr_info("sema-svc: GR hold h=0x%x entry=%u (%s; retry window %u ms)\n", + ch->h_object, + (u32)idx, + svc_gr_data_stalled ? + "data args unresolved" : + "forward busy (m156)", + hold_ms); + } + kfree(pb); + break; } - kfree(pb); - break; } /* * P6b wave 1.5: when |