diff options
Diffstat (limited to 'kernel/gsp_phys_booter.c')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 158 |
1 files changed, 144 insertions, 14 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index c5870ce..b3e095a 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -74,6 +74,25 @@ module_param_named(gsp_phys_nocat, gsp_phys_nocat_param, bool, 0644); MODULE_PARM_DESC(gsp_phys_nocat, "Log every GSP-RM NOCAT journal record (status-queue function 0x1020) individually during the stage-3e RM-init poll (default 0; the unique-record summary is always accumulated and dumped when the poll ends)"); +/* + * GRFIX FIX3 (relocation coverage, default 0): the QMD/bank rewrite scan + * counts 8-byte words that pass the rewrite's own device-VA shape test + * (>= 0x10000000, < 2^47, 256-byte aligned) but match NO tracked data page + * (req->data[x].va). Such a word is left as an unrelocated guest VA for the + * engine to walk, which is the multi-pointer QMD failure signature. With the + * parameter clear the scan only logs the per-page count (RELOC-GAP, + * rate-limited); with it set a launch carrying any such word is held + * fail-closed before publish (same backend-refusal path as the copy-in hold; + * the sema hold applies the software releases - bounded, never a hang). + * The hold is opt-in because the shape heuristic can false-positive on + * legitimate non-pointer data words; the RELOC-GAP count measures that rate. + */ +static bool gsp_phys_gr_strict_reloc_param; +module_param_named(gsp_phys_gr_strict_reloc, gsp_phys_gr_strict_reloc_param, + bool, 0644); +MODULE_PARM_DESC(gsp_phys_gr_strict_reloc, + "Hold a GR launch fail-closed when the QMD rewrite finds device-VA-shaped words that match no tracked data page (default 0: RELOC-GAP log only; 1: hold the launch)"); + bool gsp_phys_boot_enabled(void) { return gsp_phys_boot_param; @@ -16751,14 +16770,14 @@ static void gsp_phys_rmcall_gr_gpc_mmu_init(void __iomem *bar0) * the affine-relocated server-managed VA (the engine cannot walk the * client-managed guest VAs: UPDATE_PDE_2 refuses them with 0x1f). */ -static void gsp_phys_gr_rewrite_data_vars(u32 *w, u32 n_words, - const struct gsp_phys_gr_submit *req, - u32 bank, u32 *mirror) +static u32 gsp_phys_gr_rewrite_data_vars(u32 *w, u32 n_words, + const struct gsp_phys_gr_submit *req, + u32 bank, u32 *mirror, u64 *gap_first) { - u32 rew = 0, skipped = 0, k; + u32 rew = 0, skipped = 0, gap = 0, k; if (!req->n_data) - return; + return 0; /* * m117: 8-byte stride with a 256-byte alignment guard. The m103 * 4-byte stride mis-matched arbitrary adjacent 4-byte words (the @@ -16839,10 +16858,24 @@ static void gsp_phys_gr_rewrite_data_vars(u32 *w, u32 n_words, rew++; break; } + /* + * GRFIX FIX3: the inner scan ran out (x reached n_data) + * without matching req->data[x].va - the word passes the + * exact shape heuristic the match test above uses, so it + * will be left as an unrelocated guest VA in this page. + * Count it; the caller decides (RELOC-GAP log with the + * parameter off, fail-closed hold with it on). + */ + if (x == req->n_data) { + gap++; + if (gap_first && !*gap_first) + *gap_first = v; + } } if (rew) pr_info("vgsp-gr: bank %u: %u data pointers relocated\n", bank, rew); + return gap; } /* @@ -17420,10 +17453,19 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, static bool deposit_rb_done; static u16 poll_ms; static u32 poll_start_us; + /* + * GRFIX FIX3: per-launch relocation coverage. Static like the other + * submit scratch: the single-inflight ordered work item is the only + * caller, and statics keep the frame at its warning-bound size. + */ + static u32 reloc_gap; + static u64 reloc_gap_first; int ret = 0; fast_rearmed = false; deposit_rb_done = false; + reloc_gap = 0; + reloc_gap_first = 0; /* * The fault record is static scratch (frame bound): reset it per * submission so a previous launch's fault can never leak into this @@ -17748,10 +17790,11 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, } if (pg->hpa && pg->qmd_word >= 32 && pg->qmd_word <= 47 && !(pg->qmd_word & 1)) - gsp_phys_gr_rewrite_data_vars((u32 *)sh_page, + (void)gsp_phys_gr_rewrite_data_vars( + (u32 *)sh_page, GSP_PHYS_PAGE_SIZE / 4, req, (pg->qmd_word - 32) / 2, - (u32 *)__va(pg->hpa)); + (u32 *)__va(pg->hpa), NULL); if (pg->hpa && pg->qmd_word == 40u && gsp_phys_rmcall_gr_mail_patch_value()) { u32 *w = (u32 *)sh_page; @@ -17862,11 +17905,31 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, * relocated values no longer match a guest data VA. */ if (pg->hpa && pg->qmd_word >= 32 && pg->qmd_word <= 47 && - !(pg->qmd_word & 1)) - gsp_phys_gr_rewrite_data_vars((u32 *)sh_page, + !(pg->qmd_word & 1)) { + u64 gf = 0; + u32 gg; + + /* + * GRFIX FIX3: this is the final authoring + * of the engine-visible page (after the + * inline replay), so its gap count is the + * launch's relocation-coverage measure. + * Log it (rate-limited) and accumulate it + * for the strict hold below. + */ + gg = gsp_phys_gr_rewrite_data_vars( + (u32 *)sh_page, GSP_PHYS_PAGE_SIZE / 4, req, (pg->qmd_word - 32) / 2, - (u32 *)__va(pg->hpa)); + (u32 *)__va(pg->hpa), &gf); + if (gg) { + reloc_gap += gg; + if (!reloc_gap_first) + reloc_gap_first = gf; + pr_info_ratelimited("vgsp-gr: RELOC-GAP words=%u first=0x%llx\n", + gg, gf); + } + } if (copied) { u32 sh_line = boot_stage.rmcall_guest_pt_next_line; @@ -18050,6 +18113,27 @@ gr_page_unmapped: } /* + * GRFIX FIX3 strict hold: with gsp_phys_gr_strict_reloc=1, a launch + * whose rewritten pages carry device-VA-shaped words that match no + * tracked data page is refused before publish - the word stays what + * it was (an unrelocated guest VA), but the engine never sees the + * QMD, so it can never walk an untranslatable pointer. The refusal + * rides the existing backend-failure path (caller bridge fallback + + * the sema service's recorded-release hold; bounded, never a hang). + * The hold is opt-in because the shape heuristic can false-positive + * on non-pointer data words; the RELOC-GAP line (emitted with the + * parameter off as well) is the false-positive evidence. + */ + if (reloc_gap && gsp_phys_gr_strict_reloc_param) { + mutex_unlock(&gsp_phys_rm_lock); + gsp_phys_gr_refused++; + pr_err("vgsp-gr: RELOC-GAP-STRICT words=%u first=0x%llx; launch held fail-closed (unrelocated device-VA words never reach the engine)\n", + reloc_gap, reloc_gap_first); + ret = -EAGAIN; + goto out; + } + + /* * Rank-2 qmdwin (gsp_phys_rmcall_gr_qmdwin=1): relocate the inline-QMD * destination to the dedicated module page planted in the ladder's * window-0 small-PT line (VA 0x107000). The PTE was written by @@ -18245,16 +18329,36 @@ gr_page_unmapped: struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); u32 dcopied = 0, dfailed = 0, di; - gsp_phys_gr_fill_bounce(req, bounce_csum_in); + /* + * GRFIX FIX2 (copy-in fail-closed): classify every data page + * BEFORE the bounce copy. An entry with no resolved HPA, no + * bounce index or no backing bounce page cannot be freshly + * filled for this launch; publishing it would hand the engine + * a QMD data pointer into a bounce page still holding a + * previous launch's bytes (the multi-page stale-output + * class). Hold on the existing backend-refusal path: the + * caller's bridge fallback and the sema service's recorded- + * release hold apply the software releases - bounded, never + * a hang - and the engine never sees the QMD. + */ for (di = 0; di < req->n_data; di++) { u32 bi = gsp_phys_gr_data_bi(req->data[di].reloc_va); if (bi == GSP_PHYS_GR_DATA_BI_NONE || - !req->data[di].hpa) + !req->data[di].hpa || + !gsp_phys_chunk_page_va(dchunk, bi)) dfailed++; - else if (gsp_phys_chunk_page_va(dchunk, bi)) - dcopied++; } + if (dfailed) { + mutex_unlock(&gsp_phys_rm_lock); + gsp_phys_gr_refused++; + pr_err("vgsp-gr: copy-in hold: %u/%u data pages unresolved at copy-in (no fresh bounce content); launch held fail-closed, no publish\n", + dfailed, req->n_data); + ret = -EAGAIN; + goto out; + } + gsp_phys_gr_fill_bounce(req, bounce_csum_in); + dcopied = req->n_data; pr_info("vgsp-gr: bounce copy-in: %u pages, %u skipped (pre-publish)\n", dcopied, dfailed); for (di = 0; di < req->n_data && di < 3; di++) @@ -19303,6 +19407,32 @@ gr_page_unmapped: */ gsp_phys_gr_deposit_rb_log(req, qmd_dst_mem); /* + * GRFIX FIX4 (agent-A decisive experiment): bounded + * real-GR register dump at the zero-write accept, one + * line per classified launch, using the same readl + * accessor and the same offsets as the read-only + * debugfs probe (gsp_phys_reg_probe_show): FE_HWW_ESR + * 0x404000, SKED_HWW_ESR 0x407020, PGRAPH_TRAPPED_ADDR + * 0x400704 + TRAPPED_DATA_LO/HI 0x400708/70c, + * FECS_IRQSTAT 0x409008, FECS_CUR/NEW_CTX 0x409b00/04. + * No new locks or sleeps: the submit context already + * holds gsp_phys_rm_lock exactly as the surrounding + * post-mortem reads do. Zero is a legitimate value for + * several of these (no context resident), so the line + * is classified with the sentinel/faultbuf evidence, + * not on its own. + */ + if (bar0) + pr_warn("vgsp-gr: NOWRITE-REGS fe_esr=0x%08x sked_esr=0x%08x trap=0x%08x/%08x/%08x fecs_irq=0x%08x ctx=0x%08x->0x%08x\n", + readl(bar0 + 0x00404000u), + readl(bar0 + 0x00407020u), + readl(bar0 + 0x00400704u), + readl(bar0 + 0x00400708u), + readl(bar0 + 0x0040070cu), + readl(bar0 + 0x00409008u), + readl(bar0 + 0x00409b00u), + readl(bar0 + 0x00409b04u)); + /* * (c) Late-store check: a store ramp that retires after * the host's window is indistinguishable from a lost * dispatch at the accept moment. Re-scan once at +1 s |