diff options
| -rw-r--r-- | kernel/gsp_phys_booter.c | 208 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.c | 32 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.h | 5 |
3 files changed, 173 insertions, 72 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index f0f4baa..535c8ad 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -17036,7 +17036,7 @@ static u32 gsp_phys_gr_data_bi(u64 reloc_va) */ static noinline u32 gsp_phys_gr_output_retire_wait(const struct gsp_phys_gr_submit *req, - const u32 *csum_in) + const u32 *csum_in, u32 *n_tracked) { /* * B-2: one entry per page of the grown region (64 KB), static like the @@ -17052,6 +17052,14 @@ gsp_phys_gr_output_retire_wait(const struct gsp_phys_gr_submit *req, const char *why = " (TIMEOUT)"; unsigned long wait_start = jiffies; + /* + * m179: report how many pages were actually trackable. "wrote == 0" + * from a launch with n_tracked == 0 (no data entry resolves to a + * bounce page) is not a no-execution signature and must not trigger + * the no-write retry/accept path. + */ + if (n_tracked) + *n_tracked = 0; if (!req->n_data) return 0; for (di = 0; di < req->n_data; di++) { @@ -17071,6 +17079,8 @@ gsp_phys_gr_output_retire_wait(const struct gsp_phys_gr_submit *req, } if (!n_all) return 0; + if (n_tracked) + *n_tracked = n_all; for (iter = 0; iter < 300u; iter++) { u32 j; @@ -17313,15 +17323,16 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB]; void __iomem *bar0; struct gsp_phys_rmcall_ce_submit sub; - struct gsp_phys_fault_info fault = { }; /* * Static scratch: the only caller is the single-inflight sema work * item (ordered gsp-phys-work queue), so no concurrent submission can - * touch these. Keeping the 96-dword method block and the 64-word QMD - * copy off the stack keeps the frame under the warning bound. + * touch these. Keeping the 96-dword method block, the 64-word QMD + * copy and the ~370-byte fault record off the stack keeps the frame + * under the warning bound. */ static u32 methods[GSP_PHYS_RMCALL_GR_METHOD_WORDS]; static u32 qmd[GSP_PHYS_GR_QMD_WORDS]; + static struct gsp_phys_fault_info fault; unsigned long start = jiffies; unsigned long deadline; u8 *sem_va; @@ -17382,6 +17393,12 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, fast_rearmed = false; deposit_rb_done = false; /* + * The fault record is static scratch (frame bound): reset it per + * submission so a previous launch's fault can never leak into this + * one's result - the old on-stack `= { }` semantics. + */ + memset(&fault, 0, sizeof(fault)); + /* * F2: the shortened first window is part of the fast path; with * gsp_phys_gr_retry_fast=0 the launch keeps today's exact behavior * (the historical 5 s first window and ladder). @@ -18879,37 +18896,60 @@ gr_page_unmapped: * retirement wait is the only execution witness. On zero writes, * refill the bounce window from the guest inputs, re-publish the * same method block on a fresh ring entry, re-issue the schedule and - * re-poll, then re-wait - bounded by the same attempt budget as the - * timeout path. After the budget the launch proceeds to copy-out - * with whatever the engine produced: a genuinely read-only kernel is - * indistinguishable from a no-execution here and simply pays the - * bounded extra polls (safe to re-execute by construction - a kernel - * that writes nothing has no side effects to duplicate). A + * re-poll, then re-wait. + * + * m179: the budget is now DEDICATED (gsp_phys_gr_nowrite_retries, + * default 3) instead of sharing the timeout ladder's attempt counter - + * a launch that had already spent attempts on the ladder used to reach + * the accept path with no re-execution at all (the r200-r202 + * memcpy_roundtrip / multikernel all-wrong class). The trigger is + * specific: only a launch with n_tracked > 0 tracked data pages can be + * a no-execution, so a launch whose data entries resolve nothing + * legitimately reports zero writes without entering this path. After + * the budget the launch proceeds to copy-out with whatever the engine + * produced, and the accept is logged loudly (pr_warn) and classified + * by the QMD fetch sentinel (CONFIRMED lost vs AMBIGUOUS): a + * genuinely read-only kernel is indistinguishable from a + * no-execution here and simply pays the bounded extra polls (safe to + * re-execute by construction - a kernel that writes nothing has no + * side effects to duplicate). A * wrong-place write detector was tried and removed: the highest-reloc * data entry is often a UVM-internal buffer, not the output, so it * false-fired on correct launches and would re-execute * side-effecting kernels (atomics) up to 3x. */ if (req->n_data && gsp_phys_gr_data_chunk()->pages) { - u32 wrote = gsp_phys_gr_output_retire_wait(req, bounce_csum_in); + u32 n_tracked = 0; + u32 nw_attempts = 0; + u32 wrote = gsp_phys_gr_output_retire_wait(req, bounce_csum_in, + &n_tracked); - if (wrote == 0) { + /* + * m179: the zero-write test is specific to launches with a + * non-empty TRACKED data set. A launch whose data entries do + * not resolve to any bounce page legitimately reports zero + * writes; treating that as a lost launch would re-execute a + * kernel for no reason (a false positive). Only + * n_tracked > 0 makes "wrote == 0" a possible no-execution + * signature. + */ + if (wrote == 0 && n_tracked) { bool ff = gsp_phys_gr_qmd_fetched(qmd_dst_mem); - pr_info("vgsp-gr: first-attempt witness: engine fetch %s (qmd sentinel %s)\n", + pr_info("vgsp-gr: first-attempt witness: engine fetch %s (qmd sentinel %s) tracked=%u\n", ff ? "OK" : "NONE", - ff ? "overwritten" : "intact"); + ff ? "overwritten" : "intact", n_tracked); } - if (gsp_phys_rmcall_gr_nowrite_retry_value()) { - while (attempts < GSP_PHYS_RMCALL_GR_ATTEMPTS) { - if (wrote > 0) - break; - attempts++; - if (attempts > GSP_PHYS_RMCALL_GR_ATTEMPTS) - break; - pr_info("vgsp-gr: outcome retry %u/%u (wrote=%u): refill + re-publish\n", - attempts, GSP_PHYS_RMCALL_GR_ATTEMPTS, - wrote); + if (wrote == 0 && n_tracked && + gsp_phys_rmcall_gr_nowrite_retry_value()) { + u32 nw_retries = + (u32)gsp_phys_gr_nowrite_retries_value(); + + while (wrote == 0 && nw_attempts < nw_retries) { + nw_attempts++; + pr_info("vgsp-gr: outcome retry %u/%u (wrote=%u tracked=%u): refill + re-publish\n", + nw_attempts, nw_retries, + wrote, n_tracked); /* * m156 (E5): on the first zero-write retry, push the * proven stage-4 GR semaphore block before re-publishing. @@ -18923,7 +18963,7 @@ gr_page_unmapped: * (r155: 3 outcome retries + rearm all dead after the * wedge). Both outcomes are logged distinctly. */ - if (attempts == 1) { + if (nw_attempts == 1) { int sp_ret; mutex_lock(&gsp_phys_rm_lock); @@ -18931,7 +18971,7 @@ gr_page_unmapped: bar0); mutex_unlock(&gsp_phys_rm_lock); pr_info("vgsp-gr: outcome retry %u: warm sem push rc=%d (engine %s)\n", - attempts, sp_ret, + nw_attempts, sp_ret, sp_ret ? "WEDGED - skipping further retries" : "ALIVE"); if (sp_ret) { @@ -19027,6 +19067,20 @@ gr_page_unmapped: GSP_PHYS_GR_QMD_SENTINEL, qmd_dst_mem + GSP_PHYS_GR_QMD_SENTINEL_WORD * 4); + /* + * m179: order the fresh + * deposit before the + * re-publish/doorbell + * below (the first-attempt + * path has the same + * barrier; the publish tail + * also carries dma_wmb + + * wmb, this keeps the + * deposit->kick order + * explicit on the retry + * path). + */ + wmb(); } } if (qmdwin_swap || win0_moved) { @@ -19039,17 +19093,17 @@ gr_page_unmapped: } if (qmdwin_swap) pr_info("vgsp-gr: outcome retry %u: release words zeroed + fresh qmdwin bank %u dest 0x%llx + win0 bank %u%s + PB rebuilt\n", - attempts, qw_bank, + nw_attempts, qw_bank, qmd_dst, win0_bank_base, win0_moved ? " (shadow pages re-copied)" : ""); else if (win0_moved) pr_info("vgsp-gr: outcome retry %u: release words zeroed + fresh win0 bank %u (shadow pages re-copied) + PB rebuilt\n", - attempts, win0_bank_base); + nw_attempts, win0_bank_base); else pr_info("vgsp-gr: outcome retry %u: release words zeroed (win0/qmdwin off)\n", - attempts); + nw_attempts); } gsp_phys_gr_fill_bounce(req, bounce_csum_in); mutex_lock(&gsp_phys_rm_lock); @@ -19062,7 +19116,7 @@ gr_page_unmapped: mutex_unlock(&gsp_phys_rm_lock); if (ret) { pr_info("vgsp-gr: outcome retry %u publish/resched failed rc=%d (accepting current outcome)\n", - attempts, ret); + nw_attempts, ret); ret = 0; break; } @@ -19102,7 +19156,7 @@ gr_page_unmapped: msleep(GSP_PHYS_RMCALL_POLL_MS); } pr_info("vgsp-gr: outcome retry %u: fresh ack %s (qmd_rel=0x%08x want=0x%08x ch_rel=0x%08x after %u ms)\n", - attempts, + nw_attempts, acked ? "OK" : "TIMEOUT", rw0, want_qmd, rw1, jiffies_to_msecs(jiffies - @@ -19133,59 +19187,69 @@ gr_page_unmapped: msleep(5); } pr_info("vgsp-gr: outcome retry %u: engine fetch witness %s (qmd sentinel %s)\n", - attempts, + nw_attempts, fetched ? "OK" : "NONE", fetched ? "overwritten" : "intact"); } wrote = gsp_phys_gr_output_retire_wait( - req, bounce_csum_in); + req, bounce_csum_in, &n_tracked); pr_info("vgsp-gr: outcome retry %u done: engine wrote %u page(s)\n", - attempts, wrote); + nw_attempts, wrote); } + } + /* + * m160/m179: if every fresh-ack retry still produced no engine + * writes, the release words may be left zeroed from the last + * retry. Restore the expected payloads so the guest client + * observes its completion and proceeds to its own result check + * (a genuinely lost launch then surfaces as the client's own + * FAIL, never as a hang). This accept runs even when the retry + * loop is disabled (gsp_phys_rmcall_gr_nowrite_retry=0) so a + * zero-write outcome is never silent. + */ + if (wrote == 0 && n_tracked) { + bool restored = false; + bool fetched = gsp_phys_gr_qmd_fetched(qmd_dst_mem); + /* - * m160: if every fresh-ack retry still produced no - * engine writes, the release words may be left zeroed - * from the last retry. Restore the expected payloads so - * the guest client observes its completion and proceeds - * to its own result check (a genuinely lost launch then - * surfaces as the client's own FAIL, never as a hang). + * m179: the no-write accept is the last resort after + * the dedicated re-execution budget + * (gsp_phys_gr_nowrite_retries). It stays + * unconditional - the alternative is stalling the + * client forever - but it is now LOUD and classified + * by the hardware fetch witness: an intact sentinel + * means the deposit never reached the engine (a + * CONFIRMED lost launch, the sweep/hold-faked-ack + * class); an overwritten sentinel means the engine + * consumed the deposit, so the zero tracked writes + * may also be a legitimately read-only kernel + * (AMBIGUOUS). */ - if (wrote == 0) { - bool restored = false; - - /* - * C6 (GAP-2 A1): the no-write accept. Unconditional - * by design - it is the rare, load-bearing event - * (zero engine writes still accepted as success), - * so it must be visible even when no release word - * needed restoring. - */ - pr_info("vgsp-gr: NOWRITE h=0x%x push=0x%llx n_data=%u wrote=0 qmd_rel=0x%x\n", - req->h_object, req->push_va, req->n_data, - want_qmd); - - if (qmd_rel_hpa) { - u8 *rp = (u8 *)__va(qmd_rel_hpa) + - qmd_rel_off; - - if (READ_ONCE(*(u32 *)rp) != want_qmd) { - WRITE_ONCE(*(u32 *)rp, want_qmd); - restored = true; - } + pr_warn("vgsp-gr: NOWRITE-%s h=0x%x push=0x%llx n_data=%u tracked=%u wrote=0 retries=%u qmd_rel=0x%x (engine fetch %s)\n", + fetched ? "AMBIGUOUS" : "CONFIRMED", + req->h_object, req->push_va, req->n_data, + n_tracked, nw_attempts, want_qmd, + fetched ? "OK" : "NONE"); + + if (qmd_rel_hpa) { + u8 *rp = (u8 *)__va(qmd_rel_hpa) + qmd_rel_off; + + if (READ_ONCE(*(u32 *)rp) != want_qmd) { + WRITE_ONCE(*(u32 *)rp, want_qmd); + restored = true; } - if (ch_rel_hpa) { - u8 *rp = (u8 *)__va(ch_rel_hpa) + - ch_rel_off; + } + if (ch_rel_hpa) { + u8 *rp = (u8 *)__va(ch_rel_hpa) + ch_rel_off; - if (READ_ONCE(*(u32 *)rp) != want_ch) { - WRITE_ONCE(*(u32 *)rp, want_ch); - restored = true; - } + if (READ_ONCE(*(u32 *)rp) != want_ch) { + WRITE_ONCE(*(u32 *)rp, want_ch); + restored = true; } - if (restored) - pr_info("vgsp-gr: no-write accept: release words restored (qmd=0x%08x ch=0x%08x); client's own check decides\n", - want_qmd, want_ch); } + if (restored) + pr_warn("vgsp-gr: no-write accept: release words restored (qmd=0x%08x ch=0x%08x); client's own check decides\n", + want_qmd, want_ch); } } /* diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c index 95bd627..81c68b4 100644 --- a/kernel/gsp_phys_gpu.c +++ b/kernel/gsp_phys_gpu.c @@ -1021,6 +1021,38 @@ unsigned int gsp_phys_gr_first_poll_ms_value(void) EXPORT_SYMBOL_GPL(gsp_phys_gr_first_poll_ms_value); /* + * m179 no-write retry budget (gsp_phys_gr_nowrite_retries; default 3, clamped + * to [0, 4]). The m153 outcome retry loop re-executes a data-bearing launch + * whose retirement wait saw zero engine writes; it shared the timeout ladder's + * attempt counter, so a launch that had already spent attempts on the ladder + * (fast re-arm + fresh ring entries) reached the accept path with NO + * re-execution at all - the r200/r201/r202 memcpy_roundtrip / multikernel + * class (a lost launch silently accepted, client output all wrong). This + * budget is dedicated to the no-write loop and independent of the ladder; it + * is read once per submission. 0 keeps the accept-without-retry behavior + * (still logged loudly). The clamp keeps the worst-case host publishes per + * launch (initial + fast re-arm + 2 fresh entries + budget) within the + * 8-entry GPFIFO ring. + */ +static int gsp_phys_gr_nowrite_retries_param = 3; +module_param_named(gsp_phys_gr_nowrite_retries, + gsp_phys_gr_nowrite_retries_param, int, 0644); +MODULE_PARM_DESC(gsp_phys_gr_nowrite_retries, + "Dedicated re-execution budget for a data-bearing GR launch that wrote nothing tracked, clamped [0,4] (default 3)"); + +int gsp_phys_gr_nowrite_retries_value(void) +{ + int v = gsp_phys_gr_nowrite_retries_param; + + if (v < 0) + v = 0; + if (v > 4) + v = 4; + return v; +} +EXPORT_SYMBOL_GPL(gsp_phys_gr_nowrite_retries_value); + +/* * m69 window-0 shadow pool (gsp_phys_rmcall_gr_shadow_win0; default 0). When * set, the forwarded GR launch copies the QMD's program/param pages into the * window-0 shadow pool (module pages at VAs 0x108000..0x10f000, PTEs planted diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h index c058469..d344ce0 100644 --- a/kernel/gsp_phys_gpu.h +++ b/kernel/gsp_phys_gpu.h @@ -383,6 +383,11 @@ int gsp_phys_gr_retry_fast_value(void); * gsp_phys_gr_retry_fast=1 (default 2500). */ unsigned int gsp_phys_gr_first_poll_ms_value(void); +/* m179: dedicated re-execution budget for a data-bearing GR launch whose + * retirement wait saw zero engine writes, independent of the timeout ladder's + * attempt counter (default 3, clamped [0,4]). */ +int gsp_phys_gr_nowrite_retries_value(void); + /* m73: use host-visible sysmem chunks for V7 instance/RAMFC (m62 shape). */ int gsp_phys_rmcall_gr_inst_sys_value(void); |