diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 82 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 56 |
2 files changed, 110 insertions, 28 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index f60f6c5..c5870ce 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -16755,7 +16755,7 @@ static void gsp_phys_gr_rewrite_data_vars(u32 *w, u32 n_words, const struct gsp_phys_gr_submit *req, u32 bank, u32 *mirror) { - u32 rew = 0, k; + u32 rew = 0, skipped = 0, k; if (!req->n_data) return; @@ -16771,19 +16771,6 @@ static void gsp_phys_gr_rewrite_data_vars(u32 *w, u32 n_words, u64 wb; u32 x; - /* - * m135: never relocate below offset 0x100. The param-page - * header (grid/block dims and driver bookkeeping at 0x0..0xff) - * can contain server-region VAs (e.g. 0x100000100) that are - * NOT kernel parameters; relocating them corrupted the - * vadd launch's blockDim word (read as 0x170100 instead of - * 0x100, so every CTA past 0 computed out-of-range indices - * and exited: VADD bad=32512 exactly one block correct). - * Kernel parameters live at >= 0x140 (JIT sm_75 layout: - * a/b/c at 0x160/0x168/0x170, n at 0x178). - */ - if (k * 4u < 0x100u) - continue; if (v < 0x10000000ULL || v >= (1ULL << 47) || (v & 0xff)) continue; wb = v & ~0xfffULL; @@ -16791,6 +16778,34 @@ static void gsp_phys_gr_rewrite_data_vars(u32 *w, u32 n_words, if (req->data[x].va != wb || !req->data[x].reloc_va) continue; + /* + * m135: never relocate below offset 0x100. The + * param-page header (grid/block dims and driver + * bookkeeping at 0x0..0xff) can contain + * server-region VAs (e.g. 0x100000100) that are NOT + * kernel parameters; relocating them corrupted the + * vadd launch's blockDim word (read as 0x170100 + * instead of 0x100, so every CTA past 0 computed + * out-of-range indices and exited: VADD bad=32512 + * exactly one block correct). Kernel parameters live + * at >= 0x140 (JIT sm_75 layout: a/b/c at + * 0x160/0x168/0x170, n at 0x178). + * + * m181 diagnostic: a below-0x100 value whose page is + * an operand VA of this launch is still logged (the + * observed class is the driver control block at + * bank offset 0, present in green clients too); the + * guard itself is unchanged until a dereferenced + * pointer in that range is proven. + */ + if (k * 4u < 0x100u) { + if (skipped < 16) { + skipped++; + pr_info("vgsp-gr: bank %u RELOC-SKIP @0x%x: 0x%llx is operand page 0x%llx (below 0x100, left guest-VA)\n", + bank, k * 4, v, wb); + } + break; + } /* m120: log the hit site so coverage is provable. */ if (rew < 16) pr_info("vgsp-gr: bank %u hit @0x%x: 0x%llx -> 0x%llx\n", @@ -19249,16 +19264,16 @@ gr_page_unmapped: fetched ? "OK" : "NONE"); /* - * m180 (lottery discriminators, agent-order H1/H2). - * - * (a) Deposit identity without changing the sentinel - * contract: read BOTH qmdwin banks' sentinel words. - * Ours intact + the other bank overwritten means an - * OLDER deposit was consumed instead of this one - * (stale/mis-targeted deposit, H2); ours overwritten - * means the engine really fetched THIS deposit - * (lost-dispatch vs read-only kernel, H1); both - * intact means no deposit ever reached the engine. + * m180 (lottery discriminator, agent-order H1/H2). + * Read BOTH qmdwin banks' sentinel words. Facts only: + * the other bank is normally the previous launch's + * deposit (legitimately consumed), so only THIS + * launch's sentinel classifies the event: intact = + * this deposit never reached the engine; overwritten + * = the engine consumed it (or something zeroed the + * word), so the zero stores are an engine-side + * non-execution of a fetched deposit, not a lost + * deposit. */ if (qmdwin_on && boot_stage.rmcall_gr_qmdwin.pages && qmd_dst_mem) { @@ -19266,18 +19281,29 @@ gr_page_unmapped: &boot_stage.rmcall_gr_qmdwin, qw_bank ^ 1u); - pr_warn("vgsp-gr: zero-write deposit stamp: bank=%u sentinel=0x%08x other-bank=%u sentinel=%s\n", + pr_warn("vgsp-gr: zero-write deposit stamp: bank=%u sentinel=0x%08x (%s) other-bank=%u %s\n", qw_bank, get_unaligned_le32( qmd_dst_mem + GSP_PHYS_GR_QMD_SENTINEL_WORD * 4), + fetched ? "this deposit fetched" : + "this deposit NOT fetched", qw_bank ^ 1u, om ? (gsp_phys_gr_qmd_fetched(om) ? - "overwritten (older deposit consumed)" : - "intact") : "n/a"); + "sentinel overwritten (previous deposit)" : + "sentinel intact") : "n/a"); } /* - * (b) Late-store check: a store ramp that retires after + * (b) Deposit readback at the accept (m181): the same + * QMD-word dump the first-timeout path logs, so a + * zero-write launch's deposit content (w0/w4/w12/.../ + * w63) shows whether the engine consumed the header + * words or the page is untouched, without needing a + * timeout to have happened first. + */ + gsp_phys_gr_deposit_rb_log(req, qmd_dst_mem); + /* + * (c) Late-store check: a store ramp that retires after * the host's window is indistinguishable from a lost * dispatch at the accept moment. Re-scan once at +1 s * and report whether the tracked pages changed in the diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 1cb771a..ccccfec 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -4794,6 +4794,7 @@ static u32 svc_ledger_n; static u64 svc_ledger_diverges; static u64 svc_ledger_stores; static u64 svc_ledger_rewrites; +static u64 svc_ledger_reloc_diverges; static u32 svc_ledger_store_log; static inline u32 svc_ledger_home(u64 page) @@ -4881,6 +4882,7 @@ static void svc_ledger_clear(void) memset(svc_ledger, 0, sizeof(svc_ledger)); svc_ledger_n = 0; svc_ledger_rewrites = 0; + svc_ledger_reloc_diverges = 0; } static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, @@ -13962,6 +13964,8 @@ static void svc_fwd_relay_select(struct svc_fwd_req *req) * svc_gr_req_fill from its QMD pointer association and the captured inline * bytes. Overflow is a failure (fail-closed), not a silent drop. */ +static const char *svc_gr_src_probe(struct nv_vgpu_device *vgpu, u64 va, + u64 *pa_out, u32 *ap_out); static void svc_gr_page_note(struct nv_vgpu_device *vgpu, u64 va, u32 qmd_word) { u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); @@ -13986,6 +13990,58 @@ static void svc_gr_page_note(struct nv_vgpu_device *vgpu, u64 va, u32 qmd_word) i = svc_fwd_req.gr_n_pages++; svc_fwd_req.gr_pages[i].va = page; svc_fwd_req.gr_pages[i].qmd_word = qmd_word; + /* + * m181 (relocation/ledger unification): the unified landing table is + * the authority for a page a host write already landed on. The CE + * inline writes that author these launch pages (param, program, + * constant banks, inline destinations) go through + * svc_grce_inline_write(), which records its landing. The backend's + * QMD relocation reads the page content through pg->pa (win0 shadow + * copy) and the m129 mirror patches that same page, so resolving + * independently here could hand the engine a stale page whose + * pointer values are not this launch's VAs: the relocation then + * matches nothing, the engine executes against unrelocated guest VAs + * it cannot walk, and the grid aborts silently (zero stores, ack + * still delivered) - the cross-session zero-write class - or runs on + * the previous client's parameters (the multi-page wrongness). + * Resolve once, from the table, and record the independent answer as + * stale when it differs. + */ + { + u64 lpa = 0; + u32 lap = 0; + + if (!svc_ledger_lookup(page, &lpa, &lap)) { + u64 rpa = 0; + u32 rap = 0; + const char *rsrc = svc_gr_src_probe(vgpu, page, &rpa, + &rap); + + if (rpa && + ((lpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (rpa & ~(u64)(SEMA_PAGE_SIZE - 1)) || + (u32)lap != (u32)rap)) { + svc_ledger_reloc_diverges++; + if (svc_ledger_reloc_diverges <= 16) + pr_info("sema-ledger: RELOC-DIVERGE va=0x%llx ledger_pa=0x%llx %s_pa=0x%llx (ledger wins)\n", + page, + lpa & ~(u64)(SEMA_PAGE_SIZE - 1), + rsrc, rpa & + ~(u64)(SEMA_PAGE_SIZE - 1)); + } + svc_fwd_req.gr_pages[i].pa = + lpa & ~(u64)(SEMA_PAGE_SIZE - 1); + svc_fwd_req.gr_pages[i].ap = (u8)lap; + /* + * Keep the pin table consistent with the authority so + * the win0 copy path and later resolves agree (and no + * stale self-pin is planted for this VA). + */ + svc_pin_store(page, lpa & ~(u64)(SEMA_PAGE_SIZE - 1), + (int)lap); + return; + } + } if (svc_resolve_va(vgpu, page, &pa, &ap)) return; svc_fwd_req.gr_pages[i].pa = pa & ~(u64)(SEMA_PAGE_SIZE - 1); |