diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 162 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.c | 29 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.h | 5 |
3 files changed, 196 insertions, 0 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 09af326..5e16630 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -17974,6 +17974,129 @@ gsp_phys_gr_codehash(const struct gsp_phys_gr_submit *req, } /* + * M91 FAULT-CAP (2026-10-18): first-fault capture for the transient SM + * instruction-fetch class ("m91 class": the first attempt of a mid-sequence + * launch faults on SM garbage - "Illegal Instruction Encoding" / "Out Of + * Range Register" - while a retry of the identical QMD and identical + * shadow content succeeds). The submit path calls this once per launch at + * the end of the launch window (poll break or timeout) with the retry + * levers that fired; it emits one vgsp-m91: FAULT-CAP line per faulting + * launch carrying: the fault-epoch delta since the launch started, the + * forwarded QMD's address and first 64 B, the live program pair (w48/49) + * with its resolved engine-visible VA/PA, the first 64 B at that code + * address (the bytes the SM will fetch), the retry decision and outcome, + * and the newest relay-decoded 0x1006 note (exceptType/chid/errString). + * Statically inert on a fault-free launch: one READ_ONCE of the fault + * epoch and one compare, no reads of content or engine state. Full dumps + * are capped at GSP_PHYS_M91_CAP_MAX per module-load cycle, followed by a + * single suppression notice; the existing vgsp-fault lines keep decoding + * every fault either way. Static state follows the other per-launch submit + * scratch: the single-inflight ordered work item is the only caller. + */ +#define GSP_PHYS_M91_CAP_MAX 20u +static u32 gsp_phys_m91_fault_start; +static u32 gsp_phys_m91_cap_used; +static bool gsp_phys_m91_cap_suppressed; +static bool gsp_phys_m91_icache_log; +static noinline void +gsp_phys_gr_m91_fault_cap(const struct gsp_phys_gr_submit *req, + const u32 *qmd, u64 qmd_dst, const u8 *qmd_dst_mem, + u32 attempts, bool rearmed, bool fast, bool ok) +{ + u32 faults = READ_ONCE(gsp_phys_fault_epoch) - gsp_phys_m91_fault_start; + const struct gsp_phys_fault_info *fn; + const u8 *code = NULL; + const char *src = "unresolved"; + u64 pva, ppa = 0; + u32 off = 0, cn = 0, i; + u8 qbuf[64], cbuf[64]; + char qhex[2 * 64 + 1], chex[2 * 64 + 1]; + + /* INERT until a fault: fault-free launches take one compare and return. */ + if (!faults || !req || !qmd) + return; + if (gsp_phys_m91_cap_used >= GSP_PHYS_M91_CAP_MAX) { + if (!gsp_phys_m91_cap_suppressed) { + gsp_phys_m91_cap_suppressed = true; + pr_info("vgsp-m91: FAULT-CAP suppressed: %u dumps already emitted this cycle (the existing vgsp-fault lines still decode every fault)\n", + GSP_PHYS_M91_CAP_MAX); + } + return; + } + gsp_phys_m91_cap_used++; + + /* First 64 B of the forwarded QMD (qmdwin copy preferred). */ + memset(qbuf, 0, sizeof(qbuf)); + if (qmd_dst_mem) { + memcpy(qbuf, qmd_dst_mem, sizeof(qbuf)); + } else if (req->qmd_hpa) { + memcpy(qbuf, (const u8 *)__va(req->qmd_hpa), sizeof(qbuf)); + } + bin2hex(qhex, qbuf, sizeof(qbuf)); + qhex[sizeof(qhex) - 1] = '\0'; + + /* Program pair (QMD w48/49) and its engine-visible resolution. */ + pva = ((u64)(qmd[49] & 0x1ffffu) << 32) | qmd[48]; + off = (u32)(pva & (GSP_PHYS_PAGE_SIZE - 1)); + if (boot_stage.rmcall_gr_shadow_win0.pages && + pva >= GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE && + pva < GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE + + (u64)GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS * GSP_PHYS_PAGE_SIZE) { + u32 slot = (u32)((pva - GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE) / + GSP_PHYS_PAGE_SIZE); + + if (slot < boot_stage.rmcall_gr_shadow_win0.pages) { + code = gsp_phys_chunk_page_va( + &boot_stage.rmcall_gr_shadow_win0, slot); + ppa = (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_gr_shadow_win0, slot); + src = "win0"; + } + } else { + u64 base = pva & ~(u64)(GSP_PHYS_PAGE_SIZE - 1); + + for (i = 0; i < boot_stage.rmcall_gr_alias_count; i++) { + if (boot_stage.rmcall_gr_alias_va[i] == base) { + ppa = boot_stage.rmcall_gr_alias_hpa[i]; + code = (const u8 *)__va(ppa); + src = "shadow-line"; + break; + } + } + for (i = 0; req->pages && !code && i < req->n_pages; i++) { + if ((req->pages[i].guest_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) != base || + !req->pages[i].hpa) + continue; + ppa = req->pages[i].hpa; + code = (const u8 *)__va(ppa); + src = "guest-hpa"; + } + } + memset(cbuf, 0, sizeof(cbuf)); + memset(chex, 0, sizeof(chex)); + if (code) { + cn = min_t(u32, sizeof(cbuf), GSP_PHYS_PAGE_SIZE - off); + memcpy(cbuf, code + off, cn); + bin2hex(chex, cbuf, cn); + chex[sizeof(chex) - 1] = '\0'; + } + + /* Newest relay-decoded 0x1006 (valid whenever the epoch advanced). */ + fn = &gsp_phys_fault_notes[(gsp_phys_fault_note_next + + GSP_PHYS_FAULT_NOTE_MAX - 1u) % + GSP_PHYS_FAULT_NOTE_MAX]; + pr_info("vgsp-m91: FAULT-CAP ok=%u faults=%u retry=%s attempts=%u qmd=0x%llx qmd64=%s prog_w48=0x%08x w49=0x%08x prog_va=0x%llx prog_pa=0x%llx code_src=%s off=0x%x code64=%s fault_et=0x%x fault_chid=%u fault_err=\"%.48s\"\n", + ok ? 1u : 0u, faults, + fast ? "fast-rearm" : (rearmed ? "resched/fresh-entry" : "none"), + attempts, qmd_dst, qhex, qmd[48], qmd[49], pva, ppa, src, off, + cn ? chex : "unresolved", + fn->valid ? fn->except_type : 0u, + fn->valid ? fn->chid : 0u, + fn->valid ? fn->err : "no relayed 0x1006 note"); +} + +/* * CODESRC (2026-10-18, r221 follow-up): conservative refusal of a * non-plausible Tier-2 program-pair source. The program page (QMD w48/49) * is copied verbatim into the shadow the engine fetches from (win0 or the @@ -18212,6 +18335,13 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, fast_rearmed = false; deposit_rb_done = false; gsp_phys_gr_codehash_done = false; /* CODEHASH: once per launch */ + /* + * M91 FAULT-CAP: snapshot the fault epoch per launch, so the capture + * at the launch's end attributes exactly the faults consumed inside + * this window (and a refusal path that never publishes compares equal + * and stays inert). + */ + gsp_phys_m91_fault_start = READ_ONCE(gsp_phys_fault_epoch); reloc_gap = 0; reloc_gap_first = 0; /* @@ -18303,6 +18433,31 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, GSP_PHYS_RMCALL_GR_SHADOW0_BANK_SLOTS; memcpy(qmd, req->qmd, sizeof(qmd)); /* + * M91 icache coherence (gsp_phys_gr_icache_inv, default 1): the + * window-0 shadow pool reuses code VAs across launches (two 8-page + * banks alternate per submit epoch), so the bytes at a program VA + * change between launches, while every captured guest QMD carries + * the QMDV02_03 cache-control block with INVALIDATE_INSTRUCTION_CACHE + * (word 5 bit 30) clear - the same field the CUDA driver arms only + * under its forceIcacheInvalidate knob. An SM instruction-fetch line + * fetched for launch N can then survive into launch N+2 and execute + * the previous kernel's bytes at the same VA, the observed + * "first attempt faults, identical retry succeeds" class. Set the + * bit only when the QMD carries the driver's standard cache-control + * signature (word 5 bits 26..29) so an unknown QMD revision is never + * touched; 0 restores the exact pre-change QMD for a one-flip A/B. + */ + if (gsp_phys_gr_icache_inv_value() && + (qmd[5] & 0x3c000000u) == 0x3c000000u && + !(qmd[5] & (1u << 30))) { + qmd[5] |= 1u << 30; + if (!gsp_phys_m91_icache_log) { + gsp_phys_m91_icache_log = true; + pr_info("vgsp-gr: M91 icache-invalidate bit set in the forwarded QMD (w5=0x%08x; QMDV02_03 word5 bit30, the guest driver's forceIcacheInvalidate field)\n", + qmd[5]); + } + } + /* * Two-step PCAS (gsp_phys_rmcall_gr_pcas_two_step): set the QMD's * REQUIRE_SCHEDULING_PCAS (w4 bit12) so the copy PCAS does not * schedule and the trailing schedule-only PCAS does. @@ -20486,6 +20641,13 @@ gr_page_unmapped: ret = 0; out: + /* + * M91 FAULT-CAP: one line per faulting launch, emitted at the end of + * the launch window (fault epoch compared against the snapshot taken + * at submission; inert - one compare - when no fault was seen). + */ + gsp_phys_gr_m91_fault_cap(req, qmd, qmd_dst, qmd_dst_mem, attempts, + rearmed, fast_rearmed, ret == 0); if (out) { out->sem_value = poll; out->elapsed_ms = jiffies_to_msecs(jiffies - start); diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c index 81c68b4..9568d22 100644 --- a/kernel/gsp_phys_gpu.c +++ b/kernel/gsp_phys_gpu.c @@ -1021,6 +1021,35 @@ unsigned int gsp_phys_gr_first_poll_ms_value(void) EXPORT_SYMBOL_GPL(gsp_phys_gr_first_poll_ms_value); /* + * M91 icache coherence (gsp_phys_gr_icache_inv; default 1). The m91 class is + * the transient SM instruction-fetch lottery: the first attempt of a + * mid-sequence launch faults on SM garbage ("Illegal Instruction Encoding" / + * "Out Of Range Register") while a retry of the identical QMD and identical + * shadow content succeeds. The window-0 shadow pool reuses code VAs across + * launches (two 8-page banks alternate by submit epoch), so the bytes at a + * program VA change between launches, and every captured guest QMD carries + * the QMDV02_03 cache-control block with INVALIDATE_INSTRUCTION_CACHE (word 5 + * bit 30) CLEAR - the field the CUDA driver arms only under its + * forceIcacheInvalidate knob. An SM fetch line cached for launch N can then + * execute the previous kernel's bytes at the same VA in launch N+2. With this + * on, the forwarded QMD sets that bit (only when the QMD carries the driver's + * standard cache-control signature, bits 26..29 of word 5, so an unknown QMD + * revision is never touched), forcing a refetch at dispatch. 0 restores the + * exact pre-change QMD for a one-flip A/B. + */ +static int gsp_phys_gr_icache_inv_param = 1; +module_param_named(gsp_phys_gr_icache_inv, gsp_phys_gr_icache_inv_param, + int, 0644); +MODULE_PARM_DESC(gsp_phys_gr_icache_inv, + "Set QMDV02_03 INVALIDATE_INSTRUCTION_CACHE (word5 bit30) on forwarded GR launches whose QMD has the driver cache-control signature (default 1; 0 = pre-m91 QMD words)"); + +int gsp_phys_gr_icache_inv_value(void) +{ + return gsp_phys_gr_icache_inv_param; +} +EXPORT_SYMBOL_GPL(gsp_phys_gr_icache_inv_value); + +/* * m179 no-write retry budget (gsp_phys_gr_nowrite_retries; default 3, clamped * to [0, 4]). The m153 outcome retry loop re-executes a data-bearing launch * whose retirement wait saw zero engine writes; it shared the timeout ladder's diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h index d344ce0..5243eba 100644 --- a/kernel/gsp_phys_gpu.h +++ b/kernel/gsp_phys_gpu.h @@ -383,6 +383,11 @@ int gsp_phys_gr_retry_fast_value(void); * gsp_phys_gr_retry_fast=1 (default 2500). */ unsigned int gsp_phys_gr_first_poll_ms_value(void); +/* M91: set QMDV02_03 INVALIDATE_INSTRUCTION_CACHE (word5 bit30) on forwarded + * GR launches whose QMD carries the driver cache-control signature + * (default 1; 0 = pre-m91 QMD words). */ +int gsp_phys_gr_icache_inv_value(void); + /* m179: dedicated re-execution budget for a data-bearing GR launch whose * retirement wait saw zero engine writes, independent of the timeout ladder's * attempt counter (default 3, clamped [0,4]). */ |