diff options
| -rw-r--r-- | kernel/gsp_phys_booter.c | 163 |
1 files changed, 157 insertions, 6 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 9cf2072..bbb6d52 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -9631,9 +9631,38 @@ static u64 gsp_phys_rmcall_pde(u64 pa) return 0xeull | ((pa >> 12) << 8); } +/* + * r431 COHERENCY ROOT-CAUSE CANDIDATE: the leaf-PTE aperture for every host + * page this backend maps (the ladder windows, the win0 shadow pool, the + * bounce/data regions, the release/scratch shadows) was hardcoded to + * aperture 3 = SYSTEM_NON_COHERENT_MEMORY. The reference RM picks the + * aperture from the allocation's CPU cache attribute + * (fabric_vaspace.c:1123-1134): NV_MEMORY_CACHED -> SYSTEM_COHERENT (2), + * otherwise NON_COHERENT (3). Every page here is dma_alloc_coherent (CPU + * cached/coherent by construction), so 3 is wrong: the GPU's L2 does not + * snoop the CPU, which makes the engine's stores linger in L2 (the host's + * retire scan reads stale memory = the "no-execute wedge") and the host's + * re-staged copy-in content invisible to the engine (the retries re-read + * stale lines). gsp_phys_rmcall_pte_coh selects aperture 2 (default 1) for + * a one-flag A/B against the historical 3 (0); the PTEs are planted before + * the engine's first fetch, so the flag is boot-time only. + */ +static bool gsp_phys_rmcall_pte_coh_param; +module_param_named(gsp_phys_rmcall_pte_coh, gsp_phys_rmcall_pte_coh_param, + bool, 0444); +MODULE_PARM_DESC(gsp_phys_rmcall_pte_coh, + "r431: sysmem leaf-PTE aperture: 1 = SYSTEM_COHERENT (correct for dma_alloc_coherent host pages), 0 = SYSTEM_NON_COHERENT (historical)"); + static u64 gsp_phys_rmcall_pte(u64 pa) { - return 0xfull | ((pa >> 12) << 8); + /* + * Low nibble: VALID (1) | APERTURE<<1 | VOL (8). Aperture 2 = + * SYSTEM_COHERENT_MEMORY -> 1|4|8 = 0xd; aperture 3 = NON_COHERENT + * -> 1|6|8 = 0xf. + */ + u64 base = gsp_phys_rmcall_pte_coh_param ? 0xdull : 0xfull; + + return base | ((pa >> 12) << 8); } /* @@ -17913,6 +17942,82 @@ static bool gsp_phys_gr_qmd_fetched(const u8 *qmd_mem) } /* + * r427 idempotent fast-accept (CONT.86 ยง6.1). A zero-write launch is either + * the no-execute wedge OR a legitimate idempotent re-launch (the same QMD + * content with the same staged inputs, whose result is already in place) - + * the host cannot tell them apart, and both burn the 40 s + 80 s quiet holds. + * The launch identity below (QMD minus the per-launch release0 address and + * payload, folded with the staged content checksums of every tracked page) is + * recorded for every launch that WROTE; a zero-write launch whose identity + * matches an entry accepts immediately, with no retries and no quiet window. + * Safe by construction: identical code + identical staged content + a prior + * execution that produced this launch's own footprint means the correct + * result is already in the buffers; a side-effecting kernel's re-launch is + * equally wrong under the current accept (the client sees the stale result + * either way) - this only removes the stall. + */ +#define GSP_PHYS_GR_IDEM_SLOTS 64u +static u64 gsp_phys_gr_idem_fp[GSP_PHYS_GR_IDEM_SLOTS]; +static u32 gsp_phys_gr_idem_n; + +static u64 gsp_phys_gr_launch_fp(const u32 *qmd, + const struct gsp_phys_gr_submit *req, + const u32 *csum_in) +{ + u64 h = 1469598103934665603ULL; + u32 i; + + for (i = 0; i < GSP_PHYS_GR_QMD_WORDS; i++) { + /* Skip release0's address (w23/w24) and payload (w25): the + * driver allocates a fresh release slot per launch, so they + * differ even for an identical re-launch. */ + u32 w; + + if (i == 23u || i == 24u || i == 25u) + continue; + w = qmd[i]; + h ^= w; + h *= 1099511628211ULL; + } + for (i = 0; i < req->n_data; i++) { + u32 bi = gsp_phys_gr_data_bi(req->data[i].reloc_va); + + if (bi == GSP_PHYS_GR_DATA_BI_NONE) + continue; + h ^= csum_in[bi]; + h *= 1099511628211ULL; + } + return h; +} + +static bool gsp_phys_gr_idem_match(u64 fp) +{ + u32 i; + + for (i = 0; i < gsp_phys_gr_idem_n; i++) + if (gsp_phys_gr_idem_fp[i] == fp) + return true; + return false; +} + +static void gsp_phys_gr_idem_record(u64 fp) +{ + u32 i; + + for (i = 0; i < gsp_phys_gr_idem_n; i++) + if (gsp_phys_gr_idem_fp[i] == fp) + return; + if (gsp_phys_gr_idem_n < GSP_PHYS_GR_IDEM_SLOTS) { + gsp_phys_gr_idem_fp[gsp_phys_gr_idem_n++] = fp; + } else { + /* ring: drop the oldest */ + memmove(&gsp_phys_gr_idem_fp[0], &gsp_phys_gr_idem_fp[1], + (GSP_PHYS_GR_IDEM_SLOTS - 1u) * sizeof(u64)); + gsp_phys_gr_idem_fp[GSP_PHYS_GR_IDEM_SLOTS - 1u] = fp; + } +} + +/* * COPYAUDIT (2026-10-18 copy-completeness audit): per-copy sighting of the * GR operand staging copies. Every staged page is spot-checked (16 bytes * at the start and end of the page, source vs destination, right after the @@ -19710,6 +19815,20 @@ gr_page_unmapped: pr_info("vgsp-gr: qmd w0..3=%08x %08x %08x %08x w4..7=%08x %08x %08x %08x w23..25=%08x %08x %08x w32/33=%08x %08x w48/49=%08x %08x\n", qmd[0], qmd[1], qmd[2], qmd[3], qmd[4], qmd[5], qmd[6], qmd[7], qmd[23], qmd[24], qmd[25], qmd[32], qmd[33], qmd[48], qmd[49]); + /* + * r430: full 64-word deposit dump for the wedge's healthy-vs-wedged + * QMD diff (hardware-only fields w50/w51 INNER_GET/PUT, w52 span-list, + * w53 next-QMD pointer are not in the partial line above). Gated by + * the skedrec param (default on); one long line per launch. + */ + if (gsp_phys_rmcall_gr_skedrec_value()) { + char qb[64 * 9 + 1]; + int qo = 0; + + for (i = 0; i < GSP_PHYS_GR_QMD_WORDS && qo < (int)sizeof(qb) - 10; i++) + qo += scnprintf(qb + qo, sizeof(qb) - qo, "%08x ", qmd[i]); + pr_info("vgsp-gr: QMD64 %s\n", qb); + } /* m120: dump the main PT page slots 0x100..0x11f so the PTE content * the walker would read (win0 + bounce window) is visible in the log. */ { @@ -20034,8 +20153,14 @@ gr_page_unmapped: * cancel or dispatch sequence of the launch window is on the * record even when the poll times out. */ - if (gsp_phys_rmcall_gr_skedrec_value() && bar0 && - (++sked_tick & 7u) == 0u) { + if (gsp_phys_rmcall_gr_skedrec_value() && bar0) { + /* + * r430: sample EVERY poll iteration (not every 8th): + * the wedge's release word lands ~7 ms after the + * doorbell, so the old 8-iteration gate saw at most one + * sample of the dispatch window. Changes only. + */ + sked_tick++; sked_sample[0] = ioread32(bar0 + 0x00407060u); sked_sample[1] = ioread32(bar0 + 0x00407064u); sked_sample[2] = ioread32(bar0 + 0x00407068u); @@ -20051,7 +20176,7 @@ gr_page_unmapped: memcmp(sked_sample, sked_prev, sizeof(sked_sample))) { pr_info("vgsp-gr: sked rec +%ums cwd0=%08x cwd1=%08x cwd2=%08x info1=%08x info2=%08x act=%08x esr=%08x grfifo=%08x fecs_cur=%08x fecs_new=%08x ctxsw1=%08x\n", - (sked_tick / 8u) * + sked_tick * GSP_PHYS_RMCALL_POLL_MS, sked_sample[0], sked_sample[1], sked_sample[2], sked_sample[3], @@ -20587,6 +20712,7 @@ gr_page_unmapped: if (req->n_data && gsp_phys_gr_data_chunk()->pages) { u32 n_tracked = 0; u32 nw_attempts = 0; + bool idem = false; u32 wrote = gsp_phys_gr_output_retire_wait(req, bounce_csum_in, &n_tracked); @@ -20606,7 +20732,24 @@ gr_page_unmapped: ff ? "OK" : "NONE", ff ? "overwritten" : "intact", n_tracked); } - if (wrote == 0 && n_tracked && + /* + * r427 idempotent fast-accept: a zero-write launch whose identity + * (QMD minus release0 + staged content checksums) matches an + * earlier WRITING launch of this boot is a legitimate idempotent + * re-launch - the result is already in place. Accept immediately: + * no retries, no quiet holds. (See gsp_phys_gr_launch_fp.) + */ + if (wrote == 0 && n_tracked) { + u64 fp = gsp_phys_gr_launch_fp(qmd, req, bounce_csum_in); + + if (gsp_phys_gr_idem_match(fp)) { + pr_warn("vgsp-gr: IDEMPOTENT-ACCEPT h=0x%x push=0x%llx fp=0x%llx tracked=%u (same QMD+content as an earlier writing launch; no retries, no hold)\n", + req->h_object, req->push_va, fp, + n_tracked); + idem = true; + } + } + if (wrote == 0 && n_tracked && !idem && gsp_phys_rmcall_gr_nowrite_retry_value()) { u32 nw_retries = (u32)gsp_phys_gr_nowrite_retries_value(); @@ -20972,7 +21115,7 @@ gr_page_unmapped: * loop is disabled (gsp_phys_rmcall_gr_nowrite_retry=0) so a * zero-write outcome is never silent. */ - if (wrote == 0 && n_tracked) { + if (wrote == 0 && n_tracked && !idem) { bool restored = false; bool fetched = gsp_phys_gr_qmd_fetched(qmd_dst_mem); @@ -21142,6 +21285,14 @@ gr_page_unmapped: want_qmd, want_ch); } /* + * r427: record the launch identity of every launch that WROTE, + * so a later idempotent re-launch (zero writes, same identity) + * can accept immediately (see gsp_phys_gr_launch_fp). + */ + if (wrote && n_tracked) + gsp_phys_gr_idem_record( + gsp_phys_gr_launch_fp(qmd, req, bounce_csum_in)); + /* * r410: streak bookkeeping for the quiet-window trigger. A * launch with tracked data that wrote nothing extends the * zero-write streak; any launch that wrote resets it. Runs |