diff options
Diffstat (limited to 'kernel/gsp_phys_booter.c')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 461 |
1 files changed, 398 insertions, 63 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 5350f90..42a0f2a 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -140,6 +140,7 @@ static inline bool gsp_phys_gr_copywit_enabled(void) return READ_ONCE(gsp_phys_gr_copyout_dump_param); } + /* * r405 late-write probes (the r405 "partial write" class: the scale launch * of streams2 wrote 1 of its 32 expected in-place pages and every other @@ -9626,41 +9627,66 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0) * (swref/published/pascal/gp100/dev_mmu.h:97-119,157; * uvm_pascal_mmu.c:64-141,235-303). */ -static u64 gsp_phys_rmcall_pde(u64 pa) -{ - return 0xeull | ((pa >> 12) << 8); -} - /* - * r431 COHERENCY ROOT-CAUSE CANDIDATE: the leaf-PTE aperture for every host - * page this backend maps (the ladder windows, the win0 shadow pool, the + * r431/r436 COHERENCY ROOT-CAUSE CANDIDATE: the leaf-PTE aperture for every + * host page this backend maps (the ladder windows, the win0 shadow pool, the * bounce/data regions, the release/scratch shadows) was hardcoded to - * aperture 3 = SYSTEM_NON_COHERENT_MEMORY. The reference RM picks the - * aperture from the allocation's CPU cache attribute - * (fabric_vaspace.c:1123-1134): NV_MEMORY_CACHED -> SYSTEM_COHERENT (2), - * otherwise NON_COHERENT (3). Every page here is dma_alloc_coherent (CPU - * cached/coherent by construction), so 3 is wrong: the GPU's L2 does not - * snoop the CPU, which makes the engine's stores linger in L2 (the host's - * retire scan reads stale memory = the "no-execute wedge") and the host's - * re-staged copy-in content invisible to the engine (the retries re-read - * stale lines). gsp_phys_rmcall_pte_coh selects aperture 2 (default 1) for - * a one-flag A/B against the historical 3 (0); the PTEs are planted before - * the engine's first fetch, so the flag is boot-time only. - */ -static bool gsp_phys_rmcall_pte_coh_param = true; + * aperture 3 = SYSTEM_NON_COHERENT_MEMORY. The reference RM derives the + * pair from the allocation's CPU cache attribute (fabric_vaspace.c:1123-1134 + * for the aperture, mem_desc.c memdescGetVolatility for VOL: NV_MEMORY_CACHED + * -> SYSTEM_COHERENT (2) + volatile FALSE, NV_MEMORY_UNCACHED -> NON_COHERENT + * (3) + volatile TRUE). The r431 experiment used aperture 2 with VOL still 1 + * (0xd) - a pair the reference RM never produces - and it made the engine's + * truncated writes visible sooner but worsened the wedge on a fresh boot. + * r436 turns the knob into the full 2x2 matrix so each consistent encoding + * can be A/B'd: + * 0 = aperture 3 + VOL 1 (0xf) - historical, NV_MEMORY_UNCACHED + * 1 = aperture 2 + VOL 1 (0xd) - r431 experiment (inconsistent pair) + * 2 = aperture 2 + VOL 0 (0x5) - NV_MEMORY_CACHED (coherent, non-volatile) + * 3 = aperture 3 + VOL 0 (0x7) - non-coherent, non-volatile + * The PTEs are planted before the engine's first fetch, so the flag is + * boot-time only. + */ +static int gsp_phys_rmcall_pte_coh_param; module_param_named(gsp_phys_rmcall_pte_coh, gsp_phys_rmcall_pte_coh_param, - bool, 0444); + int, 0444); MODULE_PARM_DESC(gsp_phys_rmcall_pte_coh, - "r431: sysmem leaf-PTE aperture: 1 = SYSTEM_COHERENT (correct for dma_alloc_coherent host pages), 0 = SYSTEM_NON_COHERENT (historical)"); + "r436: sysmem leaf-PTE aperture/VOL mode: 0=0xf (aperture3+VOL, historical), 1=0xd (aperture2+VOL, r431), 2=0x5 (aperture2, non-volatile), 3=0x7 (aperture3, non-volatile)"); + +static u64 gsp_phys_rmcall_sysmem_base_low(void) +{ + switch (gsp_phys_rmcall_pte_coh_param) { + case 1: + return 0xdull; /* aperture 2, VOL 1 */ + case 2: + return 0x5ull; /* aperture 2, VOL 0 */ + case 3: + return 0x7ull; /* aperture 3, VOL 0 */ + default: + return 0xfull; /* aperture 3, VOL 1 (historical) */ + } +} static u64 gsp_phys_rmcall_pte(u64 pa) { /* * Low nibble: VALID (1) | APERTURE<<1 | VOL (8). Aperture 2 = * SYSTEM_COHERENT_MEMORY -> 1|4|8 = 0xd; aperture 3 = NON_COHERENT - * -> 1|6|8 = 0xf. + * -> 1|6|8 = 0xf. r436: see gsp_phys_rmcall_pte_coh_param. */ - u64 base = gsp_phys_rmcall_pte_coh_param ? 0xdull : 0xfull; + u64 base = gsp_phys_rmcall_sysmem_base_low(); + + return base | ((pa >> 12) << 8); +} + +/* + * r436: the single PDE uses the same aperture/VOL pair as the leaf PTEs but + * with IS_PDE=0 in bit 0 (the low bit of the PTE base is VALID there), so the + * PDE base is the PTE base with bit 0 cleared: 0xe/0xc/0x4/0x6. + */ +static u64 gsp_phys_rmcall_pde(u64 pa) +{ + u64 base = gsp_phys_rmcall_sysmem_base_low() & ~1ull; return base | ((pa >> 12) << 8); } @@ -15123,6 +15149,39 @@ static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va, * semaphore. On timeout the existing status-queue drain runs and the helper * returns -ETIMEDOUT; it never retries. */ +static int gsp_phys_rmcall_gr_resched_locked(void __iomem *bar0); +static int gsp_phys_rmcall_gr_sem_push_locked(void __iomem *bar0); + +/* + * r445 wedge hard re-arm: the outcome-retry's warm semaphore probe proves the + * engine refused the channel's proven GR block (the sticky post-dispatch + * no-store wedge). One bounded recovery attempt instead of an immediate + * break: force a context save/unload with a bWait TSG preempt, re-run the + * mode-35 lane-enable tail + schedules (the m155 extended re-arm), then + * repeat the warm probe. Caller holds gsp_phys_rm_lock. Returns 0 when the + * engine executed the probe after the re-arm. + */ +static int gsp_phys_gr_wedge_hard_rearm_locked(void __iomem *bar0) +{ + u32 pre[2] = { 1u, 0u }; /* bWait=1, bManualTimeout=0 */ + u32 rpc_result = 0, status = 0; + int ret; + + ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_tsg, + GSP_PHYS_RMCALL_CTRL_TSG_PREEMPT, + pre, sizeof(pre), sizeof(pre), NULL, 0, + &rpc_result, &status, + "TSG PREEMPT bWait=1 (r445 wedge hard re-arm)"); + pr_warn("vgsp-gr: WEDGE-REARM preempt status=0x%x rpc=0x%x ret=%d\n", + status, rpc_result, ret); + ret = gsp_phys_rmcall_gr_resched_locked(bar0); + pr_warn("vgsp-gr: WEDGE-REARM resched ret=%d\n", ret); + ret = gsp_phys_rmcall_gr_sem_push_locked(bar0); + pr_warn("vgsp-gr: WEDGE-REARM probe rc=%d (engine %s after re-arm)\n", + ret, ret ? "still wedged" : "ALIVE"); + return ret; +} + static int gsp_phys_rmcall_gr_sem_push_locked(void __iomem *bar0) { struct gsp_phys_boot_chunk *userd_c = @@ -16309,6 +16368,19 @@ EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_ce_copy); #define GSP_PHYS_RMCALL_GR_CWD_REFCOUNT 1536u #define GSP_PHYS_RMCALL_GR_CWD_LEVELS 64u #define GSP_PHYS_RMCALL_GR_MAX_SM_COUNT 24u +/* + * r434 CWD-slot-count path: the falcon PRI write (CUDA's + * turingFalconPRIWriteMasked: flag/data/mask through SET_MME_SHADOW_SCRATCH + * 0/1/2, then SET_FALCON04 = register offset) and the gated method. Used only + * when gsp_phys_rmcall_gr_cwd_slot >= 0. + */ +#define GSP_PHYS_RMCALL_GR_MME_SCRATCH0 0x3400u +#define GSP_PHYS_RMCALL_GR_MME_SCRATCH1 0x3404u +#define GSP_PHYS_RMCALL_GR_MME_SCRATCH2 0x3408u +#define GSP_PHYS_RMCALL_GR_SET_FALCON04 0x0510u +#define GSP_PHYS_RMCALL_GR_SET_CWD_SLOT_COUNT 0x02b0u +#define GSP_PHYS_RMCALL_PRI_SKED_DEBUG_1 0x00407040u +#define GSP_PHYS_RMCALL_PRI_SKED_DEBUG_1_EN_CWD_SLOT_COUNT 0x800u /* Method block bound: init(152) + inline run(67) + PCAS(4) + REPORT_SEM(8) + * the optional QMD-virt pair(2). */ #define GSP_PHYS_RMCALL_GR_METHOD_WORDS 512u @@ -16401,7 +16473,7 @@ static bool gsp_phys_rmcall_gr_init_emitted; * stale-QMD execution) landed after its launch was shipped. */ static u32 gsp_phys_gr_prev_out_bi; -static u32 gsp_phys_gr_prev_out_csum; +static u64 gsp_phys_gr_prev_out_csum; static bool gsp_phys_gr_prev_out_valid; /* r425: SKED/FECS/CWD register witness (one line; the healthy-vs-wedged @@ -16529,6 +16601,29 @@ static u32 *gsp_phys_rmcall_build_gr_init(u32 *p) GSP_PHYS_RMCALL_GR_MAX_SM_COUNT); p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_INIT_SM_DISP_CTRL, 0u); + /* + * r434 CWD-slot-count path (gsp_phys_rmcall_gr_cwd_slot >= 0): masked + * PRI write of SKED_DEBUG_1.ENABLE_CWD_SLOT_COUNT (bit 11) through the + * falcon scratch registers, then the gated SET_CWD_SLOT_COUNT method - + * the CUDA driver's own turingLaunchSetCwdSlotCount sequence. -1 (the + * default) emits nothing, so the historical preamble is unchanged. + */ + if (gsp_phys_rmcall_gr_cwd_slot_value() >= 0) { + p = gsp_phys_rmcall_push_gr( + p, GSP_PHYS_RMCALL_GR_MME_SCRATCH0, 0u); + p = gsp_phys_rmcall_push_gr( + p, GSP_PHYS_RMCALL_GR_MME_SCRATCH1, + GSP_PHYS_RMCALL_PRI_SKED_DEBUG_1_EN_CWD_SLOT_COUNT); + p = gsp_phys_rmcall_push_gr( + p, GSP_PHYS_RMCALL_GR_MME_SCRATCH2, + GSP_PHYS_RMCALL_PRI_SKED_DEBUG_1_EN_CWD_SLOT_COUNT); + p = gsp_phys_rmcall_push_gr( + p, GSP_PHYS_RMCALL_GR_SET_FALCON04, + GSP_PHYS_RMCALL_PRI_SKED_DEBUG_1); + p = gsp_phys_rmcall_push_gr( + p, GSP_PHYS_RMCALL_GR_SET_CWD_SLOT_COUNT, + (u32)gsp_phys_rmcall_gr_cwd_slot_value()); + } return p; } @@ -17295,6 +17390,54 @@ static struct gsp_phys_boot_chunk *gsp_phys_gr_data_chunk(void) } /* + * r452: scan the whole active GR data-bounce chunk for an 8-byte pattern + * (write "0x<pattern>" to /sys/module/vgsp/parameters/gsp_phys_gr_scan). + * Verifies by content where a lost engine store actually landed: the chunk + * is the only bulk memory the operand window PTEs map, so a store that + * executed but is absent from the writer's tracked pages must be findable + * here (a stale-PTE landing) or nowhere (a dropped store). + */ +static int gsp_phys_gr_scan_set(const char *val, + const struct kernel_param *kp) +{ + struct gsp_phys_boot_chunk *c = gsp_phys_gr_data_chunk(); + u64 pattern = 0; + u32 i, hits = 0; + + if (kstrtoull(val, 0, &pattern)) + return -EINVAL; + if (!c->pages) + return -ENODEV; + for (i = 0; i < c->pages; i++) { + const u8 *p = gsp_phys_chunk_page_va(c, i); + u32 off; + + if (!p) + continue; + for (off = 0; off + 8 <= GSP_PHYS_PAGE_SIZE; off += 8) { + if (get_unaligned_le64(p + off) == pattern) { + if (hits < 32) + pr_info("vgsp-gr: CHUNK-SCAN found 0x%llx at bi=%u off=0x%x (va=0x%llx)\n", + pattern, i, off, + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 + + ((u64)i << 12) + off); + hits++; + } + } + } + pr_info("vgsp-gr: CHUNK-SCAN 0x%llx done: %u hit(s) over %u pages\n", + pattern, hits, c->pages); + return 0; +} + +static const struct kernel_param_ops gsp_phys_gr_scan_ops = { + .set = gsp_phys_gr_scan_set, +}; +module_param_cb(gsp_phys_gr_scan, &gsp_phys_gr_scan_ops, NULL, 0200); +MODULE_PARM_DESC(gsp_phys_gr_scan, + "Scan the GR data-bounce chunk for an 8-byte pattern (hex)"); + +/* * B-2: page index of a relocated operand VA inside the engine-visible data * region, replacing the five inlined (reloc_va - 0x110000) >> 12 derivations * and their MAIL_SLOT bounds. Grown region: VAs from @@ -17744,7 +17887,7 @@ static bool gsp_phys_gr_reloc_fix_word(const struct gsp_phys_gr_submit *req, */ static noinline u32 gsp_phys_gr_scan_pages(const struct gsp_phys_gr_submit *req, - const u32 *csum_in, u32 *n_tracked, u64 *hash_out) + const u64 *csum_in, u32 *n_tracked, u64 *hash_out) { static u32 checked[GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES]; struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); @@ -17790,15 +17933,24 @@ gsp_phys_gr_scan_pages(const struct gsp_phys_gr_submit *req, *n_tracked = n_all; for (j = 0; j < n_all; j++) { const u8 *src = gsp_phys_chunk_page_va(dchunk, checked[j]); - u32 ci, csum = 0; + u64 fp; if (!src) continue; - for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) - csum += get_unaligned_le32(src + ci); - if (csum != csum_in[checked[j]]) + /* + * r452: FNV-1a per page, not a 32-bit additive sum. The + * add-sum was PROVABLY blind to a power-of-two in-place + * scale: doubling every word of a 4 KB page shifts each + * exponent by +1, i.e. +2^23 per word, and 1024 words give + * 2^33 == 0 (mod 2^32) - so c[i] *= 2.0f left the checksum + * bit-identical, the page classified "unchanged", and the + * client read the pre-scale values (the deterministic + * streams2/scale class). FNV-1a has no such structure. + */ + fp = gsp_phys_gr_fnv1a64(src, GSP_PHYS_PAGE_SIZE); + if (fp != csum_in[checked[j]]) changed++; - hash ^= csum; + hash ^= fp; hash *= 1099511628211ULL; } if (hash_out) @@ -17806,9 +17958,35 @@ gsp_phys_gr_scan_pages(const struct gsp_phys_gr_submit *req, return changed; } +/* + * r434: post-launch deposit dump (hardware-only words). The r430 QMD64 dump + * runs at submission, before the engine touches the deposit, so the + * hardware-only fields (w50 INNER_GET, w51 INNER_PUT, w52 span-list head, + * w53 next-QMD) are always zero there. Reading the same page AFTER the + * retire wait shows what the SKED actually consumed: w50/w51 are the CTA + * (INNER) get/put counters, so a wedged launch whose grid "completed" with + * zero stores is distinguishable from a genuinely dispatched grid by + * whether INNER_PUT advanced to the CTA count. One line per launch, gated + * by the skedrec param. + */ +static void gsp_phys_gr_qmd64_post(const u8 *deposit, const char *tag) +{ + char qb[64 * 9 + 1]; + int qo = 0; + u32 i; + + if (!deposit || !gsp_phys_rmcall_gr_skedrec_value()) + return; + for (i = 0; i < GSP_PHYS_GR_QMD_WORDS && qo < (int)sizeof(qb) - 10; + i++) + qo += scnprintf(qb + qo, sizeof(qb) - qo, "%08x ", + get_unaligned_le32(deposit + i * 4)); + pr_info("vgsp-gr: QMD64-POST %s %s\n", tag, qb); +} + static noinline u32 gsp_phys_gr_output_retire_wait(const struct gsp_phys_gr_submit *req, - const u32 *csum_in, u32 *n_tracked) + const u64 *csum_in, u32 *n_tracked) { u32 n_all = 0, iter; u32 changed = 0; @@ -17962,7 +18140,7 @@ static u32 gsp_phys_gr_idem_n; static u64 gsp_phys_gr_launch_fp(const u32 *qmd, const struct gsp_phys_gr_submit *req, - const u32 *csum_in) + const u64 *csum_in) { u64 h = 1469598103934665603ULL; u32 i; @@ -18040,6 +18218,22 @@ static void gsp_phys_gr_page_spot_check(const u8 *dst, const u8 *src, } /* + * r452: the legacy 32-bit additive page sum, kept ONLY for the all-zero + * filter. It is never a change-detection fingerprint any more: doubling every + * word of a 4 KB page leaves the sum bit-identical (1024 x 2^23 == 2^33 == 0 + * mod 2^32), which silently classified an in-place x2 scale as "no engine + * store" and shipped stale values (the deterministic streams2 class). + */ +static inline u32 gsp_phys_gr_page_sum32(const u8 *p) +{ + u32 ci, csum = 0; + + for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) + csum += get_unaligned_le32(p + ci); + return csum; +} + +/* * COPYWIT r226 zero filter: an all-zero page is never a meaningful alias * partner. A fresh client's fill stages long runs of zero pages * (uninitialized cudaMalloc buffers), and consecutive zero pages produced @@ -18085,7 +18279,7 @@ static u32 gsp_phys_gr_in_census_log; * again). Returns the number of pages staged. */ static u32 gsp_phys_gr_fill_bounce(const struct gsp_phys_gr_submit *req, - u32 *csum_in, u64 *staged_hpa) + u64 *csum_in, u64 *staged_hpa) { struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); struct { @@ -18122,7 +18316,9 @@ static u32 gsp_phys_gr_fill_bounce(const struct gsp_phys_gr_submit *req, "gr-bounce-copy-in"); for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) csum += get_unaligned_le32(dst + ci); - csum_in[bi] = csum; + /* r452: the change baseline is the FNV-1a fingerprint (the + * add-sum above survives only as the all-zero test). */ + csum_in[bi] = gsp_phys_gr_fnv1a64(dst, GSP_PHYS_PAGE_SIZE); zero = gsp_phys_gr_page_all_zero(csum, dst); if (!zero && gsp_phys_gr_copywit_enabled()) { u32 j; @@ -18171,7 +18367,7 @@ static u32 gsp_phys_gr_fill_bounce(const struct gsp_phys_gr_submit *req, * retry, so a retried launch starts from the guest inputs in both sets. * Returns the number of pages staged. */ -static u32 gsp_phys_gr_fill_gap_bounce(u32 *csum_in, u64 *staged_hpa) +static u32 gsp_phys_gr_fill_gap_bounce(u64 *csum_in, u64 *staged_hpa) { struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); u32 gi, copied = 0; @@ -18179,7 +18375,6 @@ static u32 gsp_phys_gr_fill_gap_bounce(u32 *csum_in, u64 *staged_hpa) for (gi = 0; gi < gsp_phys_gr_gap_n; gi++) { const u8 *src = (const u8 *)__va(gsp_phys_gr_gap[gi].hpa); u8 *dst = gsp_phys_chunk_page_va(dchunk, gsp_phys_gr_gap[gi].bi); - u32 ci, csum = 0; if (!src || !dst) continue; @@ -18189,9 +18384,9 @@ static u32 gsp_phys_gr_fill_gap_bounce(u32 *csum_in, u64 *staged_hpa) staged_hpa[gsp_phys_gr_gap[gi].bi] = gsp_phys_gr_gap[gi].hpa; gsp_phys_gr_page_spot_check(dst, src, gsp_phys_gr_gap[gi].va, "gr-bounce-copy-in"); - for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) - csum += get_unaligned_le32(dst + ci); - csum_in[gsp_phys_gr_gap[gi].bi] = csum; + /* r452: FNV-1a baseline (see gsp_phys_gr_scan_pages). */ + csum_in[gsp_phys_gr_gap[gi].bi] = + gsp_phys_gr_fnv1a64(dst, GSP_PHYS_PAGE_SIZE); copied++; } dma_wmb(); @@ -18454,6 +18649,8 @@ static u32 gsp_phys_m91_fault_start; static u32 gsp_phys_m91_cap_used; static bool gsp_phys_m91_cap_suppressed; static bool gsp_phys_m91_icache_log; +static bool gsp_phys_r449_ccache_log; +static bool gsp_phys_gr_rel_membar_log; static noinline void gsp_phys_gr_m91_fault_cap(const struct gsp_phys_gr_submit *req, const u32 *qmd, u64 qmd_dst, const u8 *qmd_dst_mem, @@ -18785,7 +18982,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, * stack). Safe as static: the only caller is the single-inflight * ordered gsp-phys-work item. */ - static u32 bounce_csum_in[GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES]; + static u64 bounce_csum_in[GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES]; /* * r378: which guest page's content each bounce slot holds this * launch (set by the copy-in fills, cleared per submission). The @@ -18963,6 +19160,52 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, } } /* + * r449 SHADER-CONSTANT-CACHE invalidate (gsp_phys_gr_ccache_inv; default + * 0 = off): the same cache-control word's bit 31 (the V01_06-layout + * SHADER_CONSTANT_CACHE position) is never set by the captured guest + * QMDs. The forwarded param page is rewritten in place between launches + * (the two-bank win0 pool), so a stale constant-cache line can serve + * the PREVIOUS launch's parameters and the current launch's output + * pages stay untouched - the post-dispatch no-store signature. Mode 1 + * sets bit 31; mode 2 forces the whole invalidate set (bits 24..31). + * Only when the QMD carries the driver's standard cache-control + * signature (bits 26..29) so an unknown QMD revision is never touched. + */ + if (gsp_phys_gr_ccache_inv_value() && + (qmd[5] & 0x3c000000u) == 0x3c000000u) { + u32 mode = gsp_phys_gr_ccache_inv_value(); + + if (mode == 2) + qmd[5] |= 0xff000000u; + else + qmd[5] |= 1u << 31; + if (!gsp_phys_r449_ccache_log) { + gsp_phys_r449_ccache_log = true; + pr_info("vgsp-gr: r449 constant-cache invalidate mode=%u applied (w5=0x%08x)\n", + mode, qmd[5]); + } + } + /* + * r433 RELEASE_MEMBAR_TYPE A/B (gsp_phys_gr_rel_membar): force the + * forwarded QMD's RELEASE_MEMBAR_TYPE to FE_SYSMEMBAR (QMDV00_06 + * word 11 bit 14) so the front-end flushes its pending system-memory + * stores before writing the release semaphore. The captured guest + * QMDs carry FE_NONE, so a grid that completes with its data still + * in the GPU's L2 releases the client while no store is host-visible + * - the post-dispatch store-loss signature. The release ordering then + * guarantees the host's retire scan and copy-out can never observe a + * completion whose data has not landed. 0 (default) keeps the + * guest's captured value. + */ + if (gsp_phys_gr_rel_membar_value() && !(qmd[11] & (1u << 14))) { + qmd[11] |= 1u << 14; + if (!gsp_phys_gr_rel_membar_log) { + gsp_phys_gr_rel_membar_log = true; + pr_info("vgsp-gr: r433 RELEASE_MEMBAR_TYPE forced to FE_SYSMEMBAR in the forwarded QMD (w11=0x%08x)\n", + qmd[11]); + } + } + /* * Two-step PCAS (gsp_phys_rmcall_gr_pcas_two_step): set the QMD's * REQUIRE_SCHEDULING_PCAS (w4 bit12) so the copy PCAS does not * schedule and the trailing schedule-only PCAS does. @@ -19789,6 +20032,27 @@ gr_page_unmapped: /* A stale release from a previous submission must not be observed. */ WRITE_ONCE(*(u32 *)sem_va, 0); WRITE_ONCE(*(u32 *)(sem_va + 4), 0); + /* + * r441: zero the GUEST-VISIBLE release words for the first attempt + * too, not only for outcome retries. The r439 evidence shows a first + * attempt whose poll broke on a STALE value (qmd_rel=0x00000005 from + * the previous launch's engine write, want=6) while the engine never + * fetched this launch's deposit (sentinel intact, "engine fetch + * NONE"): the host then shipped a completion for a launch that did + * not run. Zeroing forces the poll to wait for THIS launch's own + * engine write; when the engine does not write it, the timeout ladder + * and the software-authored completion take over (the honest path), + * and the wedge classification stops being satisfiable by a stale + * value. The write guard re-validates the target like the retry path. + */ + if (qmd_rel_hpa && qmd_rel_off < GSP_PHYS_PAGE_SIZE && + gsp_sema_gr_write_guard(req->rel_va, qmd_rel_hpa, + "gr-rel-zero-first")) + WRITE_ONCE(*(u32 *)((u8 *)__va(qmd_rel_hpa) + qmd_rel_off), 0); + if (ch_rel_hpa && ch_rel_off < GSP_PHYS_PAGE_SIZE && + gsp_sema_gr_write_guard(req->rel_va, ch_rel_hpa, + "gr-chrel-zero-first")) + WRITE_ONCE(*(u32 *)((u8 *)__va(ch_rel_hpa) + ch_rel_off), 0); wmb(); /* @@ -20717,6 +20981,14 @@ gr_page_unmapped: &n_tracked); /* + * r434: post-launch hardware-only QMD dump (w50/w51 INNER + * get/put = the SKED's CTA dispatch counters) so the + * healthy-vs-wedged diff has the fields the submission-time + * dump cannot show. + */ + gsp_phys_gr_qmd64_post(qmd_dst_mem, "retire"); + + /* * m179: the zero-write test is specific to launches with a * non-empty TRACKED data set. A launch whose data entries do * not resolve to any bounce page legitimately reports zero @@ -20805,6 +21077,26 @@ gr_page_unmapped: pr_info("vgsp-gr: outcome retry %u/%u (wrote=%u tracked=%u): refill + re-publish\n", nw_attempts, nw_retries, wrote, n_tracked); + /* + * r445 wedge hard re-arm (gsp_phys_gr_wedge_rearm, + * default off): the r445 evidence shows every + * post-dispatch no-store episode leaves the warm + * ladder sem probe ALIVE (the ladder block executes) + * while the client launch keeps writing nothing, so + * the plain refill+re-publish retries are futile. + * On the FINAL retry attempt, force one context + * save/unload (TSG PREEMPT bWait=1) and re-run the + * mode-35 lane-enable tail + schedules before the + * re-publish, so the retried launch meets a freshly + * restored context. Bounded: one preempt per launch + * (2 s RPC timeout), only on the last retry. + */ + if (nw_attempts == nw_retries && + gsp_phys_gr_wedge_rearm_value()) { + mutex_lock(&gsp_phys_rm_lock); + gsp_phys_gr_wedge_hard_rearm_locked(bar0); + mutex_unlock(&gsp_phys_rm_lock); + } if (quiet_ms) { gsp_phys_gr_wedge_quiet++; pr_warn("vgsp-gr: WEDGE-QUIET hold %u ms before outcome retry %u (push=0x%llx; %u consecutive zero-write launch(es); no submissions in between)\n", @@ -21085,6 +21377,8 @@ gr_page_unmapped: } wrote = gsp_phys_gr_output_retire_wait( req, bounce_csum_in, &n_tracked); + gsp_phys_gr_qmd64_post(qmd_dst_mem, + "retry-retire"); pr_info("vgsp-gr: outcome retry %u done: engine wrote %u page(s)\n", nw_attempts, wrote); /* r410: the quiet-window verdict. */ @@ -21222,14 +21516,12 @@ gr_page_unmapped: const u8 *pp = gsp_phys_chunk_page_va( gsp_phys_gr_data_chunk(), gsp_phys_gr_prev_out_bi); - u32 pc = 0, pi; + u64 pc; if (pp) { - for (pi = 0; pi < GSP_PHYS_PAGE_SIZE; - pi += 4) - pc += get_unaligned_le32(pp + pi); + pc = gsp_phys_gr_fnv1a64(pp, GSP_PHYS_PAGE_SIZE); if (pc != gsp_phys_gr_prev_out_csum) - pr_warn("vgsp-gr: PREV-OUT-CHANGED bi=%u csum %08x -> %08x (late store after its launch shipped)\n", + pr_warn("vgsp-gr: PREV-OUT-CHANGED bi=%u fp %016llx -> %016llx (late store after its launch shipped)\n", gsp_phys_gr_prev_out_bi, gsp_phys_gr_prev_out_csum, pc); @@ -21326,7 +21618,7 @@ gr_page_unmapped: for (wbi = 0; wbi < req->n_data; wbi++) { u32 bi = gsp_phys_gr_data_bi(req->data[wbi].reloc_va); const u8 *src; - u32 ci, csum = 0; + u64 fp; if (bi == GSP_PHYS_GR_DATA_BI_NONE || !req->data[wbi].hpa) @@ -21347,9 +21639,8 @@ gr_page_unmapped: aliased++; continue; } - for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) - csum += get_unaligned_le32(src + ci); - if (csum == bounce_csum_in[bi]) { + fp = gsp_phys_gr_fnv1a64(src, GSP_PHYS_PAGE_SIZE); + if (fp == bounce_csum_in[bi]) { /* * r378: the engine did not touch this page, * so the guest page still holds exactly this @@ -21358,6 +21649,37 @@ gr_page_unmapped: * stale-content corruption at worst - skip. */ unchanged++; + /* + * r452: witness the UNCHANGED page's chunk + * content (first 4 words) and the guest + * page's, bounded per launch. This is the + * only place the two can be compared for a + * page the csum rule classified as "engine + * did not touch" - it separates "the store + * never landed in the chunk" from "the + * store landed but the csum rule missed + * it" (or "the reader resolves a different + * page than the writer wrote"). + */ + if (gsp_phys_gr_copywit_enabled() && + outwit < GSP_PHYS_GR_COPYWIT_MAX) { + const u8 *gp = req->data[wbi].hpa ? + (const u8 *)__va(req->data[wbi].hpa) : + NULL; + + outwit++; + pr_info("vgsp-gr: OUT-UNCH bi=%u va=0x%llx fp=%016llx pre=%016llx chunk=%08x %08x %08x %08x guest=%08x %08x %08x %08x\n", + bi, req->data[wbi].va, fp, + bounce_csum_in[bi], + get_unaligned_le32(src), + get_unaligned_le32(src + 4), + get_unaligned_le32(src + 8), + get_unaligned_le32(src + 12), + gp ? get_unaligned_le32(gp) : 0, + gp ? get_unaligned_le32(gp + 4) : 0, + gp ? get_unaligned_le32(gp + 8) : 0, + gp ? get_unaligned_le32(gp + 12) : 0); + } continue; } { @@ -21408,13 +21730,13 @@ gr_page_unmapped: &rap, &rval); outwit++; - pr_info("vgsp-gr: OUT-PAGE-WIT va=0x%llx hpa=0x%llx bi=%u first8=%016llx last8=%016llx pre=%08x post=%08x res=%s pa=0x%llx ap=%u val8=%016llx vbase=%px\n", + pr_info("vgsp-gr: OUT-PAGE-WIT va=0x%llx hpa=0x%llx bi=%u first8=%016llx last8=%016llx pre=%016llx post=%016llx res=%s pa=0x%llx ap=%u val8=%016llx vbase=%px\n", req->data[wbi].va, req->data[wbi].hpa, bi, f8, (u64)get_unaligned_le64( src + GSP_PHYS_PAGE_SIZE - 8), - bounce_csum_in[bi], csum, + bounce_csum_in[bi], fp, rhave ? "ledger" : "none", rpa, rap, rval, (void *)gsp_sema_gr_shadow_base()); @@ -21429,7 +21751,8 @@ gr_page_unmapped: * evidence). */ if (gsp_phys_gr_copywit_enabled() && - !gsp_phys_gr_page_all_zero(csum, src)) { + !gsp_phys_gr_page_all_zero( + gsp_phys_gr_page_sum32(src), src)) { u32 wi; for (wi = 0; wi < gsp_phys_gr_outwit_n; @@ -21465,7 +21788,7 @@ gr_page_unmapped: */ if (changed == 0) { gsp_phys_gr_prev_out_bi = bi; - gsp_phys_gr_prev_out_csum = csum; + gsp_phys_gr_prev_out_csum = fp; gsp_phys_gr_prev_out_valid = true; } changed++; @@ -21546,8 +21869,7 @@ gr_page_unmapped: if (bi == GSP_PHYS_GR_DATA_BI_NONE) continue; wp = gsp_phys_chunk_page_va(dchunk, bi); - for (k = 0; k < GSP_PHYS_PAGE_SIZE; k += 4) - wc += get_unaligned_le32(wp + k); + (void)k; (void)wc; pr_info("vgsp-gr: window[%u] reloc=0x%llx first=%08x %08x %08x %08x changed=%u\n", bi, req->data[wi].reloc_va, @@ -21555,7 +21877,9 @@ gr_page_unmapped: get_unaligned_le32(wp + 4), get_unaligned_le32(wp + 8), get_unaligned_le32(wp + 12), - wc != bounce_csum_in[bi]); + gsp_phys_gr_fnv1a64( + wp, GSP_PHYS_PAGE_SIZE) != + bounce_csum_in[bi]); } } pr_info("vgsp-gr: bounce copy-out: %u pages written back, %u changed by engine, %u unchanged-skipped, %u alias-skipped, %u stale-skipped (first bi: %u %u %u %u %u %u %u %u)\n", @@ -21578,7 +21902,7 @@ gr_page_unmapped: for (gi = 0; gi < gsp_phys_gr_gap_n; gi++) { u32 bi = gsp_phys_gr_gap[gi].bi; const u8 *src = gsp_phys_chunk_page_va(dchunk, bi); - u32 ci, csum = 0; + u64 fp; if (!src) continue; @@ -21591,9 +21915,8 @@ gr_page_unmapped: gskipped++; continue; } - for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) - csum += get_unaligned_le32(src + ci); - if (csum == bounce_csum_in[bi]) { + fp = gsp_phys_gr_fnv1a64(src, GSP_PHYS_PAGE_SIZE); + if (fp == bounce_csum_in[bi]) { /* Engine did not touch it: nothing to ship. */ gunchanged++; continue; @@ -21662,8 +21985,20 @@ gr_page_unmapped: jiffies_to_msecs(jiffies - start), gsp_phys_gr_ok); /* m128: post-success qmdwin words so a probe that stores a value into * the qmdwin page (e.g. p6's constant-read dump at w48) is readable - * from the success log (the post-mortem only prints on timeout). */ - if (boot_stage.rmcall_gr_qmdwin.pages) { + * from the success log (the post-mortem only prints on timeout). + * r434: read the launch's OWN bank (qmd_dst_mem) when available - + * the old bank-0-only read showed a stale page for odd epochs. */ + if (qmd_dst_mem) { + const u32 *qw = (const u32 *)qmd_dst_mem; + + pr_info("vgsp-gr: post-OK qmdwin w48..59=%08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n", + get_unaligned_le32(qw + 48), get_unaligned_le32(qw + 49), + get_unaligned_le32(qw + 50), get_unaligned_le32(qw + 51), + get_unaligned_le32(qw + 52), get_unaligned_le32(qw + 53), + get_unaligned_le32(qw + 54), get_unaligned_le32(qw + 55), + get_unaligned_le32(qw + 56), get_unaligned_le32(qw + 57), + get_unaligned_le32(qw + 58), get_unaligned_le32(qw + 59)); + } else if (boot_stage.rmcall_gr_qmdwin.pages) { const u32 *qw = (const u32 *)gsp_phys_chunk_page_va( &boot_stage.rmcall_gr_qmdwin, 0); |